优化代码

fir_ser/api/management/commands/services/command.py

@@ -102,6 +102,7 @@ class BaseActionCommand(BaseCommand):
         parser.add_argument('-d', '--daemon', nargs="?", const=True)
         parser.add_argument('-up', '--uwsgi_processes', type=int, nargs="?", default=get_sys_process_num())
         parser.add_argument('-ut', '--uwsgi_threads', type=int, nargs="?", default=get_sys_thread_num())
+        parser.add_argument('-cn', '--celery_num', type=int, nargs="?", default=20)
         parser.add_argument('-usm', '--uwsgi_socket_mode', nargs="?", const=True,
                             help='run to bind socket mode, default http mode, only uwsgi service')
         parser.add_argument('-f', '--force', nargs="?", const=True)
@@ -114,6 +115,7 @@ class BaseActionCommand(BaseCommand):
             'uwsgi_processes': options.get('uwsgi_processes'),
             'uwsgi_threads': options.get('uwsgi_threads'),
             'uwsgi_socket_mode': options.get('uwsgi_socket_mode'),
+            'celery_num': options.get('celery_num'),
             'uid': options.get('uid'),
             'gid': options.get('uid') if options.get('gid') in ['root'] else options.get('gid'),
         }

fir_ser/api/management/commands/services/services/celery_base.py

@@ -4,10 +4,10 @@ from ..hands import *
 
 class CeleryBaseService(BaseService):
 
-    def __init__(self, queue, num=10, **kwargs):
+    def __init__(self, queue, celery_num=10, **kwargs):
         super().__init__(**kwargs)
         self.queue = queue
-        self.num = num
+        self.celery_num = celery_num
 
     @property
     def cmd(self):
@@ -26,7 +26,7 @@ class CeleryBaseService(BaseService):
             'celery', '-A',
             'fir_ser', 'worker',
             '-l', 'INFO',
-            '-c', str(self.num),
+            '-c', str(self.celery_num),
             '-Q', self.queue,
             '-n', f'{self.queue}@{server_hostname}'
         ]

fir_ser/api/tasks.py

@@ -13,6 +13,7 @@ from api.models import Apps, DeveloperAppID
 from api.utils.app.supersignutils import IosUtils, resign_by_app_id_and_developer
 from api.utils.crontab.ctasks import sync_download_times, auto_clean_upload_tmp_file, auto_delete_ios_mobile_tmp_file, \
     auto_check_ios_developer_active
+from api.utils.crontab.iproxy import get_best_proxy_ips
 from api.utils.geetest.geetest_utils import check_bypass_status
 from api.utils.mp.wechat import sync_wx_access_token
 from api.utils.storage.caches import MigrateStorageState
@@ -112,3 +113,8 @@ def auto_check_ios_developer_active_job():
 def sync_wx_access_token_job():
     if get_login_type().get('third', '').get('wxp'):
         sync_wx_access_token()
+
+
+@app.task
+def get_best_proxy_ips_job():
+    get_best_proxy_ips()

fir_ser/api/utils/app/iossignapi.py

@@ -195,6 +195,12 @@ def make_pem(cer_content, pem_path):
         f.write(dump_certificate(FILETYPE_PEM, cert))
 
 
+def check_error_call_back(error):
+    logger.error(error)
+    if 'Cannot connect to proxy' in error or 'Read timed out' in error or 'Max retries exceeded with' in error:
+        logger.error('access apple api failed . change proxy ip again')
+
+
 class AppDeveloperApiV2(object):
     def __init__(self, issuer_id, private_key_id, p8key, cert_id):
         self.issuer_id = issuer_id
@@ -215,6 +221,7 @@ class AppDeveloperApiV2(object):
                 return True, result
         except Exception as e:
             logger.error(f"ios developer active Failed Exception:{e}")
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
         return False, result
 
@@ -243,6 +250,7 @@ class AppDeveloperApiV2(object):
                 return True, certificates
         except Exception as e:
             logger.error(f"ios developer create cert Failed Exception:{e}")
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
         return False, result
 
@@ -258,6 +266,7 @@ class AppDeveloperApiV2(object):
                 return False, result
         except Exception as e:
             logger.error(f"ios developer get cert {self.cert_id} Failed Exception:{e}")
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
         return False, result
 
@@ -277,6 +286,7 @@ class AppDeveloperApiV2(object):
                     return True, result
         except Exception as e:
             logger.error(f"ios developer cert {self.cert_id} revoke Failed Exception:{e}")
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
         return False, result
 
@@ -297,6 +307,7 @@ class AppDeveloperApiV2(object):
             return False, result
         except Exception as e:
             logger.error(f"ios developer cert {app_dev_pem} auto get Failed Exception:{e}")
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
         return False, result
 
@@ -308,6 +319,7 @@ class AppDeveloperApiV2(object):
                 return True
         except Exception as e:
             logger.error(f"ios developer delete profile Failed Exception:{e}")
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
             return False, result
 
@@ -325,6 +337,7 @@ class AppDeveloperApiV2(object):
                 return True, result
         except Exception as e:
             logger.error("ios developer set devices status Failed Exception:%s" % e)
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
             if device_err_callback and ("There are no current ios devices" in str(e) or "Device obj is None" in str(e)):
                 with CleanErrorBundleIdSignDataState(failed_call_prefix) as state:
@@ -345,6 +358,7 @@ class AppDeveloperApiV2(object):
                 return True, devices_obj_list
         except Exception as e:
             logger.error("ios developer get device Failed Exception:%s" % e)
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
             return False, result
 
@@ -357,6 +371,7 @@ class AppDeveloperApiV2(object):
 
         except Exception as e:
             logger.error("ios developer delete app Failed Exception:%s" % e)
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
             return False, result
 
@@ -378,6 +393,7 @@ class AppDeveloperApiV2(object):
 
         except Exception as e:
             logger.error("ios developer create app Failed Exception:%s" % e)
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
             if app_id_err_callback and "There is no App ID with ID" in str(e):
                 for call_fun in app_id_err_callback:
@@ -391,6 +407,7 @@ class AppDeveloperApiV2(object):
             return True, apple_obj.register_device(device_name, device_udid)
         except Exception as e:
             logger.error("ios developer register device Failed Exception:%s" % e)
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
             if device_err_callback and "There are no current ios devices" in str(e):
 
@@ -422,6 +439,7 @@ class AppDeveloperApiV2(object):
                 return True, result
         except Exception as e:
             logger.error(f"app_id {app_obj.app_id} ios developer make profile Failed Exception:{e}")
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
             if app_id_err_callback and "There is no App ID with ID" in str(e):
                 with CleanErrorBundleIdSignDataState(failed_call_prefix) as state:
@@ -456,5 +474,6 @@ class AppDeveloperApiV2(object):
             return True, result
         except Exception as e:
             logger.error("ios developer modify_capability Failed Exception:%s" % e)
+            check_error_call_back(str(e))
             result['return_info'] = "%s" % e
             return False, result

fir_ser/api/utils/apple/appleapiv3.py

@@ -17,15 +17,18 @@ import jwt
 import requests
 from django.conf import settings
 
+from api.utils.crontab.iproxy import get_proxy_ip_from_cache
+
 logger = logging.getLogger(__name__)
 
+
 # https://developer.apple.com/documentation/appstoreconnectapi/creating_api_keys_for_app_store_connect_api
 #  https://appstoreconnect.apple.com/access/api 去申请秘钥
 #
 
-proxies = settings.APPLE_DEVELOPER_API_PROXY if settings.APPLE_DEVELOPER_API_PROXY else {}
+# proxies = settings.APPLE_DEVELOPER_API_PROXY if settings.APPLE_DEVELOPER_API_PROXY else {}
 
-timeout = settings.APPLE_DEVELOPER_API_TIMEOUT if settings.APPLE_DEVELOPER_API_TIMEOUT else 120
+# timeout = settings.APPLE_DEVELOPER_API_TIMEOUT if settings.APPLE_DEVELOPER_API_TIMEOUT else 120
 
 
 def request_format_log(req):
@@ -99,7 +102,8 @@ class DevicesAPI(object):
             for k, v in query_parameters.items():
                 params[k] = v
         return request_format_log(
-            requests.get(self.devices_url, params=params, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.get(self.devices_url, params=params, headers=self.headers, proxies=self.proxies,
+                         timeout=self.timeout))
 
     def list_enabled_devices(self):
         return self.list_devices({"filter[status]": "ENABLED"})
@@ -133,7 +137,8 @@ class DevicesAPI(object):
             }
         }
         return request_format_log(
-            requests.post(self.devices_url, json=json, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.post(self.devices_url, json=json, headers=self.headers, proxies=self.proxies,
+                          timeout=self.timeout))
 
     def read_device_information(self, device_id):
         """
@@ -149,7 +154,7 @@ class DevicesAPI(object):
             "fields[devices]": "addedDate, deviceClass, model, name, platform, status, udid",
         }
         return request_format_log(
-            requests.get(base_url, params=params, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.get(base_url, params=params, headers=self.headers, proxies=self.proxies, timeout=self.timeout))
 
     def enabled_device(self, device_id, device_name):
         return self.modify_registered_device(device_id, device_name, 'ENABLED')
@@ -181,7 +186,7 @@ class DevicesAPI(object):
             }
         }
         return request_format_log(
-            requests.patch(base_url, json=json, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.patch(base_url, json=json, headers=self.headers, proxies=self.proxies, timeout=self.timeout))
 
 
 class BundleIDsAPI(object):
@@ -214,7 +219,8 @@ class BundleIDsAPI(object):
             }
         }
         return request_format_log(
-            requests.post(self.bundle_ids_url, json=json, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.post(self.bundle_ids_url, json=json, headers=self.headers, proxies=self.proxies,
+                          timeout=self.timeout))
 
     def delete_bundle_id_by_id(self, bundle_id):
         """
@@ -229,7 +235,7 @@ class BundleIDsAPI(object):
         base_url = '%s/%s' % (self.bundle_ids_url, bundle_id)
         json = {}
         return request_format_log(
-            requests.delete(base_url, json=json, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.delete(base_url, json=json, headers=self.headers, proxies=self.proxies, timeout=self.timeout))
 
     def list_bundle_ids(self, query_parameters=None):
         """
@@ -248,7 +254,8 @@ class BundleIDsAPI(object):
             for k, v in query_parameters.items():
                 params[k] = v
         return request_format_log(
-            requests.get(self.bundle_ids_url, params=params, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.get(self.bundle_ids_url, params=params, headers=self.headers, proxies=self.proxies,
+                         timeout=self.timeout))
 
     def list_bundle_id_by_identifier(self, identifier):
         return self.list_bundle_ids({"filter[identifier]": identifier})
@@ -278,7 +285,7 @@ class BundleIDsAPI(object):
             }
         }
         return request_format_log(
-            requests.patch(base_url, json=json, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.patch(base_url, json=json, headers=self.headers, proxies=self.proxies, timeout=self.timeout))
 
 
 class BundleIDsCapabilityAPI(object):
@@ -301,7 +308,7 @@ class BundleIDsCapabilityAPI(object):
         base_url = '%s/%s_%s' % (self.bundle_ids_capability_url, bundle_id, capability_type)
         json = {}
         return request_format_log(
-            requests.delete(base_url, json=json, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.delete(base_url, json=json, headers=self.headers, proxies=self.proxies, timeout=self.timeout))
 
     def enable_capability(self, bundle_id, capability_type):
         """
@@ -331,8 +338,8 @@ class BundleIDsCapabilityAPI(object):
             }
         }
         return request_format_log(
-            requests.post(self.bundle_ids_capability_url, json=json, headers=self.headers, proxies=proxies,
+            requests.post(self.bundle_ids_capability_url, json=json, headers=self.headers, proxies=self.proxies,
-                          timeout=timeout))
+                          timeout=self.timeout))
 
 
 class ProfilesAPI(object):
@@ -385,7 +392,8 @@ class ProfilesAPI(object):
             }
         }
         return request_format_log(
-            requests.post(self.profiles_url, json=json, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.post(self.profiles_url, json=json, headers=self.headers, proxies=self.proxies,
+                          timeout=self.timeout))
 
     def delete_profile(self, profile_id):
         """
@@ -400,7 +408,7 @@ class ProfilesAPI(object):
         base_url = '%s/%s' % (self.profiles_url, profile_id)
         json = {}
         return request_format_log(
-            requests.delete(base_url, json=json, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.delete(base_url, json=json, headers=self.headers, proxies=self.proxies, timeout=self.timeout))
 
     def download_profile(self, profile_id):
         # n=base64.b64decode(profileContent)
@@ -426,7 +434,8 @@ class ProfilesAPI(object):
             for k, v in query_parameters.items():
                 params[k] = v
         return request_format_log(
-            requests.get(self.profiles_url, params=params, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.get(self.profiles_url, params=params, headers=self.headers, proxies=self.proxies,
+                         timeout=self.timeout))
 
     def list_profile_by_profile_id(self, profile_id):
         return self.list_profiles({"filter[id]": profile_id, "include": ""})
@@ -462,7 +471,8 @@ class CertificatesAPI(object):
             }
         }
         return request_format_log(
-            requests.post(self.certificates_url, json=json, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.post(self.certificates_url, json=json, headers=self.headers, proxies=self.proxies,
+                          timeout=self.timeout))
 
     def download_certificate(self, certificate_id):
         # req.json()['data'][0]['attributes']['certificateContent']
@@ -488,7 +498,8 @@ class CertificatesAPI(object):
             for k, v in query_parameters.items():
                 params[k] = v
         return request_format_log(
-            requests.get(self.certificates_url, params=params, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.get(self.certificates_url, params=params, headers=self.headers, proxies=self.proxies,
+                         timeout=self.timeout))
 
     def list_certificate_by_certificate_id(self, certificate_id):
         return self.list_certificate({"filter[id]": certificate_id, })
@@ -506,7 +517,7 @@ class CertificatesAPI(object):
         base_url = '%s/%s' % (self.certificates_url, certificate_id)
         json = {}
         return request_format_log(
-            requests.delete(base_url, json=json, headers=self.headers, proxies=proxies, timeout=timeout))
+            requests.delete(base_url, json=json, headers=self.headers, proxies=self.proxies, timeout=self.timeout))
 
 
 class BaseInfoObj(object):
@@ -672,7 +683,7 @@ class Certificates(namedtuple("Certificates",
         return filepath
 
 
-def call_function_try_attempts(try_attempts=3, sleep_time=1):
+def call_function_try_attempts(try_attempts=5, sleep_time=3):
     def decorator(func):
         @wraps(func)
         def wrapper(*args, **kwargs):
@@ -685,6 +696,10 @@ def call_function_try_attempts(try_attempts=3, sleep_time=1):
                     flag = True
                     break
                 except Exception as e:
+                    if 'Cannot connect to proxy' in str(e) or 'Read timed out' in str(
+                            e) or 'Max retries exceeded with' in str(e):
+                        logger.error('access apple api failed . change proxy ip again')
+                        get_proxy_ip_from_cache(True)
                     logger.warning(
                         f'exec {func} failed. Failed:{e} {try_attempts} times in total. now {sleep_time} later try '
                         f'again...{i}')
@@ -720,6 +735,8 @@ class AppStoreConnectApi(DevicesAPI, BundleIDsAPI, BundleIDsCapabilityAPI, Profi
         self.private_key_id = private_key_id
         self.p8_private_key = p8_private_key
         self.exp_seconds = exp_seconds
+        self.proxies = get_proxy_ip_from_cache()
+        self.timeout = settings.APPLE_DEVELOPER_API_TIMEOUT if settings.APPLE_DEVELOPER_API_TIMEOUT else 120
         self.__make_jwt_headers()
         DevicesAPI.__init__(self, self.BASE_URI, self.headers)
         BundleIDsAPI.__init__(self, self.BASE_URI, self.headers)
@@ -753,6 +770,8 @@ class AppStoreConnectApi(DevicesAPI, BundleIDsAPI, BundleIDsCapabilityAPI, Profi
         }
         jwt_encoded = jwt.encode(data, self.p8_private_key, algorithm=self.JWT_ALG, headers=jwt_headers)
         headers = {
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.39 (KHTML, like Gecko) '
+                          'Chrome/72.0.3626.109 Safari/537.39',
             'Authorization': 'Bearer %s' % jwt_encoded
         }
         self.headers = headers

fir_ser/api/utils/crontab/iproxy.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+# project: 12月 
+# author: NinEveN
+# date: 2021/12/10
+import logging
+import random
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+import requests
+from django.core.cache import cache
+
+from fir_ser.settings import CACHE_KEY_TEMPLATE, APPLE_DEVELOPER_API_PROXY_LIST, APPLE_DEVELOPER_API_PROXY
+
+logger = logging.getLogger(__name__)
+
+
+def get_best_proxy_ips(url='https://api.appstoreconnect.apple.com/agreement'):
+    active_proxy_ips = [proxy_info['proxy'] for proxy_info in APPLE_DEVELOPER_API_PROXY_LIST if
+                        proxy_info.get('active')]
+    access_ip_info = []
+    if not active_proxy_ips:
+        return
+
+    def task(proxies):
+        start_time = time.time()
+        try:
+            r = requests.get(url, proxies={'http': proxies, 'https': proxies}, timeout=30)
+            access_ip_info.append({'ip': proxies, 'time': time.time() - start_time})
+            logger.info(f"ip:{proxies} code:{r.status_code} time: {time.time() - start_time}")
+        except Exception as e:
+            logger.warning(f"ip {proxies} check failed Exception:{e}")
+
+    pools = ThreadPoolExecutor(50)
+
+    for proxy_ip in active_proxy_ips:
+        pools.submit(task, proxy_ip)
+    pools.shutdown()
+    best_sorted_ips = sorted(access_ip_info, key=lambda x: x.get('time'))[:8]
+    ip_proxy_store_key = CACHE_KEY_TEMPLATE.get("ip_proxy_store_list_key")
+    best_sorted_ips = [ip_proxy['ip'] for ip_proxy in best_sorted_ips]
+    cache.set(ip_proxy_store_key, best_sorted_ips, 24 * 60 * 60)
+    return best_sorted_ips
+
+
+def get_proxy_ip_from_cache(change_ip=False):
+    ip_proxy_store_key = CACHE_KEY_TEMPLATE.get("ip_proxy_store_list_key")
+    ip_proxy_store_active_key = CACHE_KEY_TEMPLATE.get("ip_proxy_store_active_key")
+    active_ip_proxy = cache.get(ip_proxy_store_active_key)
+    if not change_ip and active_ip_proxy:
+        logger.info(f"get ip proxy cache {active_ip_proxy}")
+        return active_ip_proxy
+
+    ip_proxy_result = cache.get(ip_proxy_store_key)
+    if not ip_proxy_result:
+        ip_proxy_result = get_best_proxy_ips()
+
+    if change_ip:
+        try:
+            ip_proxy_result.remove(active_ip_proxy)
+        except Exception as e:
+            logger.warning(f'remove bad ip proxy failed {e}')
+        logger.error(f"remove bad ip proxy {active_ip_proxy}")
+        cache.delete(ip_proxy_store_active_key)
+        cache.set(ip_proxy_store_key, ip_proxy_result, 24 * 60 * 60)
+
+    if len(ip_proxy_result) > 0:
+        proxy_ip = ip_proxy_result[random.randint(0, 2 if len(ip_proxy_result) > 2 else len(ip_proxy_result) - 1)]
+        proxy_info = {
+            'http': proxy_ip,
+            'https': proxy_ip
+        }
+    else:
+        proxy_info = APPLE_DEVELOPER_API_PROXY
+    logger.info(f"make ip proxy cache {proxy_info}")
+    cache.set(ip_proxy_store_active_key, proxy_info, 24 * 60 * 60)
+    return proxy_info

fir_ser/config.py

@@ -246,10 +246,9 @@ class SENDERCONF(object):
 
 
 class IPACONF(object):
+    APPLE_DEVELOPER_API_PROXY_LIST = []
     APPLE_DEVELOPER_API_PROXY = {
         # 代理的作用，主要是为了加快苹果api的访问，在国内会出现卡死，访问超时等问题，怀疑是被苹果服务器拦截了
-        # 'http': '47.243.172.202:17897',
-        # 'https': '47.243.172.202:17897'
     }
     APPLE_DEVELOPER_API_TIMEOUT = 120  # 访问苹果api超时时间，默认3分钟
     MOBILE_CONFIG_SIGN_SSL = {

fir_ser/fir_ser/settings.py

@@ -237,6 +237,8 @@ CACHE_KEY_TEMPLATE = {
     'wx_access_token_key': 'wx_basic_access_token',
     'wx_ticket_info_key': 'wx_ticket_info',
     'ipa_sign_udid_queue_key': 'ipa_sign_udid_queue',
+    'ip_proxy_store_list_key': 'ip_proxy_store_list',
+    'ip_proxy_store_active_key': 'ip_proxy_store_active',
 }
 
 DATA_DOWNLOAD_KEY = "d_token"
@@ -248,6 +250,7 @@ AUTH_USER_GIVE_DOWNLOAD_TIMES = 200
 
 SYNC_CACHE_TO_DATABASE = {
     'download_times': 10,  # 下载次数同步时间
+    'best_proxy_ips_times': 60 * 60,  # 代理ip 自动获取时间
     'wx_get_access_token_times': 60 * 10,  # 微信access_token 自动获取时间
     'try_login_times': (10, 12 * 60 * 60),  # 当天登录失败次数，超过该失败次数，锁定24小时
     'auto_clean_tmp_file_times': 60 * 30,  # 定时清理上传失误生成的临时文件
@@ -265,6 +268,8 @@ APPLE_DEVELOPER_API_PROXY = IPACONF.APPLE_DEVELOPER_API_PROXY
 
 APPLE_DEVELOPER_API_TIMEOUT = IPACONF.APPLE_DEVELOPER_API_TIMEOUT
 
+APPLE_DEVELOPER_API_PROXY_LIST = IPACONF.APPLE_DEVELOPER_API_PROXY_LIST
+
 DEFAULT_MOBILEPROVISION = IPACONF.DEFAULT_MOBILEPROVISION
 # DEFAULT_MOBILEPROVISION = {
 #     # 默认描述文件路径或者下载路径，用户企业签名或者超级签名 跳转 [设置 - 通用 - 描述文件|设备管理] 页面
@@ -454,6 +459,11 @@ CELERY_BEAT_SCHEDULE = {
         'schedule': SYNC_CACHE_TO_DATABASE.get("wx_get_access_token_times"),
         'args': (),
     },
+    'get_best_proxy_ips_job': {
+        'task': 'api.tasks.get_best_proxy_ips_job',
+        'schedule': SYNC_CACHE_TO_DATABASE.get("best_proxy_ips_times"),
+        'args': (),
+    },
 }
 
 MSGTEMPLATE = {