优化代码，增加cli.py 上传进度条，修复签名相关问题

fir_client/src/components/apps/FirApps.vue

@@ -819,7 +819,8 @@
                 }
                 let certinfo = {
                     'upload_key': upload_key,
-                    'upload_token': upload_token
+                    'upload_token': upload_token,
+                    'app_info': analyseappinfo
                 };
                 if (analyseappinfo.storage === 1) {
                     // eslint-disable-next-line no-unused-vars,no-unreachable

fir_client/src/components/user/FirUserProfileInfo.vue

@@ -247,7 +247,8 @@
                 </el-row>
             </el-form-item>
 
-            <div class="other-way" v-if="userinfo && userinfo.login_type && userinfo.login_type.third && JSON.stringify(userinfo.login_type.third).indexOf('true')!==-1">
+            <div class="other-way"
+                 v-if="userinfo && userinfo.login_type && userinfo.login_type.third && JSON.stringify(userinfo.login_type.third).indexOf('true')!==-1">
                 <hr>
                 <span class="info">绑定第三方账户</span>
                 <el-row :gutter="20" v-if="userinfo.login_type.third && userinfo.login_type.third.wxp">

fir_client/src/utils/index.js

@@ -125,6 +125,24 @@ export function dataURLtoFile(dataurl, filename) {//将base64转换为文件
     return new File([u8arr], filename, {type: mime});
 }
 
+function get_file_type(upload_key) {
+    if (upload_key) {
+        const key_list = upload_key.split('.');
+        if (key_list.length > 2) {
+            return key_list[key_list.length - 2]
+        }
+    }
+}
+
+function get_filename(app_info, upload_key) {
+    if (app_info) {
+        let app_type = app_info.type;
+        if (app_type === 'iOS' || app_info === 'Android') {
+            return app_info.appname + '-' + app_info.version + '-' + app_info.short + '.' + get_file_type(upload_key)
+        }
+    }
+}
+
 export function uploadaliyunoss(file, certinfo, app, successcallback, processcallback) {
     let token = certinfo.upload_token;
     let uploadFileClient = new app.oss({
@@ -163,12 +181,19 @@ export function uploadaliyunoss(file, certinfo, app, successcallback, processcal
             uploadFileClient = client;
         }
 
-        const options = {
+        let options = {
             progress,
             parallel: 10,
             partSize: 1024 * 1024,
             timeout: 600000,
         };
+        let filename = get_filename(certinfo.app_info, certinfo.upload_key);
+        if (filename) {
+            options['headers'] = {
+                'Content-Disposition': 'attachment; filename="' + encodeURIComponent(filename) + '"',
+                'Cache-Control': ''
+            }
+        }
 
         if (currentCheckpoint) {
             options.checkpoint = currentCheckpoint;

fir_ser/api/utils/baseutils.py

@@ -214,3 +214,17 @@ def check_app_password(app_password, password):
     if app_password.lower() != password.strip().lower():
         return None
     return True
+
+
+def get_filename_form_file(filename):
+    file_id_list = filename.split('.')
+    if file_id_list[-1] in ['ipa', 'apk']:
+        app_release_obj = AppReleaseInfo.objects.filter(release_id='.'.join(file_id_list[0:-1])).first()
+        if app_release_obj:
+            app_obj = app_release_obj.app_id
+            if app_obj.type == 0:
+                f_type = '.apk'
+            else:
+                f_type = '.ipa'
+            return f"{app_obj.name}-{app_release_obj.app_version}-{app_obj.short}{f_type}"
+    return filename

fir_ser/api/utils/storage/aliyunApi.py

@@ -15,6 +15,10 @@ import re
 import hashlib
 import time
 import logging
+from oss2 import SizedFileAdapter, determine_part_size
+from oss2.models import PartInfo
+
+from api.utils.baseutils import get_filename_form_file
 
 logger = logging.getLogger(__name__)
 
@@ -197,10 +201,13 @@ class AliYunOss(object):
         return self.del_file(old_filename)
 
     def upload_file(self, local_file_full_path):
+        return self.multipart_upload_file(local_file_full_path)
+
         if os.path.isfile(local_file_full_path):
             filename = os.path.basename(local_file_full_path)
             headers = {
-                'Content-Disposition': 'attachment; filename="%s"' % filename
+                'Content-Disposition': 'attachment; filename="%s"' % filename,
+                'Cache-Control': ''
             }
             self.bucket.put_object_from_file(filename, local_file_full_path, headers)
             # with open(local_file_full_path, 'rb') as fileobj:
@@ -218,3 +225,39 @@ class AliYunOss(object):
         if not os.path.exists(dir_path):
             os.makedirs(dir_path)
         return self.bucket.get_object_to_file(name, local_file_full_path)
+
+    def multipart_upload_file(self, local_file_full_path):
+        if os.path.isfile(local_file_full_path):
+            total_size = os.path.getsize(local_file_full_path)
+            # determine_part_size方法用于确定分片大小。
+            part_size = determine_part_size(total_size, preferred_size=1024 * 1024 * 10)
+            filename = os.path.basename(local_file_full_path)
+            headers = {
+                'Content-Disposition': 'attachment; filename="%s"' % get_filename_form_file(filename).encode(
+                    "utf-8").decode("latin1"),
+                'Cache-Control': ''
+            }
+            # 初始化分片。
+            # 如需在初始化分片时设置文件存储类型，请在init_multipart_upload中设置相关headers，参考如下。
+            # headers = dict()
+            # headers["x-oss-storage-class"] = "Standard"
+            upload_id = self.bucket.init_multipart_upload(filename, headers=headers).upload_id
+            parts = []
+
+            # 逐个上传分片。
+            with open(local_file_full_path, 'rb') as f:
+                part_number = 1
+                offset = 0
+                while offset < total_size:
+                    num_to_upload = min(part_size, total_size - offset)
+                    # 调用SizedFileAdapter(fileobj, size)方法会生成一个新的文件对象，重新计算起始追加位置。
+                    result = self.bucket.upload_part(filename, upload_id, part_number,
+                                                     SizedFileAdapter(f, num_to_upload))
+                    parts.append(PartInfo(part_number, result.etag))
+
+                    offset += num_to_upload
+                    part_number += 1
+            self.bucket.complete_multipart_upload(filename, upload_id, parts)
+            return True
+        else:
+            logger.error(f"file {local_file_full_path} is not file")

fir_ser/api/utils/utils.py

@@ -7,7 +7,7 @@ import os, datetime, random
 import binascii
 from fir_ser.settings import SERVER_DOMAIN, CAPTCHA_LENGTH, MEDIA_ROOT, CACHE_KEY_TEMPLATE
 from api.models import APPSuperSignUsedInfo, APPToDeveloper, \
-    UDIDsyncDeveloper, UserInfo, AppReleaseInfo, AppScreenShot, Token, DeveloperDevicesID
+    UDIDsyncDeveloper, UserInfo, AppReleaseInfo, AppScreenShot, Token, DeveloperDevicesID, Apps
 from api.utils.storage.caches import get_app_d_count_by_app_id
 from api.utils.storage.localApi import LocalStorage
 from api.utils.storage.storage import Storage
@@ -15,7 +15,6 @@ from api.utils.tempcaches import TmpCache
 from api.utils.TokenManager import generate_numeric_token_of_length, generate_alphanumeric_token_of_length, make_token, \
     verify_token
 from api.utils.sendmsg.sendmsg import SendMessage
-from django.db.models import Sum
 from captcha.models import CaptchaStore
 from captcha.helpers import captcha_image_url
 from api.utils.storage.caches import consume_user_download_times

fir_ser/api/views/download.py

@@ -21,12 +21,23 @@ from api.utils.serializer import AppsShortSerializer
 from api.models import Apps, AppReleaseInfo, APPToDeveloper, APPSuperSignUsedInfo
 from django.http import FileResponse
 import logging
-from api.utils.baseutils import get_profile_full_path, get_app_domain_name
+from api.utils.baseutils import get_profile_full_path, get_app_domain_name, get_filename_form_file
 from api.utils.throttle import VisitShortThrottle, InstallShortThrottle, InstallThrottle1, InstallThrottle2
 
 logger = logging.getLogger(__name__)
 
 
+def file_response(stream, filename, content_type):
+    return FileResponse(stream, as_attachment=True,
+                        filename=filename,
+                        content_type=content_type)
+
+
+def mobileprovision_file_response(file_path):
+    return file_response(open(file_path, 'rb'), make_random_uuid() + '.mobileprovision',
+                         "application/x-apple-aspen-config")
+
+
 class DownloadView(APIView):
     """
     文件下载接口,适用于本地存储和所有plist文件下载
@@ -64,10 +75,7 @@ class DownloadView(APIView):
                     ios_plist_bytes = make_resigned(storage.get_download_url(filename.split('.')[0] + ".ipa"),
                                                     storage.get_download_url(release_obj.icon_url), bundle_id,
                                                     app_version, name)
-                    response = FileResponse(ios_plist_bytes)
-                    response['Content-Type'] = "application/x-plist"
-                    response['Content-Disposition'] = 'attachment; filename=' + make_random_uuid()
-                    return response
+                    return file_response(ios_plist_bytes, make_random_uuid(), "application/x-plist")
                 res.msg = "plist release_id error"
             elif f_type == 'mobileconifg':
                 release_obj = AppReleaseInfo.objects.filter(release_id=filename.split('.')[0]).first()
@@ -76,10 +84,8 @@ class DownloadView(APIView):
                     app_obj = release_obj.app_id
                     ios_udid_mobile_config = make_sign_udid_mobile_config(udid_url, app_obj.app_id, app_obj.bundle_id,
                                                                           app_obj.name)
-                    response = FileResponse(ios_udid_mobile_config)
-                    response['Content-Type'] = "application/x-apple-aspen-config"
-                    response['Content-Disposition'] = 'attachment; filename=' + make_random_uuid() + '.mobileconfig'
-                    return response
+                    return file_response(ios_udid_mobile_config, make_random_uuid() + '.mobileconfig',
+                                         "application/x-apple-aspen-config")
                 res.msg = "mobile_config release_id error"
             elif f_type == 'mobileprovision':
                 release_obj = AppReleaseInfo.objects.filter(release_id=filename.split('.')[0]).first()
@@ -90,51 +96,34 @@ class DownloadView(APIView):
 
                     if not app_super_obj:
                         file_path = settings.DEFAULT_MOBILEPROVISION.get("supersign").get('path')
-                        if os.path.isfile(file_path):
-                            response = FileResponse(open(file_path, 'rb'))
-                        else:
-                            response = FileResponse()
+                        if file_path and os.path.isfile(file_path):
+                            return mobileprovision_file_response(file_path)
                     else:
                         developer_obj = app_super_obj.developerid
                         file_path = get_profile_full_path(developer_obj, release_obj.app_id)
                         if os.path.isfile(file_path):
-                            response = FileResponse(open(file_path, 'rb'))
+                            return mobileprovision_file_response(file_path)
                         else:
                             file_path = settings.DEFAULT_MOBILEPROVISION.get("supersign").get('path')
-                            if os.path.isfile(file_path):
-                                response = FileResponse(open(file_path, 'rb'))
-                            else:
-                                response = FileResponse()
-                    response['Content-Type'] = "application/x-apple-aspen-config"
-                    response['Content-Disposition'] = 'attachment; filename=' + make_random_uuid() + '.mobileprovision'
-                    return response
+                            if file_path and os.path.isfile(file_path):
+                                return mobileprovision_file_response(file_path)
+
                 res.msg = "mobile_provision release_id error"
-            elif f_type == 'dmobileprovision':
+            elif f_type == 'dmobileprovision':  # 企业签名安装信任跳转
                 release_obj = AppReleaseInfo.objects.filter(release_id=filename.split('.')[0]).first()
                 if release_obj:
-                    file_path = settings.DEFAULT_MOBILEPROVISION.get("supersign").get('path')
-                    if os.path.isfile(file_path):
-                        response = FileResponse(open(file_path, 'rb'))
-                    else:
-                        response = FileResponse()
-                    response['Content-Type'] = "application/x-apple-aspen-config"
-                    response['Content-Disposition'] = 'attachment; filename=' + make_random_uuid() + '.mobileprovision'
-                    return response
+                    file_path = settings.DEFAULT_MOBILEPROVISION.get("enterprise").get('path')
+                    if file_path and os.path.isfile(file_path):
+                        return mobileprovision_file_response(file_path)
                 res.msg = "d_mobile_provision release_id error"
-
             else:
                 file_path = os.path.join(settings.MEDIA_ROOT, filename)
                 try:
                     if os.path.isfile(file_path):
-                        response = FileResponse(open(file_path, 'rb'))
-                    else:
-                        response = FileResponse()
+                        return FileResponse(open(file_path, 'rb'), as_attachment=True,
+                                            filename=get_filename_form_file(filename))
                 except Exception as e:
                     logger.error(f"read {file_path} failed  Exception:{e}")
-                    response = FileResponse()
-                response['content_type'] = "application/octet-stream"
-                response['Content-Disposition'] = 'attachment; filename=' + filename
-                return response
 
         res.code = 1004
         res.msg = "token校验失败"

fir_ser/cli.py

@@ -10,7 +10,10 @@ import random
 import re
 import json
 import zipfile
-from requests_toolbelt import MultipartEncoder
+
+from oss2 import determine_part_size, SizedFileAdapter
+from oss2.models import PartInfo
+from requests_toolbelt.multipart.encoder import MultipartEncoderMonitor
 import oss2
 import requests
 from androguard.core.bytecodes import apk
@@ -24,24 +27,65 @@ pip install androguard
 '''
 
 
-def upload_aliyunoss(access_key_id, access_key_secret, security_token, endpoint, bucket, file_path, file_name):
+def progress(percent, width=50):
+    if percent >= 1:
+        percent = 1
+    show_str = ('[%%-%ds]' % width) % (int(width * percent) * '#')
+    print('\r%s %d%% ' % (show_str, int(100 * percent)), file=sys.stdout, flush=True, end='')
+
+
+def local_upload_callback(monitor):
+    progress(monitor.bytes_read / monitor.len, width=100)
+
+
+def qiniu_progress_callback(upload_size, total_size):
+    progress(upload_size / total_size, width=100)
+
+
+def alioss_progress_callback_fun(offset, total_size):
+    def progress_callback(upload_size, now_part_size):
+        percent = (offset + upload_size) / total_size  # 接收的比例
+        progress(percent, width=100)
+
+    return progress_callback
+
+
+def upload_aliyunoss(access_key_id, access_key_secret, security_token, endpoint, bucket, local_file_full_path, filename,
+                     headers=None):
     stsauth = oss2.StsAuth(access_key_id, access_key_secret, security_token)
     bucket = oss2.Bucket(stsauth, endpoint, bucket)
-    with open(file_path, 'rb') as fileobj:
-        bucket.put_object(file_name, fileobj)
+    total_size = os.path.getsize(local_file_full_path)
+    # determine_part_size方法用于确定分片大小。
+    part_size = determine_part_size(total_size, preferred_size=1024 * 1024 * 10)
+    upload_id = bucket.init_multipart_upload(filename, headers=headers).upload_id
+    parts = []
+    # 逐个上传分片。
+
+    with open(local_file_full_path, 'rb') as f:
+        part_number = 1
+        offset = 0
+        while offset < total_size:
+            num_to_upload = min(part_size, total_size - offset)
+            result = bucket.upload_part(filename, upload_id, part_number,
+                                        SizedFileAdapter(f, num_to_upload),
+                                        progress_callback=alioss_progress_callback_fun(offset, total_size))
+            parts.append(PartInfo(part_number, result.etag))
+            offset += num_to_upload
+            part_number += 1
+    bucket.complete_multipart_upload(filename, upload_id, parts)
     print("数据上传存储成功.")
 
 
 def upload_qiniuyunoss(key, token, file_path):
     from qiniu import put_file
-    ret, info = put_file(token, key, file_path)
+    ret, info = put_file(token, key, file_path, progress_handler=qiniu_progress_callback)
     if info.status_code == 200:
         print("数据上传存储成功.")
     else:
         raise AssertionError(info.text)
 
 
-class FLY_CLI_SER(object):
+class FLYCliSer(object):
 
     def __init__(self, fly_cli_domain, fly_cli_token):
         self.fly_cli_domain = fly_cli_domain
@@ -72,7 +116,7 @@ class FLY_CLI_SER(object):
 
     def upload_local_storage(self, upload_key, upload_token, app_id, file_path):
         url = '%s/api/v2/fir/server/upload' % self.fly_cli_domain
-        m = MultipartEncoder(fields={
+        m = MultipartEncoderMonitor.from_fields(fields={
             'file': (os.path.basename(file_path), open(file_path, 'rb'), 'application/octet-stream'),
             'certinfo': json.dumps(
                 {
@@ -82,7 +126,7 @@ class FLY_CLI_SER(object):
                     'app_id': app_id,
                 }
             ),
-        })
+        }, callback=local_upload_callback)
         header = {
             'Content-Type': m.content_type,
         }
@@ -99,6 +143,16 @@ class FLY_CLI_SER(object):
         icon_path = appobj.make_app_png(icon_path=appinfo.get("icon_path", None))
         bundle_id = appinfo.get("bundle_id")
         upcretsdata = self.get_upload_token(bundle_id, appinfo.get("type"))
+        if appinfo.get('iOS', '') == 'iOS':
+            f_type = '.ipa'
+        else:
+            f_type = '.apk'
+        filename = "%s-%s-%s%s" % (appinfo.get('labelname'), appinfo.get('version'), upcretsdata.get('short'), f_type)
+        headers = {
+            'Content-Disposition': 'attachment; filename="%s"' % filename.encode(
+                "utf-8").decode("latin1"),
+            'Cache-Control': ''
+        }
         if upcretsdata['storage'] == 1:
             # 七牛云存储
             upload_qiniuyunoss(upcretsdata['png_key'], upcretsdata['png_token'], icon_path)
@@ -113,7 +167,7 @@ class FLY_CLI_SER(object):
             file_auth = upcretsdata['upload_token']
 
             upload_aliyunoss(file_auth['access_key_id'], file_auth['access_key_secret'], file_auth['security_token'],
-                             file_auth['endpoint'], file_auth['bucket'], app_path, upcretsdata['upload_key'])
+                             file_auth['endpoint'], file_auth['bucket'], app_path, upcretsdata['upload_key'], headers)
         elif upcretsdata['storage'] == 3:
             # 本地存储
             self.upload_local_storage(upcretsdata['png_key'], upcretsdata['png_token'], upcretsdata['app_uuid'],
@@ -267,9 +321,9 @@ class AppInfo(object):
 
 
 if __name__ == '__main__':
-    fly_cli_domain = 'https://fly.harmonygames.cn'
-    fly_cli_token = '4baf717aff0011ea80c000163e069e27x2gRvmf2035rP3YWITJxvF4VVE32M55AuVHxOvSbTgdwgF54UeXFzrNLkP1lgUID'
-    fly_obj = FLY_CLI_SER(fly_cli_domain, fly_cli_token)
+    fly_cli_domain = 'https://app.hehelucky.cn'
+    fly_cli_token = 'f63d05d2bb0511eb9fa400163e069e27oTa51Q3N6rJT5yjddCrkmgiFeKRQNa0L3TctPLTYd0CrdIqgO3wfYCKsFnJysQXJ'
+    fly_obj = FLYCliSer(fly_cli_domain, fly_cli_token)
     if len(sys.argv) != 2:
         raise ValueError('参数有误')
     app_path = sys.argv[1]

fir_ser/config.py

@@ -3,10 +3,12 @@
 # project: 7月 
 # author: NinEveN
 # date: 2021/7/19
+import os
 
 API_DOMAIN = "https://app.hehelucky.cn"
 WEB_DOMAIN = "https://app.hehelucky.cn"
-MOBILEPROVISION = "https://ali-static.jappstore.com/embedded2.mobileprovision"
+MOBILEPROVISION = "https://ali-static.jappstore.com/embedded3.mobileprovision"
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
 class BASECONF(object):
@@ -231,12 +233,14 @@ class IPACONF(object):
         # 默认描述文件路径或者下载路径，用户企业签名或者超级签名 跳转 [设置 - 通用 - 描述文件|设备管理] 页面
         # 如果配置了path路径，则走路径，如果配置了url，则走URL，path 优先级大于url优先级
         'enterprise': {
-            'url': MOBILEPROVISION
+            'url': MOBILEPROVISION,
+            # 'path': os.path.join(BASE_DIR,'files', 'embedded.mobileprovision'),
         },
         'supersign': {
             # 超级签名，如果self 为True，则默认用自己的描述文件，否则同企业配置顺序一致,自己的配置文件有时候有问题
             'self': False,
-            'url': MOBILEPROVISION
+            'url': MOBILEPROVISION,
+            # 'path': os.path.join(BASE_DIR,'files', 'embedded.mobileprovision'),
         }
     }