存储切换，迁移头像数据

fir_client/src/components/FirAppInfostimeline.vue

@@ -182,7 +182,14 @@
                 this.$confirm('确认删除 ' + this.currentapp.name + '下 当前 release 版本吗?')
                     // eslint-disable-next-line no-unused-vars
                     .then(res => {
+                        let loadingobj = this.$loading({
+                            lock: true,
+                            text: '操作中，请耐心等待',
+                            spinner: 'el-icon-loading',
+                            // background: 'rgba(0, 0, 0, 0.7)'
+                        });
                         releaseapputils(data => {
+                            loadingobj.close();
                             if (data.code === 1000) {
                                 this.$message({
                                     message: '删除成功',

fir_client/src/components/FirUserProfileStorage.vue

@@ -43,6 +43,23 @@
                                       v-model="editstorageinfo.additionalparameter.endpoint"></el-input>
                         </el-form-item>
 
+                        <el-form-item label-width="110px" label="下载授权方式">
+                            <el-select v-model="editstorageinfo.additionalparameter.download_auth_type"
+                                       placeholder="下载授权方式"
+                                       style="width: 80%" :disabled="disabled">
+                                <el-option v-for="st in download_auth_type_list" :key="st.id" :label="st.name"
+                                           :value="st.id"></el-option>
+                            </el-select>
+
+                            <el-form-item label-width="120px" style="margin-top: 10px;margin-left: 70px;width: 60%"
+                                          label="CDN鉴权主KEY"
+                                          v-if="editstorageinfo.additionalparameter.download_auth_type === 2">
+                                <el-input :disabled='disabled' placeholder="CDN鉴权主KEY"
+                                          v-model="editstorageinfo.additionalparameter.cnd_auth_key"/>
+                            </el-form-item>
+
+                        </el-form-item>
+
                     </div>
 
                     <el-form-item label-width="110px" label="备注">
@@ -124,11 +141,26 @@
                                           v-model="storageinfo.additionalparameter.endpoint"></el-input>
                             </el-form-item>
 
-                            <el-form-item label-width="110px" label="备注">
-                                <el-input :disabled='Sdisabled' v-model="storageinfo.description"></el-input>
+                            <el-form-item label-width="110px" label="下载授权方式">
+                                <el-select v-model="storageinfo.additionalparameter.download_auth_type"
+                                           placeholder="下载授权方式"
+                                           style="width: 100%" :disabled="disabled">
+                                    <el-option v-for="st in download_auth_type_list" :key="st.id" :label="st.name"
+                                               :value="st.id"></el-option>
+                                </el-select>
+
+                                <el-form-item label-width="120px" style="margin-top: 10px;width: 100%" label="CDN鉴权主KEY"
+                                              v-if="storageinfo.additionalparameter.download_auth_type === 2">
+                                    <el-input :disabled='disabled' placeholder="CDN鉴权主KEY"
+                                              v-model="storageinfo.additionalparameter.cnd_auth_key"/>
+                                </el-form-item>
+
                             </el-form-item>
 
                         </div>
+                        <el-form-item label-width="110px" label="备注">
+                            <el-input :disabled='Sdisabled' v-model="storageinfo.description"></el-input>
+                        </el-form-item>
 
                     </div>
 
@@ -226,6 +258,10 @@
                 activeName: 'chostorage',
                 storage_info_lists: [],
                 is_admin_storage: false,
+                download_auth_type_list: [{id: 1, name: 'OSS模式： 需要把OSS权限开启私有模式'}, {
+                    id: 2,
+                    name: 'CDN模式： 请先配置好阿里云CDN，开启阿里云OSS私有Bucket回源，将使用鉴权A方式'
+                },]
             }
         }, methods: {
 
@@ -245,7 +281,7 @@
                 this.title = '新增存储';
                 this.disabled = false;
                 this.dialogstorageVisible = true;
-                this.editstorageinfo = {'additionalparameter': {}};
+                this.editstorageinfo = {'additionalparameter': {download_auth_type: 1}};
                 this.isaddflag = true;
             },
             // eslint-disable-next-line no-unused-vars

fir_ser/api/utils/app/supersignutils.py

@@ -11,10 +11,9 @@ from api.models import APPSuperSignUsedInfo, AppUDID, AppIOSDeveloperInfo, AppRe
     UDIDsyncDeveloper, DeveloperAppID, DeveloperDevicesID
 from api.utils.app.randomstrings import make_app_uuid, make_from_user_uuid
 from api.utils.serializer import get_developer_udided
-from api.utils.storage.localApi import LocalStorage
 from api.utils.storage.caches import del_cache_response_by_short, send_msg_over_limit
 from api.utils.utils import file_format_path, delete_app_to_dev_and_file, delete_app_profile_file, \
-    send_ios_developer_active_status, get_profile_full_path, delete_local_files
+    send_ios_developer_active_status, get_profile_full_path, delete_local_files, download_files_form_oss
 from api.utils.storage.storage import Storage
 
 logger = logging.getLogger(__file__)
@@ -28,20 +27,7 @@ def check_org_file(user_obj, org_file):
         return
 
     storage_obj = Storage(user_obj)
-    download_url = storage_obj.get_download_url(os.path.basename(org_file), 600, key='check_org_file', force_new=True)
-    req = requests.get(download_url)
-    logger.info("download user %s file %s success" % (user_obj, org_file))
-
-    with open(org_file + ".check.tmp", "wb") as f:
-        f.write(req.content)
-    try:
-        if os.path.isfile(org_file):
-            os.remove(org_file)
-        os.rename(os.path.join(org_file + ".check.tmp"), org_file)
-        return True
-    except Exception as e:
-        logger.error("check org file and mv file %s failed Exception %s" % (org_file, e))
-        return False
+    return download_files_form_oss(storage_obj, org_file)
 
 
 def resign_by_app_obj(app_obj, need_download_profile=True):

fir_ser/api/utils/storage/aliyunApi.py

@@ -8,11 +8,15 @@
 
 import json
 import os
-
 from aliyunsdkcore import client
 from aliyunsdksts.request.v20150401 import AssumeRoleRequest
-
 import oss2
+import re
+import hashlib
+import time
+import logging
+
+logger = logging.getLogger(__file__)
 
 
 # 以下代码展示了STS的用法，包括角色扮演获取临时用户的密钥、使用临时用户的密钥访问OSS。
@@ -32,6 +36,46 @@ import oss2
 #   https://oss-cn-hangzhou.aliyuncs.com
 # 分别以HTTP、HTTPS协议访问。
 
+def md5sum(src):
+    m = hashlib.md5()
+    m.update(src)
+    return m.hexdigest()
+
+
+class AliYunCdn(object):
+    def __init__(self, key, is_https, domain_name):
+        uri = 'http://'
+        if is_https:
+            uri = 'https://'
+        self.domain = uri + domain_name
+        self.key = key
+
+    # 鉴权方式A
+    def a_auth(self, uri, exp):
+        p = re.compile("^(http://|https://)?([^/?]+)(/[^?]*)?(\\?.*)?$")
+        if not p:
+            return None
+        m = p.match(uri)
+        scheme, host, path, args = m.groups()
+        if not scheme: scheme = "http://"
+        if not path: path = "/"
+        if not args: args = ""
+        rand = "0"  # "0" by default, other value is ok
+        uid = "0"  # "0" by default, other value is ok
+        sstring = "%s-%s-%s-%s-%s" % (path, exp, rand, uid, self.key)
+        hashvalue = md5sum(sstring.encode("utf-8"))
+        auth_key = "%s-%s-%s-%s" % (exp, rand, uid, hashvalue)
+        if args:
+            return "%s%s%s%s&auth_key=%s" % (scheme, host, path, args, auth_key)
+        else:
+            return "%s%s%s%s?auth_key=%s" % (scheme, host, path, args, auth_key)
+
+    def get_cdn_download_token(self, filename, expires=1800):
+        uri = "%s/%s" % (self.domain, filename)
+        exp = int(time.time()) + expires
+        return self.a_auth(uri, exp)
+
+
 class StsToken(object):
     """AssumeRole返回的临时用户密钥
     :param str access_key_id: 临时用户的access key id
@@ -53,7 +97,8 @@ class StsToken(object):
 
 class AliYunOss(object):
 
-    def __init__(self, access_key, secret_key, bucket_name, endpoint, sts_role_arn, is_https, domain_name=None):
+    def __init__(self, access_key, secret_key, bucket_name, endpoint, sts_role_arn, is_https, domain_name=None,
+                 download_auth_type=1, cnd_auth_key=None):
         self.access_key_id = access_key
         self.access_key_secret = secret_key
         self.bucket_name = bucket_name
@@ -62,6 +107,8 @@ class AliYunOss(object):
         self.region_id = '-'.join(self.endpoint.split('.')[0].split("-")[1:3])
         self.token = StsToken()
         self.is_https = is_https
+        self.download_auth_type = download_auth_type
+        self.cnd_auth_key = cnd_auth_key
         self.domain_name = domain_name
         self.get_auth_bucket('init', 900)
 
@@ -91,6 +138,7 @@ class AliYunOss(object):
         self.token.expiration = oss2.utils.to_unixtime(j['Credentials']['Expiration'], '%Y-%m-%dT%H:%M:%SZ')
         self.token.bucket = self.bucket_name
         self.token.endpoint = self.endpoint
+        logger.info("get aliyun sts token %s" % self.token.__dict__)
         return self.token.__dict__
 
     def get_upload_token(self, name, expires=1800):
@@ -103,7 +151,7 @@ class AliYunOss(object):
             uri = 'https://'
         url = self.endpoint
         is_cname = False
-        if self.domain_name:
+        if self.domain_name and self.download_auth_type == 1:
             url = self.domain_name
             is_cname = True
 

fir_ser/api/utils/storage/storage.py

@@ -5,7 +5,7 @@
 # date: 2020/3/23
 
 from api.models import UserInfo
-from .aliyunApi import AliYunOss
+from .aliyunApi import AliYunOss, AliYunCdn
 from .qiniuApi import QiNiuOss
 from .localApi import LocalStorage
 import json, time, base64
@@ -17,9 +17,9 @@ logger = logging.getLogger(__file__)
 
 
 class Storage(object):
-    def __init__(self, user, storage_obj=None):
+    def __init__(self, user, storage_obj=None, use_default_storage=False):
         try:
-            self.storage = self.get_storage(user, storage_obj)
+            self.storage = self.get_storage(user, storage_obj, use_default_storage)
         except Exception as e:
             logger.error("get %s storage failed Exception:%s" % (user, e))
             self.storage = None
@@ -36,8 +36,11 @@ class Storage(object):
             if download_val and not force_new:
                 if download_val.get("time") > now - 60:
                     return download_val.get("download_url")
-
-            download_url = self.storage.get_download_url(filename, expires, force_new=True)
+            if self.storage.__dict__.get("download_auth_type", None) == 2:
+                cdn_obj = AliYunCdn(self.storage.cnd_auth_key, self.storage.is_https, self.storage.domain_name)
+                download_url = cdn_obj.get_cdn_download_token(filename, expires)
+            else:
+                download_url = self.storage.get_download_url(filename, expires, force_new=True)
             cache.set(down_key, {"download_url": download_url, "time": now + expires}, expires)
             return download_url
 
@@ -67,11 +70,14 @@ class Storage(object):
             except Exception as e:
                 logger.error("oss upload  %s failed  Exception %s" % (local_file_full_path, e))
 
-    def get_storage(self, user, storage_obj):
+    def get_storage(self, user, storage_obj, use_default_storage):
         if storage_obj:
             self.storage_obj = storage_obj
         else:
             self.storage_obj = user.storage
+        if use_default_storage:
+            self.storage_obj = None
+
         if self.storage_obj:
             auth = self.get_storage_auth(self.storage_obj)
             storage_key = "_".join([CACHE_KEY_TEMPLATE.get('user_storage_key'), user.uid,
@@ -79,6 +85,7 @@ class Storage(object):
             storage_type = self.storage_obj.storage_type
             new_storage_obj = cache.get(storage_key)
             if new_storage_obj and not storage_obj:
+                logger.info("user %s get storage obj cache %s" % (user, new_storage_obj))
                 return new_storage_obj
             else:
                 if storage_type == 1:
@@ -87,17 +94,20 @@ class Storage(object):
                     new_storage_obj = AliYunOss(**auth)
                 else:
                     new_storage_obj = LocalStorage(**auth)
-
+                logger.info("user %s make storage obj %s" % (user, new_storage_obj))
                 new_storage_obj.storage_type = storage_type
                 cache.set(storage_key, new_storage_obj, 600)
                 return new_storage_obj
         else:
-            return self.get_default_storage(user, storage_obj)
+            logger.info("user %s has not storage obj, so get default" % user)
+            return self.get_default_storage(user, storage_obj, False)
 
-    def get_default_storage(self, user, storage_obj):
+    def get_default_storage(self, user, storage_obj, use_default_storage):
         admin_obj = UserInfo.objects.filter(is_superuser=True).order_by('pk').first()
         if admin_obj and admin_obj.storage:
-            return self.get_storage(admin_obj, storage_obj)
+            logger.info("user %s has not storage obj, from admin "
+                        "get default storage" % user)
+            return self.get_storage(admin_obj, storage_obj, use_default_storage)
         else:
             storage_lists = THIRD_PART_CONFIG.get('storage')
             for storage in storage_lists:
@@ -120,6 +130,8 @@ class Storage(object):
                             new_storage_obj = LocalStorage(**auth)
                             new_storage_obj.storage_type = 3
                         cache.set(storage_key, new_storage_obj, 600)
+                        logger.info("user %s has not storage obj, admin already has not storage obj, from settings "
+                                    "get default storage %s" % (user, new_storage_obj))
                         return new_storage_obj
         return None
 

fir_ser/api/utils/utils.py

@@ -3,10 +3,10 @@
 # project: 5月
 # author: liuyu
 # date: 2020/5/7
-import os, re
+import os, re, json, requests
 from fir_ser.settings import SUPER_SIGN_ROOT, SERVER_DOMAIN, CAPTCHA_LENGTH, MEDIA_ROOT
 from api.models import APPSuperSignUsedInfo, APPToDeveloper, \
-    UDIDsyncDeveloper, UserInfo
+    UDIDsyncDeveloper, UserInfo, AppReleaseInfo
 from api.utils.app.randomstrings import make_app_uuid
 from api.utils.storage.localApi import LocalStorage
 from api.utils.storage.storage import Storage
@@ -206,14 +206,98 @@ def send_ios_developer_active_status(user_info, msg):
         logger.warning("user %s has no email. so %s can't send!" % (user_info, msg))
 
 
+def get_filename_from_apptype(filename, apptype):
+    if apptype == 0:
+        filename = filename + '.apk'
+    else:
+        filename = filename + '.ipa'
+    return filename
+
+
 def delete_local_files(filename, apptype=None):
     storage = LocalStorage("localhost", False)
     if apptype is not None:
-        if apptype == 0:
-            filename = filename + '.apk'
-        else:
-            filename = filename + '.ipa'
+        filename = get_filename_from_apptype(filename, apptype)
     try:
         return storage.del_file(filename)
     except Exception as e:
         logger.error("delete file  %s  failed  Exception %s" % (filename, e))
+
+
+def check_storage_additionalparameter(request, res):
+    data = request.data
+    try:
+        extra_parameters = data.get('additionalparameter', '')
+        if extra_parameters:
+            if not extra_parameters.get("download_auth_type", None):
+                extra_parameters['download_auth_type'] = 1
+            if extra_parameters.get("download_auth_type", None) == 2:
+                if not extra_parameters.get("cnd_auth_key", None):
+                    logger.error("user %s add new storage failed" % (request.user))
+                    res.msg = "cdn 鉴权KEY 缺失"
+                    res.code = 1006
+                    return False, res
+        data['additionalparameters'] = json.dumps(extra_parameters)
+    except Exception as e:
+        logger.error("user:%s additionalparameters %s dumps failed Exception:%s" % (
+            request.user, data.get('additionalparameter', ''), e))
+    return True, res
+
+
+def change_storage_and_change_head_img(user_obj, new_storage_obj):
+    migrating_storage_file_data(user_obj, user_obj.head_img, new_storage_obj)
+
+
+def download_files_form_oss(storage_obj, org_file):
+    download_url = storage_obj.get_download_url(os.path.basename(org_file), 600, key='check_org_file', force_new=True)
+    req = requests.get(download_url)
+    if req.status_code == 200:
+        logger.info("download  file %s success" % org_file)
+    else:
+        logger.error("download  file %s failed %s" % (org_file, req.content))
+        return False
+
+    with open(org_file + ".check.tmp", "wb") as f:
+        f.write(req.content)
+    try:
+        if os.path.isfile(org_file):
+            os.remove(org_file)
+        os.rename(os.path.join(org_file + ".check.tmp"), org_file)
+        return True
+    except Exception as e:
+        logger.error("check org file and mv file %s failed Exception %s" % (org_file, e))
+        return False
+
+
+def migrating_storage_file_data(user_obj, filename, new_storage_obj):
+    local_file_full_path = os.path.join(MEDIA_ROOT, filename)
+    old_storage_obj = Storage(user_obj)
+    if not new_storage_obj:
+        new_storage_obj = Storage(user_obj, None, True)
+    else:
+        new_storage_obj = Storage(user_obj, new_storage_obj)
+
+    if old_storage_obj.get_storage_type() == 3:
+        if new_storage_obj.get_storage_type() == 3:
+            pass
+        else:
+            # 本地向云存储上传,并删除本地数据
+            new_storage_obj.upload_file(local_file_full_path)
+            delete_local_files(filename)
+    else:
+        if new_storage_obj.get_storage_type() == 3:
+            # 云存储下载 本地，并删除云存储
+            if download_files_form_oss(old_storage_obj, local_file_full_path):
+                old_storage_obj.delete_file(filename)
+        else:
+            # 云存储互传，先下载本地，然后上传新云存储，删除本地和老云存储
+            if download_files_form_oss(old_storage_obj, local_file_full_path):
+                new_storage_obj.upload_file(local_file_full_path)
+                delete_local_files(filename)
+                old_storage_obj.delete_file(filename)
+
+
+def migrating_storage_data(user_obj, new_storage_obj):
+    for app_release_obj in AppReleaseInfo.objects.filter(app_id__user_id=user_obj).all():
+        filename = get_filename_from_apptype(app_release_obj.release_id, app_release_obj.release_type)
+        migrating_storage_file_data(user_obj, filename, new_storage_obj)

fir_ser/api/views/storage.py

@@ -8,10 +8,10 @@ from rest_framework.views import APIView
 from api.utils.response import BaseResponse
 from api.utils.auth import ExpiringTokenAuthentication, StoragePermission
 from rest_framework.response import Response
-import json
 from api.utils.storage.caches import del_cache_storage
 from api.models import AppStorage, UserInfo
-from api.utils.utils import upload_oss_default_head_img
+from api.utils.utils import upload_oss_default_head_img, check_storage_additionalparameter, \
+    change_storage_and_change_head_img
 from api.utils.serializer import StorageSerializer
 import logging
 
@@ -59,12 +59,9 @@ class StorageView(APIView):
         res = BaseResponse()
         data = request.data
         logger.info("user %s add new storage data:%s" % (request.user, data))
-        try:
-            data['additionalparameters'] = json.dumps(data.get('additionalparameter', ''))
-        except Exception as e:
-            logger.error("user:%s additionalparameters %s dumps failed Exception:%s" % (
-                request.user, data.get('additionalparameter', ''), e))
-
+        status, msg = check_storage_additionalparameter(request, res)
+        if not status:
+            return Response(msg.dict)
         serializer = StorageSerializer(data=data, context={'user_obj': request.user})
         if serializer.is_valid():
             storage_obj = serializer.save()
@@ -96,10 +93,17 @@ class StorageView(APIView):
         if use_storage_id:
             try:
                 if use_storage_id == -1:
+                    change_storage_and_change_head_img(request.user, None)
                     UserInfo.objects.filter(pk=request.user.pk).update(storage=None)
+
                 else:
+                    new_storage_obj = AppStorage.objects.filter(pk=use_storage_id).first()
+                    change_storage_and_change_head_img(request.user, new_storage_obj)
                     UserInfo.objects.filter(pk=request.user.pk).update(storage_id=use_storage_id)
+
                 del_cache_storage(request.user)
+                # change_storage_and_change_head_img(request.user, old_storage_obj, new_storage_obj)
+
             except Exception as e:
                 logger.error("update user %s storage failed Exception:%s" % (request.user, e))
                 res.code = 1006
@@ -112,13 +116,9 @@ class StorageView(APIView):
                 res.msg = '存储正在使用中，无法修改'
                 res.code = 1007
                 return Response(res.dict)
-            try:
-                data['additionalparameters'] = json.dumps(data.get('additionalparameter', ''))
-            except Exception as e:
-                del data['additionalparameters']
-                logger.error("user:%s additionalparameters %s dumps failed Exception:%s" % (
-                    request.user, data.get('additionalparameter', ''), e))
-
+            status, msg = check_storage_additionalparameter(request, res)
+            if not status:
+                return Response(msg.dict)
             storage_obj = AppStorage.objects.filter(id=storage_id, user_id=request.user).first()
             storage_obj_bak = AppStorage.objects.filter(id=storage_id, user_id=request.user).first()
             serializer = StorageSerializer(instance=storage_obj, data=data, context={'user_obj': request.user},

fir_ser/fir_ser/settings.py

@@ -211,7 +211,8 @@ THIRD_PART_CONFIG = {
                 'sts_role_arn': 'ap-sage',
                 'endpoint': 'oss-cn-beijing.aliyuncs.com',
                 'is_https': True,
-                'domain_name': 'aoud.xin'
+                'domain_name': 'aoud.xin',
+                'download_auth_type': 1  # 1:oss 2:cdn
             },
             'active': False
         },