存储切换,迁移头像数据

pull/5/head
youngS 4 years ago
parent 4d856bf5c0
commit f902020079
  1. 7
      fir_client/src/components/FirAppInfostimeline.vue
  2. 42
      fir_client/src/components/FirUserProfileStorage.vue
  3. 18
      fir_ser/api/utils/app/supersignutils.py
  4. 56
      fir_ser/api/utils/storage/aliyunApi.py
  5. 30
      fir_ser/api/utils/storage/storage.py
  6. 94
      fir_ser/api/utils/utils.py
  7. 30
      fir_ser/api/views/storage.py
  8. 3
      fir_ser/fir_ser/settings.py

@ -182,7 +182,14 @@
this.$confirm('确认删除 ' + this.currentapp.name + '下 当前 release 版本吗?')
// eslint-disable-next-line no-unused-vars
.then(res => {
let loadingobj = this.$loading({
lock: true,
text: '操作中,请耐心等待',
spinner: 'el-icon-loading',
// background: 'rgba(0, 0, 0, 0.7)'
});
releaseapputils(data => {
loadingobj.close();
if (data.code === 1000) {
this.$message({
message: '删除成功',

@ -43,6 +43,23 @@
v-model="editstorageinfo.additionalparameter.endpoint"></el-input>
</el-form-item>
<el-form-item label-width="110px" label="下载授权方式">
<el-select v-model="editstorageinfo.additionalparameter.download_auth_type"
placeholder="下载授权方式"
style="width: 80%" :disabled="disabled">
<el-option v-for="st in download_auth_type_list" :key="st.id" :label="st.name"
:value="st.id"></el-option>
</el-select>
<el-form-item label-width="120px" style="margin-top: 10px;margin-left: 70px;width: 60%"
label="CDN鉴权主KEY"
v-if="editstorageinfo.additionalparameter.download_auth_type === 2">
<el-input :disabled='disabled' placeholder="CDN鉴权主KEY"
v-model="editstorageinfo.additionalparameter.cnd_auth_key"/>
</el-form-item>
</el-form-item>
</div>
<el-form-item label-width="110px" label="备注">
@ -124,11 +141,26 @@
v-model="storageinfo.additionalparameter.endpoint"></el-input>
</el-form-item>
<el-form-item label-width="110px" label="备注">
<el-input :disabled='Sdisabled' v-model="storageinfo.description"></el-input>
<el-form-item label-width="110px" label="下载授权方式">
<el-select v-model="storageinfo.additionalparameter.download_auth_type"
placeholder="下载授权方式"
style="width: 100%" :disabled="disabled">
<el-option v-for="st in download_auth_type_list" :key="st.id" :label="st.name"
:value="st.id"></el-option>
</el-select>
<el-form-item label-width="120px" style="margin-top: 10px;width: 100%" label="CDN鉴权主KEY"
v-if="storageinfo.additionalparameter.download_auth_type === 2">
<el-input :disabled='disabled' placeholder="CDN鉴权主KEY"
v-model="storageinfo.additionalparameter.cnd_auth_key"/>
</el-form-item>
</el-form-item>
</div>
<el-form-item label-width="110px" label="备注">
<el-input :disabled='Sdisabled' v-model="storageinfo.description"></el-input>
</el-form-item>
</div>
@ -226,6 +258,10 @@
activeName: 'chostorage',
storage_info_lists: [],
is_admin_storage: false,
download_auth_type_list: [{id: 1, name: 'OSS模式: 需要把OSS权限开启私有模式'}, {
id: 2,
name: 'CDN模式: 请先配置好阿里云CDN,开启阿里云OSS私有Bucket回源,将使用鉴权A方式'
},]
}
}, methods: {
@ -245,7 +281,7 @@
this.title = '新增存储';
this.disabled = false;
this.dialogstorageVisible = true;
this.editstorageinfo = {'additionalparameter': {}};
this.editstorageinfo = {'additionalparameter': {download_auth_type: 1}};
this.isaddflag = true;
},
// eslint-disable-next-line no-unused-vars

@ -11,10 +11,9 @@ from api.models import APPSuperSignUsedInfo, AppUDID, AppIOSDeveloperInfo, AppRe
UDIDsyncDeveloper, DeveloperAppID, DeveloperDevicesID
from api.utils.app.randomstrings import make_app_uuid, make_from_user_uuid
from api.utils.serializer import get_developer_udided
from api.utils.storage.localApi import LocalStorage
from api.utils.storage.caches import del_cache_response_by_short, send_msg_over_limit
from api.utils.utils import file_format_path, delete_app_to_dev_and_file, delete_app_profile_file, \
send_ios_developer_active_status, get_profile_full_path, delete_local_files
send_ios_developer_active_status, get_profile_full_path, delete_local_files, download_files_form_oss
from api.utils.storage.storage import Storage
logger = logging.getLogger(__file__)
@ -28,20 +27,7 @@ def check_org_file(user_obj, org_file):
return
storage_obj = Storage(user_obj)
download_url = storage_obj.get_download_url(os.path.basename(org_file), 600, key='check_org_file', force_new=True)
req = requests.get(download_url)
logger.info("download user %s file %s success" % (user_obj, org_file))
with open(org_file + ".check.tmp", "wb") as f:
f.write(req.content)
try:
if os.path.isfile(org_file):
os.remove(org_file)
os.rename(os.path.join(org_file + ".check.tmp"), org_file)
return True
except Exception as e:
logger.error("check org file and mv file %s failed Exception %s" % (org_file, e))
return False
return download_files_form_oss(storage_obj, org_file)
def resign_by_app_obj(app_obj, need_download_profile=True):

@ -8,11 +8,15 @@
import json
import os
from aliyunsdkcore import client
from aliyunsdksts.request.v20150401 import AssumeRoleRequest
import oss2
import re
import hashlib
import time
import logging
logger = logging.getLogger(__file__)
# 以下代码展示了STS的用法,包括角色扮演获取临时用户的密钥、使用临时用户的密钥访问OSS。
@ -32,6 +36,46 @@ import oss2
# https://oss-cn-hangzhou.aliyuncs.com
# 分别以HTTP、HTTPS协议访问。
def md5sum(src):
m = hashlib.md5()
m.update(src)
return m.hexdigest()
class AliYunCdn(object):
def __init__(self, key, is_https, domain_name):
uri = 'http://'
if is_https:
uri = 'https://'
self.domain = uri + domain_name
self.key = key
# 鉴权方式A
def a_auth(self, uri, exp):
p = re.compile("^(http://|https://)?([^/?]+)(/[^?]*)?(\\?.*)?$")
if not p:
return None
m = p.match(uri)
scheme, host, path, args = m.groups()
if not scheme: scheme = "http://"
if not path: path = "/"
if not args: args = ""
rand = "0" # "0" by default, other value is ok
uid = "0" # "0" by default, other value is ok
sstring = "%s-%s-%s-%s-%s" % (path, exp, rand, uid, self.key)
hashvalue = md5sum(sstring.encode("utf-8"))
auth_key = "%s-%s-%s-%s" % (exp, rand, uid, hashvalue)
if args:
return "%s%s%s%s&auth_key=%s" % (scheme, host, path, args, auth_key)
else:
return "%s%s%s%s?auth_key=%s" % (scheme, host, path, args, auth_key)
def get_cdn_download_token(self, filename, expires=1800):
uri = "%s/%s" % (self.domain, filename)
exp = int(time.time()) + expires
return self.a_auth(uri, exp)
class StsToken(object):
"""AssumeRole返回的临时用户密钥
:param str access_key_id: 临时用户的access key id
@ -53,7 +97,8 @@ class StsToken(object):
class AliYunOss(object):
def __init__(self, access_key, secret_key, bucket_name, endpoint, sts_role_arn, is_https, domain_name=None):
def __init__(self, access_key, secret_key, bucket_name, endpoint, sts_role_arn, is_https, domain_name=None,
download_auth_type=1, cnd_auth_key=None):
self.access_key_id = access_key
self.access_key_secret = secret_key
self.bucket_name = bucket_name
@ -62,6 +107,8 @@ class AliYunOss(object):
self.region_id = '-'.join(self.endpoint.split('.')[0].split("-")[1:3])
self.token = StsToken()
self.is_https = is_https
self.download_auth_type = download_auth_type
self.cnd_auth_key = cnd_auth_key
self.domain_name = domain_name
self.get_auth_bucket('init', 900)
@ -91,6 +138,7 @@ class AliYunOss(object):
self.token.expiration = oss2.utils.to_unixtime(j['Credentials']['Expiration'], '%Y-%m-%dT%H:%M:%SZ')
self.token.bucket = self.bucket_name
self.token.endpoint = self.endpoint
logger.info("get aliyun sts token %s" % self.token.__dict__)
return self.token.__dict__
def get_upload_token(self, name, expires=1800):
@ -103,7 +151,7 @@ class AliYunOss(object):
uri = 'https://'
url = self.endpoint
is_cname = False
if self.domain_name:
if self.domain_name and self.download_auth_type == 1:
url = self.domain_name
is_cname = True

@ -5,7 +5,7 @@
# date: 2020/3/23
from api.models import UserInfo
from .aliyunApi import AliYunOss
from .aliyunApi import AliYunOss, AliYunCdn
from .qiniuApi import QiNiuOss
from .localApi import LocalStorage
import json, time, base64
@ -17,9 +17,9 @@ logger = logging.getLogger(__file__)
class Storage(object):
def __init__(self, user, storage_obj=None):
def __init__(self, user, storage_obj=None, use_default_storage=False):
try:
self.storage = self.get_storage(user, storage_obj)
self.storage = self.get_storage(user, storage_obj, use_default_storage)
except Exception as e:
logger.error("get %s storage failed Exception:%s" % (user, e))
self.storage = None
@ -36,7 +36,10 @@ class Storage(object):
if download_val and not force_new:
if download_val.get("time") > now - 60:
return download_val.get("download_url")
if self.storage.__dict__.get("download_auth_type", None) == 2:
cdn_obj = AliYunCdn(self.storage.cnd_auth_key, self.storage.is_https, self.storage.domain_name)
download_url = cdn_obj.get_cdn_download_token(filename, expires)
else:
download_url = self.storage.get_download_url(filename, expires, force_new=True)
cache.set(down_key, {"download_url": download_url, "time": now + expires}, expires)
return download_url
@ -67,11 +70,14 @@ class Storage(object):
except Exception as e:
logger.error("oss upload %s failed Exception %s" % (local_file_full_path, e))
def get_storage(self, user, storage_obj):
def get_storage(self, user, storage_obj, use_default_storage):
if storage_obj:
self.storage_obj = storage_obj
else:
self.storage_obj = user.storage
if use_default_storage:
self.storage_obj = None
if self.storage_obj:
auth = self.get_storage_auth(self.storage_obj)
storage_key = "_".join([CACHE_KEY_TEMPLATE.get('user_storage_key'), user.uid,
@ -79,6 +85,7 @@ class Storage(object):
storage_type = self.storage_obj.storage_type
new_storage_obj = cache.get(storage_key)
if new_storage_obj and not storage_obj:
logger.info("user %s get storage obj cache %s" % (user, new_storage_obj))
return new_storage_obj
else:
if storage_type == 1:
@ -87,17 +94,20 @@ class Storage(object):
new_storage_obj = AliYunOss(**auth)
else:
new_storage_obj = LocalStorage(**auth)
logger.info("user %s make storage obj %s" % (user, new_storage_obj))
new_storage_obj.storage_type = storage_type
cache.set(storage_key, new_storage_obj, 600)
return new_storage_obj
else:
return self.get_default_storage(user, storage_obj)
logger.info("user %s has not storage obj, so get default" % user)
return self.get_default_storage(user, storage_obj, False)
def get_default_storage(self, user, storage_obj):
def get_default_storage(self, user, storage_obj, use_default_storage):
admin_obj = UserInfo.objects.filter(is_superuser=True).order_by('pk').first()
if admin_obj and admin_obj.storage:
return self.get_storage(admin_obj, storage_obj)
logger.info("user %s has not storage obj, from admin "
"get default storage" % user)
return self.get_storage(admin_obj, storage_obj, use_default_storage)
else:
storage_lists = THIRD_PART_CONFIG.get('storage')
for storage in storage_lists:
@ -120,6 +130,8 @@ class Storage(object):
new_storage_obj = LocalStorage(**auth)
new_storage_obj.storage_type = 3
cache.set(storage_key, new_storage_obj, 600)
logger.info("user %s has not storage obj, admin already has not storage obj, from settings "
"get default storage %s" % (user, new_storage_obj))
return new_storage_obj
return None

@ -3,10 +3,10 @@
# project: 5月
# author: liuyu
# date: 2020/5/7
import os, re
import os, re, json, requests
from fir_ser.settings import SUPER_SIGN_ROOT, SERVER_DOMAIN, CAPTCHA_LENGTH, MEDIA_ROOT
from api.models import APPSuperSignUsedInfo, APPToDeveloper, \
UDIDsyncDeveloper, UserInfo
UDIDsyncDeveloper, UserInfo, AppReleaseInfo
from api.utils.app.randomstrings import make_app_uuid
from api.utils.storage.localApi import LocalStorage
from api.utils.storage.storage import Storage
@ -206,14 +206,98 @@ def send_ios_developer_active_status(user_info, msg):
logger.warning("user %s has no email. so %s can't send!" % (user_info, msg))
def delete_local_files(filename, apptype=None):
storage = LocalStorage("localhost", False)
if apptype is not None:
def get_filename_from_apptype(filename, apptype):
if apptype == 0:
filename = filename + '.apk'
else:
filename = filename + '.ipa'
return filename
def delete_local_files(filename, apptype=None):
storage = LocalStorage("localhost", False)
if apptype is not None:
filename = get_filename_from_apptype(filename, apptype)
try:
return storage.del_file(filename)
except Exception as e:
logger.error("delete file %s failed Exception %s" % (filename, e))
def check_storage_additionalparameter(request, res):
data = request.data
try:
extra_parameters = data.get('additionalparameter', '')
if extra_parameters:
if not extra_parameters.get("download_auth_type", None):
extra_parameters['download_auth_type'] = 1
if extra_parameters.get("download_auth_type", None) == 2:
if not extra_parameters.get("cnd_auth_key", None):
logger.error("user %s add new storage failed" % (request.user))
res.msg = "cdn 鉴权KEY 缺失"
res.code = 1006
return False, res
data['additionalparameters'] = json.dumps(extra_parameters)
except Exception as e:
logger.error("user:%s additionalparameters %s dumps failed Exception:%s" % (
request.user, data.get('additionalparameter', ''), e))
return True, res
def change_storage_and_change_head_img(user_obj, new_storage_obj):
migrating_storage_file_data(user_obj, user_obj.head_img, new_storage_obj)
def download_files_form_oss(storage_obj, org_file):
download_url = storage_obj.get_download_url(os.path.basename(org_file), 600, key='check_org_file', force_new=True)
req = requests.get(download_url)
if req.status_code == 200:
logger.info("download file %s success" % org_file)
else:
logger.error("download file %s failed %s" % (org_file, req.content))
return False
with open(org_file + ".check.tmp", "wb") as f:
f.write(req.content)
try:
if os.path.isfile(org_file):
os.remove(org_file)
os.rename(os.path.join(org_file + ".check.tmp"), org_file)
return True
except Exception as e:
logger.error("check org file and mv file %s failed Exception %s" % (org_file, e))
return False
def migrating_storage_file_data(user_obj, filename, new_storage_obj):
local_file_full_path = os.path.join(MEDIA_ROOT, filename)
old_storage_obj = Storage(user_obj)
if not new_storage_obj:
new_storage_obj = Storage(user_obj, None, True)
else:
new_storage_obj = Storage(user_obj, new_storage_obj)
if old_storage_obj.get_storage_type() == 3:
if new_storage_obj.get_storage_type() == 3:
pass
else:
# 本地向云存储上传,并删除本地数据
new_storage_obj.upload_file(local_file_full_path)
delete_local_files(filename)
else:
if new_storage_obj.get_storage_type() == 3:
# 云存储下载 本地,并删除云存储
if download_files_form_oss(old_storage_obj, local_file_full_path):
old_storage_obj.delete_file(filename)
else:
# 云存储互传,先下载本地,然后上传新云存储,删除本地和老云存储
if download_files_form_oss(old_storage_obj, local_file_full_path):
new_storage_obj.upload_file(local_file_full_path)
delete_local_files(filename)
old_storage_obj.delete_file(filename)
def migrating_storage_data(user_obj, new_storage_obj):
for app_release_obj in AppReleaseInfo.objects.filter(app_id__user_id=user_obj).all():
filename = get_filename_from_apptype(app_release_obj.release_id, app_release_obj.release_type)
migrating_storage_file_data(user_obj, filename, new_storage_obj)

@ -8,10 +8,10 @@ from rest_framework.views import APIView
from api.utils.response import BaseResponse
from api.utils.auth import ExpiringTokenAuthentication, StoragePermission
from rest_framework.response import Response
import json
from api.utils.storage.caches import del_cache_storage
from api.models import AppStorage, UserInfo
from api.utils.utils import upload_oss_default_head_img
from api.utils.utils import upload_oss_default_head_img, check_storage_additionalparameter, \
change_storage_and_change_head_img
from api.utils.serializer import StorageSerializer
import logging
@ -59,12 +59,9 @@ class StorageView(APIView):
res = BaseResponse()
data = request.data
logger.info("user %s add new storage data:%s" % (request.user, data))
try:
data['additionalparameters'] = json.dumps(data.get('additionalparameter', ''))
except Exception as e:
logger.error("user:%s additionalparameters %s dumps failed Exception:%s" % (
request.user, data.get('additionalparameter', ''), e))
status, msg = check_storage_additionalparameter(request, res)
if not status:
return Response(msg.dict)
serializer = StorageSerializer(data=data, context={'user_obj': request.user})
if serializer.is_valid():
storage_obj = serializer.save()
@ -96,10 +93,17 @@ class StorageView(APIView):
if use_storage_id:
try:
if use_storage_id == -1:
change_storage_and_change_head_img(request.user, None)
UserInfo.objects.filter(pk=request.user.pk).update(storage=None)
else:
new_storage_obj = AppStorage.objects.filter(pk=use_storage_id).first()
change_storage_and_change_head_img(request.user, new_storage_obj)
UserInfo.objects.filter(pk=request.user.pk).update(storage_id=use_storage_id)
del_cache_storage(request.user)
# change_storage_and_change_head_img(request.user, old_storage_obj, new_storage_obj)
except Exception as e:
logger.error("update user %s storage failed Exception:%s" % (request.user, e))
res.code = 1006
@ -112,13 +116,9 @@ class StorageView(APIView):
res.msg = '存储正在使用中,无法修改'
res.code = 1007
return Response(res.dict)
try:
data['additionalparameters'] = json.dumps(data.get('additionalparameter', ''))
except Exception as e:
del data['additionalparameters']
logger.error("user:%s additionalparameters %s dumps failed Exception:%s" % (
request.user, data.get('additionalparameter', ''), e))
status, msg = check_storage_additionalparameter(request, res)
if not status:
return Response(msg.dict)
storage_obj = AppStorage.objects.filter(id=storage_id, user_id=request.user).first()
storage_obj_bak = AppStorage.objects.filter(id=storage_id, user_id=request.user).first()
serializer = StorageSerializer(instance=storage_obj, data=data, context={'user_obj': request.user},

@ -211,7 +211,8 @@ THIRD_PART_CONFIG = {
'sts_role_arn': 'ap-sage',
'endpoint': 'oss-cn-beijing.aliyuncs.com',
'is_https': True,
'domain_name': 'aoud.xin'
'domain_name': 'aoud.xin',
'download_auth_type': 1 # 1:oss 2:cdn
},
'active': False
},

Loading…
Cancel
Save