k
/
flyapps
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

服务器需要部署阿里云，服务器和oss存储内网互通

Browse Source
dependabot/npm_and_yarn/fir_admin/eventsource-1.1.1
nineven 3 years ago
parent b191cbae6b
commit e7858a840e
7 changed files with 93 additions and 67 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 4
      fir_client/src/components/user/FirUserStorage.vue
  2. 2
      fir_client/vue.config.js
  3. 14
      fir_ser/api/utils/modelutils.py
  4. 45
      fir_ser/api/utils/utils.py
  5. 19
      fir_ser/common/libs/storage/aliyunApi.py
  6. 24
      fir_ser/common/utils/storage.py
  7. 52
      fir_ser/xsign/utils/supersignutils.py

4
fir_client/src/components/user/FirUserStorage.vue
Unescape View File

@ -263,7 +263,7 @@
</div> </div>
<el-divider/> <el-divider/>
<el-form ref="storageinfoform" :model="storageinfo" <el-form ref="storageinfoform" :model="storageinfo"
label-width="80px" style="width: 39%;margin:0 auto;"> label-width="80px" style="width: 50%;margin:0 auto;">
<el-form-item label="存储最大容量" label-width="110px" style="text-align: left"> <el-form-item label="存储最大容量" label-width="110px" style="text-align: left">
<el-tag>存储最大容量空间 {{ diskSize(storageinfo.max_storage_capacity) }}</el-tag> <el-tag>存储最大容量空间 {{ diskSize(storageinfo.max_storage_capacity) }}</el-tag>
@ -1231,7 +1231,7 @@ export default {
}, filters: {}, }, filters: {},
watch: { watch: {
watchObj: function () { watchObj: function () {
if (this.editstorageinfo.bucket_name && this.editstorageinfo.download_auth_type === 1) { if (this.editstorageinfo.bucket_name && this.editstorageinfo.download_auth_type === 1 && this.isaddflag) {
this.editstorageinfo.domain_name = this.editstorageinfo.bucket_name + '.' + this.editstorageinfo.endpoint.replace("-internal", "") this.editstorageinfo.domain_name = this.editstorageinfo.bucket_name + '.' + this.editstorageinfo.endpoint.replace("-internal", "")
} }
} }

2
fir_client/vue.config.js
Unescape View File

@ -82,7 +82,7 @@ if (page) {
} }
} }
const version='2.3.6'; const version='2.3.8';
const pro_base_env = { const pro_base_env = {
baseUrl: 'https://flyapps.cn', baseUrl: 'https://flyapps.cn',

14
fir_ser/api/utils/modelutils.py
Unescape View File

@ -13,7 +13,7 @@ from django.db.models import Count
from rest_framework.pagination import PageNumberPagination from rest_framework.pagination import PageNumberPagination
from api.models import AppReleaseInfo, UserDomainInfo, DomainCnameInfo, UserAdDisplayInfo, RemoteClientInfo, \ from api.models import AppReleaseInfo, UserDomainInfo, DomainCnameInfo, UserAdDisplayInfo, RemoteClientInfo, \
AppBundleIdBlackList, NotifyReceiver, WeChatInfo, AppDownloadToken, Apps, StorageShareInfo, UserInfo AppBundleIdBlackList, NotifyReceiver, WeChatInfo, AppDownloadToken, Apps, StorageShareInfo, UserInfo, AppStorage
from common.base.baseutils import get_server_domain_from_request, get_user_default_domain_name, get_real_ip_address, \ from common.base.baseutils import get_server_domain_from_request, get_user_default_domain_name, get_real_ip_address, \
get_origin_domain_name get_origin_domain_name
from common.base.magic import MagicCacheData from common.base.magic import MagicCacheData
@ -295,3 +295,15 @@ def get_user_storage_capacity(user_obj):
storage_capacity = Config.STORAGE_FREE_CAPACITY storage_capacity = Config.STORAGE_FREE_CAPACITY
return storage_capacity return storage_capacity
def get_user_storage_obj(user_obj, default=False):
storage = user_obj.storage
if storage and not default:
return storage
for storage in Config.STORAGE:
if storage.get("active", None):
storage_type = storage.get('type', None)
auth = storage.get('auth', {})
storage = AppStorage(user_id=user_obj, storage_type=storage_type, **auth)
return storage

45
fir_ser/api/utils/utils.py
Unescape View File

@ -11,7 +11,7 @@ import os
from django.core.cache import cache from django.core.cache import cache
from api.models import UserInfo, AppReleaseInfo, AppScreenShot, Token, UserAdDisplayInfo from api.models import UserInfo, AppReleaseInfo, AppScreenShot, Token, UserAdDisplayInfo
from api.utils.modelutils import get_app_d_count_by_app_id from api.utils.modelutils import get_app_d_count_by_app_id, get_user_storage_obj
from api.utils.signalutils import run_xsign_migrate_data, run_xsign_clean_data from api.utils.signalutils import run_xsign_migrate_data, run_xsign_clean_data
from common.base.baseutils import get_real_ip_address from common.base.baseutils import get_real_ip_address
from common.cache.storage import UserTokenCache from common.cache.storage import UserTokenCache
@ -108,40 +108,57 @@ def check_storage_is_new_storage(user_obj, new_storage_obj):
def migrating_storage_file_data(user_obj, filename, new_storage_obj, clean_old_data=True): def migrating_storage_file_data(user_obj, filename, new_storage_obj, clean_old_data=True):
local_file_full_path = os.path.join(MEDIA_ROOT, filename) local_file_full_path = os.path.join(MEDIA_ROOT, filename)
old_storage_obj = Storage(user_obj) if filename == 'head_img.jpeg':
return True
storage_obj = get_user_storage_obj(user_obj)
if storage_obj.storage_type == 2:
storage_obj.download_auth_type = 2
old_oss_storage_obj = Storage(user_obj, storage_obj, prefix='migrate_storage')
else:
old_oss_storage_obj = Storage(user_obj, prefix='migrate_storage')
if not new_storage_obj: if not new_storage_obj:
new_storage_obj = Storage(user_obj, None, True) storage_obj = get_user_storage_obj(user_obj, default=True)
if storage_obj.storage_type == 2:
storage_obj.download_auth_type = 2
new_oss_storage_obj = Storage(user_obj, storage_obj, prefix='migrate_storage')
else:
new_oss_storage_obj = Storage(user_obj, None, True, prefix='migrate_storage')
else: else:
new_storage_obj = Storage(user_obj, new_storage_obj) if new_storage_obj.storage_type == 2:
new_storage_obj.download_auth_type = 2
new_oss_storage_obj = Storage(user_obj, new_storage_obj, prefix='migrate_storage')
else:
new_oss_storage_obj = Storage(user_obj, new_storage_obj, prefix='migrate_storage')
if old_storage_obj.get_storage_uuid() == new_storage_obj.get_storage_uuid(): if old_oss_storage_obj.get_storage_uuid() == new_oss_storage_obj.get_storage_uuid():
# 同一个存储，无需迁移数据 # 同一个存储，无需迁移数据
return True return True
if old_storage_obj.get_storage_type() == 3: if old_oss_storage_obj.get_storage_type() == 3:
if new_storage_obj.get_storage_type() == 3: if new_oss_storage_obj.get_storage_type() == 3:
# 都是本地存储，无需操作 # 都是本地存储，无需操作
return True return True
else: else:
# 本地向云存储上传,并删除本地数据 # 本地向云存储上传,并删除本地数据
res = new_storage_obj.upload_file(local_file_full_path) res = new_oss_storage_obj.upload_file(local_file_full_path)
if clean_old_data: if clean_old_data:
return delete_local_files(filename) return delete_local_files(filename)
return res return res
else: else:
if new_storage_obj.get_storage_type() == 3: if new_oss_storage_obj.get_storage_type() == 3:
# 云存储下载 本地，并删除云存储 # 云存储下载 本地，并删除云存储
if download_files_form_oss(old_storage_obj, local_file_full_path, True): if download_files_form_oss(old_oss_storage_obj, local_file_full_path, True):
if clean_old_data: if clean_old_data:
return old_storage_obj.delete_file(filename) return old_oss_storage_obj.delete_file(filename)
return True return True
else: else:
# 云存储互传，先下载本地，然后上传新云存储，删除本地和老云存储 # 云存储互传，先下载本地，然后上传新云存储，删除本地和老云存储
if download_files_form_oss(old_storage_obj, local_file_full_path, True): if download_files_form_oss(old_oss_storage_obj, local_file_full_path, True):
res1 = new_storage_obj.upload_file(local_file_full_path) res1 = new_oss_storage_obj.upload_file(local_file_full_path)
res2 = delete_local_files(filename) res2 = delete_local_files(filename)
if clean_old_data: if clean_old_data:
return old_storage_obj.delete_file(filename) return old_oss_storage_obj.delete_file(filename)
return res1 and res2 return res1 and res2

19
fir_ser/common/libs/storage/aliyunApi.py
Unescape View File

@ -200,23 +200,6 @@ class AliYunOss(object):
def upload_file(self, local_file_full_path): def upload_file(self, local_file_full_path):
return self.multipart_upload_file(local_file_full_path) return self.multipart_upload_file(local_file_full_path)
if os.path.isfile(local_file_full_path):
filename = os.path.basename(local_file_full_path)
headers = {
'Content-Disposition': f'attachment; filename="{filename}"',
'Cache-Control': ''
}
self.bucket.put_object_from_file(filename, local_file_full_path, headers)
# with open(local_file_full_path, 'rb') as fileobj:
# # Seek方法用于指定从第1000个字节位置开始读写。上传时会从您指定的第1000个字节位置开始上传，直到文件结束。
# fileobj.seek(1000, os.SEEK_SET)
# # Tell方法用于返回当前位置。
# # current = fileobj.tell()
# self.bucket.put_object(os.path.basename(local_file_full_path), fileobj)
return True
else:
logger.error(f"file {local_file_full_path} is not file")
def download_file(self, name, local_file_full_path): def download_file(self, name, local_file_full_path):
dir_path = os.path.dirname(local_file_full_path) dir_path = os.path.dirname(local_file_full_path)
if not os.path.exists(dir_path): if not os.path.exists(dir_path):
@ -236,7 +219,7 @@ class AliYunOss(object):
if os.path.isfile(local_file_full_path): if os.path.isfile(local_file_full_path):
total_size = os.path.getsize(local_file_full_path) total_size = os.path.getsize(local_file_full_path)
# determine_part_size方法用于确定分片大小。 # determine_part_size方法用于确定分片大小。
part_size = determine_part_size(total_size, preferred_size=1024 * 1024 * 10) part_size = determine_part_size(total_size, preferred_size=1024 * 1024 * 100)
filename = os.path.basename(local_file_full_path) filename = os.path.basename(local_file_full_path)
headers = { headers = {
'Content-Disposition': 'attachment; filename="%s"' % get_filename_form_file(filename).encode( 'Content-Disposition': 'attachment; filename="%s"' % get_filename_form_file(filename).encode(

24
fir_ser/common/utils/storage.py
Unescape View File

@ -25,9 +25,9 @@ def get_storage_auth(storage_obj):
@run_function_by_locker() @run_function_by_locker()
def get_storage(user, assigned_storage_obj, use_default_storage): def get_oss_storage(user, assigned_storage_obj, use_default_storage, prefix):
if use_default_storage: if use_default_storage:
return get_storage_form_conf(user) return get_storage_form_conf(user, prefix)
if assigned_storage_obj: if assigned_storage_obj:
storage_obj = assigned_storage_obj storage_obj = assigned_storage_obj
else: else:
@ -36,7 +36,7 @@ def get_storage(user, assigned_storage_obj, use_default_storage):
if storage_obj: if storage_obj:
auth = get_storage_auth(storage_obj) auth = get_storage_auth(storage_obj)
storage_type = storage_obj.storage_type storage_type = storage_obj.storage_type
storage_cache = CloudStorageCache(auth, user.uid) storage_cache = CloudStorageCache(auth, f'{user.uid}_{prefix}')
storage_key, new_storage_obj = storage_cache.get_storage_key_and_cache() storage_key, new_storage_obj = storage_cache.get_storage_key_and_cache()
if new_storage_obj and not assigned_storage_obj: if new_storage_obj and not assigned_storage_obj:
@ -56,16 +56,20 @@ def get_storage(user, assigned_storage_obj, use_default_storage):
else: else:
logger.info(f"user {user} has not storage obj, so get default") logger.info(f"user {user} has not storage obj, so get default")
# 不需要管理存储，直接从配置文件获取默认存储 # 不需要管理存储，直接从配置文件获取默认存储
return get_storage_form_conf(user) return get_storage_form_conf(user, prefix)
class Storage(object): class Storage(object):
def __init__(self, user, assigned_storage_obj=None, use_default_storage=False): def __init__(self, user, assigned_storage_obj=None, use_default_storage=False, prefix='default'):
self.user = user
self.prefix = prefix
self.assigned_storage_obj = assigned_storage_obj
self.use_default_storage = use_default_storage
try: try:
locker = { locker = {
'locker_key': f"make_storage_cache_{user.uid}", 'locker_key': f"make_storage_cache_{user.uid}_{prefix}",
"timeout": 60 * 5, "blocking_timeout": 6} "timeout": 60, "blocking_timeout": 6}
self.storage = get_storage(user, assigned_storage_obj, use_default_storage, locker=locker) self.storage = get_oss_storage(user, assigned_storage_obj, use_default_storage, prefix, locker=locker)
except Exception as e: except Exception as e:
logger.error(f"get {user} storage failed Exception:{e}") logger.error(f"get {user} storage failed Exception:{e}")
self.storage = None self.storage = None
@ -167,12 +171,12 @@ def get_local_storage(clean_cache=False):
return new_storage_obj return new_storage_obj
def get_storage_form_conf(user): def get_storage_form_conf(user, prefix):
for storage in Config.STORAGE: for storage in Config.STORAGE:
if storage.get("active", None): if storage.get("active", None):
storage_type = storage.get('type', None) storage_type = storage.get('type', None)
auth = storage.get('auth', {}) auth = storage.get('auth', {})
storage_cache = CloudStorageCache(auth, 'default') storage_cache = CloudStorageCache(auth, f'system_{prefix}')
storage_key, new_storage_obj = storage_cache.get_storage_key_and_cache() storage_key, new_storage_obj = storage_cache.get_storage_key_and_cache()
if new_storage_obj: if new_storage_obj:
logger.info(f"user {user} get default storage {storage_key} obj cache {new_storage_obj} ") logger.info(f"user {user} get default storage {storage_key} obj cache {new_storage_obj} ")

52
fir_ser/xsign/utils/supersignutils.py
Unescape View File

@ -17,6 +17,7 @@ from django.core.cache import cache
from django.db.models import Count, F from django.db.models import Count, F
from api.models import UserInfo, AppReleaseInfo, Apps from api.models import UserInfo, AppReleaseInfo, Apps
from api.utils.modelutils import get_user_storage_obj
from api.utils.response import BaseResponse from api.utils.response import BaseResponse
from api.utils.utils import delete_local_files, download_files_form_oss from api.utils.utils import delete_local_files, download_files_form_oss
from common.base.baseutils import file_format_path, delete_app_profile_file, get_profile_full_path, format_apple_date, \ from common.base.baseutils import file_format_path, delete_app_profile_file, get_profile_full_path, format_apple_date, \
@ -44,6 +45,16 @@ from xsign.utils.utils import delete_app_to_dev_and_file
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def get_sign_oss_storage(user_obj):
storage_obj = get_user_storage_obj(user_obj)
if storage_obj.storage_type == 2:
storage_obj.download_auth_type = 2
oss_storage_obj = Storage(user_obj, storage_obj, prefix='super_sign')
else:
oss_storage_obj = Storage(user_obj, prefix='super_sign')
return oss_storage_obj
def check_org_file(user_obj, org_file): def check_org_file(user_obj, org_file):
if not os.path.isdir(os.path.dirname(org_file)): if not os.path.isdir(os.path.dirname(org_file)):
os.makedirs(os.path.dirname(org_file)) os.makedirs(os.path.dirname(org_file))
@ -51,8 +62,7 @@ def check_org_file(user_obj, org_file):
if os.path.isfile(org_file): if os.path.isfile(org_file):
return True return True
storage_obj = Storage(user_obj) return download_files_form_oss(get_sign_oss_storage(user_obj), org_file)
return download_files_form_oss(storage_obj, org_file)
def get_abnormal_queryset(user_obj, udid): def get_abnormal_queryset(user_obj, udid):
@ -581,26 +591,26 @@ class IosUtils(object):
@staticmethod @staticmethod
def update_sign_file_name(user_obj, app_obj, developer_obj_id, release_obj, random_file_name): def update_sign_file_name(user_obj, app_obj, developer_obj_id, release_obj, random_file_name):
apptodev_obj = APPToDeveloper.objects.filter(developerid_id=developer_obj_id, app_id=app_obj).first() apptodev_obj = APPToDeveloper.objects.filter(developerid_id=developer_obj_id, app_id=app_obj).first()
storage_obj = Storage(user_obj) storage_obj = get_sign_oss_storage(user_obj)
logger.info(f"update sign file end, now upload {storage_obj.storage} {random_file_name}.ipa file") logger.info(f"update sign file end, now upload {storage_obj.storage} {random_file_name}.ipa file")
if storage_obj.upload_file(os.path.join(MEDIA_ROOT, random_file_name + ".ipa")): with cache.lock(f"upload_files_form_oss_{random_file_name}", timeout=60 * 30):
if apptodev_obj: if storage_obj.upload_file(os.path.join(MEDIA_ROOT, random_file_name + ".ipa")):
delete_local_files(apptodev_obj.binary_file + ".ipa") if apptodev_obj:
storage_obj.delete_file(apptodev_obj.binary_file + ".ipa") delete_local_files(apptodev_obj.binary_file + ".ipa")
apptodev_obj.binary_file = random_file_name storage_obj.delete_file(apptodev_obj.binary_file + ".ipa")
old_release_file = apptodev_obj.release_file apptodev_obj.binary_file = random_file_name
apptodev_obj.release_file = release_obj.release_id old_release_file = apptodev_obj.release_file
apptodev_obj.save(update_fields=['binary_file', 'release_file', 'updated_time']) apptodev_obj.release_file = release_obj.release_id
if storage_obj.get_storage_type() in [1, 2] and old_release_file != release_obj.release_id: apptodev_obj.save(update_fields=['binary_file', 'release_file', 'updated_time'])
logger.warning(f"update sign file , now clean ole {old_release_file}.ipa file") if storage_obj.get_storage_type() in [1, 2] and old_release_file != release_obj.release_id:
delete_local_files(old_release_file + ".ipa") logger.warning(f"update sign file , now clean ole {old_release_file}.ipa file")
else: delete_local_files(old_release_file + ".ipa")
APPToDeveloper.objects.create(developerid_id=developer_obj_id, app_id=app_obj, else:
binary_file=random_file_name, release_file=release_obj.release_id) APPToDeveloper.objects.create(developerid_id=developer_obj_id, app_id=app_obj,
if storage_obj.get_storage_type() in [1, 2]: binary_file=random_file_name, release_file=release_obj.release_id)
delete_local_files(random_file_name + ".ipa") if storage_obj.get_storage_type() in [1, 2]:
return True delete_local_files(random_file_name + ".ipa")
return True
@staticmethod @staticmethod
def update_sign_data(user_obj, app_obj, developer_obj_id, random_file_name, release_obj, udid_list): def update_sign_data(user_obj, app_obj, developer_obj_id, random_file_name, release_obj, udid_list):

Write Preview
Loading…
Cancel
Save