修复存储切换下载token过期问题

增加私有云存储下载点计算
修复配置文件获取异常
修改云存储切换为任务执行
dependabot/npm_and_yarn/fir_admin/eventsource-1.1.1
nineven 3 years ago
parent e7858a840e
commit 5f967a9b52
  1. 84
      fir_client/src/components/user/FirUserStorage.vue
  2. 11
      fir_ser/admin/views/storage.py
  3. 38
      fir_ser/api/base_views.py
  4. 5
      fir_ser/api/migrations/0006_auto_20220513_0843.py
  5. 2
      fir_ser/api/models.py
  6. 30
      fir_ser/api/tasks.py
  7. 15
      fir_ser/api/utils/ctasks.py
  8. 4
      fir_ser/api/utils/modelutils.py
  9. 8
      fir_ser/api/utils/serializer.py
  10. 22
      fir_ser/api/utils/utils.py
  11. 37
      fir_ser/api/views/storage.py
  12. 8
      fir_ser/common/cache/invalid.py
  13. 6
      fir_ser/common/cache/storage.py
  14. 26
      fir_ser/common/core/sysconfig.py
  15. 16
      fir_ser/common/utils/caches.py
  16. 4
      fir_ser/common/utils/download.py
  17. 12
      fir_ser/config.py
  18. 1
      fir_ser/fir_ser/settings.py
  19. 6
      fir_ser/xsign/tasks.py

@ -193,8 +193,8 @@
</el-dialog> </el-dialog>
<el-tabs v-model="activeName" tab-position="top" type="border-card" @tab-click="handleClick"> <el-tabs v-model="activeName" tab-position="top" type="border-card" @tab-click="handleClick">
<el-tab-pane label="存储选择" name="change">' <el-tab-pane label="存储选择" name="change">
<el-row :gutter="24"> <el-row :gutter="24" style="margin-top: 15px">
<el-col :span="14"> <el-col :span="14">
存储选择 存储选择
<el-select v-model="use_storage_id" :placeholder="selectlabel" filterable style="width: 400px" <el-select v-model="use_storage_id" :placeholder="selectlabel" filterable style="width: 400px"
@ -235,8 +235,6 @@
type="line"/> type="line"/>
</el-col> </el-col>
</el-row> </el-row>
<div v-if="use_storage_id!==org_storage_id" style="margin-top: 20px"> <div v-if="use_storage_id!==org_storage_id" style="margin-top: 20px">
<el-button style="margin-left: 10px" <el-button style="margin-left: 10px"
type="info" type="info"
@ -262,7 +260,7 @@
<!-- </el-button>--> <!-- </el-button>-->
</div> </div>
<el-divider/> <el-divider/>
<el-form ref="storageinfoform" :model="storageinfo" <el-form v-if="storageinfo" ref="storageinfoform" :model="storageinfo"
label-width="80px" style="width: 50%;margin:0 auto;"> label-width="80px" style="width: 50%;margin:0 auto;">
<el-form-item label="存储最大容量" label-width="110px" style="text-align: left"> <el-form-item label="存储最大容量" label-width="110px" style="text-align: left">
@ -666,9 +664,60 @@
</el-table> </el-table>
</el-tab-pane> </el-tab-pane>
<el-tab-pane label="存储设置" name="setting" style="text-align: center"> <el-tab-pane label="存储设置" name="setting" style="text-align: center">
<el-tag style="margin: 20px 0"> 应用版本数设置当应用历史版本超过该限制将会自动清理较老的版本</el-tag> <div v-if="migrate_progress && migrate_progress.s_time" style="text-align: left">
<el-card>
<div slot="header">
<el-tag type="danger">存储迁移中请耐心等待</el-tag>
</div>
<el-row :gutter="24">
<el-col :span="4">存储迁移进度</el-col>
<el-col :span="20">
<el-progress
:color="storage_usedColor"
:percentage="parseInt(migrate_progress.migrate_num*100/migrate_progress.release_num)"
:stroke-width="18"
:text-inside="true" status="success" style="width: 80%"
type="line"/>
</el-col>
</el-row>
<el-row :gutter="24" style="margin-top: 30px">
<el-col :span="4">总迁移应用数</el-col>
<el-col :span="20">
<el-tag>{{ migrate_progress.release_num }}</el-tag>
</el-col>
</el-row>
<el-row :gutter="24" style="margin-top: 20px">
<el-col :span="4">已经迁移成功应用数</el-col>
<el-col :span="20">
<el-tag>{{ migrate_progress.migrate_num }}</el-tag>
</el-col>
</el-row>
<el-row :gutter="24" style="margin-top: 20px">
<el-col :span="4">存储迁移开始时间</el-col>
<el-col :span="20">
<el-tag>{{ getFormatDate(migrate_progress.s_time) }}</el-tag>
</el-col>
</el-row>
</el-card>
<el-divider/>
</div>
<div style="margin: auto;width: 700px;height: 100%"> <div style="margin: auto;width: 700px;height: 100%">
<el-form ref="form" :model="storage_config" label-width="180px"> <el-form ref="form" :model="storage_config" label-width="180px">
<el-form-item v-if="storage_config.storage_status" label="存储迁移状态" style="text-align: left">
<el-tag v-if="!storage_config.storage_status">未迁移状态正常</el-tag>
<div v-else>
<el-tag>迁移中迁移时间 {{ getFormatDate(storage_config.storage_status) }}</el-tag>
<el-tag type="warning">若存储状态长时间处于迁移中可能是迁移卡死了可以尝试
<el-button plain size="mini" type="text" @click="cancelStorage">解除迁移锁</el-button>
进行解除
</el-tag>
</div>
</el-form-item>
<div v-if="!(migrate_progress && migrate_progress.s_time)">
<el-tag style="margin: 20px 0"> 应用版本数设置当应用历史版本超过该限制将会自动清理较老的版本</el-tag>
<el-form-item label="应用历史版本数" style="text-align: left"> <el-form-item label="应用历史版本数" style="text-align: left">
<el-input-number v-model="storage_config.user_history_limit" :min="1" <el-input-number v-model="storage_config.user_history_limit" :min="1"
style="width: 300px;margin: 0 10px"></el-input-number> style="width: 300px;margin: 0 10px"></el-input-number>
@ -683,18 +732,6 @@
{{ diskSize(storage_config.user_max_storage_capacity - storage_config.user_used_storage_capacity) }} {{ diskSize(storage_config.user_max_storage_capacity - storage_config.user_used_storage_capacity) }}
</el-tag> </el-tag>
</el-form-item> </el-form-item>
<el-form-item label="存储迁移状态" style="text-align: left">
<el-tag v-if="!storage_config.storage_status">未迁移状态正常</el-tag>
<div v-else>
<el-tag>迁移中迁移时间 {{ getFormatDate(storage_config.storage_status) }}</el-tag>
<el-tag type="warning">若存储状态长时间处于迁移中可能是迁移卡死了可以尝试
<el-button plain size="mini" type="text" @click="cancelStorage">解除迁移锁</el-button>
进行解除
</el-tag>
</div>
</el-form-item>
<el-form-item label="清理所有应用数据" style="text-align: left"> <el-form-item label="清理所有应用数据" style="text-align: left">
<el-button style="margin-left: 10px" <el-button style="margin-left: 10px"
type="danger" type="danger"
@ -709,6 +746,8 @@
清理应用历史版本数据只保留最新版本数据 清理应用历史版本数据只保留最新版本数据
</el-button> </el-button>
</el-form-item> </el-form-item>
</div>
</el-form> </el-form>
</div> </div>
@ -761,6 +800,7 @@ export default {
operatestatus: '', operatestatus: '',
dialogstorageVisible: false, dialogstorageVisible: false,
editstorageinfo: {}, editstorageinfo: {},
migrate_progress: {},
selectlabel: "", selectlabel: "",
storageinfo: {'used_number': 0, 'max_storage_capacity': 1}, storageinfo: {'used_number': 0, 'max_storage_capacity': 1},
storage_list: [], storage_list: [],
@ -993,6 +1033,13 @@ export default {
this.fstorage_lists = data.storage_group_list; this.fstorage_lists = data.storage_group_list;
this.org_storage_id = this.use_storage_id = data.storage; this.org_storage_id = this.use_storage_id = data.storage;
this.storage_list = data.storage_list; this.storage_list = data.storage_list;
this.migrate_progress = data.migrate_progress;
if (this.migrate_progress && this.migrate_progress.s_time) {
this.activeName = 'setting'
this.$message.warning("存储迁移中,请耐心等待")
this.refreshactiveFun()
return
}
this.getstorageinfobyid(this.use_storage_id) this.getstorageinfobyid(this.use_storage_id)
} else { } else {
this.$message.error('存储类别获取失败,' + data.msg); this.$message.error('存储类别获取失败,' + data.msg);
@ -1207,6 +1254,7 @@ export default {
configStorageData(res => { configStorageData(res => {
if (res.code === 1000) { if (res.code === 1000) {
this.storage_config = res.data this.storage_config = res.data
this.migrate_progress = res.migrate_progress
} else { } else {
this.$message.error("获取数据失败了," + res.msg) this.$message.error("获取数据失败了," + res.msg)
} }

@ -14,8 +14,8 @@ from rest_framework.views import APIView
from admin.utils.serializer import AdminStorageSerializer from admin.utils.serializer import AdminStorageSerializer
from admin.utils.utils import AppsPageNumber, BaseModelSet from admin.utils.utils import AppsPageNumber, BaseModelSet
from api.base_views import storage_change
from api.models import UserInfo, AppStorage from api.models import UserInfo, AppStorage
from api.tasks import migrate_storage_job
from common.base.baseutils import format_storage_selection from common.base.baseutils import format_storage_selection
from common.core.auth import AdminTokenAuthentication from common.core.auth import AdminTokenAuthentication
from common.core.response import ApiResponse from common.core.response import ApiResponse
@ -64,8 +64,13 @@ class StorageChangeView(APIView):
use_storage_id = data.get("use_storage_id", None) use_storage_id = data.get("use_storage_id", None)
force = data.get("force", None) force = data.get("force", None)
if use_storage_id: if use_storage_id:
if not storage_change(use_storage_id, obj, force): c_task = migrate_storage_job.apply_async((use_storage_id, obj.pk, force))
ApiResponse(code=1006, msg="修改失败") msg = c_task.get(propagate=False)
logger.info(f"run migrate storage task {obj} msg:{msg}")
if c_task.successful():
c_task.forget()
else:
ApiResponse(code=1006, msg=msg)
return ApiResponse() return ApiResponse()
return ApiResponse(code=1004, msg="数据校验失败") return ApiResponse(code=1004, msg="数据校验失败")

@ -65,44 +65,6 @@ def app_screen_delete(screen_id, apps_obj, storage):
del_cache_response_by_short(apps_obj.app_id) del_cache_response_by_short(apps_obj.app_id)
def app_release_delete(app_obj, release_id, storage):
res = BaseResponse()
user_obj = app_obj.user_id
if app_obj:
apprelease_count = AppReleaseInfo.objects.filter(app_id=app_obj).values("release_id").count()
appreleaseobj = AppReleaseInfo.objects.filter(app_id=app_obj, release_id=release_id).first()
if not appreleaseobj.is_master:
logger.info(f"delete app release {appreleaseobj}")
storage.delete_file(appreleaseobj.release_id, appreleaseobj.release_type)
delete_local_files(appreleaseobj.release_id, appreleaseobj.release_type)
storage.delete_file(appreleaseobj.icon_url)
appreleaseobj.delete()
elif appreleaseobj.is_master and apprelease_count < 2:
logger.info(f"delete app master release {appreleaseobj} and clean app {app_obj} ")
count = APPToDeveloper.objects.filter(app_id=app_obj).count()
if app_obj.issupersign or count > 0:
logger.info(f"app_id:{app_obj.app_id} is supersign ,delete this app need clean IOS developer")
IosUtils.clean_app_by_user_obj(app_obj)
storage.delete_file(appreleaseobj.release_id, appreleaseobj.release_type)
delete_local_files(appreleaseobj.release_id, appreleaseobj.release_type)
storage.delete_file(appreleaseobj.icon_url)
del_cache_by_delete_app(app_obj.app_id)
appreleaseobj.delete()
delete_app_screenshots_files(storage, app_obj)
has_combo = app_obj.has_combo
if has_combo:
app_obj.has_combo.has_combo = None
app_obj.delete()
else:
pass
del_cache_response_by_short(app_obj.app_id)
return res
def check_storage_ok(user_obj, new_storage_obj): def check_storage_ok(user_obj, new_storage_obj):
if migrating_storage_file_data(user_obj, 'head_img.jpeg', new_storage_obj, False): if migrating_storage_file_data(user_obj, 'head_img.jpeg', new_storage_obj, False):
return True return True

@ -21,6 +21,11 @@ class Migration(migrations.Migration):
name='storage_capacity', name='storage_capacity',
field=models.BigIntegerField(default=0, verbose_name='存储容量,单位byte'), field=models.BigIntegerField(default=0, verbose_name='存储容量,单位byte'),
), ),
migrations.AlterField(
model_name='userinfo',
name='download_times',
field=models.PositiveBigIntegerField(default=0, verbose_name='可用下载点次数,需要用户充值'),
),
migrations.AlterField( migrations.AlterField(
model_name='userinfo', model_name='userinfo',
name='storage', name='storage',

@ -39,7 +39,7 @@ class UserInfo(AbstractUser):
memo = models.TextField('备注', blank=True, null=True, default=None, ) memo = models.TextField('备注', blank=True, null=True, default=None, )
date_joined = models.DateTimeField(auto_now_add=True, verbose_name="注册时间") date_joined = models.DateTimeField(auto_now_add=True, verbose_name="注册时间")
download_times = models.PositiveIntegerField(default=0, verbose_name="可用下载次数,需要用户充值") download_times = models.PositiveBigIntegerField(default=0, verbose_name="可用下载次数,需要用户充值")
storage_capacity = models.BigIntegerField(default=0, verbose_name="存储容量,单位byte") storage_capacity = models.BigIntegerField(default=0, verbose_name="存储容量,单位byte")
all_download_times = models.BigIntegerField(default=0, verbose_name="总共下载次数") all_download_times = models.BigIntegerField(default=0, verbose_name="总共下载次数")
default_domain_name = models.ForeignKey(to="DomainCnameInfo", verbose_name="默认下载页域名", on_delete=models.CASCADE) default_domain_name = models.ForeignKey(to="DomainCnameInfo", verbose_name="默认下载页域名", on_delete=models.CASCADE)

@ -9,7 +9,8 @@ import logging
from captcha.models import CaptchaStore from captcha.models import CaptchaStore
from api.utils.ctasks import sync_download_times, auto_clean_upload_tmp_file, auto_clean_remote_client_log, \ from api.utils.ctasks import sync_download_times, auto_clean_upload_tmp_file, auto_clean_remote_client_log, \
notify_check_user_download_times, notify_check_apple_developer_devices, notify_check_apple_developer_cert notify_check_user_download_times, notify_check_apple_developer_devices, notify_check_apple_developer_cert, \
migrate_user_oss_storage
from api.views.login import get_login_type from api.views.login import get_login_type
from common.core.sysconfig import Config, ConfigCacheBase from common.core.sysconfig import Config, ConfigCacheBase
from common.libs.geetest.geetest_utils import check_bypass_status from common.libs.geetest.geetest_utils import check_bypass_status
@ -31,57 +32,62 @@ def start_api_sever_do_clean():
def clean_config_cache(key): def clean_config_cache(key):
ConfigCacheBase().invalid_config_cache(key) return ConfigCacheBase().invalid_config_cache(key)
@app.task @app.task
def sync_download_times_job(): def sync_download_times_job():
sync_download_times() return sync_download_times()
@app.task @app.task
def check_bypass_status_job(): def check_bypass_status_job():
if Config.LOGIN.get("geetest") or Config.CHANGER.get('geetest') or Config.REGISTER.get( if Config.LOGIN.get("geetest") or Config.CHANGER.get('geetest') or Config.REGISTER.get(
'geetest') or Config.REPORT.get('geetest'): 'geetest') or Config.REPORT.get('geetest'):
check_bypass_status() return check_bypass_status()
@app.task @app.task
def auto_clean_upload_tmp_file_job(): def auto_clean_upload_tmp_file_job():
auto_clean_upload_tmp_file() return auto_clean_upload_tmp_file()
@app.task @app.task
def auto_clean_captcha_store_job(): def auto_clean_captcha_store_job():
CaptchaStore.remove_expired() return CaptchaStore.remove_expired()
@app.task @app.task
def auto_delete_tmp_file_job(): def auto_delete_tmp_file_job():
auto_delete_ios_mobile_tmp_file() return auto_delete_ios_mobile_tmp_file()
@app.task @app.task
def sync_wx_access_token_job(): def sync_wx_access_token_job():
if get_login_type().get('third', '').get('wxp'): if get_login_type().get('third', '').get('wxp'):
sync_wx_access_token() return sync_wx_access_token()
@app.task @app.task
def auto_clean_remote_client_job(): def auto_clean_remote_client_job():
auto_clean_remote_client_log() return auto_clean_remote_client_log()
@app.task @app.task
def download_times_notify_check_job(): def download_times_notify_check_job():
notify_check_user_download_times() return notify_check_user_download_times()
@app.task @app.task
def apple_developer_devices_check_job(): def apple_developer_devices_check_job():
notify_check_apple_developer_devices() return notify_check_apple_developer_devices()
@app.task @app.task
def apple_developer_cert_notify_check_job(): def apple_developer_cert_notify_check_job():
notify_check_apple_developer_cert() return notify_check_apple_developer_cert()
@app.task
def migrate_storage_job(use_storage_id, user_pk, force):
return migrate_user_oss_storage(use_storage_id, user_pk, force)

@ -9,7 +9,9 @@ import time
from django.core.cache import cache from django.core.cache import cache
from api.base_views import storage_change
from api.models import Apps, UserInfo, RemoteClientInfo from api.models import Apps, UserInfo, RemoteClientInfo
from common.cache.state import MigrateStorageState
from common.notify.ntasks import check_user_download_times, check_apple_developer_devices, check_apple_developer_cert from common.notify.ntasks import check_user_download_times, check_apple_developer_devices, check_apple_developer_cert
from common.utils.storage import Storage from common.utils.storage import Storage
from fir_ser.settings import CACHE_KEY_TEMPLATE from fir_ser.settings import CACHE_KEY_TEMPLATE
@ -66,3 +68,16 @@ def notify_check_apple_developer_devices():
def notify_check_apple_developer_cert(): def notify_check_apple_developer_cert():
for user_obj in UserInfo.objects.filter(is_active=True, supersign_active=True).all(): for user_obj in UserInfo.objects.filter(is_active=True, supersign_active=True).all():
check_apple_developer_cert(user_obj, expire_day=7, days=[0, 1, 3, 7]) check_apple_developer_cert(user_obj, expire_day=7, days=[0, 1, 3, 7])
def migrate_user_oss_storage(use_storage_id, user_pk, force):
msg = 'success'
user_obj = UserInfo.objects.filter(pk=user_pk).first()
if user_obj:
with MigrateStorageState(user_obj.uid) as state:
if state:
if not storage_change(use_storage_id, user_obj, force):
msg = '数据迁移失败'
else:
msg = "数据迁移中,请耐心等待"
return msg

@ -22,12 +22,12 @@ from common.core.sysconfig import Config
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def get_app_d_count_by_app_id(app_id): def get_app_d_count_by_app_id(app_id, is_oss=False):
d_count = 1 d_count = 1
binary_size = AppReleaseInfo.objects.filter(is_master=True, app_id__app_id=app_id).values('binary_size').first() binary_size = AppReleaseInfo.objects.filter(is_master=True, app_id__app_id=app_id).values('binary_size').first()
if binary_size and binary_size.get('binary_size', 0) > 0: if binary_size and binary_size.get('binary_size', 0) > 0:
d_count += binary_size.get('binary_size') // 1024 // 1024 // 100 d_count += binary_size.get('binary_size') // 1024 // 1024 // 100
return d_count return d_count * (Config.APP_USE_BASE_DOWNLOAD_TIMES if not is_oss else Config.PRIVATE_OSS_DOWNLOAD_TIMES)
def get_user_domain_name(obj, domain_type=1): def get_user_domain_name(obj, domain_type=1):

@ -70,6 +70,11 @@ class UserInfoSerializer(serializers.ModelSerializer):
def get_storage_used_capacity(self, obj): def get_storage_used_capacity(self, obj):
return get_user_storage_used(obj) return get_user_storage_used(obj)
download_times = serializers.SerializerMethodField()
def get_download_times(self, obj):
return obj.download_times // Config.APP_USE_BASE_DOWNLOAD_TIMES
storage_used = serializers.SerializerMethodField() storage_used = serializers.SerializerMethodField()
def get_storage_used(self, obj): def get_storage_used(self, obj):
@ -83,7 +88,8 @@ class UserInfoSerializer(serializers.ModelSerializer):
free_download_times = serializers.SerializerMethodField() free_download_times = serializers.SerializerMethodField()
def get_free_download_times(self, obj): def get_free_download_times(self, obj):
return get_user_free_download_times(obj.id, auth_status=get_user_cert_auth_status(obj.id)) return get_user_free_download_times(obj.id, auth_status=get_user_cert_auth_status(
obj.id)) // Config.APP_USE_BASE_DOWNLOAD_TIMES
certification = serializers.SerializerMethodField() certification = serializers.SerializerMethodField()

@ -7,6 +7,7 @@ import binascii
import datetime import datetime
import logging import logging
import os import os
import time
from django.core.cache import cache from django.core.cache import cache
@ -14,7 +15,7 @@ from api.models import UserInfo, AppReleaseInfo, AppScreenShot, Token, UserAdDis
from api.utils.modelutils import get_app_d_count_by_app_id, get_user_storage_obj from api.utils.modelutils import get_app_d_count_by_app_id, get_user_storage_obj
from api.utils.signalutils import run_xsign_migrate_data, run_xsign_clean_data from api.utils.signalutils import run_xsign_migrate_data, run_xsign_clean_data
from common.base.baseutils import get_real_ip_address from common.base.baseutils import get_real_ip_address
from common.cache.storage import UserTokenCache from common.cache.storage import UserTokenCache, TaskProgressCache
from common.libs.storage.localApi import LocalStorage from common.libs.storage.localApi import LocalStorage
from common.utils.caches import consume_user_download_times from common.utils.caches import consume_user_download_times
from common.utils.storage import Storage from common.utils.storage import Storage
@ -164,12 +165,24 @@ def migrating_storage_file_data(user_obj, filename, new_storage_obj, clean_old_d
def migrating_storage_data(user_obj, new_storage_obj, clean_old_data): def migrating_storage_data(user_obj, new_storage_obj, clean_old_data):
with cache.lock(f"migrating_storage_data_{user_obj.uid}", timeout=60 * 60 * 24): with cache.lock(f"migrating_storage_data_{user_obj.uid}", timeout=60 * 60 * 24):
task_cache = TaskProgressCache('migrating_storage_data', user_obj.uid)
app_release_queryset = AppReleaseInfo.objects.filter(app_id__user_id=user_obj)
cache_data = task_cache.get_storage_cache()
if not cache_data:
cache_default_data = {
's_time': time.time(),
'release_num': app_release_queryset.count(),
'migrate_num': 0,
'target_oss_id': -1 if not new_storage_obj else new_storage_obj.pk
}
else:
logger.warning(f"migrate {user_obj} oss progress is exist. may be is migrating data {cache_data}")
auth_status = False auth_status = False
certification = getattr(user_obj, 'certification', None) certification = getattr(user_obj, 'certification', None)
if certification and certification.status == 1: if certification and certification.status == 1:
auth_status = True auth_status = True
for app_release_obj in AppReleaseInfo.objects.filter(app_id__user_id=user_obj).all(): for app_release_obj in app_release_queryset.all():
# 迁移APP数据 # 迁移APP数据
filename = get_filename_from_apptype(app_release_obj.release_id, app_release_obj.release_type) filename = get_filename_from_apptype(app_release_obj.release_id, app_release_obj.release_type)
migrating_storage_file_data(user_obj, filename, new_storage_obj, clean_old_data) migrating_storage_file_data(user_obj, filename, new_storage_obj, clean_old_data)
@ -183,8 +196,11 @@ def migrating_storage_data(user_obj, new_storage_obj, clean_old_data):
# filename = get_filename_from_apptype(apptodev_obj.binary_file, app_release_obj.release_type) # filename = get_filename_from_apptype(apptodev_obj.binary_file, app_release_obj.release_type)
# migrating_storage_file_data(user_obj, filename, new_storage_obj, clean_old_data) # migrating_storage_file_data(user_obj, filename, new_storage_obj, clean_old_data)
# 消费下载次数 # 消费下载次数
amount = get_app_d_count_by_app_id(app_release_obj.app_id.app_id) cache_default_data['migrate_num'] += 1
task_cache.set_storage_cache(cache_default_data)
amount = get_app_d_count_by_app_id(app_release_obj.app_id.app_id, False)
consume_user_download_times(user_obj.pk, app_release_obj.app_id, amount, auth_status) consume_user_download_times(user_obj.pk, app_release_obj.app_id, amount, auth_status)
task_cache.del_storage_cache()
return True return True

@ -10,8 +10,9 @@ from django.db.models import Q
from rest_framework.response import Response from rest_framework.response import Response
from rest_framework.views import APIView from rest_framework.views import APIView
from api.base_views import storage_change, app_delete from api.base_views import app_delete
from api.models import AppStorage, Apps, StorageShareInfo, UserInfo from api.models import AppStorage, Apps, StorageShareInfo, UserInfo
from api.tasks import migrate_storage_job
from api.utils.apputils import clean_history_apps from api.utils.apputils import clean_history_apps
from api.utils.modelutils import PageNumber, get_user_storage_capacity, get_user_storage_used from api.utils.modelutils import PageNumber, get_user_storage_capacity, get_user_storage_used
from api.utils.response import BaseResponse from api.utils.response import BaseResponse
@ -19,6 +20,7 @@ from api.utils.serializer import StorageSerializer, StorageShareSerializer
from api.utils.utils import upload_oss_default_head_img from api.utils.utils import upload_oss_default_head_img
from common.base.baseutils import get_choices_dict, get_choices_name_from_key, get_real_ip_address from common.base.baseutils import get_choices_dict, get_choices_name_from_key, get_real_ip_address
from common.cache.state import MigrateStorageState from common.cache.state import MigrateStorageState
from common.cache.storage import TaskProgressCache
from common.core.auth import ExpiringTokenAuthentication, StoragePermission from common.core.auth import ExpiringTokenAuthentication, StoragePermission
from common.core.sysconfig import Config from common.core.sysconfig import Config
from common.utils.caches import del_cache_storage from common.utils.caches import del_cache_storage
@ -86,6 +88,8 @@ class StorageView(APIView):
if act == 'storage_group': if act == 'storage_group':
res.storage_group_list = get_storage_group(request, res, self.storage_type, is_default == 'true') res.storage_group_list = get_storage_group(request, res, self.storage_type, is_default == 'true')
migrate_progress = TaskProgressCache('migrating_storage_data', request.user.uid).get_storage_cache()
res.migrate_progress = migrate_progress if migrate_progress else {}
return Response(res.dict) return Response(res.dict)
if pk: if pk:
if pk == '-1': if pk == '-1':
@ -188,23 +192,24 @@ class StorageView(APIView):
res.msg = "数据异常,请重试" res.msg = "数据异常,请重试"
return Response(res.dict) return Response(res.dict)
with MigrateStorageState(request.user.uid) as state: c_task = migrate_storage_job.apply_async((use_storage_id, request.user.pk, force))
if state: msg = c_task.get(propagate=False)
if not storage_change(use_storage_id, request.user, force): logger.info(f"run migrate storage task {request.user} msg:{msg}")
res.code = 1006 if c_task.successful():
res.msg = '修改失败' c_task.forget()
else: else:
res.code = 1007 res.code = 1006
res.msg = "数据迁移中,请耐心等待" res.msg = msg
return Response(res.dict) return Response(res.dict)
storage_id = data.get("id", None) storage_id = data.get("id", None)
if storage_id: if storage_id:
# if request.user.storage and storage_id == request.user.storage.id: if request.user.storage and storage_id == request.user.storage.id and MigrateStorageState(
# res.msg = '存储正在使用中,无法修改' request.user.uid).get_state():
# res.code = 1007 res.msg = '存储迁移中,无法修改'
# return Response(res.dict) res.code = 1007
return Response(res.dict)
storage_obj = AppStorage.objects.filter(id=storage_id, user_id=request.user).first() storage_obj = AppStorage.objects.filter(id=storage_id, user_id=request.user).first()
storage_obj_bak = AppStorage.objects.filter(id=storage_id, user_id=request.user).first() storage_obj_bak = AppStorage.objects.filter(id=storage_id, user_id=request.user).first()
serializer = StorageSerializer(instance=storage_obj, data=data, context={'user_obj': request.user}, serializer = StorageSerializer(instance=storage_obj, data=data, context={'user_obj': request.user},
@ -241,6 +246,10 @@ class StorageView(APIView):
storage_id = request.query_params.get("id", None) storage_id = request.query_params.get("id", None)
if storage_id: if storage_id:
try: try:
if request.user.storage and storage_id == request.user.storage.id:
res.msg = '存储使用中,无法删除'
res.code = 1007
return Response(res.dict)
AppStorage.objects.filter(user_id=request.user, id=storage_id).delete() AppStorage.objects.filter(user_id=request.user, id=storage_id).delete()
logger.error(f"user {request.user} delete storage id:{storage_id} success") logger.error(f"user {request.user} delete storage id:{storage_id} success")
except Exception as e: except Exception as e:
@ -418,12 +427,14 @@ class StorageConfigView(APIView):
def get(self, request): def get(self, request):
res = BaseResponse() res = BaseResponse()
migrate_progress = TaskProgressCache('migrating_storage_data', request.user.uid).get_storage_cache()
res.data = { res.data = {
'user_max_storage_capacity': get_user_storage_capacity(request.user), 'user_max_storage_capacity': get_user_storage_capacity(request.user),
'user_used_storage_capacity': get_user_storage_used(request.user), 'user_used_storage_capacity': get_user_storage_used(request.user),
'user_history_limit': UserInfo.objects.filter(pk=request.user.pk).first().history_release_limit, 'user_history_limit': UserInfo.objects.filter(pk=request.user.pk).first().history_release_limit,
'storage_status': MigrateStorageState(request.user.uid).get_state() 'storage_status': MigrateStorageState(request.user.uid).get_state(),
} }
res.migrate_progress = migrate_progress if migrate_progress else {}
return Response(res.dict) return Response(res.dict)
def put(self, request): def put(self, request):

@ -48,16 +48,20 @@ def invalid_short_cache(app_obj, key='ShortDownloadView'.lower()):
""" """
if not app_obj: if not app_obj:
return return
master_release_dict = AppReleaseInfo.objects.filter(app_id=app_obj, is_master=True).values('icon_url', master_release_dict = AppReleaseInfo.objects.filter(app_id=app_obj, is_master=True).values('icon_url', 'release_id',
'release_id').first() 'release_type').first()
if master_release_dict: if master_release_dict:
# 1.清理图片缓存 # 1.清理图片缓存
DownloadUrlCache(key, master_release_dict.get('icon_url')).del_storage_cache() DownloadUrlCache(key, master_release_dict.get('icon_url')).del_storage_cache()
release_id = master_release_dict.get('release_id') release_id = master_release_dict.get('release_id')
release_type = master_release_dict.get('release_type')
# 2.清理下载token缓存 # 2.清理下载token缓存
TokenManagerCache(key, release_id).del_storage_cache() TokenManagerCache(key, release_id).del_storage_cache()
TokenManagerCache(key, f'{release_id}.{"apk" if release_type == 0 else "ipa"}').del_storage_cache()
DownloadUrlCache(key, f'{release_id}.{"apk" if release_type == 0 else "ipa"}').del_storage_cache()
TokenManagerCache('', f"{release_id}.plist").del_storage_cache() TokenManagerCache('', f"{release_id}.plist").del_storage_cache()
DownloadUrlCache('', f"{release_id}.plist").del_storage_cache()
# 3.清理广告缓存 # 3.清理广告缓存
invalid_ad_pic_cache(key, app_obj.short) invalid_ad_pic_cache(key, app_obj.short)

@ -217,6 +217,12 @@ class TaskStateCache(RedisCacheBase):
super().__init__(self.cache_key) super().__init__(self.cache_key)
class TaskProgressCache(RedisCacheBase):
def __init__(self, prefix, task_id):
self.cache_key = f"{CACHE_KEY_TEMPLATE.get('task_progress_key')}_{prefix}_{task_id}"
super().__init__(self.cache_key)
class PendingStateCache(RedisCacheBase): class PendingStateCache(RedisCacheBase):
def __init__(self, locker_key): def __init__(self, locker_key):
self.cache_key = f"{CACHE_KEY_TEMPLATE.get('pending_state_key')}_{locker_key}" self.cache_key = f"{CACHE_KEY_TEMPLATE.get('pending_state_key')}_{locker_key}"

@ -8,7 +8,7 @@ import json
import logging import logging
import re import re
from django.template import Context, Template from django.template import Context, Template, TemplateSyntaxError
from django.template.base import VariableNode from django.template.base import VariableNode
from rest_framework import serializers from rest_framework import serializers
@ -58,10 +58,24 @@ class ConfigCacheBase(object):
logger.warning(f"get same render key. so continue") logger.warning(f"get same render key. so continue")
continue continue
context_dict[sys_obj_dict['key']] = sys_obj_dict['value'] context_dict[sys_obj_dict['key']] = sys_obj_dict['value']
try:
value = get_render_context(value, context_dict)
except TemplateSyntaxError as e:
res_list = re.findall("Could not parse the remainder: '{{(.*?)}}'", str(e))
for res in res_list:
r_value = get_render_context(f'{{{{{res}}}}}', context_dict)
value = value.replace(f'{{{{{res}}}}}', r_value)
value = get_render_context(value, context_dict) value = get_render_context(value, context_dict)
except Exception as e: except Exception as e:
logger.warning(f"db config - render failed {e}") logger.warning(f"db config - render failed {e}")
except Exception as e:
logger.warning(f"db config - render failed {e}")
if isinstance(value, str):
v_group = re.findall('"(.*?)"', value)
if v_group and len(v_group) == 1 and v_group[0].isdigit():
return int(v_group[0])
if value.isdigit():
return int(value)
return value return value
def get_value_from_db(self, key): def get_value_from_db(self, key):
@ -297,6 +311,14 @@ class UserDownloadTimesCache(ConfigCacheBase):
def SIGN_EXTRA_MULTIPLE(self): def SIGN_EXTRA_MULTIPLE(self):
return super().get_value('SIGN_EXTRA_MULTIPLE', DOWNLOADTIMESCONF.SIGN_EXTRA_MULTIPLE) return super().get_value('SIGN_EXTRA_MULTIPLE', DOWNLOADTIMESCONF.SIGN_EXTRA_MULTIPLE)
@property
def APP_USE_BASE_DOWNLOAD_TIMES(self):
return super().get_value('APP_USE_BASE_DOWNLOAD_TIMES', DOWNLOADTIMESCONF.APP_USE_BASE_DOWNLOAD_TIMES)
@property
def PRIVATE_OSS_DOWNLOAD_TIMES(self):
return super().get_value('PRIVATE_OSS_DOWNLOAD_TIMES', DOWNLOADTIMESCONF.PRIVATE_OSS_DOWNLOAD_TIMES)
class OssStorageConfCache(ConfigCacheBase): class OssStorageConfCache(ConfigCacheBase):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):

@ -230,10 +230,15 @@ def get_user_cert_auth_status(user_id):
def check_user_has_all_download_times(app_obj, user_obj): def check_user_has_all_download_times(app_obj, user_obj):
if user_obj: if user_obj:
user_id = user_obj.pk user_id = user_obj.pk
storage = user_obj.storage
else: else:
user_id = app_obj.user_id_id user_id = app_obj.user_id_id
storage = app_obj.user_id.storage
auth_status = get_user_cert_auth_status(user_id) auth_status = get_user_cert_auth_status(user_id)
d_count = get_app_d_count_by_app_id(app_obj.app_id) d_count = get_app_d_count_by_app_id(app_obj.app_id, storage)
a = get_user_free_download_times(user_id, auth_status=auth_status)
b = d_count
c = check_user_can_download(user_id)
return get_user_free_download_times(user_id, auth_status=auth_status) - d_count >= 0 or check_user_can_download( return get_user_free_download_times(user_id, auth_status=auth_status) - d_count >= 0 or check_user_can_download(
user_id) user_id)
@ -241,8 +246,9 @@ def check_user_has_all_download_times(app_obj, user_obj):
def consume_user_download_times_by_app_obj(app_obj): def consume_user_download_times_by_app_obj(app_obj):
user_id = app_obj.user_id_id user_id = app_obj.user_id_id
auth_status = get_user_cert_auth_status(user_id) auth_status = get_user_cert_auth_status(user_id)
amount = get_app_d_count_by_app_id(app_obj.app_id) amount = get_app_d_count_by_app_id(app_obj.app_id, False)
if consume_user_download_times(user_id, app_obj.app_id, int(amount * Config.SIGN_EXTRA_MULTIPLE), auth_status): if consume_user_download_times(user_id, app_obj.app_id, int(amount * (Config.SIGN_EXTRA_MULTIPLE - 1)),
auth_status):
return False return False
return True return True
@ -290,7 +296,7 @@ def update_order_info(user_id, out_trade_no, payment_number, payment_type, descr
order_obj.save( order_obj.save(
update_fields=["status", "payment_type", "payment_number", "pay_time", update_fields=["status", "payment_type", "payment_number", "pay_time",
"description"]) "description"])
add_user_download_times(user_id, download_times) add_user_download_times(user_id, download_times * Config.APP_USE_BASE_DOWNLOAD_TIMES)
logger.info(f"{user_obj} 订单 {out_trade_no} msg:{order_obj.description}") logger.info(f"{user_obj} 订单 {out_trade_no} msg:{order_obj.description}")
pay_success_notify(user_obj, order_obj) pay_success_notify(user_obj, order_obj)
return True return True
@ -340,6 +346,8 @@ def get_wx_ticket_login_info_cache(ticket):
def add_download_times_free_base(user_obj, amount, payment_name, description, order_type=1): def add_download_times_free_base(user_obj, amount, payment_name, description, order_type=1):
if amount <= 0:
return True
order_number = get_order_num() order_number = get_order_num()
order_obj = Order.objects.create(payment_type=2, order_number=order_number, payment_number=order_number, order_obj = Order.objects.create(payment_type=2, order_number=order_number, payment_number=order_number,
user_id=user_obj, status=1, order_type=order_type, actual_amount=0, user_id=user_obj, status=1, order_type=order_type, actual_amount=0,

@ -68,10 +68,10 @@ def get_app_instance_by_cache(app_id, limit):
app_obj_cache = app_instance_cache.get_storage_cache() app_obj_cache = app_instance_cache.get_storage_cache()
if not app_obj_cache: if not app_obj_cache:
app_obj_cache = Apps.objects.filter(app_id=app_id).values("pk", 'user_id', 'type', 'need_password', app_obj_cache = Apps.objects.filter(app_id=app_id).values("pk", 'user_id', 'type', 'need_password',
'issupersign', 'issupersign', 'user_id__storage',
'user_id__certification__status').first() 'user_id__certification__status').first()
if app_obj_cache: if app_obj_cache:
app_obj_cache['d_count'] = get_app_d_count_by_app_id(app_id) app_obj_cache['d_count'] = get_app_d_count_by_app_id(app_id, app_obj_cache.get('user_id__storage', False))
app_instance_cache.set_storage_cache(app_obj_cache, limit) app_instance_cache.set_storage_cache(app_obj_cache, limit)
if not app_obj_cache: if not app_obj_cache:
return False, '应用不存在' return False, '应用不存在'

@ -391,10 +391,14 @@ bIX1aWjPxirQX9mzaL3oEQI=
class DOWNLOADTIMESCONF(object): class DOWNLOADTIMESCONF(object):
SIGN_EXTRA_MULTIPLE = 2 # 超级签名消耗额外倍数,超级签名需要占用的服务大量资源 SIGN_EXTRA_MULTIPLE = 2 # 超级签名消耗额外倍数,超级签名需要占用的服务大量资源
USER_FREE_DOWNLOAD_TIMES = 5
AUTH_USER_FREE_DOWNLOAD_TIMES = 10 APP_USE_BASE_DOWNLOAD_TIMES = 100 # 单个应用下载消费点数
NEW_USER_GIVE_DOWNLOAD_TIMES = 100 PRIVATE_OSS_DOWNLOAD_TIMES = 4 # 私有存储单个应用下载消费点数
AUTH_USER_GIVE_DOWNLOAD_TIMES = 200
USER_FREE_DOWNLOAD_TIMES = "{% widthratio 5 1 {{APP_USE_BASE_DOWNLOAD_TIMES}} %}" # 用户每日免费下载点数
AUTH_USER_FREE_DOWNLOAD_TIMES = '{% widthratio 10 1 {{APP_USE_BASE_DOWNLOAD_TIMES}} %}' # 认证用户每日免费下载点数
NEW_USER_GIVE_DOWNLOAD_TIMES = "{% widthratio 100 1 {{APP_USE_BASE_DOWNLOAD_TIMES}} %}" # 新用户注册赠送下载点数
AUTH_USER_GIVE_DOWNLOAD_TIMES = "{% widthratio 200 1 {{APP_USE_BASE_DOWNLOAD_TIMES}} %}" # 用户认证成功赠送下载点数
class APPLEDEVELOPERCONF(object): class APPLEDEVELOPERCONF(object):

@ -261,6 +261,7 @@ CORS_ALLOW_HEADERS = (
CACHE_KEY_TEMPLATE = { CACHE_KEY_TEMPLATE = {
'config_key': 'config', 'config_key': 'config',
'task_state_key': 'task_state', 'task_state_key': 'task_state',
'task_progress_key': 'task_progress',
'pending_state_key': 'pending_state', 'pending_state_key': 'pending_state',
'wx_login_bind_key': 'wx_login_bind', 'wx_login_bind_key': 'wx_login_bind',
'notify_loop_msg_key': 'notify_loop_msg', 'notify_loop_msg_key': 'notify_loop_msg',

@ -101,14 +101,14 @@ def run_resign_task_do(app_id, developer_id, developer_app_id, need_download_pro
@app.task @app.task
def auto_check_ios_developer_active_job(): def auto_check_ios_developer_active_job():
auto_check_ios_developer_active() return auto_check_ios_developer_active()
@app.task @app.task
def get_best_proxy_ips_job(): def get_best_proxy_ips_job():
get_best_proxy_ips() return get_best_proxy_ips()
@app.task @app.task
def auto_clean_sign_log_job(): def auto_clean_sign_log_job():
auto_clean_sign_log() return auto_clean_sign_log()

Loading…
Cancel
Save