fix some bugs 增加清理临时上传文件

super_signature
nineven 5 years ago
parent 47e79acf42
commit 1dac608843
  1. 23
      fir_client/src/components/FirApps.vue
  2. 21
      fir_ser/api/utils/app/apputils.py
  3. 6
      fir_ser/api/utils/app/supersignutils.py
  4. 15
      fir_ser/api/utils/crontab/run.py
  5. 34
      fir_ser/api/utils/crontab/sync_cache.py
  6. 4
      fir_ser/api/utils/storage/aliyunApi.py
  7. 13
      fir_ser/api/utils/storage/caches.py
  8. 12
      fir_ser/api/utils/storage/localApi.py
  9. 5
      fir_ser/api/utils/storage/qiniuApi.py
  10. 8
      fir_ser/api/utils/storage/storage.py
  11. 10
      fir_ser/api/views/apps.py
  12. 6
      fir_ser/api/views/download.py
  13. 45
      fir_ser/api/views/uploads.py
  14. 12
      fir_ser/fir_ser/settings.py

@ -168,7 +168,7 @@
</el-dialog>
<el-row>
<el-col :span="4">
<el-col :span="3">
<el-radio-group v-model="searchfromtype">
<el-radio-button label="android" icon="el-icon-mobile-phone"><i
class="iconfont icon-android2"/>
@ -178,15 +178,26 @@
</el-radio-group>
</el-col>
<el-col :span="5">
<el-col :span="4">
<el-row>
<el-col :span="20">
<el-input
placeholder="请输入名称搜索"
v-model="keysearch"
@click="searchapps"
clearable prefix-icon="el-icon-search">
clearable >
</el-input>
</el-col>
<el-col :span="2" >
<el-button icon="el-icon-search" @click="searchFun">
</el-button>
</el-col>
</el-row>
</el-col>
<!-- <el-col :span="1" class="surplus-card">-->
<!-- -->
<!-- </el-col>-->
<el-col :span="7" class="surplus-card">
<el-row>
@ -435,6 +446,7 @@
updateappinfo(file) {
this.uploadsuccess += 1;
if (this.uploadsuccess === 2) {
delete this.analyseappinfo.icon;
this.$message.success(file.name + '上传成功');
analyseApps(data => {
if (data.code === 1000) {
@ -781,7 +793,10 @@
// eslint-disable-next-line no-unused-vars
keysearch: function (val, oldVal) {
// this.searchapps()
this.searchFun()
let keysearch = this.keysearch.replace(/^\s+|\s+$/g, "");
if(keysearch === ""){
this.searchFun()
}
},
// eslint-disable-next-line no-unused-vars
searchfromtype: function (val, oldVal) {

@ -112,7 +112,9 @@ def SaveAppInfos(app_file_name, user_obj, appinfo, bundle_id, app_img, short, si
##判断是否存在该app
appmobj = Apps.objects.filter(app_id=app_uuid, user_id=user_obj).first()
storage = Storage(user_obj)
newapp = False
if not appmobj:
newapp = True
appdata = {
"app_id": app_uuid,
"user_id": user_obj,
@ -126,11 +128,12 @@ def SaveAppInfos(app_file_name, user_obj, appinfo, bundle_id, app_img, short, si
appmobj = Apps.objects.create(**appdata)
except Exception as e:
print(e)
storage.delete_file(app_file_name)
storage.delete_file(app_img)
return
# storage.delete_file(app_file_name)
# storage.delete_file(app_img)
return False
else:
try:
newapp = False
del_cache_response_by_short(appmobj.short, appmobj.app_id)
appmobj.short = short
appmobj.name = appinfo["labelname"]
@ -157,13 +160,18 @@ def SaveAppInfos(app_file_name, user_obj, appinfo, bundle_id, app_img, short, si
"changelog": appinfo.get("changelog", ''),
"udid": appinfo.get("udid", ''),
}
AppReleaseInfo.objects.create(**release_data)
try:
AppReleaseInfo.objects.create(**release_data)
except Exception as e:
print(e)
if newapp:
appmobj.delete()
return False
try:
history_release_limit = int(user_obj.history_release_limit)
except Exception as e:
print(e)
return
return True
if history_release_limit == 0:
pass
@ -177,3 +185,4 @@ def SaveAppInfos(app_file_name, user_obj, appinfo, bundle_id, app_img, short, si
storage.delete_file(release_obj.release_id, appmobj.type)
storage.delete_file(release_obj.icon_url)
release_obj.delete()
return True

@ -380,10 +380,8 @@ class IosUtils(object):
except Exception as e:
print(e)
cert_id = re.findall(r'.*\n\tid=(.*),.*', cert_info)[0].replace('"', '')
developer_obj = AppIOSDeveloperInfo.objects.filter(user_id=user_obj, email=auth.get("username")).first()
developer_obj.is_actived = True
developer_obj.certid = cert_id
developer_obj.save()
AppIOSDeveloperInfo.objects.filter(user_id=user_obj, email=auth.get("username")).update(is_actived=True,
certid=cert_id)
return status, result
@staticmethod

@ -7,7 +7,11 @@
from apscheduler.schedulers.background import BackgroundScheduler
from django_apscheduler.jobstores import DjangoJobStore, register_events, register_job
from fir_ser.settings import SYNC_CACHE_TO_DATABASE
from api.utils.crontab.sync_cache import sync_download_times
from api.utils.crontab.sync_cache import sync_download_times, auto_clean_upload_tmp_file, auto_delete_job_log
import logging
logging.basicConfig()
logging.getLogger('apscheduler').setLevel(logging.ERROR)
import atexit
import fcntl
@ -34,6 +38,14 @@ try:
sync_download_times()
# @register_job(scheduler, 'cron', day_of_week='mon-fri', hour='9', minute='30', second='10',id='task_time')
@register_job(scheduler, "interval", seconds=SYNC_CACHE_TO_DATABASE.get("auto_clean_tmp_file_times"))
def auto_clean_upload_tmp_file_job():
# 这里写你要执行的任务
auto_clean_upload_tmp_file()
auto_delete_job_log()
register_events(scheduler)
scheduler.start()
except Exception as e:
@ -41,6 +53,7 @@ try:
# 有错误就停止定时器
scheduler.shutdown()
except:
pass

@ -4,9 +4,12 @@
# author: liuyu
# date: 2020/4/7
from api.models import Apps
from api.models import Apps, UserInfo
from api.utils.storage.storage import Storage
from django.core.cache import cache
from fir_ser.settings import CACHE_KEY_TEMPLATE
from fir_ser.settings import CACHE_KEY_TEMPLATE, SYNC_CACHE_TO_DATABASE
import time
from django_apscheduler.models import DjangoJobExecution
def sync_download_times():
@ -26,3 +29,30 @@ def sync_download_times_by_app_id(app_ids):
for k, v in down_times_lists.items():
app_id = k.split(CACHE_KEY_TEMPLATE.get("download_times_key"))[1].strip('_')
Apps.objects.filter(app_id=app_id).update(count_hits=v)
def auto_clean_upload_tmp_file():
upload_tem_key = CACHE_KEY_TEMPLATE.get("upload_file_tmp_name_key")
key = "_".join([upload_tem_key, '*'])
for upload_tem_file_key in cache.iter_keys(key):
data = cache.get(upload_tem_file_key)
if data:
stime = data.get("stime", None)
if stime and time.time() - stime > 60 * 60:
user_obj = UserInfo.objects.filter(pk=data.get("id")).first()
if user_obj:
storage = Storage(user_obj)
filename = data.get("filename", None)
if filename:
storage.delete_file(filename)
cache.delete(upload_tem_file_key)
def auto_delete_job_log():
job_execution = DjangoJobExecution.objects.order_by("-id").values("id").first()
if job_execution:
need_count = SYNC_CACHE_TO_DATABASE.get("auto_clean_apscheduler_log", 10000)
max_id = job_execution.get("id")
count = DjangoJobExecution.objects.count()
if count > need_count:
DjangoJobExecution.objects.filter(id__lte=max_id - need_count).delete()

@ -120,3 +120,7 @@ class AliYunOss(object):
# auth = oss2.StsAuth(self.token.access_key_id, self.token.access_key_secret, self.token.security_token)
# bucket = oss2.Bucket(auth, self.endpoint,self.bucket_name)
return self.bucket.delete_object(name)
def rename_file(self, oldfilename, newfilename):
self.bucket.copy_object(self.bucket_name, oldfilename, newfilename)
return self.del_file(oldfilename)

@ -45,7 +45,7 @@ def get_download_url_by_cache(app_obj, filename, limit, isdownload=True, key='',
else:
return ""
return local_storage.get_download_url(filename.split(".")[0]+ "." + download_url_type, limit)
return local_storage.get_download_url(filename.split(".")[0] + "." + download_url_type, limit)
down_key = "_".join([key.lower(), CACHE_KEY_TEMPLATE.get('download_url_key'), filename])
download_val = cache.get(down_key)
if download_val:
@ -169,3 +169,14 @@ def developer_auth_code(act, user_obj, developer_email, code=None):
return cache.get(key)
elif act == "del":
cache.delete(key)
def upload_file_tmp_name(act, filename, user_obj_id):
tmp_key = "_".join([CACHE_KEY_TEMPLATE.get("upload_file_tmp_name_key"), filename])
if act == "set":
cache.delete(tmp_key)
cache.set(tmp_key, {'time': time.time(), 'id': user_obj_id, "filename": filename}, 60 * 60)
elif act == "get":
return cache.get(tmp_key)
elif act == "del":
cache.delete(tmp_key)

@ -35,7 +35,17 @@ class LocalStorage(object):
def del_file(self, name):
try:
os.remove(os.path.join(settings.MEDIA_ROOT, name))
file = os.path.join(settings.MEDIA_ROOT, name)
if os.path.isfile(file):
os.remove(file)
return True
except Exception as e:
print(e)
return False
def rename_file(self, oldfilename, newfilename):
try:
os.rename(os.path.join(settings.MEDIA_ROOT, oldfilename), os.path.join(settings.MEDIA_ROOT, newfilename))
return True
except Exception as e:
print(e)

@ -50,3 +50,8 @@ class QiNiuOss(object):
# 删除bucket_name 中的文件 key
ret, info = bucket.delete(self.bucket_name, name)
return ret
def rename_file(self, oldfilename, newfilename):
bucket = BucketManager(self.qiniu_obj)
ret, info = bucket.move(self.bucket_name, oldfilename, self.bucket_name, newfilename)
return ret

@ -24,7 +24,7 @@ class Storage(object):
if self.storage:
return self.storage.get_upload_token(filename, expires)
def get_download_url(self, filename, expires=900, key='', force_new=False):
def get_download_url(self, filename, expires=900, key='', force_new=False):
if self.storage:
now = time.time()
down_key = "_".join([key.lower(), CACHE_KEY_TEMPLATE.get('download_url_key'), filename])
@ -33,7 +33,7 @@ class Storage(object):
if download_val.get("time") > now - 60:
return download_val.get("download_url")
download_url = self.storage.get_download_url(filename, expires,force_new=True)
download_url = self.storage.get_download_url(filename, expires, force_new=True)
cache.set(down_key, {"download_url": download_url, "time": now + expires}, expires)
return download_url
@ -46,6 +46,10 @@ class Storage(object):
filename = filename + '.ipa'
return self.storage.del_file(filename)
def rename_file(self, oldfilename, newfilename):
if self.storage:
return self.storage.rename_file(oldfilename, newfilename)
def get_storage(self, user):
self.storage_obj = user.storage
if self.storage_obj:

@ -15,7 +15,7 @@ from api.utils.app.supersignutils import IosUtils
from api.utils.app.randomstrings import make_from_user_uuid
from api.utils.storage.storage import Storage
from api.utils.storage.caches import del_cache_response_by_short, get_app_today_download_times
from api.models import Apps, AppReleaseInfo, APPToDeveloper
from api.models import Apps, AppReleaseInfo, APPToDeveloper, AppIOSDeveloperInfo, UserInfo
from api.utils.serializer import AppsSerializer, AppReleaseSerializer, UserInfoSerializer
from rest_framework.pagination import PageNumberPagination
@ -53,7 +53,8 @@ class AppsView(APIView):
if count_hits:
if count_hits > 0:
request.user.all_download_times += count_hits
request.user.save()
UserInfo.objects.filter(pk=request.user.id).update(
all_download_times=request.user.all_download_times + count_hits)
else:
count_hits = 0
res.hdata["all_hits_count"] = count_hits
@ -184,6 +185,11 @@ class AppInfoView(APIView):
# res.code = 1008
# res.msg = "超级签余额不足,无法开启"
# return Response(res.dict)
developer_count = AppIOSDeveloperInfo.objects.filter(user_id=request.user).count()
if developer_count == 0:
res.code = 1008
res.msg = "超级签开发者不存在,无法开启"
return Response(res.dict)
apps_obj.issupersign = data.get("issupersign", apps_obj.issupersign)
apps_obj.save()
except Exception as e:

@ -33,7 +33,7 @@ class DownloadView(APIView):
res = BaseResponse()
downtoken = request.query_params.get(settings.DATA_DOWNLOAD_KEY, None)
ftype = filename.split(".")[-1]
flag=True
flag = True
if settings.DATA_DOWNLOAD_KEY_OPEN:
if not downtoken:
res.code = 1004
@ -41,7 +41,7 @@ class DownloadView(APIView):
return Response(res.dict)
dtoken = DownloadToken()
flag=dtoken.verify_token(downtoken, filename)
flag = dtoken.verify_token(downtoken, filename)
if flag:
if ftype == 'plist':
@ -56,7 +56,7 @@ class DownloadView(APIView):
bundle_id = release_obj.app_id.bundle_id
app_version = release_obj.app_version
name = release_obj.app_id.name
ios_plist_bytes = make_resigned(storage.get_download_url(filename.split('.')[0]+".ipa"),
ios_plist_bytes = make_resigned(storage.get_download_url(filename.split('.')[0] + ".ipa"),
storage.get_download_url(release_obj.icon_url), bundle_id,
app_version, name)
response = FileResponse(ios_plist_bytes)

@ -6,8 +6,8 @@
from api.utils.app.apputils import get_random_short, SaveAppInfos
from api.utils.storage.storage import Storage
from api.utils.storage.caches import del_cache_response_by_short
from api.models import Apps, AppReleaseInfo
from api.utils.storage.caches import upload_file_tmp_name, del_cache_response_by_short
from api.models import Apps, AppReleaseInfo, UserInfo
from api.utils.app.randomstrings import make_app_uuid
from rest_framework.views import APIView
from api.utils.response import BaseResponse
@ -47,10 +47,10 @@ class AppAnalyseView(APIView):
is_new = True
short = get_random_short()
if app_type == 'iOS':
upload_key = release_id + '.ipa'
upload_key = release_id + '.ipa' + settings.FILE_UPLOAD_TMP_KEY
else:
upload_key = release_id + '.apk'
png_key = png_id + '.png'
upload_key = release_id + '.apk' + settings.FILE_UPLOAD_TMP_KEY
png_key = png_id + '.png' + settings.FILE_UPLOAD_TMP_KEY
storage = Storage(request.user)
storage_type = storage.get_storage_type()
@ -63,6 +63,8 @@ class AppAnalyseView(APIView):
upload_token = storage.get_upload_token(upload_key)
png_token = storage.get_upload_token(png_key)
upload_file_tmp_name("set", png_key, request.user.id)
upload_file_tmp_name("set", upload_key, request.user.id)
res.data = {"app_uuid": app_uuid, "short": short,
"domain_name": request.user.domain_name,
"upload_token": upload_token,
@ -90,8 +92,26 @@ class AppAnalyseView(APIView):
}
try:
SaveAppInfos(data.get("upload_key"), request.user, appinfo,
data.get("bundleid"), data.get("png_key"), data.get("short"), data.get('filesize'))
storage = Storage(request.user)
app_tmp_filename = data.get("upload_key")
app_new_filename = data.get("upload_key").strip(settings.FILE_UPLOAD_TMP_KEY)
png_tmp_filename = data.get("png_key")
png_new_filename = data.get("png_key").strip(settings.FILE_UPLOAD_TMP_KEY)
if SaveAppInfos(app_new_filename, request.user, appinfo,
data.get("bundleid"), png_new_filename, data.get("short"), data.get('filesize')):
pass
# 需要将tmp 文件修改为正式文件
storage.rename_file(app_tmp_filename, app_new_filename)
storage.rename_file(png_tmp_filename, png_new_filename)
else:
storage.delete_file(app_tmp_filename)
storage.delete_file(png_tmp_filename)
# 删除redis key
upload_file_tmp_name("del", app_tmp_filename, request.user.id)
upload_file_tmp_name("del", png_tmp_filename, request.user.id)
except Exception as e:
print(e)
res.code = 10003
@ -191,7 +211,10 @@ class UploadView(APIView):
res.msg = "文件不存在"
for file_obj in files:
try:
app_type = file_obj.name.split(".")[-1]
if app_type == "tmp":
app_type = file_obj.name.split(".")[-2]
if app_type not in ["apk", "ipa", 'png', 'jpeg', 'jpg']:
raise TypeError
except Exception as e:
@ -210,6 +233,7 @@ class UploadView(APIView):
destination.close()
except Exception as e:
print(e)
res.code = 1003
res.msg = "数据写入失败"
try:
@ -223,6 +247,10 @@ class UploadView(APIView):
return Response(res.dict)
'''
应用图片上传之后
'''
def put(self, request):
res = BaseResponse()
certinfo = request.data.get('certinfo', None)
@ -248,8 +276,7 @@ class UploadView(APIView):
res.msg = '该用户不存在'
return Response(res.dict)
old_file_key = request.user.head_img
request.user.head_img = certinfo.get("upload_key")
request.user.save()
UserInfo.objects.filter(pk=request.user.id).update(head_img=certinfo.get("upload_key"))
if old_file_key != "" or old_file_key != 'head_img.jpeg':
storage.delete_file(old_file_key)

@ -89,7 +89,7 @@ DATABASES = {
'PORT': '3306',
# 设置MySQL的驱动
# 'OPTIONS': {'init_command': 'SET storage_engine=INNODB'},
'OPTIONS': {'init_command': 'SET sql_mode="STRICT_TRANS_TABLES"','charset': 'utf8mb4'}
'OPTIONS': {'init_command': 'SET sql_mode="STRICT_TRANS_TABLES"', 'charset': 'utf8mb4'}
}
}
@ -235,13 +235,17 @@ CACHE_KEY_TEMPLATE = {
'user_auth_token_key': 'user_auth_token',
'download_today_times_key': 'download_today_times',
'developer_auth_code_key': 'developer_auth_code',
'upload_file_tmp_name_key': 'upload_file_tmp_name',
}
DATA_DOWNLOAD_KEY="d_token"
DATA_DOWNLOAD_KEY_OPEN=True #是否开启图片和应用下载token,只本地存储支持该操作
DATA_DOWNLOAD_KEY = "d_token"
DATA_DOWNLOAD_KEY_OPEN = True # 是否开启图片和应用下载token,只本地存储支持该操作
FILE_UPLOAD_TMP_KEY = ".tmp"
SYNC_CACHE_TO_DATABASE = {
'download_times': 10 # 下载次数同步时间
'download_times': 10, # 下载次数同步时间
'auto_clean_tmp_file_times': 60 * 30, # 定时清理上传失误生成的临时文件
'auto_clean_apscheduler_log': 100000, # 定时清理定时任务执行的日志,该参数为日志保留的数量
}
SERVER_DOMAIN = {

Loading…
Cancel
Save