From a8b5d4dbbe393c7b2439866819b1a1d3eaa3395e Mon Sep 17 00:00:00 2001 From: nineven Date: Tue, 7 Apr 2020 20:43:16 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=8B=E8=BD=BD=E6=AC=A1?= =?UTF-8?q?=E6=95=B0=E7=BB=9F=E8=AE=A1=EF=BC=8C=E6=94=BE=E5=88=B0=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E9=87=8C=E9=9D=A2=EF=BC=8C=E6=AF=8F10=E7=A7=92?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E5=88=B0=E6=95=B0=E6=8D=AE=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fir_ser/api/models.py | 2 +- fir_ser/api/utils/crontab/run.py | 52 +++++++++++++++++++++++++ fir_ser/api/utils/crontab/sync_cache.py | 20 ++++++++++ fir_ser/api/utils/serializer.py | 4 +- fir_ser/api/utils/storage/caches.py | 47 ++++++++++++++++++++++ fir_ser/api/views/download.py | 44 ++++++--------------- fir_ser/fir_ser/settings.py | 8 ++++ fir_ser/requirements.txt | 3 ++ 8 files changed, 145 insertions(+), 35 deletions(-) create mode 100644 fir_ser/api/utils/crontab/run.py create mode 100644 fir_ser/api/utils/crontab/sync_cache.py create mode 100644 fir_ser/api/utils/storage/caches.py diff --git a/fir_ser/api/models.py b/fir_ser/api/models.py index c9401fc..8734048 100644 --- a/fir_ser/api/models.py +++ b/fir_ser/api/models.py @@ -59,7 +59,7 @@ class Token(models.Model): """ The default authorization token model. """ - access_token = models.CharField(max_length=42, unique=True) + access_token = models.CharField(max_length=64, unique=True) user = models.ForeignKey( UserInfo, related_name='auth_token', on_delete=models.CASCADE, verbose_name="关联用户" diff --git a/fir_ser/api/utils/crontab/run.py b/fir_ser/api/utils/crontab/run.py new file mode 100644 index 0000000..16981e9 --- /dev/null +++ b/fir_ser/api/utils/crontab/run.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +# project: 4月 +# author: liuyu +# date: 2020/4/7 + +from apscheduler.schedulers.background import BackgroundScheduler +from django_apscheduler.jobstores import DjangoJobStore, register_events, register_job + +from api.utils.crontab.sync_cache import sync_download_times + +import atexit +import fcntl + +# 主要是为了防止多进程出现的多个定时任务同时执行 +f = open("scheduler.lock", "wb") +try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + + # 开启定时工作 + try: + # 实例化调度器 + scheduler = BackgroundScheduler() + # 调度器使用DjangoJobStore() + scheduler.add_jobstore(DjangoJobStore(), "default") + + # 设置定时任务，选择方式为interval，时间间隔为10s + # 另一种方式为每天固定时间执行任务，对应代码为： + # @register_job(scheduler, 'cron', day_of_week='mon-fri', hour='9', minute='30', second='10',id='task_time') + @register_job(scheduler, "interval", seconds=10) + def sync_download_times_job(): + # 这里写你要执行的任务 + sync_download_times() + + register_events(scheduler) + scheduler.start() + except Exception as e: + print(e) + # 有错误就停止定时器 + scheduler.shutdown() + +except: + pass + +def unlock(): + fcntl.flock(f, fcntl.LOCK_UN) + f.close() + +atexit.register(unlock) + + + diff --git a/fir_ser/api/utils/crontab/sync_cache.py b/fir_ser/api/utils/crontab/sync_cache.py new file mode 100644 index 0000000..7ffaabd --- /dev/null +++ b/fir_ser/api/utils/crontab/sync_cache.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +# project: 4月 +# author: liuyu +# date: 2020/4/7 + +from api.models import Apps +from django.core.cache import cache +from fir_ser.settings import CACHE_KEY_TEMPLATE + + +def sync_download_times(): + down_tem_key = CACHE_KEY_TEMPLATE.get("download_times_key") + key = "%s%s" %(down_tem_key,'*') + for app_download in cache.iter_keys(key): + app_id = app_download.split(down_tem_key)[1] + Apps.objects.filter(app_id=app_id).update(count_hits=cache.get(app_download)) + + + diff --git a/fir_ser/api/utils/serializer.py b/fir_ser/api/utils/serializer.py index e6b7a91..553d0ea 100644 --- a/fir_ser/api/utils/serializer.py +++ b/fir_ser/api/utils/serializer.py @@ -46,7 +46,7 @@ class AppsSerializer(serializers.ModelSerializer): if self.context.get("storage", None) and self.context.get("storage") != "undefined": storage = self.context.get("storage", None) - icon_url = storage.get_download_url(os.path.basename(master_release_obj.icon_url),300) + icon_url = storage.get_download_url(os.path.basename(master_release_obj.icon_url),600) datainfo = { "app_version": master_release_obj.app_version, "icon_url": icon_url, @@ -60,7 +60,7 @@ class AppsSerializer(serializers.ModelSerializer): "binary_url":master_release_obj.binary_url, } - download_token = token_obj.make_token(master_release_obj.release_id,300) + download_token = token_obj.make_token(master_release_obj.release_id,600) datainfo["download_token"] = download_token return datainfo diff --git a/fir_ser/api/utils/storage/caches.py b/fir_ser/api/utils/storage/caches.py new file mode 100644 index 0000000..09d3c68 --- /dev/null +++ b/fir_ser/api/utils/storage/caches.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +# project: 4月 +# author: liuyu +# date: 2020/4/7 + +from django.core.cache import cache +from api.models import Apps,UserInfo +import time +from fir_ser.settings import CACHE_KEY_TEMPLATE +from api.utils.storage.storage import Storage,LocalStorage +from api.utils.crontab import run + +def get_download_url_by_cache(app_obj, filename, limit, isdownload=False): + now = time.time() + download_val = cache.get("%s_%s" % ('download_url', filename)) + if download_val: + if download_val.get("time") > now - 60: + return download_val.get("download_url") + else: + if isdownload: + local_storage = LocalStorage('localhost', False) + return local_storage.get_download_url(filename, limit, 'plist') + user_obj = UserInfo.objects.filter(pk=app_obj.get("user_id")).first() + storage = Storage(user_obj) + return storage.get_download_url(filename, limit) + + +def get_app_instance_by_cache(app_id, limit): + app_obj_cache = cache.get("%s_%s" % ('app_instance', app_id)) + if not app_obj_cache: + app_obj_cache = Apps.objects.filter(app_id=app_id).values("pk", 'user_id', 'type').first() + cache.set("%s_%s" % ('app_instance', app_id), app_obj_cache, limit) + return app_obj_cache + + +def get_app_download_by_cache(app_id): + down_tem_key = CACHE_KEY_TEMPLATE.get("download_times_key") + key = "%s%s" %(down_tem_key,app_id) + download_times = cache.get(key) + if not download_times: + download_times=Apps.objects.filter(app_id=app_id).values("count_hits").first().get('count_hits') + cache.set(key, download_times + 1, 900) + else: + cache.incr(key) + return download_times + 1 + diff --git a/fir_ser/api/views/download.py b/fir_ser/api/views/download.py index 816a584..e1a80e3 100644 --- a/fir_ser/api/views/download.py +++ b/fir_ser/api/views/download.py @@ -10,13 +10,13 @@ from fir_ser import settings from api.utils.TokenManager import DownloadToken from api.utils.app.randomstrings import make_random_uuid from api.utils.app.apputils import make_resigned -from api.utils.storage.storage import Storage,LocalStorage -import os,time -from django.core.cache import cache +from api.utils.storage.storage import Storage +from api.utils.storage.caches import get_app_instance_by_cache,get_download_url_by_cache,get_app_download_by_cache +import os from rest_framework_extensions.cache.decorators import cache_response from api.utils.serializer import AppsSerializer -from api.models import Apps,AppReleaseInfo,UserInfo +from api.models import Apps,AppReleaseInfo from django.db.models import F from django.http import FileResponse class DownloadView(APIView): @@ -72,7 +72,7 @@ class ShortDownloadView(APIView): 根据下载短链接，获取应用信息 ''' - @cache_response(timeout=300-60, cache="default",key_func='calculate_cache_key',cache_errors=False) + @cache_response(timeout=600-60, cache="default",key_func='calculate_cache_key',cache_errors=False) def get(self,request,short): res = BaseResponse() release_id = request.query_params.get("release_id", None) @@ -118,23 +118,23 @@ class InstallView(APIView): dtoken = DownloadToken() if dtoken.verify_token(downtoken,release_id): # app_obj = Apps.objects.filter(app_id=app_id).values("pk",'user_id','type','short').first() - app_obj = self.get_app_instance_by_cache(app_id,900) + app_obj = get_app_instance_by_cache(app_id,900) if app_obj: - Apps.objects.filter(app_id=app_id).update(count_hits=F('count_hits') + 1) - UserInfo.objects.filter(pk=app_obj.get("user_id")).update(all_download_times=F('all_download_times') + 1) - + # Apps.objects.filter(app_id=app_id).update(count_hits=F('count_hits') + 1) + # UserInfo.objects.filter(pk=app_obj.get("user_id")).update(all_download_times=F('all_download_times') + 1) + get_app_download_by_cache(app_id) # release_obj = AppReleaseInfo.objects.filter(app_id=app_obj.get('pk')).values("release_id") # if release_obj: if app_obj.get("type") == 0: apptype = '.apk' - download_url = self.get_download_url_by_cache(app_obj,release_id + apptype,600) + download_url = get_download_url_by_cache(app_obj,release_id + apptype,600) else: apptype = '.ipa' if isdownload : - download_url = self.get_download_url_by_cache(app_obj,release_id + apptype, 600) + download_url = get_download_url_by_cache(app_obj,release_id + apptype, 600) else: - download_url = self.get_download_url_by_cache(app_obj,release_id + apptype,600,isdownload) + download_url = get_download_url_by_cache(app_obj,release_id + apptype,600,isdownload) res.data={"download_url":download_url} return Response(res.dict) @@ -147,23 +147,3 @@ class InstallView(APIView): res.msg = "该应用不存在" return Response(res.dict) - def get_download_url_by_cache(self,app_obj,filename,limit,isdownload=False): - now = time.time() - download_val = cache.get("%s_%s" % ('download_url', filename)) - if download_val: - if download_val.get("time") > now - 60: - return download_val.get("download_url") - else: - if isdownload: - local_storage = LocalStorage('localhost', False) - return local_storage.get_download_url(filename, limit, 'plist') - user_obj = UserInfo.objects.filter(pk=app_obj.get("user_id")).first() - storage = Storage(user_obj) - return storage.get_download_url(filename, limit) - - def get_app_instance_by_cache(self,app_id,limit): - app_obj_cache = cache.get("%s_%s" % ('app_instance', app_id)) - if not app_obj_cache: - app_obj_cache = Apps.objects.filter(app_id=app_id).values("pk",'user_id','type').first() - cache.set("%s_%s" % ('app_instance', app_id),app_obj_cache,limit) - return app_obj_cache diff --git a/fir_ser/fir_ser/settings.py b/fir_ser/fir_ser/settings.py index 28f93dc..30a309d 100644 --- a/fir_ser/fir_ser/settings.py +++ b/fir_ser/fir_ser/settings.py @@ -38,6 +38,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'api.apps.ApiConfig', + 'django_apscheduler',#定时执行任务 'rest_framework', ] @@ -231,3 +232,10 @@ THIRD_PART_CONFIG = { ] } + +CACHE_KEY_TEMPLATE={ + 'download_times_key':'app_download_times_' +} + +SESSION_ENGINE = "django.contrib.sessions.backends.cache" +SESSION_CACHE_ALIAS = "default" diff --git a/fir_ser/requirements.txt b/fir_ser/requirements.txt index 7cf738f..c0e28aa 100644 --- a/fir_ser/requirements.txt +++ b/fir_ser/requirements.txt @@ -3,6 +3,7 @@ aliyun-python-sdk-core-v3==2.13.11 aliyun-python-sdk-kms==2.10.1 aliyun-python-sdk-sts==3.0.1 androguard==3.3.5 +APScheduler==3.6.3 asgiref==3.2.3 asn1crypto==1.3.0 backcall==0.1.0 @@ -14,6 +15,7 @@ crcmod==1.7 cycler==0.10.0 decorator==4.4.2 Django==3.0.3 +django-apscheduler==0.3.0 django-qrcode==0.3 django-redis==4.11.0 django-rest-framework==0.1.0 @@ -52,6 +54,7 @@ requests==2.23.0 six==1.14.0 sqlparse==0.3.1 traitlets==4.3.3 +tzlocal==2.0.0 urllib3==1.25.8 uWSGI==2.0.18 wcwidth==0.1.8