diff --git a/fir_admin/src/router/index.js b/fir_admin/src/router/index.js
index 89acf09..69f9ae5 100644
--- a/fir_admin/src/router/index.js
+++ b/fir_admin/src/router/index.js
@@ -3,6 +3,8 @@ import Router from 'vue-router'
Vue.use(Router)
+const celery_flower = process.env.VUE_APP_BASE_API.split('/')
+
/* Layout */
import Layout from '@/layout'
@@ -373,6 +375,16 @@ export const constantRoutes = [
}
]
},
+ {
+ path: 'celery-flower-link',
+ component: Layout,
+ children: [
+ {
+ path: celery_flower[0] + '//' + celery_flower[2] + '/flower',
+ meta: { title: '任务监控', icon: 'link' }
+ }
+ ]
+ },
// 404 page must be placed at the end !!!
{ path: '*', redirect: '/404', hidden: true }
diff --git a/fir_admin/src/store/modules/user.js b/fir_admin/src/store/modules/user.js
index 2db5e2e..4a63d7e 100644
--- a/fir_admin/src/store/modules/user.js
+++ b/fir_admin/src/store/modules/user.js
@@ -1,7 +1,7 @@
import { login, logout, getInfo } from '@/api/user'
import { getToken, setToken, removeToken } from '@/utils/auth'
import { resetRouter } from '@/router'
-const Base64 = require('js-base64').Base64;
+const Base64 = require('js-base64').Base64
const getDefaultState = () => {
return {
diff --git a/fir_ser/admin/views/celery_flower.py b/fir_ser/admin/views/celery_flower.py
new file mode 100644
index 0000000..6ef9a61
--- /dev/null
+++ b/fir_ser/admin/views/celery_flower.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+#
+from django.http import HttpResponse
+from django.conf import settings
+from django.utils.translation import ugettext as _
+from proxy.views import proxy_view
+from rest_framework.views import APIView
+
+from api.utils.auth import AdminTokenAuthentication
+
+flower_url = f'{settings.CELERY_FLOWER_HOST}:{settings.CELERY_FLOWER_PORT}'
+
+
+class CeleryFlowerView(APIView):
+ authentication_classes = [AdminTokenAuthentication, ]
+
+ def get(self, request, path):
+ remote_url = 'http://{}/flower/{}'.format(flower_url, path)
+ try:
+ response = proxy_view(request, remote_url)
+ except Exception as e:
+ msg = _("Flower service unavailable, check it
") + \
+ '
{}
'.format(e)
+ response = HttpResponse(msg)
+ return response
+
+ def post(self, request, path):
+ return self.get(request, path)
diff --git a/fir_ser/api/management/__init__.py b/fir_ser/api/management/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/fir_ser/api/management/commands/__init__.py b/fir_ser/api/management/commands/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/fir_ser/api/management/commands/expire_caches.py b/fir_ser/api/management/commands/expire_caches.py
new file mode 100644
index 0000000..c69d090
--- /dev/null
+++ b/fir_ser/api/management/commands/expire_caches.py
@@ -0,0 +1,10 @@
+from django.core.management.base import BaseCommand
+
+from api.tasks import start_api_sever_do_clean
+
+
+class Command(BaseCommand):
+ help = 'Expire caches'
+
+ def handle(self, *args, **options):
+ start_api_sever_do_clean()
diff --git a/fir_ser/api/management/commands/restart.py b/fir_ser/api/management/commands/restart.py
new file mode 100644
index 0000000..57285f9
--- /dev/null
+++ b/fir_ser/api/management/commands/restart.py
@@ -0,0 +1,6 @@
+from .services.command import BaseActionCommand, Action
+
+
+class Command(BaseActionCommand):
+ help = 'Restart services'
+ action = Action.restart.value
diff --git a/fir_ser/api/management/commands/services/__init__.py b/fir_ser/api/management/commands/services/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/fir_ser/api/management/commands/services/command.py b/fir_ser/api/management/commands/services/command.py
new file mode 100644
index 0000000..2dd5c8c
--- /dev/null
+++ b/fir_ser/api/management/commands/services/command.py
@@ -0,0 +1,146 @@
+from django.core.management.base import BaseCommand, CommandError
+from django.db.models import TextChoices
+from .utils import ServicesUtil
+from .hands import *
+
+
+class Services(TextChoices):
+ # gunicorn = 'gunicorn', 'gunicorn'
+ uwsgi = 'uwsgi', 'uwsgi'
+ celery = 'celery', 'celery'
+ beat = 'beat', 'beat'
+ flower = 'flower', 'flower'
+ task = 'task', 'task'
+ all = 'all', 'all'
+
+ @classmethod
+ def get_service_object_class(cls, name):
+ from . import services
+ services_map = {
+ cls.flower: services.FlowerService,
+ cls.celery: services.CeleryDefaultService,
+ cls.beat: services.BeatService,
+ cls.uwsgi: services.UwsgiService
+ }
+ return services_map.get(name)
+
+ @classmethod
+ def api_services(cls):
+ return [cls.uwsgi]
+
+ @classmethod
+ def flower_services(cls):
+ return [cls.flower]
+
+ @classmethod
+ def beat_services(cls):
+ return [cls.beat]
+
+ @classmethod
+ def celery_services(cls):
+ return [cls.celery]
+
+ @classmethod
+ def task_services(cls):
+ return cls.celery_services() + cls.beat_services() + cls.flower_services()
+
+ @classmethod
+ def all_services(cls):
+ return cls.task_services() + cls.api_services()
+
+ @classmethod
+ def export_services_values(cls):
+ return [cls.all.value, cls.uwsgi.value, cls.task.value, cls.celery.value, cls.flower.value, cls.beat.value]
+
+ @classmethod
+ def get_service_objects(cls, service_names, **kwargs):
+ services = set()
+ for name in service_names:
+ method_name = f'{name}_services'
+ if hasattr(cls, method_name):
+ _services = getattr(cls, method_name)()
+ elif hasattr(cls, name):
+ _services = [getattr(cls, name)]
+ else:
+ continue
+ services.update(set(_services))
+
+ service_objects = []
+ for s in services:
+ service_class = cls.get_service_object_class(s.value)
+ if not service_class:
+ continue
+ kwargs.update({
+ 'name': s.value
+ })
+ service_object = service_class(**kwargs)
+ service_objects.append(service_object)
+ return service_objects
+
+
+class Action(TextChoices):
+ start = 'start', 'start'
+ status = 'status', 'status'
+ stop = 'stop', 'stop'
+ restart = 'restart', 'restart'
+
+
+class BaseActionCommand(BaseCommand):
+ help = 'Service Base Command'
+
+ action = None
+ util = None
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ 'services', nargs='+', choices=Services.export_services_values(), help='Service',
+ )
+ parser.add_argument('-d', '--daemon', nargs="?", const=True)
+ parser.add_argument('-up', '--uwsgi_processes', type=int, nargs="?", default=4)
+ parser.add_argument('-ut', '--uwsgi_threads', type=int, nargs="?", default=2)
+ parser.add_argument('-usm', '--uwsgi_socket_mode', nargs="?", const=True,
+ help='run to bind socket mode, default http mode, only uwsgi service')
+ parser.add_argument('-f', '--force', nargs="?", const=True)
+ parser.add_argument('-u', '--uid', nargs="?", default='root', type=str)
+ parser.add_argument('-g', '--gid', nargs="?", default='root', type=str)
+
+ def initial_util(self, *args, **options):
+ service_names = options.get('services')
+ service_kwargs = {
+ 'uwsgi_processes': options.get('uwsgi_processes'),
+ 'uwsgi_threads': options.get('uwsgi_threads'),
+ 'uwsgi_socket_mode': options.get('uwsgi_socket_mode'),
+ 'uid': options.get('uid'),
+ 'gid': options.get('gid'),
+ }
+ services = Services.get_service_objects(service_names=service_names, **service_kwargs)
+
+ kwargs = {
+ 'services': services,
+ 'run_daemon': options.get('daemon', False),
+ 'stop_daemon': self.action == Action.stop.value and Services.all.value in service_names,
+ 'force_stop': options.get('force') or False,
+ }
+ self.util = ServicesUtil(**kwargs)
+
+ def handle(self, *args, **options):
+ self.initial_util(*args, **options)
+ assert self.action in Action.values, f'The action {self.action} is not in the optional list'
+ _handle = getattr(self, f'_handle_{self.action}', lambda: None)
+ _handle()
+
+ def _handle_start(self):
+ self.util.start_and_watch()
+ os._exit(0)
+
+ def _handle_stop(self):
+ self.util.stop()
+
+ def _handle_restart(self):
+ self.util.restart()
+
+ def _handle_status(self):
+ self.util.show_status()
diff --git a/fir_ser/api/management/commands/services/hands.py b/fir_ser/api/management/commands/services/hands.py
new file mode 100644
index 0000000..4611099
--- /dev/null
+++ b/fir_ser/api/management/commands/services/hands.py
@@ -0,0 +1,28 @@
+import os
+import sys
+import logging
+from django.conf import settings
+
+from config import BASECONF
+
+try:
+ from apps.jumpserver import const
+
+ __version__ = const.VERSION
+except ImportError as e:
+ print("Not found __version__: {}".format(e))
+ print("Python is: ")
+ logging.info(sys.executable)
+ __version__ = 'Unknown'
+
+SOCKET_HOST = BASECONF.SERVER_BIND_HOST or '127.0.0.1'
+SOCKET_PORT = BASECONF.SERVER_LISTEN_PORT or 8080
+
+CELERY_FLOWER_HOST = BASECONF.CELERY_FLOWER_HOST or '127.0.0.1'
+CELERY_FLOWER_PORT = BASECONF.CELERY_FLOWER_PORT or 5555
+
+DEBUG = BASECONF.DEBUG or False
+BASE_DIR = settings.BASE_DIR
+APPS_DIR = BASE_DIR
+LOG_DIR = os.path.join(BASE_DIR, 'logs')
+TMP_DIR = os.path.join(BASE_DIR, 'tmp')
diff --git a/fir_ser/api/management/commands/services/services/__init__.py b/fir_ser/api/management/commands/services/services/__init__.py
new file mode 100644
index 0000000..5918309
--- /dev/null
+++ b/fir_ser/api/management/commands/services/services/__init__.py
@@ -0,0 +1,5 @@
+from .beat import *
+from .flower import *
+from .gunicorn import *
+from .celery_default import *
+from .uwsgi import *
diff --git a/fir_ser/api/management/commands/services/services/base.py b/fir_ser/api/management/commands/services/services/base.py
new file mode 100644
index 0000000..526231f
--- /dev/null
+++ b/fir_ser/api/management/commands/services/services/base.py
@@ -0,0 +1,233 @@
+import abc
+import time
+import shutil
+import psutil
+import datetime
+import threading
+import subprocess
+import pwd
+from ..hands import *
+
+
+def get_user_id(uid):
+ if uid.isdigit():
+ try:
+ return pwd.getpwuid(int(uid)).pw_uid
+ except:
+ ...
+ else:
+ try:
+ return pwd.getpwnam(uid).pw_uid
+ except:
+ ...
+
+
+class BaseService(object):
+
+ def __init__(self, **kwargs):
+ self.name = kwargs['name']
+ self.uid = kwargs['uid']
+ self.gid = kwargs['gid']
+ self._process = None
+ self.STOP_TIMEOUT = 10
+ self.max_retry = 0
+ self.retry = 3
+ self.LOG_KEEP_DAYS = 7
+ self.EXIT_EVENT = threading.Event()
+
+ @property
+ @abc.abstractmethod
+ def cmd(self):
+ return []
+
+ @property
+ @abc.abstractmethod
+ def cwd(self):
+ return ''
+
+ @property
+ def is_running(self):
+ if self.pid == 0:
+ return False
+ try:
+ os.kill(self.pid, 0)
+ except (OSError, ProcessLookupError):
+ return False
+ else:
+ return True
+
+ def show_status(self):
+ if self.is_running:
+ msg = f'{self.name} is running: {self.pid}.'
+ else:
+ msg = f'{self.name} is stopped.'
+ print(msg)
+
+ # -- log --
+ @property
+ def log_filename(self):
+ return f'{self.name}.log'
+
+ @property
+ def log_filepath(self):
+ return os.path.join(LOG_DIR, self.log_filename)
+
+ @property
+ def log_file(self):
+ return open(self.log_filepath, 'a')
+
+ @property
+ def log_dir(self):
+ return os.path.dirname(self.log_filepath)
+
+ # -- end log --
+
+ # -- pid --
+ @property
+ def pid_filepath(self):
+ return os.path.join(TMP_DIR, f'{self.name}.pid')
+
+ @property
+ def pid(self):
+ if not os.path.isfile(self.pid_filepath):
+ return 0
+ with open(self.pid_filepath) as f:
+ try:
+ pid = int(f.read().strip())
+ except ValueError:
+ pid = 0
+ return pid
+
+ def write_pid(self):
+ with open(self.pid_filepath, 'w') as f:
+ f.write(str(self.process.pid))
+
+ def remove_pid(self):
+ if os.path.isfile(self.pid_filepath):
+ os.unlink(self.pid_filepath)
+
+ # -- end pid --
+
+ # -- process --
+ @property
+ def process(self):
+ if not self._process:
+ try:
+ self._process = psutil.Process(self.pid)
+ except:
+ pass
+ return self._process
+
+ # -- end process --
+
+ # -- action --
+ def open_subprocess(self):
+ self.set_work_dir_owner()
+ kwargs = {'cwd': self.cwd, 'stderr': self.log_file, 'stdout': self.log_file}
+ self._process = subprocess.Popen(self.cmd, **kwargs)
+
+ def start(self):
+ if self.is_running:
+ self.show_status()
+ return
+ self.remove_pid()
+ self.open_subprocess()
+ self.write_pid()
+ self.start_other()
+
+ def start_other(self):
+ pass
+
+ def set_work_dir_owner(self):
+ uid = get_user_id(self.uid)
+ gid = get_user_id(self.gid)
+ u_gid = uid if uid else gid
+ if u_gid:
+ for (dir_path, dir_names, filenames) in os.walk(LOG_DIR):
+ for filename in filenames + dir_names:
+ os.chown(uid=u_gid, gid=u_gid, path=os.path.join(dir_path, filename))
+ else:
+ logging.error(f'uid: {self.uid} gid:{self.gid} is not exists')
+
+ def stop(self, force=False):
+ if not self.is_running:
+ self.show_status()
+ # self.remove_pid()
+ return
+
+ print(f'Stop service: {self.name}', end='')
+ sig = 9 if force else 15
+ os.kill(self.pid, sig)
+
+ if self.process is None:
+ print("\033[31m No process found\033[0m")
+ return
+ try:
+ self.process.wait(5)
+ except:
+ pass
+
+ for i in range(self.STOP_TIMEOUT):
+ if i == self.STOP_TIMEOUT - 1:
+ print("\033[31m Error\033[0m")
+ if not self.is_running:
+ print("\033[32m Ok\033[0m")
+ self.remove_pid()
+ break
+ else:
+ time.sleep(1)
+ continue
+
+ def watch(self):
+ self._check()
+ if not self.is_running:
+ self._restart()
+ self._rotate_log()
+
+ def _check(self):
+ now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+ print(f"{now} Check service status: {self.name} -> ", end='')
+ if self.process:
+ try:
+ self.process.wait(1) # 不wait，子进程可能无法回收
+ except:
+ pass
+
+ if self.is_running:
+ print(f'running at {self.pid}')
+ else:
+ print(f'stopped at {self.pid}')
+
+ def _restart(self):
+ if self.retry > self.max_retry:
+ logging.info(f"Service {self.name} start failed, exit")
+ self.EXIT_EVENT.set()
+ return
+ self.retry += 1
+ logging.info(f'> Find {self.name} stopped, retry {self.retry}, {self.pid}')
+ self.start()
+
+ def _rotate_log(self):
+ now = datetime.datetime.now()
+ _time = now.strftime('%H:%M')
+ if _time != '23:59':
+ return
+
+ backup_date = now.strftime('%Y-%m-%d')
+ backup_log_dir = os.path.join(self.log_dir, backup_date)
+ if not os.path.exists(backup_log_dir):
+ os.mkdir(backup_log_dir)
+
+ backup_log_path = os.path.join(backup_log_dir, self.log_filename)
+ if os.path.isfile(self.log_filepath) and not os.path.isfile(backup_log_path):
+ logging.info(f'Rotate log file: {self.log_filepath} => {backup_log_path}')
+ shutil.copy(self.log_filepath, backup_log_path)
+ with open(self.log_filepath, 'w') as f:
+ pass
+
+ to_delete_date = now - datetime.timedelta(days=self.LOG_KEEP_DAYS)
+ to_delete_dir = os.path.join(LOG_DIR, to_delete_date.strftime('%Y-%m-%d'))
+ if os.path.exists(to_delete_dir):
+ logging.info(f'Remove old log: {to_delete_dir}')
+ shutil.rmtree(to_delete_dir, ignore_errors=True)
+ # -- end action --
diff --git a/fir_ser/api/management/commands/services/services/beat.py b/fir_ser/api/management/commands/services/services/beat.py
new file mode 100644
index 0000000..0600f50
--- /dev/null
+++ b/fir_ser/api/management/commands/services/services/beat.py
@@ -0,0 +1,31 @@
+from ..hands import *
+from .base import BaseService
+from django.core.cache import cache
+
+__all__ = ['BeatService']
+
+
+class BeatService(BaseService):
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self.lock = cache.lock('beat-distribute-start-lock', timeout=60)
+
+ @property
+ def cmd(self):
+ scheduler = "django_celery_beat.schedulers:DatabaseScheduler"
+ print("\n- Start Beat as Periodic Task Scheduler")
+ cmd = [
+ 'celery', '-A',
+ 'fir_ser', 'beat',
+ '-l', 'INFO',
+ '--uid', self.uid,
+ '--gid', self.gid,
+ '--scheduler', scheduler,
+ '--max-interval', '60'
+ ]
+ return cmd
+
+ @property
+ def cwd(self):
+ return APPS_DIR
diff --git a/fir_ser/api/management/commands/services/services/celery_base.py b/fir_ser/api/management/commands/services/services/celery_base.py
new file mode 100644
index 0000000..eca414a
--- /dev/null
+++ b/fir_ser/api/management/commands/services/services/celery_base.py
@@ -0,0 +1,39 @@
+from ..hands import *
+from .base import BaseService
+
+
+class CeleryBaseService(BaseService):
+
+ def __init__(self, queue, num=10, **kwargs):
+ super().__init__(**kwargs)
+ self.queue = queue
+ self.num = num
+
+ @property
+ def cmd(self):
+ print('\n- Start Celery as Distributed Task Queue: {}'.format(self.queue.capitalize()))
+
+ os.environ.setdefault('PYTHONOPTIMIZE', '1')
+ os.environ.setdefault('ANSIBLE_FORCE_COLOR', 'True')
+
+ if os.getuid() == 0:
+ os.environ.setdefault('C_FORCE_ROOT', '1')
+ server_hostname = os.environ.get("SERVER_HOSTNAME")
+ if not server_hostname:
+ server_hostname = '%n'
+
+ cmd = [
+ 'celery', '-A',
+ 'fir_ser', 'worker',
+ '-l', 'INFO',
+ '--uid', self.uid,
+ '--gid', self.gid,
+ '-c', str(self.num),
+ '-Q', self.queue,
+ '-n', f'{self.queue}@{server_hostname}'
+ ]
+ return cmd
+
+ @property
+ def cwd(self):
+ return APPS_DIR
diff --git a/fir_ser/api/management/commands/services/services/celery_default.py b/fir_ser/api/management/commands/services/services/celery_default.py
new file mode 100644
index 0000000..5d3e6d7
--- /dev/null
+++ b/fir_ser/api/management/commands/services/services/celery_default.py
@@ -0,0 +1,10 @@
+from .celery_base import CeleryBaseService
+
+__all__ = ['CeleryDefaultService']
+
+
+class CeleryDefaultService(CeleryBaseService):
+
+ def __init__(self, **kwargs):
+ kwargs['queue'] = 'celery'
+ super().__init__(**kwargs)
diff --git a/fir_ser/api/management/commands/services/services/flower.py b/fir_ser/api/management/commands/services/services/flower.py
new file mode 100644
index 0000000..30ba8aa
--- /dev/null
+++ b/fir_ser/api/management/commands/services/services/flower.py
@@ -0,0 +1,36 @@
+from ..hands import *
+from .base import BaseService
+
+__all__ = ['FlowerService']
+
+
+class FlowerService(BaseService):
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @property
+ def cmd(self):
+ print("\n- Start Flower as Task Monitor")
+
+ if os.getuid() == 0:
+ os.environ.setdefault('C_FORCE_ROOT', '1')
+ cmd = [
+ 'celery', '-A',
+ 'fir_ser', 'flower',
+ '-l', 'INFO',
+ '--uid', self.uid,
+ '--gid', self.gid,
+ '--url_prefix=/flower',
+ '--auto_refresh=False',
+ '--max_tasks=1000',
+ f'--address={CELERY_FLOWER_HOST}',
+ f'--port={CELERY_FLOWER_PORT}',
+ # '--basic_auth=flower:ninevenxxx'
+ # '--tasks_columns=uuid,name,args,state,received,started,runtime,worker'
+ ]
+ return cmd
+
+ @property
+ def cwd(self):
+ return APPS_DIR
diff --git a/fir_ser/api/management/commands/services/services/gunicorn.py b/fir_ser/api/management/commands/services/services/gunicorn.py
new file mode 100644
index 0000000..f3c3d76
--- /dev/null
+++ b/fir_ser/api/management/commands/services/services/gunicorn.py
@@ -0,0 +1,35 @@
+from ..hands import *
+from .base import BaseService
+
+__all__ = ['GunicornService']
+
+
+class GunicornService(BaseService):
+
+ def __init__(self, **kwargs):
+ self.worker = kwargs['worker_gunicorn']
+ super().__init__(**kwargs)
+
+ @property
+ def cmd(self):
+ print("\n- Start Gunicorn WSGI HTTP Server")
+
+ log_format = '%(h)s %(t)s %(L)ss "%(r)s" %(s)s %(b)s '
+ bind = f'{HTTP_HOST}:{HTTP_PORT}'
+ cmd = [
+ 'gunicorn', 'fir_ser.wsgi',
+ '-b', bind,
+ '-k', 'gthread',
+ '--threads', '10',
+ '-w', str(self.worker),
+ '--max-requests', '4096',
+ '--access-logformat', log_format,
+ '--access-logfile', '-'
+ ]
+ if DEBUG:
+ cmd.append('--reload')
+ return cmd
+
+ @property
+ def cwd(self):
+ return APPS_DIR
diff --git a/fir_ser/api/management/commands/services/services/uwsgi.py b/fir_ser/api/management/commands/services/services/uwsgi.py
new file mode 100644
index 0000000..96466eb
--- /dev/null
+++ b/fir_ser/api/management/commands/services/services/uwsgi.py
@@ -0,0 +1,45 @@
+from ..hands import *
+from .base import BaseService
+
+__all__ = ['UwsgiService']
+
+
+class UwsgiService(BaseService):
+
+ def __init__(self, **kwargs):
+ self.processes = kwargs['uwsgi_processes']
+ self.threads = kwargs['uwsgi_threads']
+ self.uwsgi_socket_mode = kwargs['uwsgi_socket_mode']
+ super().__init__(**kwargs)
+
+ @property
+ def cmd(self):
+ if os.getuid() == 0:
+ os.environ.setdefault('C_FORCE_ROOT', '1')
+ print("\n- Start Uwsgi WSGI HTTP Server")
+ bind = f'{SOCKET_HOST}:{SOCKET_PORT}'
+ cmd = [
+ 'uwsgi',
+ '--uid', self.uid,
+ '--gid', self.gid,
+ '--processes', f'{self.processes}',
+ '--threads', f'{self.threads}',
+ '--wsgi-file', f"{BASE_DIR}/fir_ser/wsgi.py",
+ '--listen', '512',
+ '--chdir', BASE_DIR,
+ '--buffer-size', '65536',
+ '--vacuum',
+ '--enable-threads',
+ ]
+
+ if self.uwsgi_socket_mode:
+ cmd.extend(['--socket', bind])
+ else:
+ cmd.extend(['--http', bind])
+ if DEBUG:
+ cmd.extend(['--touch-reload', BASE_DIR])
+ return cmd
+
+ @property
+ def cwd(self):
+ return APPS_DIR
diff --git a/fir_ser/api/management/commands/services/utils.py b/fir_ser/api/management/commands/services/utils.py
new file mode 100644
index 0000000..510929e
--- /dev/null
+++ b/fir_ser/api/management/commands/services/utils.py
@@ -0,0 +1,141 @@
+import threading
+import signal
+import time
+import daemon
+from daemon import pidfile
+from .hands import *
+from .hands import __version__
+from .services.base import BaseService
+
+
+class ServicesUtil(object):
+
+ def __init__(self, services, run_daemon=False, force_stop=False, stop_daemon=False):
+ self._services = services
+ self.run_daemon = run_daemon
+ self.force_stop = force_stop
+ self.stop_daemon = stop_daemon
+ self.EXIT_EVENT = threading.Event()
+ self.check_interval = 30
+ self.files_preserve_map = {}
+
+ def restart(self):
+ self.stop()
+ time.sleep(5)
+ self.start_and_watch()
+
+ def start_and_watch(self):
+ logging.info(time.ctime())
+ logging.info(f'FlyApps version {__version__}, more see https://flyapps.cn')
+ self.start()
+ if self.run_daemon:
+ self.show_status()
+ with self.daemon_context:
+ self.watch()
+ else:
+ self.watch()
+
+ def start(self):
+ for service in self._services:
+ service: BaseService
+ service.start()
+ self.files_preserve_map[service.name] = service.log_file
+ time.sleep(1)
+
+ def stop(self):
+ for service in self._services:
+ service: BaseService
+ service.stop(force=self.force_stop)
+
+ if self.stop_daemon:
+ self._stop_daemon()
+
+ # -- watch --
+ def watch(self):
+ while not self.EXIT_EVENT.is_set():
+ try:
+ _exit = self._watch()
+ if _exit:
+ break
+ time.sleep(self.check_interval)
+ except KeyboardInterrupt:
+ print('Start stop services')
+ break
+ self.clean_up()
+
+ def _watch(self):
+ for service in self._services:
+ service: BaseService
+ service.watch()
+ if service.EXIT_EVENT.is_set():
+ self.EXIT_EVENT.set()
+ return True
+ return False
+
+ # -- end watch --
+
+ def clean_up(self):
+ if not self.EXIT_EVENT.is_set():
+ self.EXIT_EVENT.set()
+
+ self.stop()
+
+ def show_status(self):
+ for service in self._services:
+ service: BaseService
+ service.show_status()
+
+ # -- daemon --
+ def _stop_daemon(self):
+ if self.daemon_pid and self.daemon_is_running:
+ os.kill(self.daemon_pid, 15)
+ self.remove_daemon_pid()
+
+ def remove_daemon_pid(self):
+ if os.path.isfile(self.daemon_pid_filepath):
+ os.unlink(self.daemon_pid_filepath)
+
+ @property
+ def daemon_pid(self):
+ if not os.path.isfile(self.daemon_pid_filepath):
+ return 0
+ with open(self.daemon_pid_filepath) as f:
+ try:
+ pid = int(f.read().strip())
+ except ValueError:
+ pid = 0
+ return pid
+
+ @property
+ def daemon_is_running(self):
+ try:
+ os.kill(self.daemon_pid, 0)
+ except (OSError, ProcessLookupError):
+ return False
+ else:
+ return True
+
+ @property
+ def daemon_pid_filepath(self):
+ return os.path.join(TMP_DIR, 'flyapps.pid')
+
+ @property
+ def daemon_log_filepath(self):
+ return os.path.join(LOG_DIR, 'flyApps.log')
+
+ @property
+ def daemon_context(self):
+ daemon_log_file = open(self.daemon_log_filepath, 'a')
+ context = daemon.DaemonContext(
+ pidfile=pidfile.TimeoutPIDLockFile(self.daemon_pid_filepath),
+ signal_map={
+ signal.SIGTERM: lambda x, y: self.clean_up(),
+ signal.SIGHUP: 'terminate',
+ },
+ stdout=daemon_log_file,
+ stderr=daemon_log_file,
+ files_preserve=list(self.files_preserve_map.values()),
+ detach_process=True,
+ )
+ return context
+ # -- end daemon --
diff --git a/fir_ser/api/management/commands/start.py b/fir_ser/api/management/commands/start.py
new file mode 100644
index 0000000..4c078a8
--- /dev/null
+++ b/fir_ser/api/management/commands/start.py
@@ -0,0 +1,6 @@
+from .services.command import BaseActionCommand, Action
+
+
+class Command(BaseActionCommand):
+ help = 'Start services'
+ action = Action.start.value
diff --git a/fir_ser/api/management/commands/status.py b/fir_ser/api/management/commands/status.py
new file mode 100644
index 0000000..36f0d36
--- /dev/null
+++ b/fir_ser/api/management/commands/status.py
@@ -0,0 +1,6 @@
+from .services.command import BaseActionCommand, Action
+
+
+class Command(BaseActionCommand):
+ help = 'Show services status'
+ action = Action.status.value
diff --git a/fir_ser/api/management/commands/stop.py b/fir_ser/api/management/commands/stop.py
new file mode 100644
index 0000000..a79a533
--- /dev/null
+++ b/fir_ser/api/management/commands/stop.py
@@ -0,0 +1,6 @@
+from .services.command import BaseActionCommand, Action
+
+
+class Command(BaseActionCommand):
+ help = 'Stop services'
+ action = Action.stop.value
diff --git a/fir_ser/api/tasks.py b/fir_ser/api/tasks.py
index f3c80d9..41ab9b3 100644
--- a/fir_ser/api/tasks.py
+++ b/fir_ser/api/tasks.py
@@ -15,6 +15,7 @@ from api.utils.app.supersignutils import IosUtils, resign_by_app_id
from api.utils.crontab.ctasks import sync_download_times, auto_clean_upload_tmp_file, auto_delete_ios_mobile_tmp_file, \
auto_check_ios_developer_active
from api.utils.geetest.geetest_utils import check_bypass_status
+from api.views.login import get_login_type
from fir_ser.celery import app
@@ -85,4 +86,5 @@ def auto_check_ios_developer_active_job():
@app.task
def sync_wx_access_token_job():
- sync_wx_access_token()
+ if get_login_type().get('third', '').get('wxp'):
+ sync_wx_access_token()
diff --git a/fir_ser/api/utils/auth.py b/fir_ser/api/utils/auth.py
index 12001ca..7107488 100644
--- a/fir_ser/api/utils/auth.py
+++ b/fir_ser/api/utils/auth.py
@@ -9,9 +9,17 @@ from rest_framework.exceptions import AuthenticationFailed
from api.models import UserInfo
from rest_framework.permissions import BasePermission
from fir_ser.settings import CACHE_KEY_TEMPLATE
+from django.http.cookie import parse_cookie
import base64
+def get_cookie_token(request):
+ cookies = request.META.get('HTTP_COOKIE')
+ if cookies:
+ cookie_dict = parse_cookie(cookies)
+ return cookie_dict.get('auth_token')
+
+
def get_user_from_api_token(request):
if request.method == "OPTIONS":
return None
@@ -29,7 +37,8 @@ def get_user_from_request_auth(request):
if request.method == "OPTIONS":
return None
request_token = request.META.get("HTTP_AUTHORIZATION",
- request.META.get("HTTP_X_TOKEN", request.query_params.get("token", None)))
+ request.META.get("HTTP_X_TOKEN",
+ request.query_params.get("token", get_cookie_token(request))))
if request_token:
try:
format_token = base64.b64decode(request_token).decode()
diff --git a/fir_ser/api/utils/pay/util.py b/fir_ser/api/utils/pay/util.py
index ce678a5..4b4fb01 100644
--- a/fir_ser/api/utils/pay/util.py
+++ b/fir_ser/api/utils/pay/util.py
@@ -4,13 +4,13 @@
# author: NinEveN
# date: 2021/4/18
-from fir_ser.settings import PAY_CONFIG
+from fir_ser.settings import PAY_CONFIG_KEY_INFO
from api.utils.pay.ali import Alipay
from api.utils.pay.wx import Weixinpay
def get_pay_obj_form_name(pay_name):
- for pay_info in PAY_CONFIG:
+ for pay_info in PAY_CONFIG_KEY_INFO:
if pay_name == pay_info.get('NAME', '') and pay_info.get('ENABLED', False):
auth_info = pay_info.get('AUTH', None)
p_type = pay_info.get('TYPE', '')
@@ -25,7 +25,7 @@ def get_pay_obj_form_name(pay_name):
def get_enable_pay_choices():
pay_choices = []
- for pay_info in PAY_CONFIG:
+ for pay_info in PAY_CONFIG_KEY_INFO:
if pay_info.get('ENABLED', False):
pay_choices.append({'type': pay_info.get('TYPE'), 'name': pay_info.get('NAME', '')})
return pay_choices
diff --git a/fir_ser/api/utils/sendmsg/sendmsg.py b/fir_ser/api/utils/sendmsg/sendmsg.py
index 8762c75..30c05a0 100644
--- a/fir_ser/api/utils/sendmsg/sendmsg.py
+++ b/fir_ser/api/utils/sendmsg/sendmsg.py
@@ -7,7 +7,7 @@
from .aliyunApi import AliMsgSender
from .emailApi import EmailMsgSender
from .jiguangApi import JiGuangMsgSender
-from fir_ser.settings import THIRD_PART_CONFIG
+from fir_ser.settings import THIRD_PART_CONFIG_KEY_INFO
import logging
logger = logging.getLogger(__name__)
@@ -20,7 +20,7 @@ def get_default_sender(send_type):
1: 阿里云
2: 极光
"""
- sender_lists = THIRD_PART_CONFIG.get('sender')
+ sender_lists = THIRD_PART_CONFIG_KEY_INFO.get('sender_key')
for sender in sender_lists:
if sender.get("active", None):
sender_type = sender.get('type', None)
diff --git a/fir_ser/api/utils/storage/storage.py b/fir_ser/api/utils/storage/storage.py
index 587d457..3df793f 100644
--- a/fir_ser/api/utils/storage/storage.py
+++ b/fir_ser/api/utils/storage/storage.py
@@ -10,7 +10,7 @@ from .qiniuApi import QiNiuOss
from .localApi import LocalStorage
import json, time, base64
from api.utils.baseutils import get_dict_from_filter_fields
-from fir_ser.settings import THIRD_PART_CONFIG, CACHE_KEY_TEMPLATE
+from fir_ser.settings import THIRD_PART_CONFIG_KEY_INFO, CACHE_KEY_TEMPLATE
from django.core.cache import cache
import logging
@@ -146,7 +146,7 @@ class Storage(object):
def get_local_storage(clean_cache=False):
- storage_lists = THIRD_PART_CONFIG.get('storage')
+ storage_lists = THIRD_PART_CONFIG_KEY_INFO.get('storage_key')
for storage in storage_lists:
storage_type = storage.get('type', None)
if storage_type == 0:
@@ -169,7 +169,7 @@ def get_local_storage(clean_cache=False):
def get_storage_form_conf(user):
- storage_lists = THIRD_PART_CONFIG.get('storage', [])
+ storage_lists = THIRD_PART_CONFIG_KEY_INFO.get('storage_key', [])
for storage in storage_lists:
if storage.get("active", None):
storage_type = storage.get('type', None)
diff --git a/fir_ser/api/views/thirdlogin.py b/fir_ser/api/views/thirdlogin.py
index 88472e9..81eba98 100644
--- a/fir_ser/api/views/thirdlogin.py
+++ b/fir_ser/api/views/thirdlogin.py
@@ -138,13 +138,11 @@ class ValidWxChatToken(APIView):
if rec_msg.Eventkey == 'query_bind':
if wx_user_obj:
user_obj = wx_user_obj.user_id
- content = f'绑定用户 {user_obj.first_name}\n'
+ content = f'绑定用户 {user_obj.first_name} '
if user_obj.email:
- content += f' 登录邮箱：{user_obj.email}\n'
+ content += f' 登录邮箱：{user_obj.email} '
if user_obj.mobile:
content += f' 登录手机：{user_obj.mobile}'
- if content.endswith('\n'):
- content = content[0:-1]
user_cert_obj = UserCertificationInfo.objects.filter(user_id=user_obj,
status=1).first()
if user_cert_obj:
@@ -202,12 +200,13 @@ class ValidWxChatToken(APIView):
content = f'账户已经被 {wx_user_obj.user_id.first_name} 绑定'
WxTemplateMsg().bind_failed_msg(to_user, wx_user_obj.nickname, content)
else:
- wx_user_info = update_or_create_wx_userinfo(to_user, user_obj)
- content = f'账户绑定 {wx_user_obj.user_id.first_name} 成功'
- WxTemplateMsg().bind_success_msg(to_user, wx_user_info.get('nickname', ''),
- user_obj.first_name)
-
- set_wx_ticket_login_info_cache(rec_msg.Ticket, {'pk': user_obj.pk})
+ if user_obj:
+ wx_user_info = update_or_create_wx_userinfo(to_user, user_obj)
+ content = f'账户绑定 {wx_user_obj.user_id.first_name} 成功'
+ WxTemplateMsg().bind_success_msg(to_user, wx_user_info.get('nickname', ''),
+ user_obj.first_name)
+ if user_obj:
+ set_wx_ticket_login_info_cache(rec_msg.Ticket, {'pk': user_obj.pk})
reply_msg = reply.TextMsg(to_user, from_user, content)
result = reply_msg.send()
else:
diff --git a/fir_ser/config.py b/fir_ser/config.py
index 47fd06f..e6bd355 100644
--- a/fir_ser/config.py
+++ b/fir_ser/config.py
@@ -17,7 +17,17 @@ class BASECONF(object):
SECRET_KEY = 'j!g@^bc(z(a3*i&kp$_@bgb)bug&^#3=amch!3lz&1x&s6ss6t'
ALLOWED_HOSTS = ['127.0.0.1', 'synchrotron', '172.16.133.34', 'ali.cdn.flyapp.dvcloud.xin',
- 'api.src.flyapp.dvcloud.xin']
+ 'api.src.flyapp.dvcloud.xin', 'app.hehelucky.cn']
+
+ # When Django start it will bind this host and port
+ # ./manage.py runserver 127.0.0.1:8080
+ # uwsgi运行时绑定端口
+ SERVER_BIND_HOST = '127.0.0.1'
+ SERVER_LISTEN_PORT = 8898
+
+ # celery flower 任务监控配置
+ CELERY_FLOWER_PORT = 5566
+ CELERY_FLOWER_HOST = '127.0.0.1'
SERVER_DOMAIN = {
'IOS_PMFILE_DOWNLOAD_DOMAIN': {
@@ -81,7 +91,7 @@ class THIRDLOGINCONF(object):
class AUTHCONF(object):
- # 注册方式，如果启用sms或者email 需要配置 THIRD_PART_CONFIG.sender 信息
+ # 注册方式，如果启用sms或者email 需要配置 THIRD_PART_CONFIG_KEY_INFO.sender 信息
REGISTER = {
"enable": True,
"captcha": False, # 是否开启注册字母验证码
@@ -117,7 +127,7 @@ class AUTHCONF(object):
}
-class STORAGECONF(object):
+class STORAGEKEYCONF(object):
STORAGE = [
{
'name': 'local',
@@ -247,7 +257,7 @@ class IPACONF(object):
class PAYCONF(object):
PAY_SUCCESS_URL = '%s/user/orders' % WEB_DOMAIN # 前端页面，支付成功跳转页面
- PAY_CONFIG = [
+ PAY_CONFIG_KEY_INFO = [
{
'NAME': 'alipay',
'TYPE': 'ALI',
diff --git a/fir_ser/fir_ser/settings.py b/fir_ser/fir_ser/settings.py
index 488ed47..92135da 100644
--- a/fir_ser/fir_ser/settings.py
+++ b/fir_ser/fir_ser/settings.py
@@ -184,16 +184,16 @@ GEETEST_CYCLE_TIME = BASECONF.GEETEST_CYCLE_TIME
GEETEST_BYPASS_STATUS_KEY = BASECONF.GEETEST_BYPASS_STATUS_KEY
GEETEST_BYPASS_URL = BASECONF.GEETEST_BYPASS_URL
-# 注册方式，如果启用sms或者email 需要配置 THIRD_PART_CONFIG.sender 信息
+# 注册方式，如果启用sms或者email 需要配置 THIRD_PART_CONFIG_KEY_INFO.sender 信息
REGISTER = AUTHCONF.REGISTER
# 个人资料修改修改也会使用该配置
CHANGER = AUTHCONF.CHANGER
LOGIN = AUTHCONF.LOGIN
-THIRD_PART_CONFIG = {
+THIRD_PART_CONFIG_KEY_INFO = {
# APP存储配置
- 'storage': STORAGECONF.STORAGE,
- 'sender': SENDERCONF.SENDER
+ 'storage_key': STORAGEKEYCONF.STORAGE,
+ 'sender_key': SENDERCONF.SENDER
}
CACHE_KEY_TEMPLATE = {
'user_can_download_key': 'user_can_download',
@@ -256,7 +256,9 @@ SESSION_CACHE_ALIAS = "default"
CAPTCHA_TIMEOUT = 5 # Minutes
CAPTCHA_LENGTH = 6 # Chars
-BASE_LOG_DIR = os.path.join(BASE_DIR, "logs")
+BASE_LOG_DIR = os.path.join(BASE_DIR, "logs", "flyapp")
+if not os.path.isdir(BASE_LOG_DIR):
+ os.makedirs(BASE_LOG_DIR)
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
@@ -325,7 +327,7 @@ LOGGING = {
'propagate': True,
},
'pay': { # 默认的logger应用如下配置
- 'handlers': ['pay'], # 上线之后可以把'console'移除
+ 'handlers': ['pay'],
'level': 'DEBUG',
'propagate': True,
},
@@ -333,7 +335,7 @@ LOGGING = {
}
PAY_SUCCESS_URL = PAYCONF.PAY_SUCCESS_URL
-PAY_CONFIG = PAYCONF.PAY_CONFIG
+PAY_CONFIG_KEY_INFO = PAYCONF.PAY_CONFIG_KEY_INFO
# 结果存放到Django|redis
# CELERY_RESULT_BACKEND = 'django-db'
@@ -352,7 +354,7 @@ CELERY_BROKER_URL = 'redis://:%s@%s/2' % (
#: Only add pickle to this list if your broker is secured
-CELERYD_CONCURRENCY = 4 # worker并发数
+CELERYD_CONCURRENCY = 10 # worker并发数
CELERYD_FORCE_EXECV = True # 非常重要,有些情况下可以防止死
CELERY_TASK_RESULT_EXPIRES = 3600 # 任务结果过期时间
@@ -415,3 +417,10 @@ MSGTEMPLATE = {
'ERROR_DEVELOPER': '用户 %s 你好，应用 %s 签名失败了，苹果开发者 %s 信息异常，请重新检查苹果开发者状态是否正常。感谢有你!',
'AUTO_CHECK_DEVELOPER': '用户 %s 你好，苹果开发者 %s 信息异常，请重新检查苹果开发者状态是否正常。感谢有你!',
}
+
+## listen port
+
+SERVER_BIND_HOST = BASECONF.SERVER_BIND_HOST
+SERVER_LISTEN_PORT = BASECONF.SERVER_LISTEN_PORT
+CELERY_FLOWER_PORT = BASECONF.CELERY_FLOWER_PORT
+CELERY_FLOWER_HOST = BASECONF.CELERY_FLOWER_HOST
diff --git a/fir_ser/fir_ser/urls.py b/fir_ser/fir_ser/urls.py
index 7823637..93bc4fb 100644
--- a/fir_ser/fir_ser/urls.py
+++ b/fir_ser/fir_ser/urls.py
@@ -16,6 +16,8 @@ Including another URLconf
from django.contrib import admin
from django.urls import re_path, include
from django.views.static import serve
+
+from admin.views.celery_flower import CeleryFlowerView
from fir_ser import settings
from api.views.download import DownloadView, InstallView
from api.views.receiveudids import IosUDIDView, ShowUdidView
@@ -32,5 +34,5 @@ urlpatterns = [
re_path("install/(?P\w+)$", InstallView.as_view(), name="install"),
re_path("^udid/(?P\w+)$", IosUDIDView.as_view()),
re_path("^show_udid$", ShowUdidView.as_view()),
-
+ re_path(r'flower/(?P.*)', CeleryFlowerView.as_view()),
]
diff --git a/fir_ser/nginx-vhost.conf b/fir_ser/nginx-vhost.conf
index b5bcbe1..5fbe7ca 100644
--- a/fir_ser/nginx-vhost.conf
+++ b/fir_ser/nginx-vhost.conf
@@ -1,9 +1,11 @@
upstream synchrotron {
least_conn ;
- #server 127.0.0.1:8899;
- server 127.0.0.1:8898;
- #server 127.0.0.1:8897;
- #server 127.0.0.1:8896;
+ #hash $remote_addr consistent;
+
+ server 127.0.0.1:8896 max_fails=1 fail_timeout=10s;
+ server 127.0.0.1:8897 max_fails=1 fail_timeout=10s;
+ server 127.0.0.1:8898 max_fails=1 fail_timeout=10s;
+ server 127.0.0.1:8899 backup; # 将服务器标记为备份服务器。当主服务器不可用时，将传递与备份服务器的连接
}
proxy_set_header X-Forwarded-For $remote_addr;
@@ -48,7 +50,7 @@ server
uwsgi_connect_timeout 300; # 指定连接到后端uWSGI的超时时间。
uwsgi_read_timeout 300; # 指定接收uWSGI应答的超时时间，完成握手后接收uWSGI应答的超时时间。
include uwsgi_params;
- uwsgi_pass 127.0.0.1:8899;
+ uwsgi_pass synchrotron;
uwsgi_param UWSGI_SCRIPT wsgi;
}
diff --git a/fir_ser/requirements.txt b/fir_ser/requirements.txt
index f2fe256..efe2f92 100644
--- a/fir_ser/requirements.txt
+++ b/fir_ser/requirements.txt
@@ -15,4 +15,8 @@ xmltodict==0.12.0
pyOpenSSL==20.0.1
paramiko==2.7.2
PyJWT==2.1.0
-drf-extensions==0.7.0
\ No newline at end of file
+drf-extensions==0.7.0
+python-daemon==2.2.3
+psutil==5.6.6
+flower==1.0.0
+django-proxy==1.2.1
\ No newline at end of file