From 6e98c99acfcb323d555f79aa1a2a7f0e8fdca51b Mon Sep 17 00:00:00 2001 From: kelvinBen Date: Wed, 18 Nov 2020 14:22:51 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=87=AA=E5=8A=A8=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E5=8A=9F=E8=83=BD=EF=BC=8C=E5=8A=9F=E8=83=BD=E6=9C=AA?= =?UTF-8?q?=E5=AE=8C=E5=96=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 14 +++++++++ libs/core/__init__.py | 10 ++++-- libs/core/download.py | 64 ++++++++++++++++++++++++++++++++++++++ libs/task/base_task.py | 1 + libs/task/download_task.py | 25 +++++++++++++++ 5 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 libs/core/download.py create mode 100644 libs/task/download_task.py diff --git a/config.py b/config.py index ea39aaa..bdde248 100644 --- a/config.py +++ b/config.py @@ -83,3 +83,17 @@ web_file_suffix =[ "py" ] +# 配置自动下载Apk文件或者缓存HTML的请求头信息 +header = { + "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:81.0) Gecko/20100101 Firefox/81.0", + "Connection":"close" +} + +# 配置自动下载Apk文件或者缓存HTML的请求体信息 +data = { + +} + +# 配置自动下载Apk文件或者缓存HTML的请求方法信息，目前仅支持GET和POST +method = "GET" + diff --git a/libs/core/__init__.py b/libs/core/__init__.py index c83b44a..eb825c2 100644 --- a/libs/core/__init__.py +++ b/libs/core/__init__.py @@ -38,6 +38,7 @@ class Bootstrapper(object): global app_history_path global domain_history_path global excel_row + global download_path create_time = time.strftime("%Y%m%d%H%M%S", time.localtime()) @@ -60,20 +61,25 @@ class Bootstrapper(object): apktool_path = os.path.join(tools_dir, "apktool.jar") output_path = os.path.join(script_root_dir,"out") history_path = os.path.join(script_root_dir,"history") + download_path = os.path.join(script_root_dir,"download") txt_result_path = os.path.join(script_root_dir,"result_"+str(create_time)+".txt") xls_result_path = os.path.join(script_root_dir,"result_"+str(create_time)+".xls") app_history_path = os.path.join(history_path,"app_history.txt") domain_history_path = os.path.join(history_path,"domain_history.txt") - # 包名信息一致的情况不记录URL信息 ，不一致的时候记录URL信息 + def init(self): if os.path.exists(output_path): shutil.rmtree(output_path) os.makedirs(output_path) + if os.path.exists(download_path): + shutil.rmtree(download_path) + os.makedirs(download_path) + if not os.path.exists(history_path): os.makedirs(history_path) - + if os.path.exists(txt_result_path): os.remove(txt_result_path) diff --git a/libs/core/download.py b/libs/core/download.py new file mode 100644 index 0000000..ffc1507 --- /dev/null +++ b/libs/core/download.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# Author: kelvinBen +# Github: https://github.com/kelvinBen/AppInfoScanner +import re +import os +import time +import config +import threading +import requests +import libs.core as cores + +class DownloadThreads(threading.Thread): + + def __init__(self, path, type): + self.url = path + self.type = type + + def __requset__(self): + try: + create_time = time.strftime("%Y%m%d%H%M%S", time.localtime()) + session = requests.Session() + session.mount('http://', HTTPAdapter(max_retries=3)) + session.mount('https://', HTTPAdapter(max_retries=3)) + session.keep_alive =False + session.adapters.DEFAULT_RETRIES = 5 + urllib3.disable_warnings() + + if config.method.upper() == "POST": + resp = session.post(url=self.url,params=config.data ,headers=config.headers,timeout=30) + else: + resp = session.get(url=self.url,data=config.data ,headers=config.headers,timeout=30) + + if resp.status_code == requests.codes.ok: + if self.type == "apk": + count = 0 + count_tmp = 0 + time1 = time.time() + length = float(resp.headers['content-length']) + apk_path = os.path.join(cores.download_path, create_time+".apk") + with open(apk_path, "wb") as f: + for chunk in resp.iter_content(chunk_size = 512): + if chunk: + f.write(chunk) + count += len(chunk) + if time.time() - time1 > 2: + p = count / length * 100 + speed = (count - count_tmp) / 1024 / 1024 / 2 + count_tmp = count + print(name + ': ' + formatFloat(p) + '%' + ' Speed: ' + formatFloat(speed) + 'M/S') + time1 = time.time() + f.close() + else: + html_path = os.path.join(cores.download_path, create_time+".html") + html = resp.html() + with open(html_path,"w",encoding='utf-8',errors='ignore') as f: + f.write(html) + f.close() + except Exception: + break + + + def run(self): + threadLock = threading.Lock() + self.__get_Http_info__(threadLock) \ No newline at end of file diff --git a/libs/task/base_task.py b/libs/task/base_task.py index 03bc0a3..47c8811 100644 --- a/libs/task/base_task.py +++ b/libs/task/base_task.py @@ -67,6 +67,7 @@ class BaseTask(object): def __tast_control__(self): + task_info = {} # 调用Android 相关处理逻辑 if self.types == "Android": diff --git a/libs/task/download_task.py b/libs/task/download_task.py new file mode 100644 index 0000000..d67655c --- /dev/null +++ b/libs/task/download_task.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Author: kelvinBen +# Github: https://github.com/kelvinBen/AppInfoScanner +import os +import re +import config +import hashlib +from queue import Queue +import libs.core as cores + + +class DownloadTask(object): + + def __init__(self): + self.path = path + + def start(self): + input_path = self.path + if input_path.startswith("http://") or input_path.startswith("https://"): + if input_path.endswith("apk"): + # 用来下载APK或者缓存H5或者页面内容 + pass + + return input_path +