diff --git a/README.md b/README.md index f38f678..7f269dc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,158 @@ -# AppInfoScanner +### AppInfoScanner + 一款适用于(Android、iOS、WEB、H5、静态网站)，信息检索的工具，可以帮助渗透测试人员快速获取App或者WEB中的有用资产信息。 + +### 适用场景 +- 日常渗透测试中对APP中的URL地址、IP地址、关键信息进行采集 +- 大型攻防演练场景中对APP中URL地址、IP地址、关键信息进行采集 +- 对WEB网站源代码进行URL地址、IP地址、关键信息进行采集(可以是开源的代码也可以是右击网页源代码另存为) + + +### 功能介绍: +- 支持目录批量扫描 +- 支持DEX、APK、IPA、HTML、JS、Smali等文件的静态资源采集 +- 支持自定义扫描规则 +- 支持IP地址信息采集 +- 支持URL地址信息采集 +- 支持中间件信息采集 +- 支持多线程 +- 支持忽略资源文件采集 +- 支持Android包名采集 + +### 环境说明 + +- Apk文件解析需要使用JAVA环境,JAVA版本1.8及以下 +- Python3的运行环境 + +### 目录说明 + +AppInfoScanner + |-- libs 程序的核心代码 + |-- core + |-- parses.py 用于解析文件中的静态信息 + |-- task + |-- android_task.py 用于处理Android相关的文件 + |-- ios_task.py 用于处理iOS相关的文件 + |-- web_task.py 用于处理Web相关的文件，比如网页右键源代码、H5相关的静态信息 + |-- tools 程序需要依赖的工具 + |-- apktool.jar 用于反编译apk文件 + |-- baksmali.jar 用于反编译dex文件 + |-- strings.exe 用于windows 32下获取iPA的字符串信息 + |-- strings64.exe 用于windows 64的系统获取iPA的字符串信息 + |-- app.py 主运行程序 + |-- config.py 用于自定义相关规则 + |-- readme.md 程序使用说明 + + +### Android 相关操作说明 + +#### 扫描指定的apk + +``` +python3 app.py android -i +``` + +#### 扫描指定的dex + +``` +python3 app.py android -i +``` + +#### 扫描一个目录下所有的APK或者dex + +``` +python3 app.py android -i +``` + +#### 扫描指定关键字(临时) + +``` +python3 app.py android -i -r +``` + +#### 扫描的时候忽略资源文件 + +``` +python3 app.py android -i -n +``` + +#### 扫描指定包下的内容 +``` +python3 app.py android -i -p +``` + +#### 扫描所有的字符串 + +``` +python3 app.py android -i -a +``` + +#### 指定线程数量 + +``` +python3 app.py android -i -t 10 +``` + +### iOS 相关操作说明 + +#### 扫描指定的iPA文件 + +``` +python3 app.py ios -i +``` + +#### 扫描指定关键字(临时) + +``` +python3 app.py ios -i -r +``` + +#### 扫描的时候忽略资源文件 + +``` +python3 app.py ios -i -n +``` + +#### 输出所有的字符串 + +``` +python3 app.py ios -i -a +``` + +#### 指定线程数量 + +``` +python3 app.py ios -i -t 10 +``` + +### Web 相关操作说明 + +#### 扫描指定的Web网站目录或者html相关文件 +``` +python3 app.py web -i +``` + +#### 扫描指定关键字(临时) + +``` +python3 app.py web -i -r +``` + +#### 输出所有的字符串 +``` +python3 app.py web -i -a +``` + + +#### 指定线程数量 +``` +python3 app.py web -i -t 10 +``` + + +### 常见问题 + +- 1. 信息检索垃圾数据过多？ + +> 方法1： 根据实际情况调整config.py中的规则信息 +> 方法2： 忽略资源文件 diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..00571f9 --- /dev/null +++ b/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- +# Author: kelvinBen +# Github: https://github.com/kelvinBen/AppInfoScanner diff --git a/config.py b/config.py new file mode 100644 index 0000000..26d8b1d --- /dev/null +++ b/config.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Author: kelvinBen +# Github: https://github.com/kelvinBen/AppInfoScanner + + +# 此处用于搜索组件信息，如fastjson、gson等 +filter_components = [ + 'com.alibaba.fastjson', + 'com.google.gson', + 'com.fasterxml.jackson', + 'net.sf.json' +] + +# 此处目前支持过滤 +# 1. https://以及http://开头的 +# 2. IPv4的ip地址 +# 3. URI地址 +filter_strs =[ + r'.*(http://.*)', + r'.*(https://.*)', + r'.*((?:[0-9]{1,3}\.){3}[0-9]{1,3}).*', + # r'/[a-z0-9A-Z]+/.*' +] + +# 过滤无用的内容 +filter_no = [ + u'127.0.0.1', + u'0.0.0.0', + u'localhost', + u'http://schemas.android.com/apk/res/android', + u"https://", + u"http://", + r"^http://www.w3.org" + r"L.*/", + r"/.*;", + r"/.*<" +] + +# 此处配置壳信息 +shell_list =[ + 'com.stub.StubApp', + 's.h.e.l.l.S', + 'com.Kiwisec.KiwiSecApplication', + 'com.Kiwisec.ProxyApplication', + 'com.secshell.secData.ApplicationWrapper', + 'com.secneo.apkwrapper.ApplicationWrapper', + 'com.tencent.StubShell.TxAppEntry', + 'c.b.c.b', + 'MyWrapperProxyApplication', + 'cn.securitystack.stee.AppStub', + 'com.linchaolong.apktoolplus.jiagu.ProxyApplication', + 'com.coral.util.StubApplication', + 'com.mogosec.AppMgr' +] + +# 此处配置需要扫描的web文件后缀 +web_file_suffix =[ + "html", + "js", + "html", + "xml", + "php", + "jsp", + "class", + "asp", + "aspx", + "py" +] diff --git a/libs/__init__.py b/libs/__init__.py new file mode 100644 index 0000000..00571f9 --- /dev/null +++ b/libs/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- +# Author: kelvinBen +# Github: https://github.com/kelvinBen/AppInfoScanner diff --git a/libs/core/__init__.py b/libs/core/__init__.py new file mode 100644 index 0000000..37d84eb --- /dev/null +++ b/libs/core/__init__.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# Author: kelvinBen +# Github: https://github.com/kelvinBen/AppInfoScanner + +import platform +import os +import shutil + +# smali 所在路径 +smali_path = "" + +# backsmli 所在路径 +backsmali_path = "" + +# apktool 所在路径 +apktool_path = "" + +# 系统类型 +os_type = "" + +# 输出路径 +output_path = "" + +class Bootstrapper(object): + + def __init__(self, path): + global smali_path + global backsmali_path + global apktool_path + global os_type + global output_path + global script_root_dir + global result_path + global strings_path + + script_root_dir = os.path.dirname(os.path.abspath(path)) + tools_dir = os.path.join(script_root_dir,"tools") + + if platform.system() == "Windows": + machine2bits = {'AMD64':64, 'x86_64': 64, 'i386': 32, 'x86': 32} + machine2bits.get(platform.machine()) + + if platform.machine() == 'i386' or platform.machine() == 'x86': + strings_path = os.path.join(script_root_dir,"strings.exe") + else: + strings_path = os.path.join(script_root_dir,"strings64.exe") + else: + strings_path ="strings" + # os_type = "win" + # smali_str = "smali.bat" + # back_smali_str = "backsmali.bat" + # apktool_path_str = "apktool.bat" + # elif platform.system() == "Linux": + # os_type = "lin" + # smali_str = "smali" + # back_smali_str = "backsmali" + # apktool_path_str = "apktool" + # else: + # os_type = "mac" + # smali_str = "smali" + + + # smali_path = os.path.join(tools_dir,str(os_type) + os.sep + smali_str) + backsmali_path = os.path.join(tools_dir,"baksmali.jar") + apktool_path = os.path.join(tools_dir, "apktool.jar") + output_path = os.path.join(script_root_dir,"out") + result_path = os.path.join(script_root_dir,"result.txt") + + def init(self): + if os.path.exists(output_path): + shutil.rmtree(output_path) + os.makedirs(output_path) + + if os.path.exists(result_path): + os.remove(result_path) + + + + + + \ No newline at end of file diff --git a/libs/core/parses.py b/libs/core/parses.py new file mode 100644 index 0000000..9852efe --- /dev/null +++ b/libs/core/parses.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# Author: kelvinBen +# Github: https://github.com/kelvinBen/AppInfoScanner + + +import threading +import config +import re +import os +import libs.core as cores + +class ParsesThreads(threading.Thread): + + def __init__(self,threadID,name,file_queue,all,result_dict): + threading.Thread.__init__(self) + self.file_queue = file_queue + self.name = name + self.threadID = threadID + self.result_list = [] + self.all = all + self.result_dict=result_dict + + def __regular_parse__(self,threadLock): + while True: + try: + file_path = self.file_queue.get(timeout = 5) + scan_str = ("Scan file : %s" % file_path) + print(scan_str) + + try: + os.path.basename(file_path).split(".")[1] + except Exception as e: + self.__get_string__(file_path,threadLock) + continue + self.__file_parse__(file_path,threadLock) + + result_set = set(self.result_list) + if len(result_set) !=0: + self.result_dict[file_path] = result_set + + if self.file_queue.empty(): + break + except Exception as e: + break + + def __file_parse__(self,file_path,threadLock): + with open(file_path,"r",encoding="utf8") as file : + file_content = file.read() + # 获取到所有的字符串 + pattern = re.compile(r'\"(.*?)\"') + results = pattern.findall(file_content) + + # 遍历所有的字符串 + for result in set(results): + self.__parse_string__(result,threadLock) + + def __get_string__(self,dir_file_path,threadLock): + temp = os.path.join(cores.output_path,"temp.txt") + cmd_str = ("%s %s > %s") % (cores.strings_path,dir_file_path,temp) + if os.system(cmd_str) == 0: + with open(temp,"r") as f: + lines = f.readlines() + for line in lines: + self.__parse_string__(line,threadLock) + + def __parse_string__(self,result,threadLock): + # 通过正则筛选需要过滤的字符串 + for filter_str in config.filter_strs: + filter_str_pat = re.compile(filter_str) + filter_resl = filter_str_pat.findall(result) + # print(filter_resl) + # 过滤掉未搜索到的内容 + if len(filter_resl)!=0: + # 提取第一个字符 + resl_str = filter_resl[0] + # 过滤 + if self.__filter__(resl_str) == 0: + continue + + threadLock.acquire() + self.result_list.append(filter_resl[0]) + threadLock.release() + continue + + def __filter__(self,resl_str): + resl_str = resl_str.replace("\r","").replace("\n","").replace(" ","") + if len(resl_str) == 0: + return 0 + + for filte in config.filter_no: + resl_str = resl_str.replace(filte,"") + if len(resl_str) == 0: + return 0 + + if re.match(filte,resl_str): + return 0 + return 1 + + def run(self): + threadLock = threading.Lock() + self.__regular_parse__(threadLock) diff --git a/libs/task/__init__.py b/libs/task/__init__.py new file mode 100644 index 0000000..00571f9 --- /dev/null +++ b/libs/task/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- +# Author: kelvinBen +# Github: https://github.com/kelvinBen/AppInfoScanner diff --git a/libs/task/android_task.py b/libs/task/android_task.py new file mode 100644 index 0000000..327a0ad --- /dev/null +++ b/libs/task/android_task.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +# Author: kelvinBen +# Github: https://github.com/kelvinBen/AppInfoScanner + + +import os +import re +import config +import threading +from queue import Queue +import libs.core as cores +from libs.core.parses import ParsesThreads + +class AndroidTask(object): + comp_list =[] + thread_list =[] + result_dict = {} + + def __init__(self, input, rules, net_sniffer,no_resource,package,all,threads): + self.net_sniffer = net_sniffer + self.path = input + if rules: + config.filter_strs.append(r'.*'+str(rules)+'.*') + self.no_resource = no_resource + self.package = package + self.all = all + self.threads = threads + self.file_queue = Queue() + self.shell_falg=False + self.packagename="" + + + def start(self): + # 检查java环境是否存在 + if os.system("java -version") !=0 : + raise Exception("Please install the Java environment!") + + # 根据不同的文件后缀进行文件解析 + if os.path.isfile(self.path): + if self.path.split(".")[1] == "apk": + self.__decode_apk__(self.path) + elif self.path.split(".")[1] == "dex": + self.__decode_dex__(self.path) + else: + # 抛出异常 + raise Exception("Retrieval of this file type is not supported. Select APK file or DEX file.") + else: + self.__get_file_type__(self.path) + self.__start_threads() + + for thread in self.thread_list: + thread.join() + + self.__print__() + + # if self.net_sniffer: + # self.__start_net__() + + # 分解apk + def __decode_apk__(self,path): + if self.no_resource: + self.__decode_dex__(path) + else: + cmd_str = ("java -jar %s d -f %s -o %s") % (cores.apktool_path,path,cores.output_path) + if os.system(cmd_str) == 0: + self.__scanner_file_by_apktool__(cores.output_path) + else: + raise Exception("The Apktool tool was not found.") + + # 分解dex + def __decode_dex__(self,path): + cmd_str = ("java -jar %s d %s") % (cores.backsmali_path,path) + if os.system(cmd_str) == 0: + self.__get_scanner_file__(cores.output_path,"smali") + else: + raise Exception("The baksmali tool was not found.") + + + # 初始化检测文件信息 + def __scanner_file_by_apktool__(self,output): + + # shell检测 + self.__shell_test__(output) + + scanner_dir_list = ["smali","assets"] + scanner_file_suffix = ["smali","js","xml"] + + for dir in scanner_dir_list: + scanner_dir = os.path.join(output,dir) + self.__get_scanner_file__(scanner_dir,scanner_file_suffix) + + def __get_scanner_file__(self,scanner_dir,file_suffix): + dir_or_files = os.listdir(scanner_dir) + for dir_file in dir_or_files: + dir_file_path = os.path.join(scanner_dir,dir_file) + if os.path.isdir(dir_file_path): + self.__get_scanner_file__(dir_file_path,file_suffix) + else: + if "." not in dir_file: + continue + if len(dir_file.split("."))>1: + if dir_file.split(".")[1] in file_suffix: + self.file_queue.put(dir_file_path) + for component in config.filter_components: + comp = component.replace(".","/") + if( comp in dir_file_path): + if(component not in self.comp_list): + self.comp_list.append(component) + + + def __get_file_type__(self,root_path): + dir_or_files = os.listdir(root_path) + for dir_file in dir_or_files: + dir_file_path = os.path.join(root_path,dir_file) + if os.path.isdir(dir_file_path): + self.__get_file_type__(dir_file_path) + else: + if dir_file.split(".")[1] == "apk": + self.__decode_apk__(dir_file) + elif dir_file.split(".")[1] == "dex": + self.__decode_dex__(dir_file) + else: + continue + + def __print__(self): + if self.packagename: + print("========= The package name of this APP is: ===============") + print(self.packagename) + + if len(self.comp_list) != 0: + print("========= Component information is as follows :===============") + for json in self.comp_list: + print(json) + + print("=========The result set for the static scan is shown below:===============") + with open(cores.result_path,"a+") as f: + for key,value in self.result_dict.items(): + f.write(key+"\r") + for result in value: + print(result) + f.write("\t"+result+"\r") + print("For more information about the search, see: %s" %(cores.result_path)) + + if self.shell_falg: + print('\033[1;33mWarning: This application has shell, the retrieval results may not be accurate, Please remove the shell and try again!') + + def __start_threads(self): + for threadID in range(1,self.threads) : + name = "Thread - " + str(threadID) + thread = ParsesThreads(threadID,name,self.file_queue,self.all,self.result_dict) + thread.start() + self.thread_list.append(thread) + + def __shell_test__(self,output): + am_path = os.path.join(output,"AndroidManifest.xml") + + with open(am_path,"r") as f: + am_str = f.read() + + am_package= re.compile(r'=1): + self.packagename = apackage + + am_name = re.compile(r'') + aname = am_name.findall(am_str) + if aname[0] in config.shell_list: + self.shell_falg = True + + + + diff --git a/libs/task/ios_task.py b/libs/task/ios_task.py new file mode 100644 index 0000000..26480ca --- /dev/null +++ b/libs/task/ios_task.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +# Author: kelvinBen +# Github: https://github.com/kelvinBen/AppInfoScanner + + +import os +import re +import zipfile +import libs.core as cores +from queue import Queue +from libs.core.parses import ParsesThreads + + +class iOSTask(object): + thread_list =[] + result_dict = {} + + def __init__(self,input, rules, net_sniffer,no_resource,all,threads): + self.path = input + self.rules = rules + self.net_sniffer = net_sniffer + self.no_resource = no_resource + self.all = all + self.threads = threads + if rules: + config.filter_strs.append(r'.*'+str(rules)+'.*') + self.file_queue = Queue() + self.shell_falg=False + + def start(self): + # ipa 文件 + if self.path.split(".")[1] == 'ipa': + # 对ipa进行解包 + self.__decode_ipa__(cores.output_path) + + #文件提取 + self.__scanner_file_by_ipa__(cores.output_path) + + self.__start_threads() + + for thread in self.thread_list: + thread.join() + + self.__print__() + + + def __scanner_file_by_ipa__(self,output): + scanner_file_suffix = ["plist","js","xml","html"] + scanner_dir = os.path.join(output,"Payload") + self.__get_scanner_file__(scanner_dir,scanner_file_suffix) + + def __get_scanner_file__(self,scanner_dir,file_suffix): + dir_or_files = os.listdir(scanner_dir) + for dir_file in dir_or_files: + dir_file_path = os.path.join(scanner_dir,dir_file) + if os.path.isdir(dir_file_path): + if dir_file.endswith(".app"): + self.elf_file_name = dir_file.split(".")[0] + self.__get_scanner_file__(dir_file_path,file_suffix) + else: + if self.elf_file_name == dir_file: + self.file_queue.put(dir_file_path) + continue + if self.no_resource: + dir_file_suffix = dir_file.split(".") + if len(dir_file_suffix) > 1: + if dir_file_suffix[1] in file_suffix: + self.file_queue.put(dir_file_path) + + def __decode_ipa__(self,output_path): + zip_files = zipfile.ZipFile(self.path) + for zip_file in zip_files.filelist: + zip_files.extract(zip_file.filename,output_path) + + def __start_threads(self): + for threadID in range(1,self.threads) : + name = "Thread - " + str(threadID) + thread = ParsesThreads(threadID,name,self.file_queue,self.all,self.result_dict) + thread.start() + self.thread_list.append(thread) + + + def __print__(self): + print("=========The result set for the static scan is shown below:===============") + with open(cores.result_path,"a+") as f: + for key,value in self.result_dict.items(): + f.write(key+"\r") + for result in value: + print(result) + f.write("\t"+result+"\r") + print("For more information about the search, see: %s" %(cores.result_path)) + + if self.shell_falg: + print('\033[1;33mWarning: This application has shell, the retrieval results may not be accurate, Please remove the shell and try again!') \ No newline at end of file diff --git a/libs/task/web_task.py b/libs/task/web_task.py new file mode 100644 index 0000000..09ad443 --- /dev/null +++ b/libs/task/web_task.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# Author: kelvinBen +# Github: https://github.com/kelvinBen/AppInfoScanner + + +import os +import re +import config +import threading +from queue import Queue +import libs.core as cores +from libs.core.parses import ParsesThreads + + +class WebTask(object): + thread_list =[] + result_dict = {} + + def __init__(self, input, rules,all,threads): + self.path = input + if rules: + config.filter_strs.append(r'.*'+str(rules)+'.*') + self.all = all + self.threads = threads + self.file_queue = Queue() + self.shell_falg=False + + def start(self): + # 此处判断是文件还是目录 + # 文件判断后缀 html，js,css,htm,xml等 + + if len(config.web_file_suffix) <=0: + scanner_file_suffix = ["html","js","html","xml"] + + scanner_file_suffix = config.web_file_suffix + if os.path.isdir(self.path): # 目录的话就提取 + self.__get_scanner_file__(self.path,scanner_file_suffix) + + else: + if not (self.path.split(".")[1] in scanner_file_suffix): # 内容包含进行下步处理 + err_info = ("Retrieval of this file type is not supported. Select a file or directory with a suffix of %s" % ",".join(scanner_file_suffix)) + raise Exception(err_info) + self.file_queue.put(self.path) + + self.__start_threads() + + for thread in self.thread_list: + thread.join() + + self.__print__() + + def __get_scanner_file__(self,scanner_dir,file_suffix): + dir_or_files = os.listdir(scanner_dir) + for dir_file in dir_or_files: + dir_file_path = os.path.join(scanner_dir,dir_file) + if os.path.isdir(dir_file_path): + self.__get_scanner_file__(dir_file_path,file_suffix) + else: + if len(dir_file.split("."))>1: + if dir_file.split(".")[1] in file_suffix: + self.file_queue.put(dir_file_path) + + def __print__(self): + print("=========The result set for the static scan is shown below:===============") + with open(cores.result_path,"a+") as f: + for key,value in self.result_dict.items(): + f.write(key+"\r") + for result in value: + print(result) + f.write("\t"+result+"\r") + print("For more information about the search, see: %s" %(cores.result_path)) + + def __start_threads(self): + for threadID in range(1,self.threads) : + name = "Thread - " + str(threadID) + thread = ParsesThreads(threadID,name,self.file_queue,self.all,self.result_dict) + thread.start() + self.thread_list.append(thread) \ No newline at end of file diff --git a/tools/apktool.jar b/tools/apktool.jar new file mode 100755 index 0000000..2eca808 Binary files /dev/null and b/tools/apktool.jar differ diff --git a/tools/baksmali.jar b/tools/baksmali.jar new file mode 100644 index 0000000..5bc0188 Binary files /dev/null and b/tools/baksmali.jar differ diff --git a/tools/strings.exe b/tools/strings.exe new file mode 100644 index 0000000..cf73d7d Binary files /dev/null and b/tools/strings.exe differ diff --git a/tools/strings64.exe b/tools/strings64.exe new file mode 100644 index 0000000..643d770 Binary files /dev/null and b/tools/strings64.exe differ diff --git a/update.md b/update.md new file mode 100644 index 0000000..e69de29