add code to git

README.md

@@ -1,2 +1,158 @@
-# AppInfoScanner
+### AppInfoScanner
+
 一款适用于(Android、iOS、WEB、H5、静态网站)，信息检索的工具，可以帮助渗透测试人员快速获取App或者WEB中的有用资产信息。
+
+### 适用场景
+- 日常渗透测试中对APP中的URL地址、IP地址、关键信息进行采集
+- 大型攻防演练场景中对APP中URL地址、IP地址、关键信息进行采集
+- 对WEB网站源代码进行URL地址、IP地址、关键信息进行采集(可以是开源的代码也可以是右击网页源代码另存为)
+
+
+### 功能介绍:
+- 支持目录批量扫描
+- 支持DEX、APK、IPA、HTML、JS、Smali等文件的静态资源采集
+- 支持自定义扫描规则
+- 支持IP地址信息采集
+- 支持URL地址信息采集
+- 支持中间件信息采集
+- 支持多线程
+- 支持忽略资源文件采集
+- 支持Android包名采集
+
+### 环境说明
+
+- Apk文件解析需要使用JAVA环境,JAVA版本1.8及以下
+- Python3的运行环境
+
+### 目录说明
+
+AppInfoScanner
+    |-- libs  程序的核心代码
+        |-- core
+            |-- parses.py 用于解析文件中的静态信息
+        |-- task
+            |-- android_task.py 用于处理Android相关的文件
+            |-- ios_task.py 用于处理iOS相关的文件
+            |-- web_task.py 用于处理Web相关的文件，比如网页右键源代码、H5相关的静态信息
+    |-- tools 程序需要依赖的工具
+        |-- apktool.jar 用于反编译apk文件
+        |-- baksmali.jar 用于反编译dex文件
+        |-- strings.exe 用于windows 32下获取iPA的字符串信息
+        |-- strings64.exe 用于windows 64的系统获取iPA的字符串信息
+    |-- app.py 主运行程序
+    |-- config.py 用于自定义相关规则
+    |-- readme.md  程序使用说明
+
+
+### Android 相关操作说明
+
+#### 扫描指定的apk
+
+```
+python3 app.py android -i <Your apk file>  
+```
+
+#### 扫描指定的dex
+
+```
+python3 app.py android -i <Your dex file> 
+```
+
+#### 扫描一个目录下所有的APK或者dex
+
+```
+python3 app.py android -i <Your apk or dex directory> 
+```
+
+#### 扫描指定关键字(临时)
+
+```
+python3 app.py android -i <Your apk or dex directory> -r <the keyword>
+```
+
+#### 扫描的时候忽略资源文件
+
+```
+python3 app.py android -i <Your apk or dex directory> -n
+```
+
+#### 扫描指定包下的内容
+```
+python3 app.py android -i <Your apk or dex directory> -p <package1.package2>
+```
+
+#### 扫描所有的字符串
+
+```
+python3 app.py android -i <Your apk or dex directory>  -a
+```
+
+#### 指定线程数量
+
+```
+python3 app.py android -i <Your apk or dex directory> -t 10
+``` 
+
+### iOS 相关操作说明
+
+#### 扫描指定的iPA文件
+
+```
+python3 app.py ios -i <Your ipa file>
+```
+
+#### 扫描指定关键字(临时)
+
+```
+python3 app.py ios -i <Your ipa file> -r  <the keyword>
+```
+
+#### 扫描的时候忽略资源文件
+
+```
+python3 app.py ios -i <Your ipa file> -n
+```
+
+#### 输出所有的字符串
+
+```
+python3 app.py ios -i <Your ipa file> -a
+```
+
+#### 指定线程数量
+
+```
+python3 app.py ios -i <Your ipa file> -t 10
+```
+
+### Web 相关操作说明
+
+#### 扫描指定的Web网站目录或者html相关文件
+```
+python3 app.py web -i <Your website directory> 
+```
+
+#### 扫描指定关键字(临时)
+
+```
+python3 app.py web -i  <Your website directory> -r <the keyword>
+```
+
+#### 输出所有的字符串
+```
+python3 app.py web -i  <Your website directory> -a
+```
+
+
+#### 指定线程数量
+```
+python3 app.py web -i <Your website directory> -t 10
+```
+
+
+### 常见问题
+
+- 1. 信息检索垃圾数据过多？
+
+> 方法1： 根据实际情况调整config.py中的规则信息
+> 方法2： 忽略资源文件

__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# Author: kelvinBen
+# Github: https://github.com/kelvinBen/AppInfoScanner

config.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+# Author: kelvinBen
+# Github: https://github.com/kelvinBen/AppInfoScanner
+
+
+# 此处用于搜索组件信息，如fastjson、gson等
+filter_components = [
+    'com.alibaba.fastjson',
+    'com.google.gson',
+    'com.fasterxml.jackson',
+    'net.sf.json'
+]
+
+# 此处目前支持过滤
+# 1. https://以及http://开头的
+# 2. IPv4的ip地址
+# 3. URI地址
+filter_strs =[
+    r'.*(http://.*)',
+    r'.*(https://.*)', 
+    r'.*((?:[0-9]{1,3}\.){3}[0-9]{1,3}).*',
+    # r'/[a-z0-9A-Z]+/.*'
+]
+
+# 过滤无用的内容
+filter_no = [
+    u'127.0.0.1',
+    u'0.0.0.0',
+    u'localhost',
+    u'http://schemas.android.com/apk/res/android',
+    u"https://",
+    u"http://",
+    r"^http://www.w3.org"
+    r"L.*/",
+    r"/.*;",
+    r"/.*<"
+]
+
+# 此处配置壳信息
+shell_list =[
+    'com.stub.StubApp',
+    's.h.e.l.l.S',
+    'com.Kiwisec.KiwiSecApplication',
+    'com.Kiwisec.ProxyApplication',
+    'com.secshell.secData.ApplicationWrapper',
+    'com.secneo.apkwrapper.ApplicationWrapper',
+    'com.tencent.StubShell.TxAppEntry',
+    'c.b.c.b',
+    'MyWrapperProxyApplication',
+    'cn.securitystack.stee.AppStub',
+    'com.linchaolong.apktoolplus.jiagu.ProxyApplication',
+    'com.coral.util.StubApplication',
+    'com.mogosec.AppMgr'
+]
+
+# 此处配置需要扫描的web文件后缀
+web_file_suffix =[
+    "html",
+    "js",
+    "html",
+    "xml",
+    "php",
+    "jsp",
+    "class",
+    "asp",
+    "aspx",
+    "py"
+]

libs/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# Author: kelvinBen
+# Github: https://github.com/kelvinBen/AppInfoScanner

libs/core/__init__.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+# Author: kelvinBen
+# Github: https://github.com/kelvinBen/AppInfoScanner
+
+import platform
+import os
+import shutil
+
+# smali 所在路径
+smali_path = ""
+
+# backsmli 所在路径
+backsmali_path = ""
+
+# apktool 所在路径
+apktool_path = ""
+
+# 系统类型
+os_type = ""
+
+# 输出路径
+output_path = ""
+
+class Bootstrapper(object):
+
+    def __init__(self, path):
+        global smali_path
+        global backsmali_path
+        global apktool_path
+        global os_type
+        global output_path
+        global script_root_dir
+        global result_path
+        global strings_path
+
+        script_root_dir =  os.path.dirname(os.path.abspath(path))
+        tools_dir = os.path.join(script_root_dir,"tools")
+
+        if platform.system() == "Windows":
+            machine2bits = {'AMD64':64, 'x86_64': 64, 'i386': 32, 'x86': 32}
+            machine2bits.get(platform.machine())
+
+            if platform.machine() == 'i386' or platform.machine() == 'x86':
+                strings_path = os.path.join(script_root_dir,"strings.exe")
+            else:
+                strings_path = os.path.join(script_root_dir,"strings64.exe")
+        else:
+            strings_path ="strings"
+        #     os_type = "win"
+        #     smali_str = "smali.bat"
+        #     back_smali_str = "backsmali.bat"
+        #     apktool_path_str = "apktool.bat"
+        # elif platform.system() == "Linux":
+        #     os_type = "lin"
+        #     smali_str = "smali"
+        #     back_smali_str = "backsmali"
+        #     apktool_path_str = "apktool"
+        # else:
+        #     os_type = "mac"
+        #     smali_str = "smali"
+        
+
+        # smali_path = os.path.join(tools_dir,str(os_type) + os.sep + smali_str)
+        backsmali_path = os.path.join(tools_dir,"baksmali.jar")
+        apktool_path = os.path.join(tools_dir, "apktool.jar")
+        output_path = os.path.join(script_root_dir,"out")
+        result_path = os.path.join(script_root_dir,"result.txt")
+
+    def init(self):
+        if os.path.exists(output_path):
+            shutil.rmtree(output_path)
+        os.makedirs(output_path)
+
+        if os.path.exists(result_path):
+            os.remove(result_path)
+
+        
+            
+
+
+        

libs/core/parses.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+# Author: kelvinBen
+# Github: https://github.com/kelvinBen/AppInfoScanner
+
+
+import threading
+import config
+import re
+import os
+import libs.core as cores
+
+class ParsesThreads(threading.Thread):
+
+    def __init__(self,threadID,name,file_queue,all,result_dict):
+        threading.Thread.__init__(self) 
+        self.file_queue = file_queue
+        self.name = name
+        self.threadID = threadID
+        self.result_list = []
+        self.all = all
+        self.result_dict=result_dict
+            
+    def __regular_parse__(self,threadLock):
+        while True:
+            try:
+                file_path = self.file_queue.get(timeout = 5)
+                scan_str = ("Scan file : %s" % file_path)
+                print(scan_str)
+
+                try:
+                    os.path.basename(file_path).split(".")[1]
+                except Exception as e:
+                    self.__get_string__(file_path,threadLock)
+                    continue
+                self.__file_parse__(file_path,threadLock)
+
+                result_set =  set(self.result_list)
+                if len(result_set) !=0:
+                    self.result_dict[file_path] = result_set
+
+                if self.file_queue.empty():
+                    break
+            except Exception as e:
+                break
+
+    def __file_parse__(self,file_path,threadLock):
+        with open(file_path,"r",encoding="utf8") as file :
+            file_content =  file.read()
+            # 获取到所有的字符串
+            pattern = re.compile(r'\"(.*?)\"') 
+            results = pattern.findall(file_content)
+
+            # 遍历所有的字符串
+            for result in set(results): 
+                self.__parse_string__(result,threadLock)
+
+    def __get_string__(self,dir_file_path,threadLock):
+        temp =  os.path.join(cores.output_path,"temp.txt")
+        cmd_str = ("%s %s > %s") % (cores.strings_path,dir_file_path,temp)
+        if os.system(cmd_str) == 0:
+            with open(temp,"r") as f:
+                lines = f.readlines()
+                for line in lines:
+                    self.__parse_string__(line,threadLock)
+
+    def __parse_string__(self,result,threadLock):
+        # 通过正则筛选需要过滤的字符串
+        for filter_str in config.filter_strs:
+            filter_str_pat = re.compile(filter_str) 
+            filter_resl = filter_str_pat.findall(result)
+            # print(filter_resl)
+            # 过滤掉未搜索到的内容
+            if len(filter_resl)!=0:
+                # 提取第一个字符
+                resl_str = filter_resl[0]
+                # 过滤
+                if self.__filter__(resl_str) == 0:
+                    continue
+
+                threadLock.acquire()
+                self.result_list.append(filter_resl[0])
+                threadLock.release()
+            continue
+
+    def __filter__(self,resl_str):
+        resl_str = resl_str.replace("\r","").replace("\n","").replace(" ","")
+        if len(resl_str) == 0:
+            return 0
+
+        for filte in config.filter_no:
+            resl_str = resl_str.replace(filte,"")
+            if len(resl_str) == 0:
+                return 0
+            
+            if re.match(filte,resl_str):
+                return 0
+        return 1  
+
+    def run(self):
+        threadLock = threading.Lock()
+        self.__regular_parse__(threadLock)

libs/task/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# Author: kelvinBen
+# Github: https://github.com/kelvinBen/AppInfoScanner

libs/task/android_task.py

@@ -0,0 +1,172 @@
+# -*- coding: utf-8 -*-
+# Author: kelvinBen
+# Github: https://github.com/kelvinBen/AppInfoScanner
+
+
+import os
+import re
+import config
+import threading
+from queue import Queue
+import libs.core as cores
+from libs.core.parses import ParsesThreads
+
+class AndroidTask(object):
+    comp_list =[]
+    thread_list =[]
+    result_dict = {}
+
+    def __init__(self, input, rules, net_sniffer,no_resource,package,all,threads):
+        self.net_sniffer = net_sniffer
+        self.path = input
+        if rules:
+            config.filter_strs.append(r'.*'+str(rules)+'.*')
+        self.no_resource = no_resource
+        self.package = package
+        self.all = all
+        self.threads = threads
+        self.file_queue = Queue()
+        self.shell_falg=False
+        self.packagename=""
+        
+
+    def start(self):
+        # 检查java环境是否存在
+        if os.system("java -version") !=0 :
+            raise Exception("Please install the Java environment!")
+        
+        # 根据不同的文件后缀进行文件解析
+        if os.path.isfile(self.path):
+            if self.path.split(".")[1] == "apk":
+                self.__decode_apk__(self.path)
+            elif self.path.split(".")[1] == "dex":
+                self.__decode_dex__(self.path)
+            else:
+                # 抛出异常
+                raise Exception("Retrieval of this file type is not supported. Select APK file or DEX file.")
+        else:
+            self.__get_file_type__(self.path)
+        self.__start_threads()
+        
+        for thread in self.thread_list:
+            thread.join()
+
+        self.__print__()
+        
+        # if self.net_sniffer:
+        #     self.__start_net__()
+
+    # 分解apk
+    def __decode_apk__(self,path):
+        if self.no_resource:
+            self.__decode_dex__(path)
+        else:
+            cmd_str = ("java -jar %s d -f %s -o %s") % (cores.apktool_path,path,cores.output_path)
+            if os.system(cmd_str) == 0:
+                self.__scanner_file_by_apktool__(cores.output_path)
+            else:
+                raise Exception("The Apktool tool was not found.")
+
+    # 分解dex
+    def __decode_dex__(self,path):
+        cmd_str = ("java -jar %s d %s") % (cores.backsmali_path,path)
+        if os.system(cmd_str) == 0:
+            self.__get_scanner_file__(cores.output_path,"smali")
+        else:
+            raise Exception("The baksmali tool was not found.")
+    
+
+    # 初始化检测文件信息
+    def __scanner_file_by_apktool__(self,output):
+
+        # shell检测
+        self.__shell_test__(output)
+
+        scanner_dir_list =  ["smali","assets"]    
+        scanner_file_suffix = ["smali","js","xml"]
+
+        for dir in scanner_dir_list:
+            scanner_dir =  os.path.join(output,dir)
+            self.__get_scanner_file__(scanner_dir,scanner_file_suffix)
+
+    def __get_scanner_file__(self,scanner_dir,file_suffix):
+        dir_or_files = os.listdir(scanner_dir)
+        for dir_file in dir_or_files:
+            dir_file_path = os.path.join(scanner_dir,dir_file)
+            if os.path.isdir(dir_file_path):
+                self.__get_scanner_file__(dir_file_path,file_suffix)
+            else:
+                if "." not in dir_file:
+                    continue
+                if len(dir_file.split("."))>1:
+                    if dir_file.split(".")[1] in file_suffix:
+                        self.file_queue.put(dir_file_path)
+                        for component in config.filter_components:
+                            comp = component.replace(".","/")
+                            if( comp in dir_file_path):
+                                if(component not in self.comp_list):
+                                    self.comp_list.append(component)
+
+
+    def __get_file_type__(self,root_path):
+        dir_or_files = os.listdir(root_path)
+        for dir_file in dir_or_files:
+            dir_file_path = os.path.join(root_path,dir_file)
+            if os.path.isdir(dir_file_path):
+                self.__get_file_type__(dir_file_path)
+            else:
+                if dir_file.split(".")[1] == "apk":
+                    self.__decode_apk__(dir_file)
+                elif dir_file.split(".")[1] == "dex":
+                    self.__decode_dex__(dir_file)
+                else:
+                    continue
+
+    def __print__(self):
+        if self.packagename: 
+            print("=========  The package name of this APP is: ===============")
+            print(self.packagename)
+
+        if len(self.comp_list) != 0:
+            print("========= Component information is as follows :===============")
+            for json in self.comp_list:
+                print(json)
+
+        print("=========The result set for the static scan is shown below:===============")
+        with open(cores.result_path,"a+") as f:
+            for key,value in self.result_dict.items():
+                f.write(key+"\r")
+                for result in value:
+                    print(result)
+                    f.write("\t"+result+"\r")
+        print("For more information about the search, see: %s" %(cores.result_path))
+
+        if self.shell_falg:
+            print('\033[1;33mWarning: This application has shell, the retrieval results may not be accurate, Please remove the shell and try again!')
+
+    def __start_threads(self):
+        for threadID in range(1,self.threads) : 
+            name = "Thread - " + str(threadID)
+            thread =  ParsesThreads(threadID,name,self.file_queue,self.all,self.result_dict)
+            thread.start()
+            self.thread_list.append(thread)
+
+    def __shell_test__(self,output):
+        am_path = os.path.join(output,"AndroidManifest.xml")
+        
+        with open(am_path,"r") as f:
+            am_str = f.read()
+
+            am_package=  re.compile(r'<manifest.*package=\"(.*?)\".*')
+            apackage = am_package.findall(am_str)
+            if len(apackage >=1):
+                self.packagename = apackage
+
+            am_name = re.compile(r'<application.*android:name=\"(.*?)\".*>') 
+            aname = am_name.findall(am_str)
+            if aname[0] in config.shell_list:
+                self.shell_falg = True
+                
+
+        
+

libs/task/ios_task.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+# Author: kelvinBen
+# Github: https://github.com/kelvinBen/AppInfoScanner
+
+
+import os
+import re
+import zipfile
+import libs.core as cores
+from queue import Queue
+from libs.core.parses import ParsesThreads
+
+
+class iOSTask(object):
+    thread_list =[]
+    result_dict = {}
+
+    def __init__(self,input, rules, net_sniffer,no_resource,all,threads):
+        self.path = input
+        self.rules = rules
+        self.net_sniffer = net_sniffer
+        self.no_resource = no_resource
+        self.all = all
+        self.threads = threads
+        if rules:
+            config.filter_strs.append(r'.*'+str(rules)+'.*')
+        self.file_queue = Queue()
+        self.shell_falg=False
+
+    def start(self):
+        # ipa 文件
+        if self.path.split(".")[1] == 'ipa':
+            # 对ipa进行解包
+            self.__decode_ipa__(cores.output_path)
+
+            #文件提取
+            self.__scanner_file_by_ipa__(cores.output_path)
+        
+        self.__start_threads()
+        
+        for thread in self.thread_list:
+            thread.join()
+
+        self.__print__()
+        
+
+    def __scanner_file_by_ipa__(self,output):   
+        scanner_file_suffix = ["plist","js","xml","html"]
+        scanner_dir =  os.path.join(output,"Payload")
+        self.__get_scanner_file__(scanner_dir,scanner_file_suffix)
+
+    def __get_scanner_file__(self,scanner_dir,file_suffix):
+        dir_or_files = os.listdir(scanner_dir)
+        for dir_file in dir_or_files:
+            dir_file_path = os.path.join(scanner_dir,dir_file)
+            if os.path.isdir(dir_file_path):
+                if dir_file.endswith(".app"):
+                    self.elf_file_name = dir_file.split(".")[0]
+                self.__get_scanner_file__(dir_file_path,file_suffix)
+            else:
+                if self.elf_file_name == dir_file:
+                    self.file_queue.put(dir_file_path)
+                    continue
+                if self.no_resource:    
+                    dir_file_suffix =  dir_file.split(".")
+                    if len(dir_file_suffix) > 1:
+                        if dir_file_suffix[1] in file_suffix:
+                            self.file_queue.put(dir_file_path)
+
+    def __decode_ipa__(self,output_path):
+        zip_files = zipfile.ZipFile(self.path)
+        for zip_file in zip_files.filelist:
+            zip_files.extract(zip_file.filename,output_path)
+
+    def __start_threads(self):
+        for threadID in range(1,self.threads) : 
+            name = "Thread - " + str(threadID)
+            thread =  ParsesThreads(threadID,name,self.file_queue,self.all,self.result_dict)
+            thread.start()
+            self.thread_list.append(thread)
+    
+
+    def __print__(self):
+        print("=========The result set for the static scan is shown below:===============")
+        with open(cores.result_path,"a+") as f:
+            for key,value in self.result_dict.items():
+                f.write(key+"\r")
+                for result in value:
+                    print(result)
+                    f.write("\t"+result+"\r")
+        print("For more information about the search, see: %s" %(cores.result_path))
+
+        if self.shell_falg:
+            print('\033[1;33mWarning: This application has shell, the retrieval results may not be accurate, Please remove the shell and try again!')

libs/task/web_task.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Author: kelvinBen
+# Github: https://github.com/kelvinBen/AppInfoScanner
+
+
+import os
+import re
+import config
+import threading
+from queue import Queue
+import libs.core as cores
+from libs.core.parses import ParsesThreads
+
+
+class WebTask(object):
+    thread_list =[]
+    result_dict = {}
+
+    def __init__(self, input, rules,all,threads):
+        self.path = input
+        if rules:
+            config.filter_strs.append(r'.*'+str(rules)+'.*')
+        self.all = all
+        self.threads = threads
+        self.file_queue = Queue()
+        self.shell_falg=False
+
+    def start(self):
+        # 此处判断是文件还是目录
+        # 文件判断后缀 html，js,css,htm,xml等
+        
+        if len(config.web_file_suffix) <=0:
+            scanner_file_suffix = ["html","js","html","xml"]
+            
+        scanner_file_suffix = config.web_file_suffix
+        if os.path.isdir(self.path): # 目录的话就提取
+            self.__get_scanner_file__(self.path,scanner_file_suffix)
+
+        else:
+            if not (self.path.split(".")[1] in scanner_file_suffix): # 内容包含进行下步处理
+                err_info = ("Retrieval of this file type is not supported. Select a file or directory with a suffix of %s" % ",".join(scanner_file_suffix))
+                raise Exception(err_info)
+            self.file_queue.put(self.path)
+        
+        self.__start_threads()
+        
+        for thread in self.thread_list:
+            thread.join()
+
+        self.__print__()
+
+    def __get_scanner_file__(self,scanner_dir,file_suffix):
+        dir_or_files = os.listdir(scanner_dir)
+        for dir_file in dir_or_files:
+            dir_file_path = os.path.join(scanner_dir,dir_file)
+            if os.path.isdir(dir_file_path):
+                self.__get_scanner_file__(dir_file_path,file_suffix)
+            else:
+                if len(dir_file.split("."))>1:
+                    if dir_file.split(".")[1] in file_suffix:
+                        self.file_queue.put(dir_file_path)
+    
+    def __print__(self):
+        print("=========The result set for the static scan is shown below:===============")
+        with open(cores.result_path,"a+") as f:
+            for key,value in self.result_dict.items():
+                f.write(key+"\r")
+                for result in value:
+                    print(result)
+                    f.write("\t"+result+"\r")
+        print("For more information about the search, see: %s" %(cores.result_path))
+
+    def __start_threads(self):
+        for threadID in range(1,self.threads) : 
+            name = "Thread - " + str(threadID)
+            thread =  ParsesThreads(threadID,name,self.file_queue,self.all,self.result_dict)
+            thread.start()
+            self.thread_list.append(thread)