优化指纹识别模块

config/setting.py

@@ -18,7 +18,9 @@ temp_save_dir = result_save_dir.joinpath('temp')
 enable_check_version = True  # 开启最新版本检查
 enable_dns_resolve = True  # 使用DNS解析子域(默认True)
 enable_http_request = True  # 使用HTTP请求子域(默认True)
-enable_finder_module = True  # 开启finder模块(默认True)
+enable_finder_module = True  # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True)
+enable_cdn_check = True  # 开启cdn检查模块(默认True)
+enable_banner_identify = False  # 开启WEB指纹识别模块(默认True)
 enable_takeover_check = False  # 开启子域接管风险检查(默认False)
 # 参数port可选值有'default', 'small', 'large'
 http_request_port = 'default'  # HTTP请求子域(默认'default'，探测80端口)

modules/banner.py

@@ -1,43 +1,41 @@
 import re
-import json
-import time
 import os
-import hashlib
+import time
 import json
 import string
+import hashlib
 
-from urllib import parse
 from bs4 import BeautifulSoup
 from http.cookies import SimpleCookie
 
 from common import utils
-from common import resolve
-from common import request
 from common.module import Module
 from config import setting
 from config.log import logger
 
 
-class Webanalyzer(Module):
+class Identify(Module):
     def __init__(self):
         Module.__init__(self)
-        self.module = 'Webanalyzer'
-        self.source = 'Webanalyzer'
+        self.module = 'Identify'
+        self.source = 'Identify'
         self.start = time.time()  # 模块开始执行时间
         self.rule_dir = setting.data_storage_dir.joinpath('rules')
         self._targets = {}
         self.load_rules()
         self._cond_parser = Condition()
+        self.url = ''
 
     def run(self, data):
-        logger.log('INFOR', f'Start Webanalyzer module')
+        logger.log('INFOR', f'Start Identify module')
         for index, item in enumerate(data):
-            if item.get('request') == 1:
+            if not item.get('request'):
+                continue
             self.url = item.get('url')
             implies = set()
             excludes = set()
             self.parse(index, item)
-                item['banner'] = []
+            banners = []
             for name, rule in RULES.items():
                 # print(name)
                 r = self._check_rule(rule)
@@ -56,7 +54,7 @@ class Webanalyzer(Module):
 
                     if r['name'] in excludes:
                         continue
-                        item['banner'].append(r)
+                    banners.append(r)
 
             for imply in implies:
                 _result = {
@@ -75,7 +73,8 @@ class Webanalyzer(Module):
                             excludes.update(rule['excludes'])
                 if _result['name'] in excludes:
                     continue
-                    item['banner'].append(_result)
+                banners.append(_result)
+            item['banner'] = json.dumps(banners, ensure_ascii=False)
         self.end = time.time()
         self.elapse = self.end - self.start
 
@@ -94,7 +93,7 @@ class Webanalyzer(Module):
                 if not i.endswith('.json'):
                     continue
 
-                with open(os.path.join(rule_type_dir, i)) as fd:
+                with open(os.path.join(rule_type_dir, i), encoding='utf-8') as fd:
                     try:
                         data = json.load(fd)
                         for match in data['matches']:
@@ -117,7 +116,7 @@ class Webanalyzer(Module):
     def parse(self, index, item) -> hash:
         script = []
         meta = {}
-        if item.get('response') != None:
+        if item.get('response'):
             p = BeautifulSoup(item.get('response'), "html.parser")
             for i in p.find_all("script"):
                 script_src = i.get("src")
@@ -147,7 +146,7 @@ class Webanalyzer(Module):
             "raw_cookies": cookies,
             "raw_response": item.get('header') + item.get('response'),
             "raw_headers": item.get('header'),
-            "md5": hashlib.md5(item.get('response').encode('utf')),
+            "md5": hashlib.md5(item.get('response').encode('utf-8')),
         }
 
     def _check_match(self, match: hash) -> (bool, str):
@@ -259,9 +258,10 @@ class Webanalyzer(Module):
         if self._cond_parser.parse(rule['condition'], cond_map):
             return result
 
-    def save_db(self, domain, data):
-        logger.log('INFOR', f'Saving Webanalyzer results')
-        utils.save_db(domain, data, 'webanalyzer')
+
+def save_db(domain, data):
+    logger.log('INFOR', f'Saving Identify results')
+    utils.save_db(domain, data, 'Identify')
 
 
 __all__ = ["Condition", "ParseException"]

oneforall.py

@@ -18,7 +18,7 @@ from common import utils, resolve, request
 from common.database import Database
 from modules.collect import Collect
 from modules.finder import Finder
-from modules import iscdn, webanalyzer
+from modules import iscdn, banner
 from config import setting
 from config.log import logger
 from takeover import Takeover
@@ -33,7 +33,7 @@ end = '\033[0m'
 version = 'v0.3.0'
 message = white + '{' + red + version + ' #dev' + white + '}'
 
-banner = f"""
+oneforall_banner = f"""
 OneForAll is a powerful subdomain integration tool{yellow}
              ___             _ _ 
  ___ ___ ___|  _|___ ___ ___| | | {message}{green}
@@ -216,14 +216,16 @@ class OneForAll(object):
             finder = Finder()
             self.data = finder.run(self.domain, self.data, self.port)
 
-        # check cdn
+        # check cdn module
+        if setting.enable_cdn_check:
             self.data = iscdn.check_cdn(self.data)
             iscdn.save_db(self.domain, self.data)
 
-        # webanalyzer
-        analyzer = webanalyzer.Webanalyzer()
-        self.data = analyzer.run(self.data)
-        analyzer.save_db(self.domain,self.data)
+        # Identify banner module
+        if setting.enable_banner_identify:
+            identifier = banner.Identify()
+            self.data = identifier.run(self.data)
+            banner.save_db(self.domain, self.data)
 
         # Add the final result list to the total data list
         self.datas.extend(self.data)
@@ -245,7 +247,7 @@ class OneForAll(object):
         :return: All subdomain results
         :rtype: list
         """
-        print(banner)
+        print(oneforall_banner)
         dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
         print(f'[*] Starting OneForAll @ {dt}\n')
         utils.check_env()
@@ -269,7 +271,7 @@ class OneForAll(object):
         """
         Print version information and exit
         """
-        print(banner)
+        print(oneforall_banner)
         exit(0)
 
     @staticmethod