diff --git a/config/setting.py b/config/setting.py index 6bceb6e..21b5ce7 100644 --- a/config/setting.py +++ b/config/setting.py @@ -18,7 +18,9 @@ temp_save_dir = result_save_dir.joinpath('temp') enable_check_version = True # 开启最新版本检查 enable_dns_resolve = True # 使用DNS解析子域(默认True) enable_http_request = True # 使用HTTP请求子域(默认True) -enable_finder_module = True # 开启finder模块(默认True) +enable_finder_module = True # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True) +enable_cdn_check = True # 开启cdn检查模块(默认True) +enable_banner_identify = False # 开启WEB指纹识别模块(默认True) enable_takeover_check = False # 开启子域接管风险检查(默认False) # 参数port可选值有'default', 'small', 'large' http_request_port = 'default' # HTTP请求子域(默认'default'，探测80端口) diff --git a/modules/webanalyzer.py b/modules/banner.py similarity index 84% rename from modules/webanalyzer.py rename to modules/banner.py index 52bf3d7..58899d0 100644 --- a/modules/webanalyzer.py +++ b/modules/banner.py @@ -1,81 +1,80 @@ import re -import json -import time import os -import hashlib +import time import json import string +import hashlib -from urllib import parse from bs4 import BeautifulSoup from http.cookies import SimpleCookie from common import utils -from common import resolve -from common import request from common.module import Module from config import setting from config.log import logger -class Webanalyzer(Module): +class Identify(Module): def __init__(self): Module.__init__(self) - self.module = 'Webanalyzer' - self.source = 'Webanalyzer' + self.module = 'Identify' + self.source = 'Identify' self.start = time.time() # 模块开始执行时间 self.rule_dir = setting.data_storage_dir.joinpath('rules') self._targets = {} self.load_rules() self._cond_parser = Condition() + self.url = '' def run(self, data): - logger.log('INFOR', f'Start Webanalyzer module') + logger.log('INFOR', f'Start Identify module') for index, item in enumerate(data): - if item.get('request') == 1: - self.url = item.get('url') - implies = set() - excludes = set() - self.parse(index, item) - item['banner'] = [] - for name, rule in RULES.items(): - # print(name) - r = self._check_rule(rule) - if r: - if 'implies' in rule: - if isinstance(rule['implies'], str): - implies.add(rule['implies']) - else: - implies.update(rule['implies']) + if not item.get('request'): + continue + self.url = item.get('url') + implies = set() + excludes = set() + self.parse(index, item) + banners = [] + for name, rule in RULES.items(): + # print(name) + r = self._check_rule(rule) + if r: + if 'implies' in rule: + if isinstance(rule['implies'], str): + implies.add(rule['implies']) + else: + implies.update(rule['implies']) - if 'excludes' in rule: - if isinstance(rule['excludes'], str): - excludes.add(rule['excludes']) - else: - excludes.update(rule['excludes']) + if 'excludes' in rule: + if isinstance(rule['excludes'], str): + excludes.add(rule['excludes']) + else: + excludes.update(rule['excludes']) - if r['name'] in excludes: - continue - item['banner'].append(r) - - for imply in implies: - _result = { - 'name': imply, - "origin": 'implies' - } - for rule_type in RULE_TYPES: - rule_name = '%s_%s' % (rule_type, imply) - rule = RULES.get(rule_name) - if not rule: - continue - if 'excludes' in rule: - if isinstance(rule['excludes'], str): - excludes.add(rule['excludes']) - else: - excludes.update(rule['excludes']) - if _result['name'] in excludes: + if r['name'] in excludes: continue - item['banner'].append(_result) + banners.append(r) + + for imply in implies: + _result = { + 'name': imply, + "origin": 'implies' + } + for rule_type in RULE_TYPES: + rule_name = '%s_%s' % (rule_type, imply) + rule = RULES.get(rule_name) + if not rule: + continue + if 'excludes' in rule: + if isinstance(rule['excludes'], str): + excludes.add(rule['excludes']) + else: + excludes.update(rule['excludes']) + if _result['name'] in excludes: + continue + banners.append(_result) + item['banner'] = json.dumps(banners, ensure_ascii=False) self.end = time.time() self.elapse = self.end - self.start @@ -94,7 +93,7 @@ class Webanalyzer(Module): if not i.endswith('.json'): continue - with open(os.path.join(rule_type_dir, i)) as fd: + with open(os.path.join(rule_type_dir, i), encoding='utf-8') as fd: try: data = json.load(fd) for match in data['matches']: @@ -117,7 +116,7 @@ class Webanalyzer(Module): def parse(self, index, item) -> hash: script = [] meta = {} - if item.get('response') != None: + if item.get('response'): p = BeautifulSoup(item.get('response'), "html.parser") for i in p.find_all("script"): script_src = i.get("src") @@ -147,7 +146,7 @@ class Webanalyzer(Module): "raw_cookies": cookies, "raw_response": item.get('header') + item.get('response'), "raw_headers": item.get('header'), - "md5": hashlib.md5(item.get('response').encode('utf')), + "md5": hashlib.md5(item.get('response').encode('utf-8')), } def _check_match(self, match: hash) -> (bool, str): @@ -259,9 +258,10 @@ class Webanalyzer(Module): if self._cond_parser.parse(rule['condition'], cond_map): return result - def save_db(self, domain, data): - logger.log('INFOR', f'Saving Webanalyzer results') - utils.save_db(domain, data, 'webanalyzer') + +def save_db(domain, data): + logger.log('INFOR', f'Saving Identify results') + utils.save_db(domain, data, 'Identify') __all__ = ["Condition", "ParseException"] diff --git a/oneforall.py b/oneforall.py index 325c2b6..08a8f42 100644 --- a/oneforall.py +++ b/oneforall.py @@ -18,7 +18,7 @@ from common import utils, resolve, request from common.database import Database from modules.collect import Collect from modules.finder import Finder -from modules import iscdn, webanalyzer +from modules import iscdn, banner from config import setting from config.log import logger from takeover import Takeover @@ -33,7 +33,7 @@ end = '\033[0m' version = 'v0.3.0' message = white + '{' + red + version + ' #dev' + white + '}' -banner = f""" +oneforall_banner = f""" OneForAll is a powerful subdomain integration tool{yellow} ___ _ _ ___ ___ ___| _|___ ___ ___| | | {message}{green} @@ -216,14 +216,16 @@ class OneForAll(object): finder = Finder() self.data = finder.run(self.domain, self.data, self.port) - # check cdn - self.data = iscdn.check_cdn(self.data) - iscdn.save_db(self.domain, self.data) + # check cdn module + if setting.enable_cdn_check: + self.data = iscdn.check_cdn(self.data) + iscdn.save_db(self.domain, self.data) - # webanalyzer - analyzer = webanalyzer.Webanalyzer() - self.data = analyzer.run(self.data) - analyzer.save_db(self.domain,self.data) + # Identify banner module + if setting.enable_banner_identify: + identifier = banner.Identify() + self.data = identifier.run(self.data) + banner.save_db(self.domain, self.data) # Add the final result list to the total data list self.datas.extend(self.data) @@ -245,7 +247,7 @@ class OneForAll(object): :return: All subdomain results :rtype: list """ - print(banner) + print(oneforall_banner) dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S') print(f'[*] Starting OneForAll @ {dt}\n') utils.check_env() @@ -269,7 +271,7 @@ class OneForAll(object): """ Print version information and exit """ - print(banner) + print(oneforall_banner) exit(0) @staticmethod