From 3fbbd1dc252fa651f2e71b3bcc4ee8b2c8028e99 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Mon, 9 Nov 2020 09:01:06 +0800 Subject: [PATCH 01/10] =?UTF-8?q?1.=E9=87=8D=E6=9E=84=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=EF=BC=8C=E8=BE=B9=E8=AF=B7=E6=B1=82=E8=BE=B9=E5=AD=98=E5=85=A5?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=EF=BC=8C=E8=A7=A3=E5=86=B3=E5=86=85?= =?UTF-8?q?=E5=AD=98=E5=8D=A0=E7=94=A8=E8=BF=87=E5=A4=A7=E9=97=AE=E9=A2=98?= =?UTF-8?q?=E3=80=82=202.format=E5=8F=82=E6=95=B0=E6=94=B9=E4=B8=BAfmt?= =?UTF-8?q?=E3=80=82=203.=E6=B7=BB=E5=8A=A0=E4=BF=A1=E6=81=AF=E5=AF=8C?= =?UTF-8?q?=E5=8C=96=E6=A8=A1=E5=9D=97=E3=80=82=204.=E4=BC=98=E5=8C=96iscd?= =?UTF-8?q?n=E6=A8=A1=E5=9D=97=E4=BB=A3=E7=A0=81=E3=80=82=205.=E7=A7=BB?= =?UTF-8?q?=E9=99=A4=E6=95=B0=E6=8D=AE=E5=BA=93new=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 +- brute.py | 15 +-- common/database.py | 70 +++++++---- common/module.py | 6 +- common/request.py | 111 ++++++++--------- common/resolve.py | 26 ---- common/utils.py | 61 +++++++--- config/default.py | 2 +- data/cdn_asn_list.json | 266 ++++++++++++++++++++--------------------- dbexport.py | 24 ++-- docs/en-us/README.md | 7 +- modules/altdns.py | 6 +- modules/enrich.py | 72 +++++++++++ modules/finder.py | 25 ++-- modules/iscdn.py | 68 ++++++----- oneforall.py | 113 +++++------------ takeover.py | 18 +-- 17 files changed, 473 insertions(+), 424 deletions(-) create mode 100644 modules/enrich.py diff --git a/README.md b/README.md index 8a4ca9e..3653f54 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ DESCRIPTION python3 oneforall.py --target example.com --valid None run python3 oneforall.py --target example.com --brute True run python3 oneforall.py --target example.com --port small run - python3 oneforall.py --target example.com --format csv run + python3 oneforall.py --target example.com --fmt csv run python3 oneforall.py --target example.com --dns False run python3 oneforall.py --target example.com --req False run python3 oneforall.py --target example.com --takeover False run @@ -179,7 +179,7 @@ DESCRIPTION Note: 参数alive可选值True，False分别表示导出存活，全部子域结果 参数port可选值有'default', 'small', 'large', 详见config.py配置 - 参数format可选格式有 'csv','json' + 参数fmt可选格式有 'csv','json' 参数path默认None使用OneForAll结果目录生成路径 ARGUMENTS @@ -199,7 +199,7 @@ FLAGS 请求验证子域的端口范围(默认只探测80端口) --valid=VALID 只导出存活的子域结果(默认False) - --format=FORMAT + --fmt=FMT 结果保存格式(默认csv) --path=PATH 结果保存路径(默认None) @@ -269,7 +269,6 @@ FLAGS ## ⌛后续计划 - [ ] 各模块持续优化和完善 -- [x] 子域监控（标记每次新发现的子域） - [x] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域） - [ ] 操作强大交互人性的前端界面实现 diff --git a/brute.py b/brute.py index fbcaa4e..4269982 100644 --- a/brute.py +++ b/brute.py @@ -384,7 +384,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl): info = dict() cnames = list() ips = list() - public = list() times = list() ttls = list() is_valid_flags = list() @@ -401,7 +400,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl): cnames.append(cname) # 去除最右边的`.`点号 ip = answer.get('data') ips.append(ip) - public.append(utils.ip_is_public(ip)) num = ip_times.get(ip) times.append(num) isvalid, reason = is_valid_subdomain(ip, ttl, num, wc_ips, wc_ttl, cname) @@ -416,7 +414,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl): info['ttl'] = ttls info['cname'] = cnames info['ip'] = ips - info['public'] = public info['times'] = times info['resolver'] = resolver infos[qname] = info @@ -559,7 +556,7 @@ class Brute(Module): brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run Note: - --format csv/json (result format) + --fmt csv/json (result fmt) --path Result path (default None, automatically generated) @@ -578,13 +575,13 @@ class Brute(Module): :param str rule: Specify the regexp rules used in fuzz mode (required if use fuzz mode) :param str fuzzlist: Dictionary path used in fuzz mode (default use ./config/default.py) :param bool export: Export the results (default True) - :param str format: Result format (default csv) + :param str fmt: Result format (default csv) :param str path: Result directory (default None) """ def __init__(self, target=None, targets=None, process=None, concurrent=None, word=False, wordlist=None, recursive=False, depth=None, nextlist=None, fuzz=False, place=None, rule=None, fuzzlist=None, export=True, - alive=True, format='csv', path=None): + alive=True, fmt='csv', path=None): Module.__init__(self) self.module = 'Brute' self.source = 'Brute' @@ -603,7 +600,7 @@ class Brute(Module): self.fuzzlist = fuzzlist or settings.fuzz_list self.export = export self.alive = alive - self.format = format + self.fmt = fmt self.path = path self.bulk = False # 是否是批量爆破场景 self.domains = list() # 待爆破的所有域名集合 @@ -761,7 +758,7 @@ class Brute(Module): logger.log('INFOR', f'Finished {self.source} module to brute {self.domain}') if not self.path: - name = f'{self.domain}_brute_result.{self.format}' + name = f'{self.domain}_brute_result.{self.fmt}' self.path = settings.result_save_dir.joinpath(name) # 数据库导出 if self.export: @@ -770,7 +767,7 @@ class Brute(Module): alive=self.alive, limit='resolve', path=self.path, - format=self.format) + fmt=self.fmt) if __name__ == '__main__': diff --git a/common/database.py b/common/database.py index 983a6ca..25244b7 100644 --- a/common/database.py +++ b/common/database.py @@ -57,7 +57,6 @@ class Database(object): f'alive int,' f'request int,' f'resolve int,' - f'new int,' f'url text,' f'subdomain text,' f'port int,' @@ -86,6 +85,20 @@ class Database(object): f'elapse float,' f'find int)') + def insert_table(self, table_name, result): + table_name = table_name.replace('.', '_') + self.conn.query( + f'insert into "{table_name}" ' + f'(id, alive, resolve, request, url, subdomain, port, level,' + f'cname, ip, public, cdn, status, reason, title, banner, header,' + f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,' + f'module, source, elapse, find) ' + f'values (:id, :alive, :resolve, :request, :url,' + f':subdomain, :port, :level, :cname, :ip, :public, :cdn,' + f':status, :reason, :title, :banner, :header, :history, :response,' + f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,' + f':source, :elapse, :find)', **result) + def save_db(self, table_name, results, module_name=None): """ Save the results of each module in the database @@ -101,11 +114,11 @@ class Database(object): try: self.conn.bulk_query( f'insert into "{table_name}" ' - f'(id, alive, resolve, request, new, url, subdomain, port, level,' + f'(id, alive, resolve, request, url, subdomain, port, level,' f'cname, ip, public, cdn, status, reason, title, banner, header,' f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,' f'module, source, elapse, find) ' - f'values (:id, :alive, :resolve, :request, :new, :url,' + f'values (:id, :alive, :resolve, :request, :url,' f':subdomain, :port, :level, :cname, :ip, :public, :cdn,' f':status, :reason, :title, :banner, :header, :history, :response,' f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,' @@ -178,7 +191,7 @@ class Database(object): def deduplicate_subdomain(self, table_name): """ - Deduplicates of subdomains in the table + Deduplicate subdomains in the table :param str table_name: table name """ @@ -199,17 +212,6 @@ class Database(object): self.query(f'delete from "{table_name}" where ' f'subdomain is null or resolve == 0') - def deal_table(self, deal_table_name, backup_table_name): - """ - Process the table when the collection task is complete - - :param str deal_table_name: Pending table name - :param str backup_table_name: Table name for backup - """ - self.copy_table(deal_table_name, backup_table_name) - self.remove_invalid(deal_table_name) - self.deduplicate_subdomain(deal_table_name) - def get_data(self, table_name): """ Get all the data in the table @@ -218,7 +220,7 @@ class Database(object): """ table_name = table_name.replace('.', '_') logger.log('TRACE', f'Get all the data from {table_name} table') - return self.query(f'select * from "{table_name}"') + return self.query(f'select * from {table_name}') def export_data(self, table_name, alive, limit): """ @@ -229,18 +231,42 @@ class Database(object): :param str limit: limit value """ table_name = table_name.replace('.', '_') - query = f'select id, new, alive, request, resolve, url, subdomain, level,' \ - f'cname, ip, public, cdn, port, status, reason, title, banner,' \ - f'cidr, asn, org, addr, isp, source from "{table_name}"' + sql = f'select id, alive, request, resolve, url, subdomain, level,' \ + f'cname, ip, public, cdn, port, status, reason, title, banner,' \ + f'cidr, asn, org, addr, isp, source from {table_name} order by subdomain' if alive and limit: if limit in ['resolve', 'request']: where = f' where {limit} = 1' - query += where + sql += where elif alive: where = f' where alive = 1' - query += where + sql += where logger.log('TRACE', f'Get the data from {table_name} table') - return self.query(query) + return self.query(sql) + + def count_alive(self, table_name): + table_name = table_name.replace('.', '_') + sql = f'select count() from {table_name} where alive = 1' + return self.query(sql) + + def get_resp_by_url(self, table_name, url): + table_name = table_name.replace('.', '_') + sql = f'select response from {table_name} where url = "{url}"' + logger.log('TRACE', f'Get response data from {url}') + return self.query(sql).scalar() + + def get_data_by_fields(self, table_name, fields): + table_name = table_name.replace('.', '_') + field_str = ', '.join(fields) + sql = f"select {field_str} from {table_name}" + logger.log('TRACE', f'Get specified field data {fields} from {table_name} table') + return self.query(sql) + + def update_data_by_url(self, table_name, info, url): + table_name = table_name.replace('.', '_') + field_str = ', '.join(map(lambda kv: f'{kv[0]} = "{kv[1]}"', info.items())) + sql = f'update {table_name} set {field_str} where url = "{url}"' + return self.query(sql) def close(self): """ diff --git a/common/module.py b/common/module.py index 29dac45..194757d 100644 --- a/common/module.py +++ b/common/module.py @@ -275,7 +275,6 @@ class Module(object): 'alive': None, 'request': None, 'resolve': None, - 'new': None, 'url': None, 'subdomain': None, 'port': None, @@ -315,25 +314,22 @@ class Module(object): ip = info.get('ip') times = info.get('times') ttl = info.get('ttl') - public = info.get('public') if isinstance(cname, list): cname = ','.join(cname) ip = ','.join(ip) times = ','.join([str(num) for num in times]) ttl = ','.join([str(num) for num in ttl]) - public = ','.join([str(num) for num in public]) result = {'id': None, 'alive': info.get('alive'), 'request': info.get('request'), 'resolve': info.get('resolve'), - 'new': None, 'url': url, 'subdomain': subdomain, 'port': 80, 'level': level, 'cname': cname, 'ip': ip, - 'public': public, + 'public': info.get('public'), 'cdn': info.get('cdn'), 'status': None, 'reason': info.get('reason'), diff --git a/common/request.py b/common/request.py index c6fe98c..8346884 100644 --- a/common/request.py +++ b/common/request.py @@ -1,7 +1,6 @@ import json from threading import Thread from queue import Queue -from operator import attrgetter import tqdm import requests @@ -9,6 +8,7 @@ from bs4 import BeautifulSoup from common import utils from config.log import logger +from common.database import Database from config import settings @@ -127,7 +127,7 @@ def get_progress_bar(total): return bar -def get(url, resp_list, session): +def get_resp(url, session): timeout = settings.request_timeout_second redirect = settings.request_allow_redirect proxy = utils.get_proxy() @@ -136,13 +136,14 @@ def get(url, resp_list, session): except Exception as e: logger.log('DEBUG', e.args) resp = e - resp_list.append((url, resp)) + return resp -def request(urls_queue, resp_list, session): +def request(urls_queue, resp_queue, session): while not urls_queue.empty(): - url = urls_queue.get() - get(url, resp_list, session) + index, url = urls_queue.get() + resp = get_resp(url, session) + resp_queue.put((index, resp)) urls_queue.task_done() @@ -168,31 +169,6 @@ def get_session(): return session -def bulk_request(urls): - logger.log('INFOR', 'Requesting urls in bulk') - resp_list = list() - urls_queue = Queue() - for url in urls: - urls_queue.put(url) - total = len(urls) - session = get_session() - thread_count = req_thread_count() - bar = get_progress_bar(total) - - progress_thread = Thread(target=progress, name='ProgressThread', - args=(bar, total, urls_queue), daemon=True) - progress_thread.start() - - for i in range(thread_count): - request_thread = Thread(target=request, name=f'RequestThread-{i}', - args=(urls_queue, resp_list, session), daemon=True) - request_thread.start() - - urls_queue.join() - - return resp_list - - def gen_new_info(info, resp): if isinstance(resp, Exception): info['reason'] = str(resp.args) @@ -220,13 +196,54 @@ def gen_new_info(info, resp): return info -def gen_new_data(data, resp_list): - new_data = list() - for url, resp in resp_list: - for info in data: - if info.get('url') == url: - new_data.append(gen_new_info(info, resp)) - return new_data +def save(name, total, req_data, resp_queue): + db = Database() + db.create_table(name) + i = 0 + while True: + if not resp_queue.empty(): + i += 1 + index, resp = resp_queue.get() + old_info = req_data[index] + new_info = gen_new_info(old_info, resp) + db.insert_table(name, new_info) + resp_queue.task_done() + if i >= total: + break + db.close() + + +def bulk_request(domain, req_data, ret=False): + logger.log('INFOR', 'Requesting urls in bulk') + resp_queue = Queue() + urls_queue = Queue() + total = len(req_data) + for index, info in enumerate(req_data): + try: + url = info.get('url') + except Exception: + pass + urls_queue.put((index, url)) + session = get_session() + thread_count = req_thread_count() + bar = get_progress_bar(total) + + progress_thread = Thread(target=progress, name='ProgressThread', + args=(bar, total, urls_queue), daemon=True) + progress_thread.start() + + for i in range(thread_count): + request_thread = Thread(target=request, name=f'RequestThread-{i}', + args=(urls_queue, resp_queue, session), daemon=True) + request_thread.start() + if ret: + urls_queue.join() + return resp_queue + save_thread = Thread(target=save, name=f'SaveThread', + args=(domain, total, req_data, resp_queue), daemon=True) + save_thread.start() + urls_queue.join() + save_thread.join() def run_request(domain, data, port): @@ -242,20 +259,6 @@ def run_request(domain, data, port): data = utils.set_id_none(data) ports = get_port_seq(port) req_data, req_urls = gen_req_data(data, ports) - resp_list = bulk_request(req_urls) - new_data = gen_new_data(req_data, resp_list) - count = utils.count_alive(new_data) + bulk_request(domain, req_data) + count = utils.count_alive(domain) logger.log('INFOR', f'Found that {domain} has {count} alive subdomains') - sorted_data = utils.sort_by_subdomain(new_data) - return sorted_data - - -def save_db(name, data): - """ - Save request results to database - - :param str name: table name - :param list data: data to be saved - """ - logger.log('INFOR', f'Saving requested results') - utils.save_db(name, data, 'request') diff --git a/common/resolve.py b/common/resolve.py index 3345796..ff52553 100644 --- a/common/resolve.py +++ b/common/resolve.py @@ -4,12 +4,6 @@ import json from config.log import logger from config import settings from common import utils -from common.ipasn import IPAsnInfo -from common.ipreg import IpRegData - - -ip_asn = IPAsnInfo() -ip_reg = IpRegData() def filter_subdomain(data): @@ -79,13 +73,7 @@ def gen_infos(data, qname, info, infos): flag = False cname = list() ips = list() - public = list() ttl = list() - cidr = list() - asn = list() - org = list() - addr = list() - isp = list() answers = data.get('answers') for answer in answers: if answer.get('type') == 'A': @@ -94,25 +82,11 @@ def gen_infos(data, qname, info, infos): ip = answer.get('data') ips.append(ip) ttl.append(str(answer.get('ttl'))) - public.append(str(utils.ip_is_public(ip))) - asn_info = ip_asn.find(ip) - cidr.append(asn_info.get('cidr')) - asn.append(asn_info.get('asn')) - org.append(asn_info.get('org')) - ip_info = ip_reg.query(ip) - addr.append(ip_info.get('addr')) - isp.append(ip_info.get('isp')) info['resolve'] = 1 info['reason'] = 'OK' info['cname'] = ','.join(cname) info['ip'] = ','.join(ips) - info['public'] = ','.join(public) info['ttl'] = ','.join(ttl) - info['cidr'] = ','.join(cidr) - info['asn'] = ','.join(asn) - info['org'] = ','.join(org) - info['addr'] = ','.join(addr) - info['isp'] = ','.join(isp) infos[qname] = info if not flag: info['alive'] = 0 diff --git a/common/utils.py b/common/utils.py index 74c03c4..1f12a1f 100644 --- a/common/utils.py +++ b/common/utils.py @@ -176,16 +176,16 @@ def check_dir(dir_path): dir_path.mkdir(parents=True, exist_ok=True) -def check_path(path, name, format): +def check_path(path, name, fmt): """ 检查结果输出目录路径 :param path: 保存路径 :param name: 导出名字 - :param format: 保存格式 + :param fmt: 保存格式 :return: 保存路径 """ - filename = f'{name}.{format}' + filename = f'{name}.{fmt}' default_path = settings.result_save_dir.joinpath(filename) if isinstance(path, str): path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠 @@ -204,19 +204,18 @@ def check_path(path, name, format): return path -def check_format(format, count): +def check_format(fmt): """ 检查导出格式 - :param format: 传入的导出格式 - :param count: 数量 + :param fmt: 传入的导出格式 :return: 导出格式 """ formats = ['csv', 'json', ] - if format in formats: - return format + if fmt in formats: + return fmt else: - logger.log('ALERT', f'Does not support {format} format') + logger.log('ALERT', f'Does not support {fmt} format') logger.log('ALERT', 'So use csv format by default') return 'csv' @@ -333,8 +332,8 @@ def remove_invalid_string(string): return re.sub(r'[\000-\010]|[\013-\014]|[\016-\037]', r'', string) -def export_all_results(path, name, format, datas): - path = check_path(path, name, format) +def export_all_results(path, name, fmt, datas): + path = check_path(path, name, fmt) logger.log('ALERT', f'The subdomain result for all main domains: {path}') row_list = list() for row in datas: @@ -346,7 +345,7 @@ def export_all_results(path, name, format, datas): values = row.values() row_list.append(Record(keys, values)) rows = RecordCollection(iter(row_list)) - content = rows.export(format) + content = rows.export(fmt) save_data(path, content) @@ -366,19 +365,19 @@ def export_all_subdomains(alive, path, name, datas): save_data(path, data) -def export_all(alive, format, path, datas): +def export_all(alive, fmt, path, datas): """ 将所有结果数据导出 :param bool alive: 只导出存活子域结果 - :param str format: 导出文件格式 + :param str fmt: 导出文件格式 :param str path: 导出文件路径 :param list datas: 待导出的结果数据 """ - format = check_format(format, len(datas)) + fmt = check_format(fmt) timestamp = get_timestring() name = f'all_subdomain_result_{timestamp}' - export_all_results(path, name, format, datas) + export_all_results(path, name, fmt, datas) export_all_subdomains(alive, path, name, datas) @@ -430,10 +429,18 @@ def python_version(): return sys.version -def count_alive(data): +def calc_alive(data): return len(list(filter(lambda item: item.get('alive') == 1, data))) +def count_alive(name): + db = Database() + result = db.count_alive(name) + count = result.scalar() + db.close() + return count + + def get_subdomains(data): return set(map(lambda item: item.get('subdomain'), data)) @@ -833,3 +840,23 @@ def looks_like_ip(maybe_ip): return True except socket.error: return False + + +def deal_data(domain): + db = Database() + db.remove_invalid(domain) + db.deduplicate_subdomain(domain) + db.close() + + +def get_data(domain): + db = Database() + data = db.get_data(domain).as_dict() + db.close() + return data + + +def clear_data(domain): + db = Database() + db.drop_table(domain) + db.close() diff --git a/config/default.py b/config/default.py index 902ea7c..11b859f 100644 --- a/config/default.py +++ b/config/default.py @@ -24,7 +24,7 @@ enable_dns_resolve = True # 使用DNS解析子域(默认True) enable_http_request = True # 使用HTTP请求子域(默认True) enable_finder_module = True # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True) enable_altdns_module = False # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True) -enable_cdn_check = True # 开启cdn检查模块(默认True) +enable_enrich_module = True # 开启enrich模块，开启会富化出信息，如ip的cdn，cidr，asn，org，addr和isp等信息 enable_banner_identify = True # 开启WEB指纹识别模块(默认True) enable_takeover_check = False # 开启子域接管风险检查(默认False) # 参数可选值有'small', 'medium', 'large' diff --git a/data/cdn_asn_list.json b/data/cdn_asn_list.json index b6e1260..ad6ff1a 100644 --- a/data/cdn_asn_list.json +++ b/data/cdn_asn_list.json @@ -1,134 +1,134 @@ [ - "10576", - "10762", - "11748", - "131099", - "132601", - "133496", - "134409", - "135295", - "136764", - "137187", - "13777", - "13890", - "14103", - "14520", - "17132", - "199251", - "200013", - "200325", - "200856", - "201263", - "202294", - "203075", - "203139", - "204248", - "204286", - "204545", - "206227", - "206734", - "206848", - "206986", - "207158", - "208559", - "209403", - "21030", - "21257", - "23327", - "23393", - "23637", - "23794", - "24997", - "26492", - "268843", - "28709", - "29264", - "30282", - "30637", - "328126", - "36408", - "38107", - "397192", - "40366", - "43303", - "44907", - "46071", - "46177", - "47542", - "49287", - "49689", - "51286", - "55082", - "55254", - "56636", - "57363", - "58127", - "59730", - "59776", - "60068", - "60626", - "60922", - "61107", - "61159", - "62026", - "62229", - "63062", - "64232", - "8868", - "9053", - "55770", - "49846", - "49249", - "48163", - "45700", - "43639", - "39836", - "393560", - "393234", - "36183", - "35994", - "35993", - "35204", - "34850", - "34164", - "33905", - "32787", - "31377", - "31110", - "31109", - "31108", - "31107", - "30675", - "24319", - "23903", - "23455", - "23454", - "22207", - "21399", - "21357", - "21342", - "20940", - "20189", - "18717", - "18680", - "17334", - "16702", - "16625", - "12222", - "209101", - "201585", - "135429", - "395747", - "394536", - "209242", - "203898", - "202623", - "14789", - "133877", - "13335", - "132892", - "21859", - "6185", - "47823", - "4134" -] \ No newline at end of file + "AS10576", + "AS10762", + "AS11748", + "AS131099", + "AS132601", + "AS133496", + "AS134409", + "AS135295", + "AS136764", + "AS137187", + "AS13777", + "AS13890", + "AS14103", + "AS14520", + "AS17132", + "AS199251", + "AS200013", + "AS200325", + "AS200856", + "AS201263", + "AS202294", + "AS203075", + "AS203139", + "AS204248", + "AS204286", + "AS204545", + "AS206227", + "AS206734", + "AS206848", + "AS206986", + "AS207158", + "AS208559", + "AS209403", + "AS21030", + "AS21257", + "AS23327", + "AS23393", + "AS23637", + "AS23794", + "AS24997", + "AS26492", + "AS268843", + "AS28709", + "AS29264", + "AS30282", + "AS30637", + "AS328126", + "AS36408", + "AS38107", + "AS397192", + "AS40366", + "AS43303", + "AS44907", + "AS46071", + "AS46177", + "AS47542", + "AS49287", + "AS49689", + "AS51286", + "AS55082", + "AS55254", + "AS56636", + "AS57363", + "AS58127", + "AS59730", + "AS59776", + "AS60068", + "AS60626", + "AS60922", + "AS61107", + "AS61159", + "AS62026", + "AS62229", + "AS63062", + "AS64232", + "AS8868", + "AS9053", + "AS55770", + "AS49846", + "AS49249", + "AS48163", + "AS45700", + "AS43639", + "AS39836", + "AS393560", + "AS393234", + "AS36183", + "AS35994", + "AS35993", + "AS35204", + "AS34850", + "AS34164", + "AS33905", + "AS32787", + "AS31377", + "AS31110", + "AS31109", + "AS31108", + "AS31107", + "AS30675", + "AS24319", + "AS23903", + "AS23455", + "AS23454", + "AS22207", + "AS21399", + "AS21357", + "AS21342", + "AS20940", + "AS20189", + "AS18717", + "AS18680", + "AS17334", + "AS16702", + "AS16625", + "AS12222", + "AS209101", + "AS201585", + "AS135429", + "AS395747", + "AS394536", + "AS209242", + "AS203898", + "AS202623", + "AS14789", + "AS133877", + "AS13335", + "AS132892", + "AS21859", + "AS6185", + "AS47823", + "AS4134" +] diff --git a/dbexport.py b/dbexport.py index 7892a81..62e073a 100644 --- a/dbexport.py +++ b/dbexport.py @@ -15,17 +15,17 @@ from common.database import Database from config.log import logger -def export(target, type='target', db=None, alive=False, limit=None, path=None, format='csv', show=False): +def export(target, type='target', db=None, alive=False, limit=None, path=None, fmt='csv', show=False): """ OneForAll export from database module Example: - python3 dbexport.py --target name --format csv --dir= ./result.csv + python3 dbexport.py --target name --fmt csv --dir= ./result.csv python3 dbexport.py --db result.db --target name --show False python3 dbexport.py --target table_name --tb True --show False Note: - --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) + --fmt csv/json (result format) --path Result directory (default directory is ./results) :param str target: Table to be exported @@ -33,7 +33,7 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f :param str db: Database path to be exported (default ./results/result.sqlite3) :param bool alive: Only export the results of alive subdomains (default False) :param str limit: Export limit (default None) - :param str format: Result format (default csv) + :param str fmt: Result format (default csv) :param str path: Result directory (default None) :param bool show: Displays the exported data in terminal (default False) """ @@ -48,29 +48,29 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f rows = database.export_data(table_name, alive, limit) if rows is None: continue - data = export_data(format, path, rows, show, table_name, target) + data = export_data(fmt, path, rows, show, table_name, target) datas.extend(data) database.close() if len(domains) > 1: - utils.export_all(alive, format, path, datas) + utils.export_all(alive, fmt, path, datas) elif type == 'table': database = Database(db) rows = database.export_data(target, alive, limit) - data = export_data(format, path, rows, show, target, target) + data, _, _ = export_data(fmt, path, rows, show, target, target) database.close() return data -def export_data(format, path, rows, show, table_name, target): - format = utils.check_format(format, len(rows)) - path = utils.check_path(path, target, format) +def export_data(fmt, path, rows, show, table_name, target): + fmt = utils.check_format(fmt) + path = utils.check_path(path, target, fmt) if show: print(rows.dataset) - data = rows.export(format) + data = rows.export(fmt) utils.save_data(path, data) logger.log('ALERT', f'The subdomain result for {table_name}: {path}') data = rows.as_dict() - return data, format, path + return data, fmt, path def domain_to_table(table): diff --git a/docs/en-us/README.md b/docs/en-us/README.md index 71ea40b..af35b6a 100644 --- a/docs/en-us/README.md +++ b/docs/en-us/README.md @@ -164,7 +164,7 @@ DESCRIPTION python3 oneforall.py --target example.com --alive False run python3 oneforall.py --target example.com --brute True run python3 oneforall.py --target example.com --port medium run - python3 oneforall.py --target example.com --format csv run + python3 oneforall.py --target example.com --fmt csv run python3 oneforall.py --target example.com --dns False run python3 oneforall.py --target example.com --req False run python3 oneforall.py --target example.com --takeover False run @@ -173,7 +173,7 @@ DESCRIPTION Note: --alive True/False Only export alive subdomains or not (default False) --port default/small/large See details in ./config/setting.py(default port 80) - --format csv/json (result format) + --fmt csv/json (result format) --path Result directory (default directory is ./results) ARGUMENTS @@ -193,7 +193,7 @@ FLAGS The port range request to the subdomains (default port 80) --alive=ALIVE Only export alive subdomains (default False) - --format=FORMAT + --fmt=FMT Result format (default csv) --path=PATH Result directory (default None) @@ -270,7 +270,6 @@ The project uses [SemVer](https://semver.org/) for version management, and you c ## ⌛Follow-up plan - [ ] Continuous optimize and improve of each module -- [x] Subdomain monitoring (mark newly discovered subdomain) - [x] Subdomain collection crawler (collect subdomains from static files such as JS) - [ ] Implementation of front-end interface for powerful interaction diff --git a/modules/altdns.py b/modules/altdns.py index 0cc7a92..62b222b 100644 --- a/modules/altdns.py +++ b/modules/altdns.py @@ -193,9 +193,7 @@ class Altdns(Module): self.end = time.time() self.elapse = round(self.end - self.start, 1) self.gen_result() - temp_data = resolve.run_resolve(self.domain, self.results) - fina_data = request.run_request(self.domain, temp_data, port) - data = data + fina_data + resolved_data = resolve.run_resolve(self.domain, self.results) + request.run_request(self.domain, resolved_data, port) logger.log('INFOR', f'Saving altdns results') utils.save_db(self.domain, data, 'altdns') - return data diff --git a/modules/enrich.py b/modules/enrich.py new file mode 100644 index 0000000..e4a801b --- /dev/null +++ b/modules/enrich.py @@ -0,0 +1,72 @@ +from modules import iscdn +from common import utils +from common.database import Database +from common.ipasn import IPAsnInfo +from common.ipreg import IpRegData + + +def get_ips(info): + ip = info.get('ip') + if not ip: + return None + ips = ip.split(',') + return ips + + +def enrich_info(data): + ip_asn = IPAsnInfo() + ip_reg = IpRegData() + for index, info in enumerate(data): + ips = get_ips(info) + if not ips: + continue + public = list() + cidr = list() + asn = list() + org = list() + addr = list() + isp = list() + for ip in ips: + public.append(str(utils.ip_is_public(ip))) + asn_info = ip_asn.find(ip) + cidr.append(asn_info.get('cidr')) + asn.append(asn_info.get('asn')) + org.append(asn_info.get('org')) + ip_info = ip_reg.query(ip) + addr.append(ip_info.get('addr')) + isp.append(ip_info.get('isp')) + data[index]['public'] = ','.join(public) + data[index]['cidr'] = ','.join(cidr) + data[index]['asn'] = ','.join(asn) + data[index]['org'] = ','.join(org) + data[index]['addr'] = ','.join(addr) + data[index]['isp'] = ','.join(isp) + return data + + +class Enrich(object): + def __init__(self, domain): + self.domain = domain + + def get_data(self): + db = Database() + fields = ['url', 'cname', 'ip', 'public', 'cdn', 'header', + 'cidr', 'asn', 'org', 'addr', 'isp'] + results = db.get_data_by_fields(self.domain, fields) + return results.as_dict() + + def save_db(self, data): + db = Database() + for info in data: + url = info.pop('url') + info.pop('cname') + info.pop('ip') + info.pop('header') + db.update_data_by_url(self.domain, info, url) + db.close() + + def run(self): + data = self.get_data() + data = enrich_info(data) + data = iscdn.do_check(data) + self.save_db(data) diff --git a/modules/finder.py b/modules/finder.py index 304d7b6..abc86c4 100644 --- a/modules/finder.py +++ b/modules/finder.py @@ -7,6 +7,7 @@ from common import utils from common import resolve from common import request from common.module import Module +from common.database import Database from config import settings from config.log import logger @@ -25,16 +26,11 @@ class Finder(Module): new_subdomains = found_subdomains - existing_subdomains if not len(new_subdomains): self.finish() # 未发现新的子域就直接返回 - return data self.subdomains = new_subdomains self.finish() self.gen_result() - temp_data = resolve.run_resolve(domain, self.results) - fina_data = request.run_request(domain, temp_data, port) - data = data + fina_data - logger.log('INFOR', f'Saving finder results') - utils.save_db(domain, data, 'finder') - return data + resolved_data = resolve.run_resolve(domain, self.results) + request.run_request(domain, resolved_data, port) file_path = settings.data_storage_dir.joinpath('common_js_library.json') @@ -177,22 +173,31 @@ def find_js_urls(domain, req_url, rsp_html): return js_urls +def convert_to_dict(url_list): + url_dict = [] + for url in url_list: + url_dict.append({'url': url}) + return url_dict + def find_subdomains(domain, data): subdomains = set() js_urls = set() + db = Database() for infos in data: jump_history = infos.get('history') req_url = infos.get('url') subdomains.update(find_in_history(domain, req_url, jump_history)) - rsp_html = infos.get('response') + rsp_html = db.get_resp_by_url(domain, req_url) if not rsp_html: logger.log('DEBUG', f'an abnormal response occurred in the request {req_url}') continue subdomains.update(find_in_resp(domain, req_url, rsp_html)) js_urls.update(find_js_urls(domain, req_url, rsp_html)) - resp_data = request.bulk_request(js_urls) - for _, resp in resp_data: + req_data = convert_to_dict(js_urls) + resp_data = request.bulk_request(domain, req_data, ret=True) + while not resp_data.empty(): + _, resp = resp_data.get() if not isinstance(resp, Response): continue text = utils.decode_resp_text(resp) diff --git a/modules/iscdn.py b/modules/iscdn.py index 10920ef..276b43f 100644 --- a/modules/iscdn.py +++ b/modules/iscdn.py @@ -3,6 +3,7 @@ import ipaddress from config import settings from common import utils +from common.database import Database from config.log import logger data_dir = settings.data_storage_dir @@ -18,6 +19,8 @@ cdn_header_key = utils.load_json(data_dir.joinpath('cdn_header_keys.json')) def check_cname_keyword(cname): + if not cname: + return False names = cname.lower().split(',') for name in names: for keyword in cdn_cname_keyword.keys(): @@ -26,14 +29,22 @@ def check_cname_keyword(cname): def check_header_key(header): - header = set(map(lambda x: x.lower(), header.keys())) - for key in cdn_header_key: - if key in header: - return True + if isinstance(header, str): + header = json.loads(header) + if isinstance(header, dict): + header = set(map(lambda x: x.lower(), header.keys())) + for key in cdn_header_key: + if key in header: + return True + else: + return False -def check_cdn_cidr(content): - ips = set(content.split(',')) +def check_cdn_cidr(ips): + if isinstance(ips, str): + ips = set(ips.split(',')) + else: + return False for ip in ips: try: ip = ipaddress.ip_address(ip) @@ -46,39 +57,30 @@ def check_cdn_cidr(content): def check_cdn_asn(asn): - if str(asn) in cdn_asn_list: - return True + if isinstance(asn, str): + if asn in cdn_asn_list: + return True + return False -def check_cdn(data): - logger.log('DEBUG', f'Start cdn check module') +def do_check(data): + logger.log('DEBUG', f'Checking cdn') for index, item in enumerate(data): cname = item.get('cname') - if cname: - if check_cname_keyword(cname): - data[index]['cdn'] = 1 - continue + if check_cname_keyword(cname): + data[index]['cdn'] = 1 + continue header = item.get('header') - if header: - header = json.loads(header) - if check_header_key(header): - data[index]['cdn'] = 1 - continue + if check_header_key(header): + data[index]['cdn'] = 1 + continue ip = item.get('ip') - if ip: - if check_cdn_cidr(ip): - data[index]['cdn'] = 1 - continue + if check_cdn_cidr(ip): + data[index]['cdn'] = 1 + continue asn = item.get('asn') - if asn: - asn = asn[2:] # 去除AS - if check_cdn_asn(asn): - data[index]['cdn'] = 1 - continue + if check_cdn_asn(asn): + data[index]['cdn'] = 1 + continue data[index]['cdn'] = 0 return data - - -def save_db(name, data): - logger.log('DEBUG', f'Saving cdn check results') - utils.save_db(name, data, 'cdn') diff --git a/oneforall.py b/oneforall.py index cb84655..13cbbd1 100644 --- a/oneforall.py +++ b/oneforall.py @@ -15,12 +15,11 @@ from datetime import datetime import dbexport from brute import Brute from common import utils, resolve, request -from common.database import Database from modules.collect import Collect from modules.srv import BruteSRV from modules.finder import Finder from modules.altdns import Altdns -from modules import iscdn +from modules.enrich import Enrich from config import settings from config.log import logger from takeover import Takeover @@ -60,7 +59,7 @@ class OneForAll(object): python3 oneforall.py --target example.com --alive False run python3 oneforall.py --target example.com --brute False run python3 oneforall.py --target example.com --port medium run - python3 oneforall.py --target example.com --format csv run + python3 oneforall.py --target example.com --fmt csv run python3 oneforall.py --target example.com --dns False run python3 oneforall.py --target example.com --req False run python3 oneforall.py --target example.com --takeover False run @@ -68,22 +67,22 @@ class OneForAll(object): Note: --port small/medium/large See details in ./config/setting.py(default small) - --format csv/json (result format) + --fmt csv/json (result format) --path Result path (default None, automatically generated) :param str target: One domain (target or targets must be provided) :param str targets: File path of one domain per line - :param bool brute: Use brute module (default True) - :param bool dns: Use DNS resolution (default True) - :param bool req: HTTP request subdomains (default True) + :param bool brute: Use brute module (default True) + :param bool dns: Use DNS resolution (default True) + :param bool req: HTTP request subdomains (default True) :param str port: The port range to request (default small port is 80,443) - :param bool alive: Only export alive subdomains (default False) - :param str format: Result format (default csv) + :param bool alive: Only export alive subdomains (default False) + :param str fmt: Result format (default csv) :param str path: Result path (default None, automatically generated) - :param bool takeover: Scan subdomain takeover (default False) + :param bool takeover: Scan subdomain takeover (default False) """ def __init__(self, target=None, targets=None, brute=None, dns=None, req=None, - port=None, alive=None, format=None, path=None, takeover=None): + port=None, alive=None, fmt=None, path=None, takeover=None): self.target = target self.targets = targets self.brute = brute @@ -91,17 +90,13 @@ class OneForAll(object): self.req = req self.port = port self.alive = alive - self.format = format + self.fmt = fmt self.path = path self.takeover = takeover self.domain = str() # The domain currently being collected self.domains = set() # All domains that are to be collected self.data = list() # The subdomain results of the current domain self.datas = list() # All subdomain results of the domain - self.old_table = str() # The table name of the last result - self.new_table = str() # The table name of the current result - self.origin_table = str() # The table name of the origin result - self.resolve_table = str() # The table name of the resolute result def config_param(self): """ @@ -119,8 +114,8 @@ class OneForAll(object): self.port = settings.http_request_port if self.alive is None: self.alive = bool(settings.result_export_alive) - if self.format is None: - self.format = settings.result_save_format + if self.fmt is None: + self.fmt = settings.result_save_format if self.path is None: self.path = settings.result_save_path @@ -134,47 +129,13 @@ class OneForAll(object): def export(self, table): """ - Export data from the database and do some follow-up processing + Export data from the database :param table: table name :return: export data :rtype: list """ - db = Database() - data = dbexport.export(table, type='table', alive=self.alive, format=self.format) - db.drop_table(self.new_table) - db.rename_table(self.domain, self.new_table) - db.close() - return data - - def deal_db(self): - """ - Process the data when the collection task is completed - """ - db = Database() - db.deal_table(self.domain, self.origin_table) - db.close() - - def mark(self): - """ - Mark the new discovered subdomain - - :return: marked data - :rtype: list - """ - db = Database() - old_data = list() - now_data = db.get_data(self.domain).as_dict() - # Database pre-processing when it is not the first time to collect this subdomain - if db.exist_table(self.new_table): - # If there is the last collection result table, delete it first - db.drop_table(self.old_table) - # Rename the new table to the old table - db.rename_table(self.new_table, self.old_table) - old_data = db.get_data(self.old_table).as_dict() - db.close() - marked_data = utils.mark_subdomain(old_data, now_data) - return marked_data + return dbexport.export(table, type='table', alive=self.alive, fmt=self.fmt) def main(self): """ @@ -183,11 +144,6 @@ class OneForAll(object): :return: subdomain results :rtype: list """ - self.old_table = self.domain + '_old_result' - self.new_table = self.domain + '_now_result' - self.origin_table = self.domain + '_origin_result' - self.resolve_table = self.domain + '_resolve_result' - collect = Collect(self.domain) collect.run() @@ -202,49 +158,44 @@ class OneForAll(object): brute.quite = True brute.run() - # Database processing - self.deal_db() - # Mark the new discovered subdomain - self.data = self.mark() - + utils.deal_data(self.domain) # Export results without resolve if not self.dns: return self.export(self.domain) + self.data = utils.get_data(self.domain) + # Resolve subdomains + utils.clear_data(self.domain) self.data = resolve.run_resolve(self.domain, self.data) # Save resolve results - resolve.save_db(self.resolve_table, self.data) + resolve.save_db(self.domain, self.data) # Export results without HTTP request if not self.req: - return self.export(self.resolve_table) + return self.export(self.domain) # HTTP request - self.data = request.run_request(self.domain, self.data, self.port) - # Save HTTP request result - request.save_db(self.domain, self.data) + utils.clear_data(self.domain) + request.run_request(self.domain, self.data, self.port) # Finder module if settings.enable_finder_module: finder = Finder() - self.data = finder.run(self.domain, self.data, self.port) + finder.run(self.domain, self.data, self.port) # altdns module if settings.enable_altdns_module: - finder = Altdns(self.domain) - self.data = finder.run(self.data, self.port) + altdns = Altdns(self.domain) + altdns.run(self.data, self.port) - # check cdn module - if settings.enable_cdn_check: - self.data = iscdn.check_cdn(self.data) - iscdn.save_db(self.domain, self.data) - - # Add the final result list to the total data list - self.datas.extend(self.data) + # Information enrichment module + if settings.enable_enrich_module: + enrich = Enrich(self.domain) + enrich.run() # Export - self.export(self.domain) + self.datas.extend(self.export(self.domain)) # Scan subdomain takeover if self.takeover: @@ -277,7 +228,7 @@ class OneForAll(object): for domain in self.domains: self.domain = utils.get_main_domain(domain) self.main() - utils.export_all(self.alive, self.format, self.path, self.datas) + utils.export_all(self.alive, self.fmt, self.path, self.datas) else: logger.log('FATAL', 'Failed to obtain domain') logger.log('INFOR', 'Finished OneForAll') diff --git a/takeover.py b/takeover.py index 721b2c4..094ed63 100644 --- a/takeover.py +++ b/takeover.py @@ -45,21 +45,21 @@ class Takeover(Module): OneForAll subdomain takeover module Example: - python3 takeover.py --target www.example.com --format csv run + python3 takeover.py --target www.example.com --fmt csv run python3 takeover.py --targets ./subdomains.txt --thread 10 run Note: - --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) + --fmt txt/csv/json (result format) --path Result directory (default directory is ./results) :param str target: One domain (target or targets must be provided) :param str targets: File path of one domain per line :param int thread: threads number (default 20) - :param str format: Result format (default csv) + :param str fmt: Result format (default csv) :param str path: Result directory (default None) """ - def __init__(self, target=None, targets=None, thread=20, path=None, format='csv'): + def __init__(self, target=None, targets=None, thread=20, path=None, fmt='csv'): Module.__init__(self) self.subdomains = set() self.module = 'Check' @@ -68,7 +68,7 @@ class Takeover(Module): self.targets = targets self.thread = thread self.path = path - self.format = format + self.fmt = fmt self.fingerprints = None self.subdomainq = Queue() self.cnames = list() @@ -76,10 +76,10 @@ class Takeover(Module): def save(self): logger.log('DEBUG', 'Saving results') - if self.format == 'txt': + if self.fmt == 'txt': data = str(self.results) else: - data = self.results.export(self.format) + data = self.results.export(self.fmt) utils.save_data(self.path, data) def compare(self, subdomain, cname, responses): @@ -131,10 +131,10 @@ class Takeover(Module): self.subdomains = self.targets else: self.subdomains = utils.get_domains(self.target, self.targets) - self.format = utils.check_format(self.format, len(self.subdomains)) + self.fmt = utils.check_format(self.fmt) timestamp = utils.get_timestamp() name = f'takeover_check_result_{timestamp}' - self.path = utils.check_path(self.path, name, self.format) + self.path = utils.check_path(self.path, name, self.fmt) if self.subdomains: logger.log('INFOR', f'Checking subdomain takeover') self.fingerprints = get_fingerprint() From 1bd78c2469fbb055ae286a5db468259e6640d0a1 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Mon, 9 Nov 2020 09:01:41 +0800 Subject: [PATCH 02/10] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/search/baidu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/search/baidu.py b/modules/search/baidu.py index 8552e49..16ecae4 100644 --- a/modules/search/baidu.py +++ b/modules/search/baidu.py @@ -57,7 +57,7 @@ class Baidu(Search): self.subdomains.update(subdomains) self.page_num += self.per_page_num # 搜索页面没有出现下一页时停止搜索 - if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text: + if f'&pn={self.page_num}&' not in resp.text: break if self.page_num >= self.limit_num: # 搜索条数限制 break From 4ef874e4ae99ba547b03c189656db9c5cf3c569c Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Mon, 9 Nov 2020 10:46:29 +0800 Subject: [PATCH 03/10] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Pipfile.lock | 109 ++++++++++++++++++++++++++--------------------- requirements.txt | 20 ++++----- 2 files changed, 70 insertions(+), 59 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index dd1c207..65f6f6d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -18,11 +18,11 @@ "default": { "beautifulsoup4": { "hashes": [ - "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7", - "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8", - "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c" + "sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35", + "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25", + "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666" ], - "version": "==4.9.1" + "version": "==4.9.3" }, "bs4": { "hashes": [ @@ -47,11 +47,11 @@ }, "colorama": { "hashes": [ - "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", - "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1" + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" ], "markers": "sys_platform == 'win32'", - "version": "==0.4.3" + "version": "==0.4.4" }, "dnspython": { "hashes": [ @@ -79,6 +79,7 @@ "hashes": [ "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.18.2" }, "idna": { @@ -86,6 +87,7 @@ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.10" }, "loguru": { @@ -118,6 +120,7 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "soupsieve": { @@ -125,45 +128,52 @@ "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" ], + "markers": "python_version >= '3.0'", "version": "==2.0.1" }, "sqlalchemy": { "hashes": [ - "sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb", - "sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804", - "sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6", - "sha256:26c5ca9d09f0e21b8671a32f7d83caad5be1f6ff45eef5ec2f6fd0db85fc5dc0", - "sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe", - "sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de", - "sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36", - "sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e", - "sha256:465c999ef30b1c7525f81330184121521418a67189053bcf585824d833c05b66", - "sha256:51064ee7938526bab92acd049d41a1dc797422256086b39c08bafeffb9d304c6", - "sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc", - "sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d", - "sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce", - "sha256:6557af9e0d23f46b8cd56f8af08eaac72d2e3c632ac8d5cf4e20215a8dca7cea", - "sha256:73a40d4fcd35fdedce07b5885905753d5d4edf413fbe53544dd871f27d48bd4f", - "sha256:8280f9dae4adb5889ce0bb3ec6a541bf05434db5f9ab7673078c00713d148365", - "sha256:83469ad15262402b0e0974e612546bc0b05f379b5aa9072ebf66d0f8fef16bea", - "sha256:860d0fe234922fd5552b7f807fbb039e3e7ca58c18c8d38aa0d0a95ddf4f6c23", - "sha256:883c9fb62cebd1e7126dd683222b3b919657590c3e2db33bdc50ebbad53e0338", - "sha256:8afcb6f4064d234a43fea108859942d9795c4060ed0fbd9082b0f280181a15c1", - "sha256:96f51489ac187f4bab588cf51f9ff2d40b6d170ac9a4270ffaed535c8404256b", - "sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e", - "sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba", - "sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02", - "sha256:b6ff91356354b7ff3bd208adcf875056d3d886ed7cef90c571aef2ab8a554b12", - "sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86", - "sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d", - "sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7", - "sha256:e49947d583fe4d29af528677e4f0aa21f5e535ca2ae69c48270ebebd0d8843c0", - "sha256:eb1d71643e4154398b02e88a42fc8b29db8c44ce4134cf0f4474bfc5cb5d4dac", - "sha256:f2e8a9c0c8813a468aa659a01af6592f71cd30237ec27c4cc0683f089f90dcfc", - "sha256:fe7fe11019fc3e6600819775a7d55abc5446dda07e9795f5954fdbf8a49e1c37" + "sha256:009e8388d4d551a2107632921320886650b46332f61dc935e70c8bcf37d8e0d6", + "sha256:0157c269701d88f5faf1fa0e4560e4d814f210c01a5b55df3cab95e9346a8bcc", + "sha256:0a92745bb1ebbcb3985ed7bda379b94627f0edbc6c82e9e4bac4fb5647ae609a", + "sha256:0cca1844ba870e81c03633a99aa3dc62256fb96323431a5dec7d4e503c26372d", + "sha256:166917a729b9226decff29416f212c516227c2eb8a9c9f920d69ced24e30109f", + "sha256:1f5f369202912be72fdf9a8f25067a5ece31a2b38507bb869306f173336348da", + "sha256:2909dffe5c9a615b7e6c92d1ac2d31e3026dc436440a4f750f4749d114d88ceb", + "sha256:2b5dafed97f778e9901b79cc01b88d39c605e0545b4541f2551a2fd785adc15b", + "sha256:2e9bd5b23bba8ae8ce4219c9333974ff5e103c857d9ff0e4b73dc4cb244c7d86", + "sha256:3aa6d45e149a16aa1f0c46816397e12313d5e37f22205c26e06975e150ffcf2a", + "sha256:4bdbdb8ca577c6c366d15791747c1de6ab14529115a2eb52774240c412a7b403", + "sha256:53fd857c6c8ffc0aa6a5a3a2619f6a74247e42ec9e46b836a8ffa4abe7aab327", + "sha256:5cdfe54c1e37279dc70d92815464b77cd8ee30725adc9350f06074f91dbfeed2", + "sha256:5d92c18458a4aa27497a986038d5d797b5279268a2de303cd00910658e8d149c", + "sha256:632b32183c0cb0053194a4085c304bc2320e5299f77e3024556fa2aa395c2a8b", + "sha256:7c735c7a6db8ee9554a3935e741cf288f7dcbe8706320251eb38c412e6a4281d", + "sha256:7cd40cb4bc50d9e87b3540b23df6e6b24821ba7e1f305c1492b0806c33dbdbec", + "sha256:84f0ac4a09971536b38cc5d515d6add7926a7e13baa25135a1dbb6afa351a376", + "sha256:8dcbf377529a9af167cbfc5b8acec0fadd7c2357fc282a1494c222d3abfc9629", + "sha256:950f0e17ffba7a7ceb0dd056567bc5ade22a11a75920b0e8298865dc28c0eff6", + "sha256:9e379674728f43a0cd95c423ac0e95262500f9bfd81d33b999daa8ea1756d162", + "sha256:b15002b9788ffe84e42baffc334739d3b68008a973d65fad0a410ca5d0531980", + "sha256:b6f036ecc017ec2e2cc2a40615b41850dc7aaaea6a932628c0afc73ab98ba3fb", + "sha256:bad73f9888d30f9e1d57ac8829f8a12091bdee4949b91db279569774a866a18e", + "sha256:bbc58fca72ce45a64bb02b87f73df58e29848b693869e58bd890b2ddbb42d83b", + "sha256:bca4d367a725694dae3dfdc86cf1d1622b9f414e70bd19651f5ac4fb3aa96d61", + "sha256:be41d5de7a8e241864189b7530ca4aaf56a5204332caa70555c2d96379e18079", + "sha256:bf53d8dddfc3e53a5bda65f7f4aa40fae306843641e3e8e701c18a5609471edf", + "sha256:c092fe282de83d48e64d306b4bce03114859cdbfe19bf8a978a78a0d44ddadb1", + "sha256:c3ab23ee9674336654bf9cac30eb75ac6acb9150dc4b1391bec533a7a4126471", + "sha256:ce64a44c867d128ab8e675f587aae7f61bd2db836a3c4ba522d884cd7c298a77", + "sha256:d05cef4a164b44ffda58200efcb22355350979e000828479971ebca49b82ddb1", + "sha256:d2f25c7f410338d31666d7ddedfa67570900e248b940d186b48461bd4e5569a1", + "sha256:d3b709d64b5cf064972b3763b47139e4a0dc4ae28a36437757f7663f67b99710", + "sha256:e32e3455db14602b6117f0f422f46bc297a3853ae2c322ecd1e2c4c04daf6ed5", + "sha256:ed53209b5f0f383acb49a927179fa51a6e2259878e164273ebc6815f3a752465", + "sha256:f605f348f4e6a2ba00acb3399c71d213b92f27f2383fc4abebf7a37368c12142", + "sha256:fcdb3755a7c355bc29df1b5e6fb8226d5c8b90551d202d69d0076a8a5649d68b" ], "index": "pypi", - "version": "==1.3.19" + "version": "==1.3.20" }, "tenacity": { "hashes": [ @@ -181,11 +191,11 @@ }, "tqdm": { "hashes": [ - "sha256:8f3c5815e3b5e20bc40463fa6b42a352178859692a68ffaa469706e6d38342a5", - "sha256:faf9c671bd3fad5ebaeee366949d969dca2b2be32c872a7092a1e1a9048d105b" + "sha256:9ad44aaf0fc3697c06f6e05c7cf025dd66bc7bcb7613c66d85f4464c47ac8fad", + "sha256:ef54779f1c09f346b2b5a8e5c61f96fbcb639929e640e59f8cf810794f406432" ], "index": "pypi", - "version": "==4.49.0" + "version": "==4.51.0" }, "treelib": { "hashes": [ @@ -196,18 +206,19 @@ }, "urllib3": { "hashes": [ - "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", - "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" + "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2", + "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e" ], - "version": "==1.25.10" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.25.11" }, "win32-setctime": { "hashes": [ - "sha256:02b4c5959ca0b195f45c98115826c6e8a630b7cf648e724feaab1a5aa6250640", - "sha256:47aa7c43548c1fc0a4f026d1944b748b37036df116c7c4cf908e82638d854313" + "sha256:4e88556c32fdf47f64165a2180ba4552f8bb32c1103a2fafd05723a0bd42bd4b", + "sha256:dc925662de0a6eb987f0b01f599c01a8236cb8c62831c22d9cada09ad958243e" ], "markers": "sys_platform == 'win32'", - "version": "==1.0.2" + "version": "==1.0.3" } }, "develop": {} diff --git a/requirements.txt b/requirements.txt index 9aba14b..85ed146 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,23 @@ -i https://mirrors.aliyun.com/pypi/simple/ -beautifulsoup4==4.9.1 +beautifulsoup4==4.9.3 bs4==0.0.1 certifi==2020.6.20 chardet==3.0.4 -colorama==0.4.3 ; sys_platform == 'win32' +colorama==0.4.4; sys_platform == 'win32' dnspython==2.0.0 exrex==0.10.5 fire==0.3.1 -future==0.18.2 -idna==2.10 +future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' +idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' loguru==0.5.3 pysocks==1.7.1 requests==2.24.0 -six==1.15.0 -soupsieve==2.0.1 -sqlalchemy==1.3.19 +six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +soupsieve==2.0.1; python_version >= '3.0' +sqlalchemy==1.3.20 tenacity==6.2.0 termcolor==1.1.0 -tqdm==4.49.0 +tqdm==4.51.0 treelib==1.6.1 -urllib3==1.25.10 -win32-setctime==1.0.2 ; sys_platform == 'win32' \ No newline at end of file +urllib3==1.25.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' +win32-setctime==1.0.3; sys_platform == 'win32' From 0f2b365a25e7fa5cb53c612af21b3ede3007a69e Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Mon, 9 Nov 2020 11:00:34 +0800 Subject: [PATCH 04/10] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8F=AA=E5=AF=8C?= =?UTF-8?q?=E5=8C=96=E4=B8=80=E6=9D=A1=E6=95=B0=E6=8D=AE=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/enrich.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/enrich.py b/modules/enrich.py index e4a801b..52db30b 100644 --- a/modules/enrich.py +++ b/modules/enrich.py @@ -41,7 +41,7 @@ def enrich_info(data): data[index]['org'] = ','.join(org) data[index]['addr'] = ','.join(addr) data[index]['isp'] = ','.join(isp) - return data + return data class Enrich(object): From fec72d20824afa58c382ae1c1409d6a4cb36d8f8 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Mon, 9 Nov 2020 18:43:16 +0800 Subject: [PATCH 05/10] =?UTF-8?q?=E9=87=8D=E6=9E=84=E7=BD=91=E7=BB=9C?= =?UTF-8?q?=E5=88=A4=E6=96=AD=E5=92=8C=E5=9C=B0=E5=9D=80=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 - brute.py | 20 +-- common/resolve.py | 3 +- common/utils.py | 134 +++++------------- config/default.py | 3 - config/setting.py | 4 - data/common_nameservers.txt | 30 ---- data/nameservers.txt | 30 ++++ ...{cn_nameservers.txt => nameservers_cn.txt} | 0 docs/en-us/README.md | 2 - oneforall.py | 20 ++- 11 files changed, 84 insertions(+), 164 deletions(-) delete mode 100644 data/common_nameservers.txt rename data/{cn_nameservers.txt => nameservers_cn.txt} (100%) diff --git a/README.md b/README.md index 3653f54..b6c00a3 100644 --- a/README.md +++ b/README.md @@ -97,8 +97,6 @@ docker run -it --rm -v ~/results:/OneForAll/results -v ~/.config:/OneForAll/conf
✨使用演示 -如果你的主机不在中国，请把 [setting](https://github.com/shmilylty/OneForAll/blob/master/config/setting.py#L46) 中`brute_nameservers_path`选项的`cn_nameservers.txt`修改为`nameservers.txt`。 - 如果你是通过pip3安装的依赖则使用以下命令运行示例： ```bash python3 oneforall.py --target example.com run diff --git a/brute.py b/brute.py index 4269982..c74f050 100644 --- a/brute.py +++ b/brute.py @@ -349,18 +349,6 @@ def collect_wildcard_record(domain, authoritative_ns): return ips, ttl -def get_nameservers_path(enable_wildcard, ns_ip_list): - path = settings.brute_nameservers_path - if not enable_wildcard: - return path - if not ns_ip_list: - return path - path = settings.authoritative_dns_path - ns_data = '\n'.join(ns_ip_list) - utils.save_data(path, ns_data) - return path - - def check_dict(): if not settings.enable_check_dict: return @@ -607,8 +595,8 @@ class Brute(Module): self.domain = str() # 当前正在进行爆破的域名 self.ips_times = dict() # IP集合出现次数 self.enable_wildcard = False # 当前域名是否使用泛解析 - self.check_env = True self.quite = False + self.in_china = None def gen_brute_dict(self, domain): logger.log('INFOR', f'Generating dictionary for {domain}') @@ -684,7 +672,7 @@ class Brute(Module): if self.enable_wildcard: wildcard_ips, wildcard_ttl = collect_wildcard_record(domain, ns_ip_list) - ns_path = get_nameservers_path(self.enable_wildcard, ns_ip_list) + ns_path = utils.get_ns_path(self.in_china, self.enable_wildcard, ns_ip_list) dict_set = self.gen_brute_dict(domain) @@ -727,8 +715,8 @@ class Brute(Module): def run(self): logger.log('INFOR', f'Start running {self.source} module') - if self.check_env: - utils.check_env() + if self.in_china is None: + _, self.in_china = utils.get_net_env() self.domains = utils.get_domains(self.target, self.targets) for self.domain in self.domains: self.results = list() # 置空 diff --git a/common/resolve.py b/common/resolve.py index ff52553..df1bb0e 100644 --- a/common/resolve.py +++ b/common/resolve.py @@ -155,8 +155,7 @@ def run_resolve(domain, data): output_name = f'resolved_result_{domain}_{timestring}.json' output_path = temp_dir.joinpath(output_name) log_path = result_dir.joinpath('massdns.log') - - ns_path = settings.brute_nameservers_path + ns_path = utils.get_ns_path() logger.log('INFOR', f'Running massdns to resolve subdomains') utils.call_massdns(massdns_path, save_path, ns_path, diff --git a/common/utils.py b/common/utils.py index 1f12a1f..5038f5c 100644 --- a/common/utils.py +++ b/common/utils.py @@ -518,15 +518,16 @@ def delete_file(*paths): @tenacity.retry(stop=tenacity.stop_after_attempt(3)) def check_net(): - logger.log('INFOR', 'Checking Internet environment') - urls = ['http://www.baidu.com', 'http://www.bing.com', - 'http://www.apple.com', 'http://www.microsoft.com'] + urls = ['http://ipinfo.io/json', 'http://ipconfig.io/json'] url = random.choice(urls) - logger.log('INFOR', f'Trying to access {url}') + header = {'User_Agent': 'curl'} + timeout = settings.request_timeout_second + verify = settings.request_ssl_verify + logger.log('DEBUG', f'Trying to access {url}') session = requests.Session() session.trust_env = False try: - rsp = session.get(url, proxies=get_proxy()) + rsp = session.get(url, headers=header, timeout=timeout, verify=verify) except Exception as e: logger.log('ERROR', e.args) logger.log('ALERT', 'Can not access Internet, retrying') @@ -536,10 +537,16 @@ def check_net(): f'{rsp.status_code} {rsp.reason}') logger.log('ALERT', 'Can not access Internet normally, retrying') raise tenacity.TryAgain - logger.log('INFOR', 'Access to Internet OK') + logger.log('DEBUG', 'Access to Internet OK') + country = rsp.json().get('country').lower() + if country in ['cn', 'china']: + logger.log('DEBUG', f'The host in china') + return True, True + else: + return True, False -def check_pre(): +def check_dep(): logger.log('INFOR', 'Checking dependent environment') implementation = platform.python_implementation() version = platform.python_version() @@ -551,15 +558,15 @@ def check_pre(): exit(1) -def check_env(): - logger.log('INFOR', 'Checking the environment') +def get_net_env(): + logger.log('INFOR', 'Checking network environment') try: - check_net() + result = check_net() except Exception as e: logger.log('DEBUG', e.args) - logger.log('FATAL', 'Can not access Internet') - exit(1) - check_pre() + logger.log('ALERT', 'Can not access Internet') + return False, None + return result def check_version(local): @@ -577,7 +584,7 @@ def check_version(local): resp_json = resp.json() latest = resp_json['tag_name'] except Exception as e: - logger.log('ERROR', 'An error occurred while checking the latest version') + logger.log('ALERT', 'An error occurred while checking the latest version') logger.log('DEBUG', e.args) return if latest > local: @@ -743,90 +750,6 @@ def sort_by_subdomain(data): return sorted(data, key=lambda item: item.get('subdomain')) -def ping(host, path): - param = '-n' if platform.system().lower() == 'windows' else '-c' - command = ['ping', param, '5', host] - with open(path, "w") as f: - return subprocess.call(command, stdout=f, stderr=f) - - -def ping_avg_time(nameserver): - check_dir(settings.temp_save_dir) - temp_path = settings.temp_save_dir.joinpath('ping') - ping(nameserver, path=temp_path) - with open(temp_path, 'r') as f: - text = f.read() - if '100.0% packet loss' in text or '100% packet loss' in text or '100% 丢失' in text: - logger.log('ALERT', f'100.0% packet loss, ping {nameserver} failed.') - return None - elif platform.system() in ('Darwin', 'Linux'): - try: - avg_time = re.findall(r'(?:min/avg/max/.+ )(?:\d+\.\d+)/(\d+\.\d+)/', text)[0] - logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.') - except IndexError: - return None - return avg_time - elif platform.system() == 'Windows': - try: - avg_time = re.findall(r'(?:Average|平均).+(\d.?)ms', text)[0] - logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.') - except IndexError: - return None - return avg_time - else: - logger.log('ALERT', f'{text}') - return None - - -def auto_select_nameserver(): - logger.log('INFOR', f'Ping test start, to select nameservers.') - avg_time1 = ping_avg_time('114.114.114.114') - avg_time2 = ping_avg_time('8.8.8.8') - if avg_time1 and avg_time2: - if avg_time1 < avg_time2: - change_nameservers_file('cn') - logger.log('INFOR', f'Ping test finished, use cn nameservers.') - else: - change_nameservers_file('common') - logger.log('INFOR', f'Ping test finished, use common nameservers.') - elif avg_time1 and not avg_time2: - change_nameservers_file('cn') - logger.log('INFOR', f'Ping test finished, use cn nameservers.') - elif not avg_time1 and avg_time1: - change_nameservers_file('common') - logger.log('INFOR', f'Ping test finished, use common nameservers.') - elif not avg_time1 and not avg_time1: - change_nameservers_file('default') - logger.log('INFOR', f'Ping test finished, use default nameservers.') - return - - -def change_nameservers_file(option): - text = '' - if option == 'cn': - with open(settings.data_storage_dir.joinpath('cn_nameservers.txt'), 'r') as f: - text = f.read() - elif option == 'common': - with open(settings.data_storage_dir.joinpath('common_nameservers.txt'), 'r') as f: - text = f.read() - elif option == 'default': - for n in default_nameserver(): - text = '\n'.join(n) - with open(settings.data_storage_dir.joinpath('nameservers.txt'), 'w') as f: - f.write(text) - return - - -def default_nameserver(): - try: - resolver = dns.resolver.Resolver() - return resolver.nameservers - except dns.resolver.NoResolverConfiguration: - logger.log('ERROR', 'Resolver configuration could not be read ' - 'or specified no nameservers.') - exit(1) - - def looks_like_ip(maybe_ip): """Does the given str look like an IP address?""" if not maybe_ip[0].isdigit(): @@ -860,3 +783,18 @@ def clear_data(domain): db = Database() db.drop_table(domain) db.close() + + +def get_ns_path(in_china=None, enable_wildcard=None, ns_ip_list=None): + data_dir = settings.data_storage_dir + path = data_dir.joinpath('nameservers.txt') + if in_china: + path = data_dir.joinpath('nameservers_cn.txt') + if not enable_wildcard: + return path + if not ns_ip_list: + return path + path = settings.authoritative_dns_path + ns_data = '\n'.join(ns_ip_list) + save_data(path, ns_data) + return path diff --git a/config/default.py b/config/default.py index 11b859f..7b4a3b3 100644 --- a/config/default.py +++ b/config/default.py @@ -58,9 +58,6 @@ brute_socket_num = 1 # 爆破时每个进程下的socket数量 brute_resolve_num = 15 # 解析失败时尝试换名称服务器重查次数 # 爆破所使用的字典路径 默认data/subdomains.txt brute_wordlist_path = data_storage_dir.joinpath('subnames.txt') -# 爆破所使用的字典路径 默认data/cn_nameservers.txt -# 如果你不在中国请改为nameservers.txt -brute_nameservers_path = data_storage_dir.joinpath('cn_nameservers.txt') # 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询 authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt') enable_recursive_brute = False # 是否使用递归爆破(默认False) diff --git a/config/setting.py b/config/setting.py index 6a35079..a331d49 100644 --- a/config/setting.py +++ b/config/setting.py @@ -38,10 +38,6 @@ enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_mo brute_concurrent_num = 2000 # 爆破时并发查询数量(默认2000，最大推荐10000) # 爆破所使用的字典路径 默认data/subdomains.txt brute_wordlist_path = data_storage_dir.joinpath('subnames.txt') -# 爆破所使用的DNS服务器路径 默认data/cn_nameservers.txt 如果你不在中国请改为nameservers.txt -# DNS resolve server file path default data/nameservers.txt -# If your computer's location are not in China, change `cn_nameservers.txt` to `nameservers.txt` plz. -brute_nameservers_path = data_storage_dir.joinpath('cn_nameservers.txt') # 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询 authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt') enable_recursive_brute = False # 是否使用递归爆破(默认False) diff --git a/data/common_nameservers.txt b/data/common_nameservers.txt deleted file mode 100644 index f74fb12..0000000 --- a/data/common_nameservers.txt +++ /dev/null @@ -1,30 +0,0 @@ -8.8.8.8 -8.8.4.4 -9.9.9.9 -9.9.9.10 -149.112.112.112 -4.2.2.1 -4.2.2.2 -4.2.2.3 -4.2.2.4 -4.2.2.5 -4.2.2.6 -1.1.1.1 -1.0.0.1 -1.0.0.2 -1.0.0.3 -1.0.0.19 -208.67.222.222 -208.67.220.220 -8.26.56.26 -8.20.247.20 -84.200.69.80 -84.200.70.40 -185.228.168.9 -185.228.169.9 -64.6.64.6 -64.6.65.6 -198.101.242.72 -23.253.163.53 -176.103.130.130 -176.103.130.131 \ No newline at end of file diff --git a/data/nameservers.txt b/data/nameservers.txt index e69de29..f74fb12 100644 --- a/data/nameservers.txt +++ b/data/nameservers.txt @@ -0,0 +1,30 @@ +8.8.8.8 +8.8.4.4 +9.9.9.9 +9.9.9.10 +149.112.112.112 +4.2.2.1 +4.2.2.2 +4.2.2.3 +4.2.2.4 +4.2.2.5 +4.2.2.6 +1.1.1.1 +1.0.0.1 +1.0.0.2 +1.0.0.3 +1.0.0.19 +208.67.222.222 +208.67.220.220 +8.26.56.26 +8.20.247.20 +84.200.69.80 +84.200.70.40 +185.228.168.9 +185.228.169.9 +64.6.64.6 +64.6.65.6 +198.101.242.72 +23.253.163.53 +176.103.130.130 +176.103.130.131 \ No newline at end of file diff --git a/data/cn_nameservers.txt b/data/nameservers_cn.txt similarity index 100% rename from data/cn_nameservers.txt rename to data/nameservers_cn.txt diff --git a/docs/en-us/README.md b/docs/en-us/README.md index af35b6a..24b7f78 100644 --- a/docs/en-us/README.md +++ b/docs/en-us/README.md @@ -89,8 +89,6 @@ Result will be saved in `~/results`.
✨OneForAll usage -If your computer are not in China, change [setting](https://github.com/shmilylty/OneForAll/blob/master/config/setting.py#L46) `brute_nameservers_path` param `cn_nameservers.txt` to `nameservers.txt` plz. - If you are use pip3, run the following command: ```bash diff --git a/oneforall.py b/oneforall.py index 13cbbd1..5609be1 100644 --- a/oneforall.py +++ b/oneforall.py @@ -97,6 +97,8 @@ class OneForAll(object): self.domains = set() # All domains that are to be collected self.data = list() # The subdomain results of the current domain self.datas = list() # All subdomain results of the domain + self.in_china = None + self.access_internet = False def config_param(self): """ @@ -144,8 +146,12 @@ class OneForAll(object): :return: subdomain results :rtype: list """ - collect = Collect(self.domain) - collect.run() + if not self.access_internet: + logger.log('ALERT', 'Because it cannot access the Internet, ' + 'OneForAll will not execute the subdomain collection module!') + if self.access_internet: + collect = Collect(self.domain) + collect.run() srv = BruteSRV(self.domain) srv.run() @@ -154,7 +160,7 @@ class OneForAll(object): # Due to there will be a large number of dns resolution requests, # may cause other network tasks to be error brute = Brute(self.domain, word=True, export=False) - brute.check_env = False + brute.in_china = self.in_china brute.quite = True brute.run() @@ -214,12 +220,12 @@ class OneForAll(object): print(oneforall_banner) dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S') print(f'[*] Starting OneForAll @ {dt}\n') - utils.check_env() - utils.auto_select_nameserver() - if settings.enable_check_version: - utils.check_version(version) logger.log('DEBUG', 'Python ' + utils.python_version()) logger.log('DEBUG', 'OneForAll ' + version) + utils.check_dep() + self.access_internet, self.in_china = utils.get_net_env() + if self.access_internet and settings.enable_check_version: + utils.check_version(version) logger.log('INFOR', 'Start running OneForAll') self.config_param() self.check_param() From 5f14d492cae30df6f01b8b917c1432a8e6bb2b2c Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Mon, 9 Nov 2020 18:49:17 +0800 Subject: [PATCH 06/10] =?UTF-8?q?=E6=96=87=E4=BB=B6=E9=87=8D=E5=91=BD?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/default.py | 4 ++-- config/setting.py | 4 ++-- data/{big_subnames.7z => subnames_big.7z} | Bin 13969773 -> 13969773 bytes ...edium_subnames.txt => subnames_medium.txt} | 0 data/{next_subnames.txt => subnames_next.txt} | 0 docs/directory_structure.md | 8 ++++---- 6 files changed, 8 insertions(+), 8 deletions(-) rename data/{big_subnames.7z => subnames_big.7z} (99%) rename data/{medium_subnames.txt => subnames_medium.txt} (100%) rename data/{next_subnames.txt => subnames_next.txt} (100%) diff --git a/config/default.py b/config/default.py index 7b4a3b3..3b501b5 100644 --- a/config/default.py +++ b/config/default.py @@ -62,8 +62,8 @@ brute_wordlist_path = data_storage_dir.joinpath('subnames.txt') authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt') enable_recursive_brute = False # 是否使用递归爆破(默认False) brute_recursive_depth = 2 # 递归爆破深度(默认2层) -# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt -recursive_nextlist_path = data_storage_dir.joinpath('next_subnames.txt') +# 爆破下一层子域所使用的字典路径 默认data/subnames_next.txt +recursive_nextlist_path = data_storage_dir.joinpath('subnames_next.txt') enable_check_dict = False # 是否开启字典配置检查提示(默认False) delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True) delete_massdns_result = True # 是否删除爆破时massdns输出的解析结果 (默认True) diff --git a/config/setting.py b/config/setting.py index a331d49..a3130a1 100644 --- a/config/setting.py +++ b/config/setting.py @@ -42,8 +42,8 @@ brute_wordlist_path = data_storage_dir.joinpath('subnames.txt') authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt') enable_recursive_brute = False # 是否使用递归爆破(默认False) brute_recursive_depth = 2 # 递归爆破深度(默认2层) -# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt -recursive_nextlist_path = data_storage_dir.joinpath('next_subnames.txt') +# 爆破下一层子域所使用的字典路径 默认data/subnames_next.txt +recursive_nextlist_path = data_storage_dir.joinpath('subnames_next.txt') enable_check_dict = False # 是否开启字典配置检查提示(默认False) delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True) # 是否删除爆破时massdns输出的解析结果 (默认True) diff --git a/data/big_subnames.7z b/data/subnames_big.7z similarity index 99% rename from data/big_subnames.7z rename to data/subnames_big.7z index 04ee18bddbcec30368a95792b5445fc44cc22e9d..d66591ae7fef743d5287064aba9be2b517df9961 100644 GIT binary patch delta 713 zcmW-ZXIF>;07hSAZy_UGva++X$(Fsh6iFc?8JXeQdu8wJy+`)mq<(>~`VZnc=iYPf zm!}{%uS7v1L}%pOXc8WT+->`3`0u9q!M~;acMxPpMa5(yMWm<{lMpE`C8VU3lG0K} z%1SvYFBPPsRFcY4MM9;jRFmowCN-p{)RNi~E_I}?)RV&h`qDrmq@gsD#?nNZN~AQC z=F&o1N-JqCZ6r$CN;_#U9i*c~ODE|pU8JjYlkU<(dP*;ek>1ir`bt0PF9T$t43bzG zEJI|d43pt9LPp9c87*UEtc;WKGC?NFB$+HzWU5S)IGHZ-k{~l=rX>-D%)hc z?2w(ZOLog1*(>{GzZ{T*k|l@aupE)2a!iiP2{|dJOrFaNc`2{twY-tH@=o5%2l*(U;1|3TdI+w9NHHlcC8VU3k`O5^Wu&Z>lk!qQ zDoP~@mC8~@s!Ettlj>4K!lkCvlG;*7>Pm#vlloHhZy*gNQW{BPX(CY)Eiuwmnn`nM zAuXkqw3ar~R@zB>=^!1YlXR9Y(p9=itaO(i(o=d#Z|Ng_rJwYd0Wweq$zX|-Au?2k z$#5AVBW09~mN7C`#>se@AQNShOqMA!Ri?>wiI*8NQxasB%$7u%BXeb*%$EhSP!>s& zB+Fu1B1>hNq{wnvAuDB-td=#BDr;q(td}&|AR8rJHpyn$BFa|TCfj9)?37)yTlUCa z*(dwufE<)Va#)T?hGfc7IVQ*Dgq)O9a$3&FSve=?<$_$4OLAGR$W^%}*X4#}$xXQ> z*>YR%$X&T7_a#Rj$U}J~kL8Ixm0Woy&*g=@lvnav-pE^dC-3EhnDB&PelRzP4^n~^K|+um%nXu(#2_UzE~X&zFJ$-kegFUf diff --git a/data/medium_subnames.txt b/data/subnames_medium.txt similarity index 100% rename from data/medium_subnames.txt rename to data/subnames_medium.txt diff --git a/data/next_subnames.txt b/data/subnames_next.txt similarity index 100% rename from data/next_subnames.txt rename to data/subnames_next.txt diff --git a/docs/directory_structure.md b/docs/directory_structure.md index 94b82e5..f2c6ea3 100644 --- a/docs/directory_structure.md +++ b/docs/directory_structure.md @@ -47,11 +47,11 @@ D:. | +---data 存放一些所需数据 | authoritative_dns.txt 临时存放开启了泛解析域名的权威DNS名称服务器IP地址 -| big_subnames.txt 子域爆破超大字典 -| cn_nameservers.txt 中国主流名称服务器IP地址 +| subnames_big.7z 子域爆破超大字典 +| nameservers_cn.txt 中国主流名称服务器IP地址 | fingerprints.json 检查子域接管风险的指纹 | nameservers.txt 全球主流名称服务器IP地址 -| next_subnames.txt 下一层子域字典 +| subnames_next.txt 下一层子域字典 | public_suffix_list.dat 顶级域名后缀 | srv_prefixes.json 常见SRV记录前缀名 | subnames.txt 子域爆破常见字典 @@ -175,4 +175,4 @@ D:. cyggcc_s-1.dll cygwin1.dll massdns_windows_i686.exe -``` \ No newline at end of file +``` From ef054085cd3eb22f2bfdf589ae1bafb5bbf29bb1 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Tue, 10 Nov 2020 10:35:01 +0800 Subject: [PATCH 07/10] =?UTF-8?q?=E8=B7=AF=E5=BE=84=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/datasets/robtex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/datasets/robtex.py b/modules/datasets/robtex.py index 5a93cd3..15a1c16 100644 --- a/modules/datasets/robtex.py +++ b/modules/datasets/robtex.py @@ -17,7 +17,7 @@ class Robtex(Query): """ self.header = self.get_header() self.proxy = self.get_proxy(self.source) - base_addr = 'https://freeapi.robtex.com/pdns/' + base_addr = 'https://freeapi.robtex.com/pdns' url = f'{base_addr}/forward/{self.domain}' resp = self.get(url) if not resp: From 504a9c88e83036d181bcab2726720dfea69649cc Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Tue, 10 Nov 2020 12:57:40 +0800 Subject: [PATCH 08/10] =?UTF-8?q?=E7=A7=BB=E9=99=A4phonebook=E6=A8=A1?= =?UTF-8?q?=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/datasets/phonebook.py | 61 ----------------------------------- 1 file changed, 61 deletions(-) delete mode 100644 modules/datasets/phonebook.py diff --git a/modules/datasets/phonebook.py b/modules/datasets/phonebook.py deleted file mode 100644 index ff2fdb6..0000000 --- a/modules/datasets/phonebook.py +++ /dev/null @@ -1,61 +0,0 @@ -from config.log import logger -from common.query import Query - - -class PhoneBook(Query): - def __init__(self, domain): - Query.__init__(self) - self.domain = domain - self.module = 'Dataset' - self.source = 'PhoneBookQuery' - - def query(self): - """ - 向接口查询子域并做子域匹配 - """ - self.header = self.get_header() - self.proxy = self.get_proxy(self.source) - self.header.update({'Referer': 'https://phonebook.cz/', - 'Origin': 'https://phonebook.cz'}) - addr = 'https://public.intelx.io/phonebook/search' - key = 'd7d1ed06-f0c5-49d4-a9ca-a167e6d2ffab' - url = f'{addr}?k={key}' - data = {"term": self.domain, "maxresults": 10000, - "media": 0, "target": 1, - "terminate": [], "timeout": 20} - resp = self.post(url, json=data) - if not resp: - return - json = resp.json() - ids = json.get('id') - if not ids: - logger.log('ALERT', f'Get PhoneBook id fail') - return - url = f'{addr}/result?k={key}&id={ids}&limit=10000' - resp = self.get(url) - self.subdomains = self.collect_subdomains(resp) - - def run(self): - """ - 类执行入口 - """ - self.begin() - self.query() - self.finish() - self.save_json() - self.gen_result() - self.save_db() - - -def run(domain): - """ - 类统一调用入口 - - :param str domain: 域名 - """ - query = PhoneBook(domain) - query.run() - - -if __name__ == '__main__': - run('freebuf.com') From 926a3955f62c8475319f3df6d595176b91f76ae9 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Tue, 10 Nov 2020 13:31:17 +0800 Subject: [PATCH 09/10] =?UTF-8?q?=E5=8F=82=E6=95=B0=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/default.py | 4 ++-- config/setting.py | 14 ++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/config/default.py b/config/default.py index 3b501b5..e403082 100644 --- a/config/default.py +++ b/config/default.py @@ -56,13 +56,13 @@ brute_process_num = 1 # 默认1 brute_concurrent_num = 2000 # 并发查询数量(默认2000，最大推荐10000) brute_socket_num = 1 # 爆破时每个进程下的socket数量 brute_resolve_num = 15 # 解析失败时尝试换名称服务器重查次数 -# 爆破所使用的字典路径 默认data/subdomains.txt +# 爆破所使用的字典路径 自定义字典请使用绝对路径 brute_wordlist_path = data_storage_dir.joinpath('subnames.txt') # 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询 authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt') enable_recursive_brute = False # 是否使用递归爆破(默认False) brute_recursive_depth = 2 # 递归爆破深度(默认2层) -# 爆破下一层子域所使用的字典路径 默认data/subnames_next.txt +# 爆破下一层子域所使用的字典路径 自定义字典请使用绝对路径 recursive_nextlist_path = data_storage_dir.joinpath('subnames_next.txt') enable_check_dict = False # 是否开启字典配置检查提示(默认False) delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True) diff --git a/config/setting.py b/config/setting.py index a3130a1..419436b 100644 --- a/config/setting.py +++ b/config/setting.py @@ -18,8 +18,8 @@ enable_finder_module = True # 开启finder模块,开启会从响应体和JS中 enable_cdn_check = True # 开启cdn检查模块(默认True) enable_banner_identify = True # 开启WEB指纹识别模块(默认True) enable_takeover_check = False # 开启子域接管风险检查(默认False) -# 参数可选值有'small', 'medium', 'large' -http_request_port = 'small' # HTTP请求子域(默认'small'，探测80,443端口) +# HTTP请求子域的端口范围 参数可选值有 'small', 'medium', 'large' +http_request_port = 'small' # 请求端口范围(默认 'small'，表示请求子域的80,443端口) # 参数可选值True，False分别表示导出存活，全部子域结果 result_export_alive = False # 只导出存活的子域结果(默认False) result_save_format = 'csv' # 子域结果保存文件格式(默认csv) @@ -36,14 +36,12 @@ enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_mo # 爆破模块设置 brute_concurrent_num = 2000 # 爆破时并发查询数量(默认2000，最大推荐10000) -# 爆破所使用的字典路径 默认data/subdomains.txt -brute_wordlist_path = data_storage_dir.joinpath('subnames.txt') -# 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询 -authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt') +# 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径) +brute_wordlist_path = None enable_recursive_brute = False # 是否使用递归爆破(默认False) brute_recursive_depth = 2 # 递归爆破深度(默认2层) -# 爆破下一层子域所使用的字典路径 默认data/subnames_next.txt -recursive_nextlist_path = data_storage_dir.joinpath('subnames_next.txt') +# 爆破下一层子域所使用的字典路径(默认None则使用data/subnames_next.txt，自定义字典请使用绝对路径) +recursive_nextlist_path = None enable_check_dict = False # 是否开启字典配置检查提示(默认False) delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True) # 是否删除爆破时massdns输出的解析结果 (默认True) From 8f988a441c0e8dfa79f303dcd7da8ba818f03c59 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Tue, 10 Nov 2020 15:26:43 +0800 Subject: [PATCH 10/10] =?UTF-8?q?=E8=B7=AF=E5=BE=84=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- brute.py | 8 ++++++++ config/default.py | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/brute.py b/brute.py index c74f050..e3423ef 100644 --- a/brute.py +++ b/brute.py @@ -653,6 +653,13 @@ class Brute(Module): logger.log('FATAL', f'Incorrect domain for fuzz') exit(1) + def init_dict_path(self): + data_dir = settings.data_storage_dir + if self.wordlist is None: + self.wordlist = settings.brute_wordlist_path or data_dir.joinpath('subnames.txt') + if self.recursive_nextlist is None: + self.recursive_nextlist = settings.recursive_nextlist_path or data_dir.joinpath('subnames_next.txt') + def main(self, domain): start = time.time() logger.log('INFOR', f'Blasting {domain} ') @@ -721,6 +728,7 @@ class Brute(Module): for self.domain in self.domains: self.results = list() # 置空 all_subdomains = list() + self.init_dict_path() self.check_brute_params() if self.recursive_brute: logger.log('INFOR', f'Start recursively brute the 1 layer subdomain' diff --git a/config/default.py b/config/default.py index e403082..62e7e01 100644 --- a/config/default.py +++ b/config/default.py @@ -56,14 +56,14 @@ brute_process_num = 1 # 默认1 brute_concurrent_num = 2000 # 并发查询数量(默认2000，最大推荐10000) brute_socket_num = 1 # 爆破时每个进程下的socket数量 brute_resolve_num = 15 # 解析失败时尝试换名称服务器重查次数 -# 爆破所使用的字典路径 自定义字典请使用绝对路径 -brute_wordlist_path = data_storage_dir.joinpath('subnames.txt') +# 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径) +brute_wordlist_path = None # 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询 authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt') enable_recursive_brute = False # 是否使用递归爆破(默认False) brute_recursive_depth = 2 # 递归爆破深度(默认2层) -# 爆破下一层子域所使用的字典路径 自定义字典请使用绝对路径 -recursive_nextlist_path = data_storage_dir.joinpath('subnames_next.txt') +# 爆破下一层子域所使用的字典路径(默认None则使用data/subnames_next.txt，自定义字典请使用绝对路径) +recursive_nextlist_path = None enable_check_dict = False # 是否开启字典配置检查提示(默认False) delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True) delete_massdns_result = True # 是否删除爆破时massdns输出的解析结果 (默认True)