From d91e8f429dacea5934fd2cf5f06d1b3fb924f52e Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Sat, 25 Apr 2020 18:12:10 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84=E5=AD=90=E5=9F=9F=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- oneforall/brute.py | 104 ++---------- oneforall/common/request.py | 13 +- oneforall/common/resolve.py | 286 +++++++++++++------------------- oneforall/common/utils.py | 59 ++++++- oneforall/example.py | 4 +- oneforall/modules/check/axfr.py | 4 +- oneforall/oneforall.py | 2 +- oneforall/takeover.py | 4 +- 8 files changed, 199 insertions(+), 277 deletions(-) diff --git a/oneforall/brute.py b/oneforall/brute.py index 7f3e75c..016f825 100644 --- a/oneforall/brute.py +++ b/oneforall/brute.py @@ -10,11 +10,8 @@ OneForAll子域爆破模块 import gc import json import time -import stat import random import secrets -import platform -import subprocess import exrex import fire @@ -24,7 +21,7 @@ from dns.resolver import NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers import config import dbexport -from common import resolve, utils +from common import utils from common.module import Module from config import logger @@ -76,7 +73,7 @@ def detect_wildcard(domain, authoritative_ns): logger.log('INFOR', f'正在探测{domain}是否使用泛解析') token = secrets.token_hex(4) random_subdomain = f'{token}.{domain}' - resolver = resolve.dns_resolver() + resolver = utils.dns_resolver() resolver.nameservers = authoritative_ns resolver.rotate = True resolver.cache = None @@ -114,15 +111,6 @@ def gen_fuzz_subdomains(expression, rule): return subdomains -def gen_domains(iterable, place): - subdomains = set() - for _ in range(place.count('*')): - for item in iterable: - subdomain = place.replace('*', item) - subdomains.add(subdomain) - return subdomains - - def gen_word_subdomains(expression, path): """ 生成基于word模式的爆破子域 @@ -151,7 +139,7 @@ def query_domain_ns_a(ns_list): if not isinstance(ns_list, list): return list() ns_ip_list = [] - resolver = resolve.dns_resolver() + resolver = utils.dns_resolver() for ns in ns_list: try: answer = resolver.query(ns, 'A') @@ -168,7 +156,7 @@ def query_domain_ns_a(ns_list): def query_domain_ns(domain): domain = utils.get_maindomain(domain) - resolver = resolve.dns_resolver() + resolver = utils.dns_resolver() try: answer = resolver.query(domain, 'NS') except Exception as e: @@ -218,7 +206,7 @@ def collect_wildcard_record(domain, authoritative_ns): logger.log('INFOR', f'正在收集{domain}的泛解析记录') if not authoritative_ns: return list(), int() - resolver = resolve.dns_resolver() + resolver = utils.dns_resolver() resolver.nameservers = authoritative_ns resolver.rotate = True resolver.cache = None @@ -259,28 +247,6 @@ def get_nameservers_path(enable_wildcard, ns_ip_list): return path -def get_massdns_path(massdns_dir): - path = config.brute_massdns_path - if path: - return path - system = platform.system().lower() - machine = platform.machine().lower() - name = f'massdns_{system}_{machine}' - if system == 'windows': - name = name + '.exe' - if machine == 'amd64': - massdns_dir = massdns_dir.joinpath('windows', 'x64') - else: - massdns_dir = massdns_dir.joinpath('windows', 'x84') - path = massdns_dir.joinpath(name) - path.chmod(stat.S_IXUSR) - if not path.exists(): - logger.log('FATAL', '暂无该系统平台及架构的massdns') - logger.log('INFOR', '请尝试自行编译massdns并在配置里指定路径') - exit(0) - return path - - def check_dict(): if not config.enable_check_dict: return @@ -294,40 +260,6 @@ def check_dict(): exit(0) -def do_brute(massdns_path, dict_path, ns_path, output_path, log_path, - query_type='A', process_num=1, concurrent_num=10000, - quiet_mode=False): - quiet = '' - if quiet_mode: - quiet = '--quiet' - status_format = config.brute_status_format - socket_num = config.brute_socket_num - resolve_num = config.brute_resolve_num - cmd = f'{massdns_path} {quiet} --status-format {status_format} ' \ - f'--processes {process_num} --socket-count {socket_num} ' \ - f'--hashmap-size {concurrent_num} --resolvers {ns_path} ' \ - f'--resolve-count {resolve_num} --type {query_type} ' \ - f'--flush --output J --outfile {output_path} ' \ - f'--error-log {log_path} {dict_path}' - logger.log('INFOR', f'执行命令 {cmd}') - subprocess.run(args=cmd, shell=True) - - -def read_result(result_path): - result = list() - with open(result_path) as fd: - for line in fd: - line = line.strip() - try: - record = json.loads(line) - except Exception as e: - logger.log('ERROR', e.args) - logger.log('ERROR', f'解析行{line}出错跳过解析该行') - continue - result.append(record) - return result - - def gen_records(items, records, subdomains, ip_times, wc_ips, wc_ttl): qname = items.get('name')[:-1] # 去出最右边的`.`点号 reason = items.get('status') @@ -474,8 +406,9 @@ def is_valid_subdomain(ip, ttl, times, wc_ips, wc_ttl): return 1, 'OK' -def save_brute_dict(path, data): - if not utils.save_data(path, data): +def save_brute_dict(dict_path, dict_set): + dict_data = '\n'.join(dict_set) + if not utils.save_data(dict_path, dict_data): logger.log('FATAL', '保存生成的字典出错') exit(1) @@ -611,7 +544,7 @@ class Brute(Module): result_dir = config.result_save_dir temp_dir = result_dir.joinpath('temp') utils.check_dir(temp_dir) - massdns_path = get_massdns_path(massdns_dir) + massdns_path = utils.get_massdns_path(massdns_dir) timestring = utils.get_timestring() wildcard_ips = list() # 泛解析IP列表 @@ -627,14 +560,11 @@ class Brute(Module): dict_set = self.gen_brute_dict(domain) dict_len = len(dict_set) - dict_data = '\n'.join(dict_set) - del dict_set - gc.collect() dict_name = f'generated_subdomains_{domain}_{timestring}.txt' dict_path = temp_dir.joinpath(dict_name) - save_brute_dict(dict_path, dict_data) - del dict_data + save_brute_dict(dict_path, dict_set) + del dict_set gc.collect() output_name = f'resolved_result_{domain}_{timestring}.json' @@ -642,11 +572,9 @@ class Brute(Module): log_path = result_dir.joinpath('massdns.log') check_dict() - logger.log('INFOR', f'开始执行massdns') - do_brute(massdns_path, dict_path, ns_path, output_path, log_path, - process_num=self.process_num, - concurrent_num=self.concurrent_num) - logger.log('INFOR', f'结束执行massdns') + utils.call_massdns(massdns_path, dict_path, ns_path, output_path, + log_path, process_num=self.process_num, + concurrent_num=self.concurrent_num) ip_times = stat_ip_times(output_path) self.records, self.subdomains = deal_output(output_path, ip_times, @@ -703,7 +631,3 @@ class Brute(Module): if __name__ == '__main__': fire.Fire(Brute) - # domain = 'example.com' - # result = queue.Queue() - # brute = AIOBrute(domain) - # brute.run(result) diff --git a/oneforall/common/request.py b/oneforall/common/request.py index ad225a8..004ceaa 100644 --- a/oneforall/common/request.py +++ b/oneforall/common/request.py @@ -22,7 +22,7 @@ def get_limit_conn(): def get_ports(port): - logger.log('DEBUG', f'正在获取请求探测端口范围') + logger.log('DEBUG', f'正在获取请求端口范围') ports = set() if isinstance(port, (set, list, tuple)): ports = port @@ -30,12 +30,12 @@ def get_ports(port): if 0 <= port <= 65535: ports = {port} elif port in {'default', 'small', 'large'}: - logger.log('DEBUG', f'探测{port}等端口范围') + logger.log('DEBUG', f'请求{port}等端口范围') ports = config.ports.get(port) if not ports: # 意外情况 - logger.log('ERROR', f'指定探测端口范围有误') + logger.log('ERROR', f'指定请求端口范围有误') ports = {80} - logger.log('INFOR', f'探测端口范围：{ports}') + logger.log('INFOR', f'请求端口范围：{ports}') return set(ports) @@ -202,7 +202,7 @@ async def bulk_request(data, port): no_req_data = utils.get_filtered_data(data) to_req_data = gen_req_data(data, ports) method = config.request_method - logger.log('INFOR', f'使用{method}请求方法') + logger.log('INFOR', f'请求使用{method}方法') logger.log('INFOR', f'正在进行异步子域请求') connector = get_connector() header = get_header() @@ -224,8 +224,6 @@ async def bulk_request(data, port): desc='Request Progress', ncols=80): await future - - logger.log('INFOR', f'完成异步进行子域的GET请求') return to_req_data + no_req_data @@ -239,6 +237,7 @@ def run_request(domain, data, port): :return: 请求后得到的结果列表 :rtype: list """ + logger.log('INFOR', f'开始执行子域请求模块') loop = asyncio.get_event_loop() asyncio.set_event_loop(loop) data = utils.set_id_none(data) diff --git a/oneforall/common/resolve.py b/oneforall/common/resolve.py index aa35662..3f500ef 100644 --- a/oneforall/common/resolve.py +++ b/oneforall/common/resolve.py @@ -1,110 +1,44 @@ -import socket -import asyncio -import functools -import sys -from multiprocessing import Manager - -import tqdm -import aiomultiprocess as aiomp -from dns.resolver import Resolver +import gc +import json import config from config import logger from common import utils from common.database import Database -socket.setdefaulttimeout(20) - -def dns_resolver(): - """ - dns解析器 - """ - resolver = Resolver() - resolver.nameservers = config.resolver_nameservers - resolver.timeout = config.resolver_timeout - resolver.lifetime = config.resolver_lifetime - return resolver - - -async def aio_resolve_a(hostname, loop=None): - """ - 异步解析A记录 - - :param str hostname: 主机名 - :param loop: 事件循环 - :return: 查询结果 - """ - if loop is None: - loop = asyncio.get_event_loop() - try: - answer = await loop.run_in_executor(None, - socket.gethostbyname_ex, - hostname) - except BaseException as e: - logger.log('TRACE', e.args) - answer = e - return hostname, answer - - -def convert_results(result_list): - """ - 将结果列表类型转换为结果字典类型 - - :param result_list: 待转换的结果列表 - :return: 转换后的结果字典 - """ - result_dict = {} - for result in result_list: - hostname, answer = result - value_dict = {'content': None, 'reason': None, 'resolve': None, - 'public': None, 'valid': None} - if isinstance(answer, tuple): - ip_list = answer[2] - value_dict['content'] = ','.join(ip_list) - value_dict['public'] = utils.check_ip_public(ip_list) - value_dict['resolve'] = 1 - result_dict[hostname] = value_dict - elif isinstance(answer, Exception): - value_dict['reason'] = str(answer.args) - value_dict['resolve'] = 0 - result_dict[hostname] = value_dict - else: - value_dict['resolve'] = 0 - result_dict[hostname] = value_dict - return result_dict - - -def filter_subdomain(data_list): +def filter_subdomain(data): """ 过滤出无解析内容的子域到新的子域列表 - :param list data_list: 待过滤的数据列表 + :param list data: 待过滤的数据列表 :return: 符合条件的子域列表 """ + logger.log('DEBUG', f'正在过滤出待解析的子域') subdomains = [] - for data in data_list: + for data in data: if not data.get('content'): subdomain = data.get('subdomain') subdomains.append(subdomain) return subdomains -def update_data(data_list, results_dict): +def update_data(data, records): """ 更新解析结果 - :param list data_list: 待更新的数据列表 - :param dict results_dict: 解析结果字典 + :param list data: 待更新的数据列表 + :param dict records: 解析结果字典 :return: 更新后的数据列表 """ - for index, data in enumerate(data_list): - if not data.get('content'): - subdomain = data.get('subdomain') - value_dict = results_dict.get(subdomain) - data.update(value_dict) - data_list[index] = data - return data_list + logger.log('DEBUG', f'正在更新解析结果') + for index, items in enumerate(data): + if not items.get('content'): + subdomain = items.get('subdomain') + record = records.get(subdomain) + items.update(record) + data[index] = items + return data def save_data(name, data): @@ -114,6 +48,7 @@ def save_data(name, data): :param str name: 保存表名 :param list data: 待保存的数据 """ + logger.log('INFOR', f'正在保存解析结果') db = Database() db.drop_table(name) db.create_table(name) @@ -121,104 +56,111 @@ def save_data(name, data): db.close() -def resolve_progress_func(done_obj, total_num): - """ - 解析进度函数 - - :param done_obj: 进程间共享的Value对象 - :param int total_num: 待解析的子域个数 - """ - bar = tqdm.tqdm() - bar.total = total_num - bar.desc = 'Resolve Progress' - bar.ncols = 80 - bar.smoothing = 0 - while True: - done_num = done_obj.value - bar.n = done_num - bar.update() - if done_num == total_num: - break - bar.close() +def save_subdomains(save_path, subdomain_list): + logger.log('DEBUG', f'正在保存待解析的子域') + subdomain_data = '\n'.join(subdomain_list) + if not utils.save_data(save_path, subdomain_data): + logger.log('FATAL', '保存待解析的子域出错') + exit(1) -async def do_resolve(done_obj, hostname): - """ - 异步解析主机名的A记录 - - :param done_obj: 进程间共享的Value对象 - :param str hostname: 主机名 - :return: 查询结果 - """ - loop = asyncio.get_event_loop() - result = await aio_resolve_a(hostname, loop) - done_obj.value += 1 - return result +def deal_output(output_path): + logger.log('INFOR', f'正在处理解析结果') + records = dict() # 用来记录所有域名解析数据 + with open(output_path) as fd: + for line in fd: + line = line.strip() + try: + items = json.loads(line) + except Exception as e: + logger.log('ERROR', e.args) + logger.log('ERROR', f'解析行{line}出错跳过解析该行') + continue + record = dict() + record['resolver'] = items.get('resolver') + qname = items.get('name')[:-1] # 去出最右边的`.`点号 + status = items.get('status') + if status != 'NOERROR': + record['alive'] = 0 + record['resolve'] = 0 + record['reason'] = status + records[qname] = record + continue + data = items.get('data') + if 'answers' not in data: + record['alive'] = 0 + record['resolve'] = 0 + record['reason'] = 'NOANSWER' + records[qname] = record + continue + flag = False + cname = list() + ips = list() + public = list() + ttls = list() + answers = data.get('answers') + for answer in answers: + if answer.get('type') == 'A': + flag = True + cname.append(answer.get('name')[:-1]) # 去出最右边的`.`点号 + ip = answer.get('data') + ips.append(ip) + ttl = answer.get('ttl') + ttls.append(str(ttl)) + is_public = utils.ip_is_public(ip) + public.append(str(is_public)) + record['resolve'] = 1 + record['reason'] = status + record['cname'] = ','.join(cname) + record['content'] = ','.join(ips) + record['public'] = ','.join(public) + record['ttl'] = ','.join(ttls) + records[qname] = record + if not flag: + record['alive'] = 0 + record['resolve'] = 0 + record['reason'] = 'NOARECORD' + records[qname] = record + return records -async def aio_resolve(subdomain_list, process_num, coroutine_num): - """ - 异步解析子域A记录 - - :param list subdomain_list: 待解析的子域列表 - :param int process_num: 解析进程数 - :param int coroutine_num: 每个解析进程下的协程数 - :return: 解析结果 - """ - m = Manager() - done_obj = m.Value('done', 0) # 创建一个进程间可以共享的值 - loop = asyncio.get_event_loop() - loop.run_in_executor(None, resolve_progress_func, - done_obj, len(subdomain_list)) - wrapped_resolve_func = functools.partial(do_resolve, done_obj) - result_list = list() - # macOS上队列大小不能超过2**15 - 1 = 32767 - # https://stackoverflow.com/questions/5900985/multiprocessing-queue-maxsize-limit-is-32767 - if sys.platform == 'darwin': - split_subdomain_list = utils.split_list(subdomain_list, 32767) - for current_subdomain_list in split_subdomain_list: - async with aiomp.Pool(processes=process_num, - childconcurrency=coroutine_num) as pool: - result = await pool.map(wrapped_resolve_func, - current_subdomain_list) - result_list.extend(result) - return result_list - async with aiomp.Pool(processes=process_num, - childconcurrency=coroutine_num) as pool: - result_list = await pool.map(wrapped_resolve_func, subdomain_list) - return result_list - - -async def run_aio_resolve(subdomain_list): - """ - 异步解析子域A记录 - - :param list subdomain_list: 待解析的子域列表 - :return: 解析得到的结果列表 - """ - process_num = utils.get_process_num() - coroutine_num = utils.get_coroutine_num() - logger.log('INFOR', '正在异步查询子域的A记录') - result_list = await aio_resolve(subdomain_list, process_num, coroutine_num) - logger.log('INFOR', '完成异步查询子域的A记录') - return result_list - - -def run_resolve(data): +def run_resolve(domain, data): """ 调用子域解析入口函数 + :param str domain: 待解析的主域 :param list data: 待解析的子域数据列表 :return: 解析得到的结果列表 :rtype: list """ - loop = asyncio.get_event_loop() - asyncio.set_event_loop(loop) - need_resolve_subdomains = filter_subdomain(data) - if not need_resolve_subdomains: + logger.log('INFOR', f'开始解析{domain}的子域') + subdomains = filter_subdomain(data) + if not subdomains: return data - resolve_coroutine = run_aio_resolve(need_resolve_subdomains) - results_list = loop.run_until_complete(resolve_coroutine) - results_dict = convert_results(results_list) - resolved_data = update_data(data, results_dict) - return resolved_data + + massdns_dir = config.third_party_dir.joinpath('massdns') + result_dir = config.result_save_dir + temp_dir = result_dir.joinpath('temp') + utils.check_dir(temp_dir) + massdns_path = utils.get_massdns_path(massdns_dir) + timestring = utils.get_timestring() + + save_name = f'collected_subdomains_{domain}_{timestring}.txt' + save_path = temp_dir.joinpath(save_name) + save_subdomains(save_path, subdomains) + del subdomains + gc.collect() + + output_name = f'resolved_result_{domain}_{timestring}.json' + output_path = temp_dir.joinpath(output_name) + log_path = result_dir.joinpath('massdns.log') + + ns_path = config.brute_nameservers_path + + utils.call_massdns(massdns_path, save_path, ns_path, + output_path, log_path, quiet_mode=True) + + records = deal_output(output_path) + data = update_data(data, records) + logger.log('INFOR', f'结束解析{domain}的子域') + return data diff --git a/oneforall/common/utils.py b/oneforall/common/utils.py index b934d7e..4a17dde 100644 --- a/oneforall/common/utils.py +++ b/oneforall/common/utils.py @@ -4,13 +4,16 @@ import sys import time import random import platform +import subprocess from ipaddress import IPv4Address, ip_address +from stat import S_IXUSR import psutil import config from pathlib import Path from records import Record, RecordCollection +from dns.resolver import Resolver from common.domain import Domain from config import logger @@ -335,6 +338,17 @@ def export_all(format, path, datas): save_data(path, content) +def dns_resolver(): + """ + dns解析器 + """ + resolver = Resolver() + resolver.nameservers = config.resolver_nameservers + resolver.timeout = config.resolver_timeout + resolver.lifetime = config.resolver_lifetime + return resolver + + def dns_query(qname, qtype): """ 查询域名DNS记录 @@ -344,7 +358,7 @@ def dns_query(qname, qtype): :return: 查询结果 """ logger.log('TRACE', f'尝试查询{qname}的{qtype}记录') - resolver = resolve.dns_resolver() + resolver = dns_resolver() try: answer = resolver.query(qname, qtype) except Exception as e: @@ -485,3 +499,46 @@ def check_env(): def get_maindomain(domain): return Domain(domain).registered() + + +def call_massdns(massdns_path, dict_path, ns_path, output_path, log_path, + query_type='A', process_num=1, concurrent_num=10000, + quiet_mode=False): + logger.log('INFOR', f'开始执行massdns') + quiet = '' + if quiet_mode: + quiet = '--quiet' + status_format = config.brute_status_format + socket_num = config.brute_socket_num + resolve_num = config.brute_resolve_num + cmd = f'{massdns_path} {quiet} --status-format {status_format} ' \ + f'--processes {process_num} --socket-count {socket_num} ' \ + f'--hashmap-size {concurrent_num} --resolvers {ns_path} ' \ + f'--resolve-count {resolve_num} --type {query_type} ' \ + f'--flush --output J --outfile {output_path} ' \ + f'--error-log {log_path} {dict_path}' + logger.log('INFOR', f'执行命令 {cmd}') + subprocess.run(args=cmd, shell=True) + logger.log('INFOR', f'结束执行massdns') + + +def get_massdns_path(massdns_dir): + path = config.brute_massdns_path + if path: + return path + system = platform.system().lower() + machine = platform.machine().lower() + name = f'massdns_{system}_{machine}' + if system == 'windows': + name = name + '.exe' + if machine == 'amd64': + massdns_dir = massdns_dir.joinpath('windows', 'x64') + else: + massdns_dir = massdns_dir.joinpath('windows', 'x84') + path = massdns_dir.joinpath(name) + path.chmod(S_IXUSR) + if not path.exists(): + logger.log('FATAL', '暂无该系统平台及架构的massdns') + logger.log('INFOR', '请尝试自行编译massdns并在配置里指定路径') + exit(0) + return path diff --git a/oneforall/example.py b/oneforall/example.py index 7516817..558ccac 100644 --- a/oneforall/example.py +++ b/oneforall/example.py @@ -8,8 +8,8 @@ import oneforall if __name__ == '__main__': - test = oneforall.OneForAll(target='example.com') - test.brute = True + test = oneforall.OneForAll(target='freebuf.com') + test.brute = False test.takeover = True test.run() result = test.datas diff --git a/oneforall/modules/check/axfr.py b/oneforall/modules/check/axfr.py index cc0318d..df6b2b2 100644 --- a/oneforall/modules/check/axfr.py +++ b/oneforall/modules/check/axfr.py @@ -10,7 +10,7 @@ DNS域传送(DNS zone transfer)指的是一台备用域名服务器使用来自 import dns.resolver import dns.zone -from common import resolve, utils +from common import utils from common.module import Module from config import logger @@ -57,7 +57,7 @@ class CheckAXFR(Module): """ 正则匹配响应头中的内容安全策略字段以发现子域名 """ - resolver = resolve.dns_resolver() + resolver = utils.dns_resolver() try: answers = resolver.query(self.domain, "NS") except Exception as e: diff --git a/oneforall/oneforall.py b/oneforall/oneforall.py index 5e7eade..caa6bc2 100644 --- a/oneforall/oneforall.py +++ b/oneforall/oneforall.py @@ -189,7 +189,7 @@ class OneForAll(object): return self.export(self.domain) # 解析子域 - self.data = resolve.run_resolve(self.data) + self.data = resolve.run_resolve(self.domain, self.data) # 保存解析结果 resolve.save_data(self.resolve_table, self.data) diff --git a/oneforall/takeover.py b/oneforall/takeover.py index 498de3b..62c140c 100644 --- a/oneforall/takeover.py +++ b/oneforall/takeover.py @@ -18,7 +18,7 @@ from tqdm import tqdm import config from config import logger -from common import resolve, utils +from common import utils from common.module import Module from common.domain import Domain @@ -31,7 +31,7 @@ def get_fingerprint(): def get_cname(subdomain): - resolver = resolve.dns_resolver() + resolver = utils.dns_resolver() try: answers = resolver.query(subdomain, 'CNAME') except Exception as e: