重构子域爆破模块

oneforall/aiobrute.py

@@ -1,328 +0,0 @@
-#!/usr/bin/python3
-# coding=utf-8
-
-"""
-OneForAll多进程多协程异步子域爆破模块
-
-:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
-:license: GNU General Public License v3.0, see LICENSE for more details.
-"""
-
-import time
-import queue
-import asyncio
-import secrets
-
-import exrex
-import fire
-
-import config
-import dbexport
-from common import resolve, utils
-from common.module import Module
-from common.database import Database
-from config import logger
-
-
-def detect_wildcard(domain):
-    """
-    探测域名是否使用泛解析
-
-    :param str domain: 域名
-    :return: 如果没有使用泛解析返回False 反之返回泛解析的IP集合和ttl整型值
-    """
-    logger.log('INFOR', f'正在探测{domain}是否使用泛解析')
-    token = secrets.token_hex(4)
-    random_subdomain = f'{token}.{domain}'
-    try:
-        resolver = resolve.dns_resolver()
-        answers = resolver.query(random_subdomain, 'A')
-    # 如果查询随机域名A记录出错 说明不存在随机子域的A记录 即没有开启泛解析
-    except Exception as e:
-        logger.log('DEBUG', e)
-        logger.log('INFOR', f'{domain}没有使用泛解析')
-        return False, None, None
-    ttl = answers.ttl
-    name = answers.name
-    ips = {item.address for item in answers}
-    logger.log('ALERT', f'{domain}使用了泛解析')
-    logger.log('ALERT', f'{random_subdomain} 解析到域名: {name} '
-                        f'IP: {ips} TTL: {ttl}')
-    return True, ips, ttl
-
-
-def wildcard_by_compare(ips, ttl, wildcard_ips, wildcard_ttl):
-    """
-    通过与泛解析返回的IP集合和返回的TTL值进行对比判断发现的子域是否是泛解析子域
-
-    :param set ips: 子域A记录查询出的IP集合
-    :param int ttl: 子域A记录查询出的TTL整型值
-    :param set wildcard_ips: 泛解析的IP集合
-    :param int wildcard_ttl: 泛解析的TTL整型值
-    :return: 判断结果
-    :rtype bool
-    """
-    # 参考：http://sh3ll.me/archives/201704041222.txt
-    if not ips.issubset(wildcard_ips):
-        return False
-    if ttl != wildcard_ttl and ttl % 60 == 0 and wildcard_ttl % 60 == 0:
-        return False
-    return True
-
-
-def wildcard_by_times(ips, ips_times):
-    """
-    对ips出现次数进行判断泛解析
-
-    :param set ips: 发现子域的IP集合
-    :param ips_times: 子域IP集合出现次数统计字典
-    :return: 判断结果
-    """
-    times = ips_times.get(str(ips))
-    if times > config.ips_appear_maximum:
-        return True
-    return False
-
-
-def gen_fuzz_domains(domain, rule):
-    """
-    生成fuzz模式下即将用于爆破的子域集合
-
-    :param str domain: 待爆破的主域
-    :param str rule: 用于爆破的正则规则
-    :return: 用于爆破的子域集合
-    """
-    domains = set()
-    if '[fuzz]' not in domain:
-        logger.log('FATAL', f'没有指定fuzz位置')
-        return domains
-    if not rule:
-        logger.log('FATAL', f'没有指定fuzz规则')
-        return domains
-    fuzz_count = exrex.count(rule)
-    if fuzz_count > 2000000:
-        logger.log('FATAL', f'fuzz规则范围太大：{fuzz_count} > 2000000')
-        return domains
-    logger.log('INFOR', f'fuzz字典大小：{fuzz_count}')
-    for i in range(3):
-        random_domain = domain.replace('[fuzz]', exrex.getone(rule))
-        logger.log('ALERT', f'请注意检查随机生成的{random_domain}是否正确')
-    logger.log('ALERT', f'你有6秒检查时间退出使用`CTRL+C`')
-    try:
-        time.sleep(6)
-    except KeyboardInterrupt:
-        logger.log('INFOR', '爆破终止')
-        exit(0)
-    parts = domain.split('[fuzz]')
-    for fuzz in exrex.generate(rule):
-        fuzz_domain = parts[0] + fuzz + parts[1]
-        domains.add(fuzz_domain)
-    return domains
-
-
-def gen_brute_domains(domain, path):
-    """
-    生成基于字典爆破的子域数据
-
-    :param str domain: 待爆破的主域
-    :param str path: 字典路径
-    :return: 用于爆破的子域集合
-    """
-    domains = set()
-    with open(path, encoding='utf-8', errors='ignore') as file:
-        for line in file:
-            brute_domain = line.strip() + '.' + domain
-            domains.add(brute_domain)
-    logger.log('INFOR', f'爆破字典大小：{len(domains)}')
-    return domains
-
-
-class AIOBrute(Module):
-    """
-    OneForAll多进程多协程异步子域爆破模块
-
-    Example：
-        python3 aiobrute.py --target subdomain.com run
-        python3 aiobrute.py --target ./subdomains.txt run
-        python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
-        python3 aiobrute.py --target example.com --wordlist subnames.txt run
-        python3 aiobrute.py --target example.com --recursive True --depth 2 run
-        python3 aiobrute.py --target m.[fuzz].bz --fuzz True --rule '[a-z]' run
-
-    Note:
-        参数segment的设置受CPU性能，网络带宽，运营商限制等限制，默认500个子域为任务组，
-        当你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
-        十万字典建议设置为5000，百万字典设置为50000
-        参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
-        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
-                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
-        参数path默认None使用OneForAll结果目录自动生成路径
-
-    :param str target:       单个域名或者每行一个域名的文件路径
-    :param int process:      爆破的进程数(默认CPU核心数)
-    :param int coroutine:    每个爆破进程下的协程数(默认64)
-    :param str wordlist:     指定爆破所使用的字典路径(默认使用config.py配置)
-    :param bool recursive:   是否使用递归爆破(默认False)
-    :param int depth:        递归爆破的深度(默认2)
-    :param str namelist:     指定递归爆破所使用的字典路径(默认使用config.py配置)
-    :param bool fuzz:        是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
-    :param str rule:         fuzz模式使用的正则规则(默认使用config.py配置)
-    :param bool export:      是否导出爆破结果(默认True)
-    :param bool valid:       只导出有效的子域结果(默认False)
-    :param str format:       导出格式(默认csv)
-    :param str path:         导出路径(默认None)
-    :param bool show:        终端显示导出数据(默认False)
-    """
-
-    def __init__(self, target, process=None, coroutine=None, wordlist=None,
-                 recursive=False, depth=None, namelist=None, fuzz=False,
-                 rule=None, export=True, valid=None, format='csv', path=None,
-                 show=False):
-        Module.__init__(self)
-        self.domains = set()
-        self.domain = str()
-        self.module = 'Brute'
-        self.source = 'AIOBrute'
-        self.target = target
-        self.process = process or utils.get_process_num()
-        self.coroutine = coroutine or utils.get_coroutine_num()
-        self.wordlist = wordlist or config.brute_wordlist_path
-        self.recursive_brute = recursive or config.enable_recursive_brute
-        self.recursive_depth = depth or config.brute_recursive_depth
-        self.recursive_namelist = namelist or config.recursive_namelist_path
-        self.fuzz = fuzz or config.enable_fuzz
-        self.rule = rule or config.fuzz_rule
-        self.export = export
-        self.valid = valid
-        self.format = format
-        self.path = path
-        self.show = show
-        self.nameservers = config.resolver_nameservers
-        self.ips_times = dict()  # IP集合出现次数
-        self.enable_wildcard = False  # 当前域名是否使用泛解析
-        self.wildcard_check = config.enable_wildcard_check
-        self.wildcard_deal = config.enable_wildcard_deal
-        self.wildcard_ips = set()  # 泛解析IP集合
-        self.wildcard_ttl = int()  # 泛解析TTL整型值
-
-    def gen_tasks(self, domain):
-        # 如果domain不是self.subdomain，而是self.domain的子域 生成递归爆破字典
-        if self.domain != domain:
-            logger.log('INFOR', f'使用{self.recursive_namelist}字典')
-            domains = gen_brute_domains(domain, self.recursive_namelist)
-        elif self.fuzz and self.rule:  # 开启fuzz模式并指定了fuzz正则规则
-            logger.log('INFOR', f'正在生成{domain}的fuzz字典')
-            domains = gen_fuzz_domains(domain, self.rule)
-        else:
-            logger.log('INFOR', f'使用{self.wordlist}字典')
-            domains = gen_brute_domains(domain, self.wordlist)
-        domains = list(domains)
-        return domains
-
-    def deal_results(self, results):
-        for result in results:
-            hostname, answer = result
-            if answer is None:
-                continue
-            if isinstance(answer, Exception):
-                # logger.log('DEBUG', f'爆破{subdomain}时出错 {str(answers)}')
-                continue
-            name, alias, ips = answer
-            if name.endswith('.'):
-                name = name[0:-1]
-            # 取值 如果是首次出现的IP集合 出现次数先赋值0
-            value = self.ips_times.setdefault(str(ips), 0)
-            self.ips_times[str(ips)] = value + 1
-            # 目前域名开启了泛解析
-            if self.enable_wildcard and self.wildcard_deal:
-                # 通过对比查询的子域和响应的子域来判断真实子域
-                # 去掉解析到CDN的情况
-                if 'cdn' in name or 'waf' in name:
-                    continue
-                if not name.endswith(self.domain):
-                    continue
-                # 通过对比解析到的IP集合的次数来判断真实子域
-                if wildcard_by_times(ips, self.ips_times):
-                    continue
-            # 只添加没有出现过的子域
-            if hostname not in self.subdomains:
-                logger.log('INFOR', f'发现{self.domain}的子域: {hostname} '
-                                    f'解析到: {name} IP: {ips}')
-                self.subdomains.add(hostname)
-                self.records[hostname] = ','.join(ips)
-
-    async def main(self, domain, rx_queue):
-        if not self.fuzz:  # fuzz模式不探测域名是否使用泛解析
-            if self.wildcard_check:
-                self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl \
-                    = detect_wildcard(domain)
-        tasks = self.gen_tasks(domain)
-        logger.log('INFOR', f'正在爆破{domain}的域名')
-        results = await resolve.aio_resolve(tasks, self.process, self.coroutine)
-        self.deal_results(results)
-        self.save_json()
-        self.gen_result()
-        rx_queue.put(self.results)
-
-    def run(self, rx_queue=None):
-        if rx_queue is None:
-            rx_queue = queue.Queue()
-        self.domains = utils.get_domains(self.target)
-        loop = asyncio.get_event_loop()
-        asyncio.set_event_loop(loop)
-        for self.domain in self.domains:
-            start = time.time()
-            db = Database()
-            db.create_table(self.domain)
-            logger.log('INFOR', f'开始执行{self.source}模块爆破域名{self.domain}')
-            logger.log('INFOR', f'使用{self.process}进程乘{self.coroutine}协程')
-            # fuzz模式不使用递归爆破
-            if self.recursive_brute and not self.fuzz:
-                logger.log('INFOR', f'开始递归爆破{self.domain}的第1层子域')
-            loop.run_until_complete(self.main(self.domain, rx_queue))
-
-            # 递归爆破下一层的子域
-            # fuzz模式不使用递归爆破
-            if self.recursive_brute and not self.fuzz:
-                for layer_num in range(1, self.recursive_depth):
-                    # 之前已经做过1层子域爆破 当前实际递归层数是layer+1
-                    logger.log('INFOR', f'开始递归爆破{self.domain}的'
-                                        f'第{layer_num + 1}层子域')
-                    for subdomain in self.subdomains.copy():
-                        # 进行下一层子域爆破的限制条件
-                        if subdomain.count('.') - self.domain.count('.') \
-                                == layer_num:
-                            loop.run_until_complete(self.main(subdomain,
-                                                              rx_queue))
-            # 队列不空就一直取数据存数据库
-            while not rx_queue.empty():
-                results = rx_queue.get()
-                # 将结果存入数据库中
-                db.save_db(self.domain, results, self.source)
-
-            end = time.time()
-            self.elapse = round(end - start, 1)
-            logger.log('INFOR', f'结束执行{self.source}模块爆破域名{self.domain}')
-            length = len(self.subdomains)
-            logger.log('INFOR', f'{self.source}模块耗时{self.elapse}秒'
-                                f'发现{self.domain}的域名{length}个')
-            logger.log('DEBUG', f'{self.source}模块发现{self.domain}的域名:\n'
-                                f'{self.subdomains}')
-            if not self.path:
-                name = f'{self.domain}_brute_result.{self.format}'
-                self.path = config.result_save_dir.joinpath(name)
-            # 数据库导出
-            if self.export:
-                dbexport.export(self.domain,
-                                valid=self.valid,
-                                path=self.path,
-                                format=self.format,
-                                show=self.show)
-
-
-if __name__ == '__main__':
-    fire.Fire(AIOBrute)
-    # domain = 'example.com'
-    # result = queue.Queue()
-    # brute = AIOBrute(domain)
-    # brute.run(result)

oneforall/brute.py

@@ -0,0 +1,613 @@
+#!/usr/bin/python3
+# coding=utf-8
+
+"""
+OneForAll子域爆破模块
+
+:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
+:license: GNU General Public License v3.0, see LICENSE for more details.
+"""
+import json
+import time
+import queue
+import asyncio
+import random
+import secrets
+import platform
+import subprocess
+
+import exrex
+import fire
+
+import config
+import dbexport
+from common import resolve, utils
+from common.module import Module
+from common.database import Database
+from config import logger
+
+
+def detect_wildcard(domain):
+    """
+    探测域名是否使用泛解析
+
+    :param str domain: 域名
+    :return: 如果没有使用泛解析返回False 反之返回泛解析的IP集合和ttl整型值
+    """
+    logger.log('INFOR', f'正在探测{domain}是否使用泛解析')
+    token = secrets.token_hex(4)
+    random_subdomain = f'{token}.{domain}'
+    resolver = resolve.dns_resolver()
+    try:
+        answer = resolver.query(random_subdomain, 'A')
+    # 如果查询随机域名A记录出错 说明不存在随机子域的A记录 即没有开启泛解析
+    except Exception as e:
+        logger.log('DEBUG', e.args)
+        logger.log('INFOR', f'{domain}没有使用泛解析')
+        return False
+    ttl = answer.ttl
+    name = answer.name
+    ips = {item.address for item in answer}
+    logger.log('ALERT', f'{domain}使用了泛解析')
+    logger.log('ALERT', f'{random_subdomain} 解析到域名: {name} '
+                        f'IP: {ips} TTL: {ttl}')
+    return True
+
+
+def gen_fuzz_subdomains(expression, rule):
+    """
+    生成基于fuzz模式的爆破子域
+
+    :param str expression: 子域域名生成表达式
+    :param str rule: 生成子域所需的正则规则
+    :return: 用于爆破的子域
+    """
+    subdomains = list()
+    fuzz_count = exrex.count(rule)
+    if fuzz_count > 10000000:
+        logger.log('ALERT', f'请注意该规则生成的字典太大：{fuzz_count} > 10000000')
+    logger.log('DEBUG', f'fuzz模式下生成的字典大小：{fuzz_count}')
+    for fuzz_string in exrex.generate(rule):
+        fuzz_string = fuzz_string.lower()
+        if not fuzz_string.isalnum():
+            continue
+        fuzz_domain = expression.replace('*', fuzz_string)
+        subdomains.append(fuzz_domain)
+    random_domain = random.choice(subdomains)
+    logger.log('ALERT', f'请注意检查基于fuzz模式生成的{random_domain}是否正确')
+    return subdomains
+
+
+def gen_domains(iterable, place):
+    subdomains = set()
+    for _ in range(place.count('*')):
+        for item in iterable:
+            subdomain = place.replace('*', item)
+            subdomains.add(subdomain)
+    return subdomains
+
+
+def gen_word_subdomains(expression, path):
+    """
+    生成基于word模式的爆破子域
+
+    :param str expression: 子域域名生成表达式
+    :param str path: 字典路径
+    :return: 用于爆破的子域
+    """
+    subdomains = list()
+    with open(path, encoding='utf-8', errors='ignore') as fd:
+        for line in fd:
+            word = line.strip().lower()
+            if not word.isalnum():
+                continue
+            if word.endswith('.'):
+                word = word[:-1]
+            subdomain = expression.replace('*', word)
+            subdomains.append(subdomain)
+    random_domain = random.choice(subdomains)
+    logger.log('DEBUG', f'fuzz模式下生成的字典大小：{len(subdomains)}')
+    logger.log('ALERT', f'请注意检查基于word模式生成的{random_domain}是否正确')
+    return subdomains
+
+
+def query_domain_ns_a(ns_list):
+    if not isinstance(ns_list, list):
+        return list()
+    ns_ip_list = []
+    resolver = resolve.dns_resolver()
+    for ns in ns_list:
+        try:
+            answer = resolver.query(ns, 'A')
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            logger.log('ERROR', f'查询权威DNS名称服务器{ns}的A记录出错')
+            continue
+        if answer:
+            for item in answer:
+                ns_ip_list.append(item.address)
+    logger.log('INFOR', f'权威DNS名称服务器对应A记录 {ns_ip_list}')
+    return ns_ip_list
+
+
+def query_domain_ns(domain):
+    resolver = resolve.dns_resolver()
+    try:
+        answer = resolver.query(domain, 'NS')
+    except Exception as e:
+        logger.log('ERROR', e.args)
+        logger.log('ERROR', f'查询{domain}的NS记录出错')
+        return list()
+    ns = [item.to_text() for item in answer]
+    logger.log('INFOR', f'{domain}的权威DNS名称服务器 {ns}')
+    return ns
+
+
+def get_wildcard_record(domain, authoritative_ns):
+    if not authoritative_ns:
+        return list(), int()
+    resolver = resolve.dns_resolver()
+    resolver.nameservers = authoritative_ns
+    token = secrets.token_hex(4)
+    random_subdomain = f'{token}.{domain}'
+    logger.log('INFOR', f'查询{random_subdomain}在权威DNS名称服务器的泛解析记录')
+    try:
+        answer = resolver.query(random_subdomain, 'A')
+    except Exception as e:
+        logger.log('ERROR', e.args)
+        logger.log('ERROR', f'查询{random_subdomain}在权威DNS名称服务器泛解析记录出错')
+        return None, None
+    name = answer.name
+    ips = {item.address for item in answer}
+    ttl = answer.ttl
+    logger.log('INFOR', f'{random_subdomain} 在权威DNS上解析到域名: {name} '
+                        f'IP: {ips} TTL: {ttl}')
+    return ips, ttl
+
+
+def get_nameservers_path(enable_wildcard, ns_ip_list):
+    path = config.brute_nameservers_path
+    if not enable_wildcard:
+        return path
+    if not ns_ip_list:
+        return path
+    path = config.authoritative_dns_path
+    ns_data = '\n'.join(ns_ip_list)
+    utils.save_data(path, ns_data)
+    return path
+
+
+def get_massdns_path(massdns_dir):
+    path = config.brute_massdns_path
+    if path:
+        return path
+    system = platform.system().lower()
+    machine = platform.machine().lower()
+    name = f'massdns_{system}_{machine}'
+    if system == 'windows':
+        name = name + '.exe'
+        if machine == 'amd64':
+            massdns_dir = massdns_dir.joinpath('windows', 'x64')
+        else:
+            massdns_dir = massdns_dir.joinpath('windows', 'x84')
+    path = massdns_dir.joinpath(name)
+    if not path.exists():
+        logger.log('FATAL', '暂无该系统平台及架构的massdns')
+        logger.log('INFOR', '请尝试自行编译massdns并在配置里指定路径')
+        exit(0)
+    return path
+
+
+def check_dict():
+    if not config.enable_check_dict:
+        return
+    sec = config.check_time
+    logger.log('ALERT', f'你有{sec}秒时间检查爆破配置是否正确')
+    logger.log('ALERT', f'退出爆破请使用`Ctrl+C`')
+    try:
+        time.sleep(sec)
+    except KeyboardInterrupt:
+        logger.log('INFOR', '爆破配置有误退出爆破')
+        exit(0)
+
+
+def do_brute(massdns_path, dict_path, ns_path, output_path, log_path,
+             query_type='A', process_num=1, concurrent_num=10000,
+             quiet_mode=False):
+    quiet = ''
+    if quiet_mode:
+        quiet = '--quiet'
+    status_format = config.brute_status_format
+    socket_num = config.brute_socket_num
+    resolve_num = config.brute_resolve_num
+    #
+    cmd = f'{massdns_path} {quiet} --status-format {status_format} ' \
+          f'--processes {process_num} --socket-count {socket_num} ' \
+          f'--hashmap-size {concurrent_num} --resolvers {ns_path} ' \
+          f'--resolve-count {resolve_num} --type {query_type} ' \
+          f'--flush --output J --outfile {output_path} ' \
+          f'--error-log {log_path} {dict_path}'
+    logger.log('INFOR', f'执行命令 {cmd}')
+    subprocess.run(args=cmd, shell=False)
+
+
+def read_result(result_path):
+    result = list()
+    with open(result_path) as fd:
+        for line in fd:
+            line = line.strip()
+            try:
+                record = json.loads(line)
+            except Exception as e:
+                logger.log('ERROR', e.args)
+                logger.log('ERROR', f'解析行{line}出错跳过解析该行')
+                continue
+            result.append(record)
+    return result
+
+
+def deal_result(result_list):
+    logger.log('INFOR', f'正在处理解析结果')
+    records = dict()  # 用来记录域名解析数据
+    times = dict()  # 用来统计IP出现次数
+    for items in result_list:
+        record = dict()
+        qname = items.get('name')[:-1]  # 去出最右边的`.`点号
+        record['resolver'] = items.get('resolver')
+        status = items.get('status')
+        record['reason'] = status
+        records[qname] = record
+        if status != 'NOERROR':
+            record['reason'] = status
+            record['resolve'] = 0
+            record['alive'] = 0
+            records[qname] = record
+            continue
+        data = items.get('data')
+        if 'answers' not in data:
+            record['reason'] = 'NOANSWER'
+            record['resolve'] = 0
+            record['alive'] = 0
+            records[qname] = record
+            continue
+        answers = data.get('answers')
+        flag = False
+        cname = list()
+        ips = list()
+        public = list()
+        ttl = list()
+        for answer in answers:
+            if answer.get('type') == 'A':
+                flag = True
+                ttl.append(answer.get('ttl'))
+                cname.append(answer.get('name')[:-1])  # 去出最右边的`.`点号
+                ip = answer.get('data')
+                ips.append(ip)
+                public.append(utils.ip_is_public(ip))
+                record['ttl'] = ttl
+                record['cname'] = cname
+                record['content'] = ips
+                record['public'] = public
+                records[qname] = record
+                # 取值 如果是首次出现的IP集合 出现次数先赋值0
+                value = times.setdefault(ip, 0)
+                times[ip] = value + 1
+        if not flag:
+            record['reason'] = 'NOA'
+            record['resolve'] = 0
+            record['alive'] = 0
+            records[qname] = record
+    return records, times
+
+
+def add_times(records, ip_times):
+    for name, record in records.items():
+        times = list()
+        ips = record.get('content')
+        if not ips:
+            continue
+        for ip in ips:
+            times.append(ip_times.get(ip))
+            record['times'] = times
+        records[name] = record
+    return records
+
+
+def check_validity(records, ip_times, wildcard_ips, wildcard_ttl):
+    valid_subdomains = list()
+    for name, record in records.items():
+        if record.get('resolve') is None:
+            ips = record['content']
+            ttl = record['ttl']
+            status, reason = is_valid_subdomain(ips, ttl, ip_times,
+                                                wildcard_ips, wildcard_ttl)
+            record['resolve'], record['reason'] = status, reason
+        records[name] = record
+        # 在打了有效性标签后 暂且把除无效子域的子域都认为是有效子域
+        if record.get('resolve') != 0:
+            valid_subdomains.append(name)
+    return records, valid_subdomains
+
+
+def check_by_compare(ips, ttl, wildcard_ips, wildcard_ttl):
+    """
+    通过与泛解析返回的IP集合和返回的TTL值进行对比判断发现的子域是否是泛解析子域
+
+    :param set ips: 子域A记录查询出的IP集合
+    :param int ttl: 子域A记录查询出的TTL
+    :param set wildcard_ips: 泛解析的IP集合
+    :param int wildcard_ttl: 泛解析的TTL
+    :return: 判断结果
+    """
+    # 参考：http://sh3ll.me/archives/201704041222.txt
+    if not ips.intersection(wildcard_ips):
+        return False  # 子域IP集合与泛解析IP集合无任何交集则不是泛解析
+    if ttl != wildcard_ttl and ttl % 60 == 0 and wildcard_ttl % 60 == 0:
+        return False
+    return True
+
+
+def check_ip_times(ips, times):
+    """
+    根据ip出现次数判断是否为泛解析
+
+    :param set ips: 子域IP集合
+    :param times: 子域IP出现次数统计字典
+    :return: 判断结果
+    """
+    for ip in ips:
+        num = times.get(ip)
+        if num > config.ip_appear_maximum:
+            # 解析得到IPS集合有任意IP出现次数大于指定值都标记为非法(泛解析)子域
+            return True
+    return False
+
+
+def is_valid_subdomain(ips, ttl, times, wildcard_ips, wildcard_ttl):
+    ip_blacklist = config.brute_ip_blacklist
+    ips = set(ips)
+    if ips.intersection(ip_blacklist):  # 解析ip与黑名单ip有交集则标记为非法子域
+        return 0, 'IP blacklist'
+    if len(set(ttl)) == 1:  # 只有一个相同TTL才进行对比
+        ttl = ttl[0]
+        if all([wildcard_ttl, wildcard_ttl]):  # 有泛解析记录才进行对比
+            if check_by_compare(ips, ttl, wildcard_ips, wildcard_ttl):
+                return 0, 'IP wildcard '
+    if check_ip_times(ips, times):
+        return 0, 'IP exceeded'
+    return 1, None
+
+
+def save_brute_dict(path, data):
+    if not utils.save_data(path, data):
+        logger.log('FATAL', '保存生成的字典出错')
+        exit(1)
+
+
+def delete_file(dict_path, output_path):
+    if config.delete_generated_dict:
+        dict_path.unlink()
+    if config.delete_massdns_result:
+        output_path.unlink()
+
+
+class Brute(Module):
+    """
+    OneForAll子域爆破模块
+
+    Example：
+        brute.py --target domain.com --word True run
+        brute.py --target ./domains.txt --word True run
+        brute.py --target domain.com --word True --process 1 run
+        brute.py --target domain.com --word True --wordlist subnames.txt run
+        brute.py --target domain.com --word True --recursive True --depth 2 run
+        brute.py --target d.com --fuzz True --place m.*.d.com --rule '[a-z]' run
+
+    Note:
+        参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+        参数path默认None使用OneForAll结果目录自动生成路径
+
+    :param str target:       单个域名或者每行一个域名的文件路径
+    :param int process:      爆破进程数(默认1)
+    :param int concurrent:   并发爆破数量(默认10000)
+    :param bool word:        是否使用word模式进行爆破(默认False)
+    :param str wordlist:     word模式爆破使用的字典路径(默认使用config.py配置)
+    :param bool recursive:   是否使用递归进行爆破(默认False)
+    :param int depth:        递归爆破的深度(默认2)
+    :param str nextlist:     递归爆破所使用的字典路径(默认使用config.py配置)
+    :param bool fuzz:        是否使用fuzz模式进行爆破(默认False)
+    :param str place:        指定爆破位置(开启fuzz模式时必需指定此参数)
+    :param str rule:         指定fuzz模式爆破使用的正则规则(开启fuzz模式时必需指定此参数)
+    :param bool export:      是否导出爆破结果(默认True)
+    :param bool valid:       只导出有效的子域结果(默认False)
+    :param str format:       结果导出格式(默认csv)
+    :param str path:         结果导出路径(默认None)
+    """
+
+    def __init__(self, target, process=None, concurrent=None, word=False,
+                 wordlist=None, recursive=False, depth=None, nextlist=None,
+                 fuzz=False, place=None, rule=None, export=True, valid=True,
+                 format='csv', path=None):
+        Module.__init__(self)
+        self.module = 'Brute'
+        self.source = 'Brute'
+        self.target = target
+        self.process_num = process or utils.get_process_num()
+        self.concurrent_num = concurrent or config.brute_concurrent_num
+        self.word = word
+        self.wordlist = wordlist or config.brute_wordlist_path
+        self.recursive_brute = recursive or config.enable_recursive_brute
+        self.recursive_depth = depth or config.brute_recursive_depth
+        self.recursive_nextlist = nextlist or config.recursive_nextlist_path
+        self.fuzz = fuzz or config.enable_fuzz
+        self.place = place or config.fuzz_place
+        self.rule = rule or config.fuzz_rule
+        self.export = export
+        self.valid = valid
+        self.format = format
+        self.path = path
+        self.bulk = False  # 是否是批量爆破场景
+        self.domains = list()  # 待爆破的所有域名集合
+        self.domain = str()  # 当前正在进行爆破的域名
+        self.ips_times = dict()  # IP集合出现次数
+        self.enable_wildcard = False  # 当前域名是否使用泛解析
+        self.wildcard_check = config.enable_wildcard_check
+        self.wildcard_deal = config.enable_wildcard_deal
+
+    def gen_brute_dict(self, domain):
+        logger.log('INFOR', f'正在为{domain}生成爆破字典')
+        dict_set = set()
+        # 如果domain不是self.subdomain 而是self.domain的子域则生成递归爆破字典
+        if self.place is None:
+            self.place = '*.' + domain
+        wordlist = self.wordlist
+        main_domain = self.register(domain)
+        if domain != main_domain:
+            wordlist = self.recursive_nextlist
+        if self.word:
+            word_subdomains = gen_word_subdomains(self.place, wordlist)
+            # set可以合并list
+            dict_set = dict_set.union(word_subdomains)
+        if self.fuzz:
+            fuzz_subdomains = gen_fuzz_subdomains(self.place, self.rule)
+            dict_set = dict_set.union(fuzz_subdomains)
+        # logger.log('INFOR', f'正在去重爆破字典')
+        # dict_set = utils.uniq_dict_list(dict_set)
+        count = len(dict_set)
+        logger.log('INFOR', f'生成的爆破字典大小为{count}')
+        if count > 10000000:
+            logger.log('ALERT', f'注意生成的爆破字典太大：{count} > 10000000')
+        return dict_set
+
+    def check_brute_params(self):
+        if len(self.domains) > 1:
+            self.bulk = True
+        if self.fuzz:
+            if not (self.word or self.fuzz):
+                logger.log('FATAL', f'请至少指定一种爆破模式')
+                exit(1)
+            if self.place is None or self.rule is None:
+                logger.log('FATAL', f'没有指定fuzz位置或规则')
+                exit(1)
+            if self.bulk:
+                logger.log('FATAL', f'批量爆破的场景下不能使用fuzz模式')
+                exit(1)
+            if self.recursive_brute:
+                logger.log('FATAL', f'使用fuzz模式下不能使用递归爆破')
+                exit(1)
+            fuzz_count = self.place.count('*')
+            if fuzz_count < 1:
+                logger.log('FATAL', f'没有指定fuzz位置')
+                exit(1)
+            if fuzz_count > 1:
+                logger.log('FATAL', f'只能指定1个fuzz位置')
+                exit(1)
+            if self.domain not in self.place:
+                logger.log('FATAL', f'指定fuzz的域名有误')
+                exit(1)
+
+    def main(self, domain):
+        start = time.time()
+        logger.log('INFOR', f'正在爆破域名{domain}')
+        massdns_dir = config.third_party_dir.joinpath('massdns')
+        result_dir = config.result_save_dir
+        temp_dir = result_dir.joinpath('temp')
+        utils.check_dir(temp_dir)
+        massdns_path = get_massdns_path(massdns_dir)
+        timestring = utils.get_timestring()
+        self.enable_wildcard = detect_wildcard(domain)
+
+        wildcard_ips = list()  # 泛解析IP列表
+        wildcard_ttl = int()  # 泛解析TTL整型值
+        ns_ip_list = list()  # DNS权威名称服务器对应A记录列表
+        if self.enable_wildcard:
+            ns_list = query_domain_ns(self.domain)
+            ns_ip_list = query_domain_ns_a(ns_list)
+            wildcard_ips, wildcard_ttl = get_wildcard_record(domain, ns_ip_list)
+        ns_path = get_nameservers_path(self.enable_wildcard, ns_ip_list)
+
+        dict_set = self.gen_brute_dict(domain)
+        self.subdomains = dict_set
+        dict_data = '\n'.join(dict_set)
+        dict_name = f'generated_subdomains_{domain}_{timestring}.txt'
+        dict_path = temp_dir.joinpath(dict_name)
+        save_brute_dict(dict_path, dict_data)
+
+        output_name = f'resolved_result_{domain}_{timestring}.json'
+        output_path = temp_dir.joinpath(output_name)
+
+        log_path = result_dir.joinpath('massdns.log')
+        check_dict()
+
+        logger.log('INFOR', f'开始执行massdns')
+        do_brute(massdns_path, dict_path, ns_path, output_path, log_path,
+                 process_num=self.process_num,
+                 concurrent_num=self.concurrent_num)
+        logger.log('INFOR', f'结束执行massdns')
+
+        result_data = read_result(output_path)
+        delete_file(dict_path, output_path)
+        resolved_records, ip_times = deal_result(result_data)
+        added_records = add_times(resolved_records, ip_times)
+        checked_records, valid_subdomains = check_validity(added_records,
+                                                           ip_times,
+                                                           wildcard_ips,
+                                                           wildcard_ttl)
+        self.records = checked_records
+        end = time.time()
+        self.elapse = round(end - start, 1)
+        logger.log('INFOR', f'{self.source}模块耗时{self.elapse}秒'
+                            f'发现{domain}的子域{len(valid_subdomains)}个')
+        logger.log('DEBUG', f'{self.source}模块发现{domain}的子域:\n'
+                            f'{valid_subdomains}')
+        self.gen_result(brute=len(self.subdomains), valid=len(valid_subdomains))
+        self.save_db()
+        return valid_subdomains
+
+    def run(self):
+        logger.log('INFOR', f'开始执行{self.source}模块')
+        self.domains = utils.get_domains(self.target)
+        all_subdomains = list()
+        for self.domain in self.domains:
+            self.check_brute_params()
+            if self.recursive_brute:
+                logger.log('INFOR', f'开始递归爆破{self.domain}的第1层子域')
+            valid_subdomains = self.main(self.domain)
+            all_subdomains.extend(valid_subdomains)
+
+            # 递归爆破下一层的子域
+            # fuzz模式不使用递归爆破
+            if self.recursive_brute:
+                for layer_num in range(1, self.recursive_depth):
+                    # 之前已经做过1层子域爆破 当前实际递归层数是layer+1
+                    logger.log('INFOR', f'开始递归爆破{self.domain}的'
+                                        f'第{layer_num + 1}层子域')
+                    for subdomain in all_subdomains:
+                        self.place = '*.' + subdomain
+                        # 进行下一层子域爆破的限制条件
+                        num = subdomain.count('.') - self.domain.count('.')
+                        if num == layer_num:
+                            valid_subdomains = self.main(subdomain)
+                            all_subdomains.extend(valid_subdomains)
+
+            logger.log('INFOR', f'结束执行{self.source}模块爆破域名{self.domain}')
+            if not self.path:
+                name = f'{self.domain}_brute_result.{self.format}'
+                self.path = config.result_save_dir.joinpath(name)
+            # 数据库导出
+            if self.export:
+                dbexport.export(self.domain,
+                                valid=self.valid,
+                                limit='resolve',
+                                path=self.path,
+                                format=self.format)
+
+
+if __name__ == '__main__':
+    fire.Fire(Brute)
+    # domain = 'example.com'
+    # result = queue.Queue()
+    # brute = AIOBrute(domain)
+    # brute.run(result)

oneforall/common/database.py

@@ -58,24 +58,32 @@ class Database(object):
         self.query(f'create table "{table_name}" ('
                    f'id integer primary key,'
                    f'type text,'
-                   f'valid int,'
+                   f'alive int,'
+                   f'request int,'
+                   f'resolve int,'
                    f'new int,'
                    f'url text,'
                    f'subdomain text,'
+                   f'port int,'
                    f'level int,'
+                   f'cname text,'
                    f'content text,'
                    f'public int,'
-                   f'port int,'
                    f'status int,'
                    f'reason text,'
                    f'title text,'
                    f'banner text,'
                    f'header text,'
                    f'response text,'
+                   f'times text,'
+                   f'ttl text,'
+                   f'resolver text,'
                    f'module text,'
                    f'source text,'
                    f'elapse float,'
-                   f'count int)')
+                   f'find int,'
+                   f'brute int,'
+                   f'valid int)')
 
     def save_db(self, table_name, results, module_name=None):
         """
@@ -92,15 +100,17 @@ class Database(object):
             try:
                 self.conn.bulk_query(
                     f'insert into "{table_name}" ('
-                    f'id, type, valid, new, url, subdomain, port, level, content,'
+                    f'id, type, alive, resolve, request, new, url, subdomain,'
-                    f'public, status, reason, title, banner, header, response,'
+                    f'port, level, cname, content, public, status, reason,'
-                    f'module, source, elapse, count)'
+                    f'title, banner, header, response, times, ttl, resolver,'
-                    f'values (:id, :type, :valid, :new, :url, :subdomain,'
+                    f'module, source, elapse, find, brute, valid) '
-                    f':port, :level, :content, :public, :status, :reason,'
+                    f'values (:id, :type, :alive, :resolve, :request, :new,'
-                    f':title, :banner, :header, :response, :module, :source,'
+                    f':url, :subdomain, :port, :level, :cname, :content,'
-                    f':elapse, :count)', results)
+                    f':public, :status, :reason, :title, :banner, :header,'
+                    f':response, :times, :ttl, :resolver, :module, :source,'
+                    f':elapse, :find, :brute, :valid)', results)
             except Exception as e:
-                logger.log('ERROR', e.args)
+                logger.log('ERROR', e)
 
     def exist_table(self, table_name):
         """
@@ -187,7 +197,7 @@ class Database(object):
         table_name = table_name.replace('.', '_')
         logger.log('TRACE', f'正在去除{table_name}表中的无效子域')
         self.query(f'delete from "{table_name}" where '
-                   f'subdomain is null or valid == 0')
+                   f'subdomain is null or resolve == 0')
 
     def deal_table(self, deal_table_name, backup_table_name):
         """
@@ -210,19 +220,25 @@ class Database(object):
         logger.log('TRACE', f'获取{table_name}表中的所有数据')
         return self.query(f'select * from "{table_name}"')
 
-    def export_data(self, table_name, valid):
+    def export_data(self, table_name, valid, limit):
         """
         获取表中的部分数据
 
         :param str table_name: 表名
         :param any valid: 有效性
+        :param str limit: 限制字段
         """
         table_name = table_name.replace('.', '_')
-        query = f'select id, type, valid, new, url, subdomain, level, ' \
+        query = f'select id, type, new, alive, request, resolve, url, ' \
-                f'content, public, port, status, reason, title, banner ' \
+                f'subdomain, level, cname, content, public, port, status, ' \
-                f'from "{table_name}"'
+                f'reason, title, banner, times, ttl, resolver, module, ' \
-        if valid:
+                f'source, elapse, find, brute, valid from "{table_name}"'
-            where = f' where valid = 1'
+        if valid and limit:
+            if limit in ['resolve', 'request']:
+                where = f' where {limit} = 1'
+                query += where
+        else:
+            where = f' where alive = 1'
             query += where
         logger.log('TRACE', f'获取{table_name}表中的所有数据')
         return self.query(query)

oneforall/common/lookup.py

@@ -18,7 +18,7 @@ class Lookup(Module):
         if answer is None:
             return None
         for item in answer:
-            record = str(item)
+            record = item.to_text()
             subdomains = utils.match_subdomain(self.domain, record)
             self.subdomains = self.subdomains.union(subdomains)
             self.gen_record(subdomains, record)

oneforall/common/module.py

@@ -236,7 +236,7 @@ class Module(object):
                       'name': self.module,
                       'source': self.source,
                       'elapse': self.elapse,
-                      'count': len(self.subdomains),
+                      'find': len(self.subdomains),
                       'subdomains': list(self.subdomains),
                       'records': self.records}
             json.dump(result, file, ensure_ascii=False, indent=4)
@@ -246,21 +246,27 @@ class Module(object):
         """
         生成记录字典
         """
+        item = dict()
+        item['content'] = record
         for subdomain in subdomains:
-            self.records[subdomain] = record
+            self.records[subdomain] = item
 
-    def gen_result(self):
+    def gen_result(self, find=0, brute=None, valid=0):
         """
         生成结果
         """
+        logger.log('DEBUG', f'正在生成最终结果')
         if not len(self.subdomains):  # 该模块一个子域都没有发现的情况
             result = {'id': None,
                       'type': self.type,
-                      'valid': None,
+                      'alive': None,
+                      'request': None,
+                      'resolve': None,
                       'new': None,
                       'url': None,
                       'subdomain': None,
                       'level': None,
+                      'cname': None,
                       'content': None,
                       'public': None,
                       'port': None,
@@ -270,39 +276,72 @@ class Module(object):
                       'banner': None,
                       'header': None,
                       'response': None,
+                      'times': None,
+                      'ttl': None,
+                      'resolver': None,
                       'module': self.module,
                       'source': self.source,
                       'elapse': self.elapse,
-                      'count': 0}
+                      'find': find,
+                      'brute': brute,
+                      'valid': valid}
             self.results.append(result)
         else:
             for subdomain in self.subdomains:
-                valid = None
-                if self.type != 'A':  # 不是利用的DNS记录的A记录查询子域默认都有效
-                    valid = 1
                 url = 'http://' + subdomain
                 level = subdomain.count('.') - self.domain.count('.')
-                content = self.records.get(subdomain)
+                record = self.records.get(subdomain)
+                if record is None:
+                    record = dict()
+                resolve = record.get('resolve')
+                request = record.get('request')
+                alive = record.get('alive')
+                if self.type != 'A':  # 不是利用的DNS记录的A记录查询子域默认都有效
+                    resolve = 1
+                    request = 1
+                    alive = 1
+                reason = record.get('reason')
+                resolver = record.get('resolver')
+                cname = record.get('cname')
+                content = record.get('content')
+                times = record.get('times')
+                ttl = record.get('ttl')
+                public = record.get('public')
+                if isinstance(cname, list):
+                    cname = ','.join(cname)
+                    content = ','.join(content)
+                    times = ','.join([str(num) for num in times])
+                    ttl = ','.join([str(num) for num in ttl])
+                    public = ','.join([str(num) for num in public])
                 result = {'id': None,
                           'type': self.type,
-                          'valid': valid,
+                          'alive': alive,
+                          'request': request,
+                          'resolve': resolve,
                           'new': None,
                           'url': url,
                           'subdomain': subdomain,
                           'level': level,
+                          'cname': cname,
                           'content': content,
-                          'public': None,
+                          'public': public,
-                          'port': None,
+                          'port': 80,
                           'status': None,
-                          'reason': None,
+                          'reason': reason,
                           'title': None,
                           'banner': None,
-                          'module': self.module,
                           'header': None,
                           'response': None,
+                          'times': times,
+                          'ttl': ttl,
+                          'resolver': resolver,
+                          'module': self.module,
                           'source': self.source,
                           'elapse': self.elapse,
-                          'count': len(self.subdomains)}
+                          'find': find,
+                          'brute': brute,
+                          'valid': valid,
+                          }
                 self.results.append(result)
 
     def save_db(self):
@@ -310,6 +349,7 @@ class Module(object):
         将模块结果存入数据库中
 
         """
+        logger.log('DEBUG', f'正在将结果存入到数据库')
         lock.acquire()
         db = Database()
         db.create_table(self.domain)

oneforall/common/request.py

@@ -43,10 +43,9 @@ def gen_req_data(data, ports):
     logger.log('INFOR', f'正在生成请求地址')
     new_data = []
     for data in data:
-        valid = data.get('valid')
+        resolve = data.get('resolve')
-        # 无效(0)和有效子域(1)不进行http请求探测
+        # 解析失败(0)的子域不进行http请求探测
-        # 有效性待确认(None)的子域才进行http请求探测
+        if resolve == 0:
-        if valid == 0 or valid == 1:
             continue
         subdomain = data.get('subdomain')
         for port in ports:
@@ -162,15 +161,18 @@ def request_callback(future, index, datas):
         logger.log('TRACE', result.args)
         name = utils.get_classname(result)
         datas[index]['reason'] = name + ' ' + str(result)
-        datas[index]['valid'] = 0
+        datas[index]['request'] = 0
+        datas[index]['alive'] = 0
     elif isinstance(result, tuple):
         resp, text = result
         datas[index]['reason'] = resp.reason
         datas[index]['status'] = resp.status
         if resp.status == 400 or resp.status >= 500:
-            datas[index]['valid'] = 0
+            datas[index]['request'] = 0
+            datas[index]['alive'] = 0
         else:
-            datas[index]['valid'] = 1
+            datas[index]['request'] = 1
+            datas[index]['alive'] = 1
             headers = resp.headers
             datas[index]['banner'] = utils.get_sample_banner(headers)
             datas[index]['header'] = str(dict(headers))[1:-1]
@@ -244,8 +246,8 @@ def run_request(domain, data, port):
     data = loop.run_until_complete(request_coroutine)
     # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
     loop.run_until_complete(asyncio.sleep(0.25))
-    count = utils.count_valid(data)
+    count = utils.count_alive(data)
-    logger.log('INFOR', f'经验证{domain}有效子域{count}个')
+    logger.log('INFOR', f'经验证{domain}存活子域{count}个')
     return data
 
 

oneforall/common/resolve.py

@@ -214,6 +214,8 @@ def run_resolve(data):
     loop = asyncio.get_event_loop()
     asyncio.set_event_loop(loop)
     need_resolve_subdomains = filter_subdomain(data)
+    if not need_resolve_subdomains:
+        return data
     resolve_coroutine = run_aio_resolve(need_resolve_subdomains)
     results_list = loop.run_until_complete(resolve_coroutine)
     results_dict = convert_results(results_list)

oneforall/common/utils.py

@@ -135,7 +135,11 @@ def get_domains(target):
                         domains.append(domain)
         elif Domain(target).match():
             domains = [target]
-    logger.log('INFOR', f'获取到{len(domains)}个域名')
+    count = len(domains)
+    if count == 0:
+        logger.log('FATAL', f'获取到{count}个域名')
+        exit(1)
+    logger.log('INFOR', f'获取到{count}个域名')
     return domains
 
 
@@ -154,6 +158,12 @@ def get_semaphore():
         return 800
 
 
+def check_dir(dir_path):
+    if not dir_path.exists():
+        logger.log('INFOR', f'不存在{dir_path}目录将会新建')
+        dir_path.mkdir(parents=True, exist_ok=True)
+
+
 def check_path(path, name, format):
     """
     检查结果输出目录路径
@@ -206,7 +216,7 @@ def check_format(format, count):
 
 def save_data(path, data):
     """
-    保存结果数据到文件
+    保存数据到文件
 
     :param path: 保存路径
     :param data: 待存数据
@@ -283,12 +293,14 @@ def remove_invalid_string(string):
 
 
 def check_value(values):
-    for i, value in enumerate(values):
+    if not isinstance(values, dict):
+        return values
+    for key, value in values.items():
         if value is None:
             continue
         if isinstance(value, str) and len(value) > 32767:
             # Excel文件中单元格值长度不能超过32767
-            values[i] = value[:32767]
+            values[key] = value[:32767]
     return values
 
 
@@ -301,18 +313,12 @@ def export_all(format, path, datas):
     :param list datas: 待导出的结果数据
     """
     format = check_format(format, len(datas))
-    timestamp = get_timestamp()
+    timestamp = get_timestring()
     name = f'all_subdomain_result_{timestamp}'
     path = check_path(path, name, format)
     logger.log('INFOR', f'所有主域的子域结果 {path}')
     row_list = list()
     for row in datas:
-        row.pop('header')
-        row.pop('response')
-        row.pop('module')
-        row.pop('source')
-        row.pop('elapse')
-        row.pop('count')
         keys = row.keys()
         values = row.values()
         if format in {'xls', 'xlsx'}:
@@ -348,6 +354,10 @@ def get_timestamp():
     return int(time.time())
 
 
+def get_timestring():
+    return time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time()))
+
+
 def get_classname(classobj):
     return classobj.__class__.__name__
 
@@ -356,8 +366,8 @@ def python_version():
     return sys.version
 
 
-def count_valid(data):
+def count_alive(data):
-    return len(list(filter(lambda item: item.get('valid') == 1, data)))
+    return len(list(filter(lambda item: item.get('alive') == 1, data)))
 
 
 def get_subdomains(data):
@@ -404,16 +414,23 @@ def check_ip_public(ip_list):
     return 1
 
 
+def ip_is_public(ip_str):
+    ip = ip_address(ip_str)
+    if not ip.is_global:
+        return 0
+    return 1
+
+
 def get_process_num():
     process_num = config.brute_process_num
     if isinstance(process_num, int):
-        return max(1, process_num)
+        return min(os.cpu_count(), process_num)
     else:
-        return os.cpu_count()
+        return 1
 
 
 def get_coroutine_num():
-    coroutine_num = config.brute_coroutine_num
+    coroutine_num = config.resolve_coroutine_num
     if isinstance(coroutine_num, int):
         return max(64, coroutine_num)
     elif coroutine_num is None:
@@ -434,3 +451,15 @@ def get_coroutine_num():
             return 2048
     else:
         return 64
+
+
+def uniq_dict_list(dict_list):
+    return list(filter(lambda name: dict_list.count(name) == 1, dict_list))
+
+
+def delete_file(*paths):
+    for path in paths:
+        try:
+            path.unlink()
+        except Exception as e:
+            logger.log('ERROR', e.args)

oneforall/config.py

@@ -12,6 +12,7 @@ from loguru import logger
 # 路径设置
 relative_directory = pathlib.Path(__file__).parent  # OneForAll代码相对路径
 module_dir = relative_directory.joinpath('modules')  # OneForAll模块目录
+third_party_dir = relative_directory.joinpath('thirdparty')  # 三方工具目录
 data_storage_dir = relative_directory.joinpath('data')  # 数据存放目录
 result_save_dir = relative_directory.joinpath('results')  # 结果保存目录
 
@@ -44,19 +45,35 @@ module_thread_timeout = 360.0  # 每个收集模块线程超时时间(默认6分
 enable_brute_module = False  # 使用爆破模块(默认False)
 enable_wildcard_check = True  # 开启泛解析检测(默认True)
 enable_wildcard_deal = True  # 开启泛解析处理(默认True)
-# 爆破时使用的进程数(根据计算机中CPU数量情况设置 不宜大于CPU数量)
+brute_massdns_path = None  # 默认None自动选择 如需填写请填写绝对路径
-brute_process_num = None  # 默认None为系统中的CPU数量
+brute_status_format = 'ansi'  # 爆破时状态输出格式（默认asni，可选json）
-# 爆破时每个进程下的协程数(根据计算机中内存大小情况设置 默认为系统中的CPU数量)
+# 爆破时使用的进程数(根据计算机中CPU数量情况设置 不宜大于逻辑CPU个数)
-brute_coroutine_num = None  # 默认None根据内存大小设置
+brute_process_num = 1  # 默认1
+brute_concurrent_num = 10000  # 并发查询数量(默认10000)
+brute_socket_num = 1  # 爆破时每个进程下的socket数量
+brute_resolve_num = 50  # 解析失败时尝试换名称服务器重查次数
 # 爆破所使用的字典路径 默认data/subdomains.txt
 brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
-enable_recursive_brute = False  # 是否使用递归爆破(默认禁用)
+brute_nameservers_path = data_storage_dir.joinpath('cn_nameservers.txt')
+# 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询
+authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt')
+enable_recursive_brute = False  # 是否使用递归爆破(默认False)
 brute_recursive_depth = 2  # 递归爆破深度(默认2层)
 # 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt
-recursive_namelist_path = data_storage_dir.joinpath('next_subnames.txt')
+recursive_nextlist_path = data_storage_dir.joinpath('next_subnames.txt')
+enable_check_dict = False  # 是否开启字典配置检查提示(默认False)
+delete_generated_dict = True  # 是否删除爆破时临时生成的字典(默认True)
+# 是否删除爆破时massdns输出的解析结果 (默认True)
+#  massdns输出的结果中包含更详细解析结果
+#  注意: 当爆破的字典较大或使用递归爆破或目标域名存在泛解析时生成的文件可能会很大
+delete_massdns_result = True
+only_save_valid = True  # 是否在处理爆破结果时只存入解析成功的子域
+check_time = 10  # 检查字典配置停留时间(默认10秒)
 enable_fuzz = False  # 是否使用fuzz模式枚举域名
-fuzz_rule = ''  # fuzz域名的正则 示例：[a-z][0-9] 第一位是字母 第二位是数字
+fuzz_place = None  # 指定爆破的位置 指定的位置用`@`表示 示例：www.@.example.com
-ips_appear_maximum = 10  # 同一IP集合出现次数超过10认为是泛解析
+fuzz_rule = None  # fuzz域名的正则 示例：'[a-z][0-9]' 表示第一位是字母 第二位是数字
+brute_ip_blacklist = {'0.0.0.0', '0.0.0.1'}  # IP黑名单 子域解析到IP黑名单则标记为非法子域
+ip_appear_maximum = 100  # 多个子域解析到同一IP次数超过100次则标记为非法(泛解析)子域
 
 # 代理设置
 enable_proxy = False  # 是否使用代理(全局开关)
@@ -84,6 +101,7 @@ enable_recursive_search = False  # 递归搜索子域
 search_recursive_times = 2  # 递归搜索层数
 
 # DNS解析设置
+resolve_coroutine_num = 64
 resolver_nameservers = [
     '119.29.29.29', '182.254.116.116',  # DNSPod
     '180.76.76.76',  # Baidu DNS

oneforall/dbexport.py

@@ -15,7 +15,7 @@ from common.database import Database
 from config import logger
 
 
-def export(table, db=None, valid=False, path=None, format='csv', show=False):
+def export(table, db=None, valid=False, limit=None, path=None, format='csv', show=False):
     """
     OneForAll数据库导出模块
 
@@ -31,13 +31,14 @@ def export(table, db=None, valid=False, path=None, format='csv', show=False):
     :param str table:   要导出的表
     :param str db:      要导出的数据库路径(默认为results/result.sqlite3)
     :param bool valid:  只导出有效的子域结果(默认False)
+    :param str limit:   导出限制条件(默认None)
     :param str format:  导出文件格式(默认csv)
     :param str path:    导出文件路径(默认None)
     :param bool show:   终端显示导出数据(默认False)
     """
 
     database = Database(db)
-    rows = database.export_data(table, valid)
+    rows = database.export_data(table, valid, limit)
     format = utils.check_format(format, len(rows))
     path = utils.check_path(path, table, format)
     if show:

oneforall/oneforall.py

@@ -15,7 +15,7 @@ import dbexport
 from datetime import datetime
 from config import logger
 from collect import Collect
-from aiobrute import AIOBrute
+from brute import Brute
 from common import utils, resolve, request
 from common.database import Database
 from takeover import Takeover
@@ -176,7 +176,7 @@ class OneForAll(object):
         collect.run()
         if self.brute:
             # 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
-            brute = AIOBrute(self.domain, export=False)
+            brute = Brute(self.domain, word=True, export=False)
             brute.run()
 
         # 有关数据库处理