v0.0.1

oneforall/__init__.py

@@ -0,0 +1 @@
+# coding=utf-8

oneforall/aiobrute.py

@@ -0,0 +1,275 @@
+# coding=utf-8
+import os
+import time
+import queue
+import signal
+import pathlib
+import asyncio
+import fire
+import tqdm
+import exrex
+import secrets
+import aiomultiprocess
+import config
+from config import logger
+from common import utils, database, resolve
+from common.module import Module
+
+
+def init_worker():
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+
+def get_wordlist(name):
+    return config.data_storage_path.joinpath(name)
+
+
+def detect_wildcard(domain):
+    """
+    探测域名是否使用泛解析
+
+    :param str domain: 域名
+    :return: 如果没有使用泛解析返回False 使用返回泛解析的IP集合和ttl整型值
+    """
+    logger.log('INFOR', f'正在探测{domain}是否使用泛解析')
+    token = secrets.token_hex(16)
+    random_subdomain = f'{token}.{domain}'
+    try:
+        answers = resolve.dns_query_a(random_subdomain)
+    except Exception as e:  # 如果查询随机域名A记录出错 说明不存在随机子域的A记录 即没有开启泛解析
+        logger.log('DEBUG', e)
+        logger.log('INFOR', f'{domain}没有使用泛解析')
+        return False, None, None
+    ttl = answers.ttl
+    ips = {item.address for item in answers}
+    logger.log('ALERT', f'{domain}使用了泛解析')
+    logger.log('ALERT', f'{random_subdomain} 解析到IP: {ips} TTL: {ttl}')
+    return True, ips, ttl
+
+
+def wildcard_by_compare(ips, ttl, wildcard_ips, wildcard_ttl):
+    """
+    通过与泛解析返回的IP集合和判TTL值进行对比判断发现的子域是否是泛解析子域
+
+    :param set ips: 子域A记录查询出的IP集合
+    :param int ttl: 子域A记录查询出的TTL整型值
+    :param set wildcard_ips: 泛解析的IP集合
+    :param int wildcard_ttl: 泛解析的TTL整型值
+    :return: 判断结果
+    :rtype bool
+    """
+    # 参考：http://sh3ll.me/archives/201704041222.txt
+    if not ips.issubset(wildcard_ips):
+        return False
+    if ttl != wildcard_ttl and ttl % 60 == 0 and wildcard_ttl % 60 == 0:
+        return False
+    return True
+
+
+def wildcard_by_times(ips, ips_times):
+    """
+    对ips出现次数进行判断泛解析
+
+    :param set ips: 发现子域的IP集合
+    :param ips_times: 子域IP集合出现次数统计字典
+    :return: 判断结果
+    """
+    times = ips_times.get(str(ips))
+    if times > config.ips_appear_maximum:
+        return True
+    return False
+
+
+def gen_fuzz_domains(domain, rule):
+    domains = set()
+    if '{fuzz}' not in domain:
+        logger.log('FATAL', f'没有指定fuzz位置')
+        return domains
+    if not rule:
+        logger.log('FATAL', f'没有指定fuzz规则')
+        return domains
+    fuzz_count = exrex.count(rule)
+    if fuzz_count > 2000000:
+        logger.log('FATAL', f'fuzz规则范围太大：{fuzz_count} > 2000000')
+        return domains
+    logger.log('INFOR', f'fuzz字典大小：{fuzz_count}')
+    for i in range(3):
+        random_domain = domain.replace('{fuzz}', exrex.getone(rule))
+        logger.log('ALERT', f'请注意检查随机生成的{random_domain}是否正确')
+    logger.log('ALERT', f'你有10秒检查时间退出使用`CTRL+C`')
+    try:
+        time.sleep(5)
+    except KeyboardInterrupt:
+        logger.log('INFOR', '爆破终止')
+        exit(0)
+    parts = domain.split('{fuzz}')
+    for fuzz in exrex.generate(rule):
+        fuzz_domain = parts[0] + fuzz + parts[1]
+        domains.add(fuzz_domain)
+    return domains
+
+
+def gen_brute_domains(domain, path):
+    domains = set()
+    with open(path) as file:
+        for line in file:
+            brute_domain = line.strip() + '.' + domain
+            domains.add(brute_domain)
+    logger.log('INFOR', f'爆破字典大小：{len(domains)}')
+    return domains
+
+
+class AIOBrute(Module):
+    """
+    多进程多协程异步子域爆破
+
+    :param str target:       单个域名或者每行一个域名的文件路径
+    :param int processes:    爆破的进程数（默认CPU核心数）
+    :param int coroutine:    每个爆破进程下的协程数（默认16）
+    :param str wordlist:     指定爆破所使用的字典路径（默认使用config.py配置）
+    :param bool recursive:   是否使用递归爆破（默认禁用）
+    :param int depth:        递归爆破的深度（默认2）
+    :param str namelist:     指定递归爆破所使用的字典路径（默认使用config.py配置）
+    :param bool fuzz:        是否使用fuzz模式进行爆破（默认禁用，开启必须指定fuzz正则规则）
+    :param str rule:         fuzz模式使用的正则规则（默认使用config.py配置）
+
+    Example：
+    python aiobrute.py --target example.com run
+    python aiobrute.py --target ./domains.txt run
+    python aiobrute.py --target example.com --processes 4 --coroutine 64 --wordlist data/subdomains.txt run
+    python aiobrute.py --target example.com --recursive True --depth 2 --namelist data/next_subdomains.txt run
+    python aiobrute.py --target www.{fuzz}.example.com --fuzz True --rule [a-z][0-9] run
+    """
+
+    def __init__(self, target, processes=None, coroutine=64, wordlist=None,
+                 recursive=False, depth=2, namelist=None, fuzz=False, rule=None):
+        Module.__init__(self)
+        self.domains = set()
+        self.domain = str()
+        self.module = 'Brute'
+        self.source = 'AIOBrute'
+        self.target = target
+        self.processes = processes or config.brute_processes_num or os.cpu_count()
+        self.coroutine = coroutine or config.brute_coroutine_num
+        self.wordlist = wordlist or config.brute_wordlist_path or get_wordlist('subdomains.txt')
+        self.recursive_brute = recursive or config.enable_recursive_brute
+        self.recursive_depth = depth or config.brute_recursive_depth
+        self.recursive_namelist = namelist or config.recursive_namelist_path or get_wordlist('next_subdomains.txt')
+        self.fuzz = fuzz or config.enable_fuzz
+        self.rule = rule or config.fuzz_rule
+        self.nameservers = config.resolver_nameservers
+        self.ips_times = dict()  # IP集合出现次数
+        self.enable_wildcard = False  # 当前域名是否使用泛解析
+        self.wildcard_ips = set()  # 泛解析IP集合
+        self.wildcard_ttl = int()  # 泛解析TTL整型值
+
+    def gen_tasks(self, domain):
+        logger.log('INFOR', f'正在生成{domain}的字典')
+        if self.domain != domain:  # 如果domain不是self.domain，而是self.domain的子域 生成递归爆破字典
+            domains = gen_brute_domains(domain, self.recursive_namelist)
+        elif self.fuzz and self.rule:  # 开启fuzz模式并指定了fuzz正则规则
+            domains = gen_fuzz_domains(domain, self.rule)
+        else:
+            domains = gen_brute_domains(domain, self.wordlist)
+        domains = list(domains)
+        return utils.split_list(domains, 500)  # 分割任务组 500个子域为一组任务
+
+    def deal_results(self, results):
+        for result in results:
+            if isinstance(result, Exception):
+                # logger.log('DEBUG', f'爆破{subdomain}时出错 {str(answers)}')
+                continue
+            if isinstance(result, tuple):
+                subdomain, answers = result
+                ips = {record.host for record in answers}
+                value = self.ips_times.setdefault(str(ips), 0)  # 取值 如果是首次出现的IP集合 出现次数先赋值0
+                self.ips_times[str(ips)] = value + 1
+                ttl = answers[0].ttl
+                if self.enable_wildcard:
+                    if wildcard_by_compare(ips, ttl, self.wildcard_ips, self.wildcard_ttl):
+                        continue
+                if wildcard_by_times(ips, self.ips_times):
+                    continue
+                logger.log('INFOR', f'发现{self.domain}的子域: {subdomain} 解析IP: {ips} TTL: {ttl}')
+                self.subdomains.add(subdomain)
+                self.records[subdomain] = str(ips)
+
+    async def main(self, domain, rx_queue):
+        if not self.fuzz:  # fuzz模式不探测域名是否使用泛解析
+            self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl = detect_wildcard(domain)
+        tasks = self.gen_tasks(domain)
+        logger.log('INFOR', f'正在爆破{domain}的域名')
+        for task in tqdm.tqdm(tasks, desc='Progress', smoothing=1.0, ncols=True):
+            async with aiomultiprocess.Pool(processes=self.processes, initializer=init_worker,
+                                            childconcurrency=self.coroutine) as pool:
+                try:
+                    results = await pool.map(resolve.aiodns_query_a, task)
+                except KeyboardInterrupt:
+                    logger.log('ALERT', '爆破终止正在退出')
+                    pool.terminate()  # 关闭pool，结束工作进程，不在处理未完成的任务。
+                    self.save_json()
+                    self.gen_result()
+                    rx_queue.put(self.results)
+                    return
+                else:
+                    self.deal_results(results)
+            self.save_json()
+            self.gen_result()
+            rx_queue.put(self.results)
+
+    def run(self, rx_queue=None):
+        self.domains = utils.get_domains(self.target)
+        while self.domains:
+            self.domain = self.domains.pop()
+            start = time.time()
+            db_conn = database.connect_db()
+            table_name = self.domain.replace('.', '_')
+            database.create_table(db_conn, table_name)
+            if not rx_queue:
+                rx_queue = queue.Queue()
+            logger.log('INFOR', f'开始执行{self.source}模块爆破域名{self.domain}')
+            logger.log('INFOR', f'{self.source}模块使用{self.processes}个进程乘{self.coroutine}个协程')
+            # logger.log('INFOR', f'{self.source}模块使用个进程乘{self.coroutine}个协程')
+            if self.recursive_brute and not self.fuzz:  # fuzz模式不使用递归爆破
+                logger.log('INFOR', f'开始递归爆破{self.domain}的第1层子域')
+            loop = asyncio.get_event_loop()
+            asyncio.set_event_loop(loop)
+            loop.run_until_complete(self.main(self.domain, rx_queue))
+
+            # 递归爆破下一层的子域
+            if self.recursive_brute and not self.fuzz:  # fuzz模式不使用递归爆破
+                for layer_num in range(1, self.recursive_depth):  # 之前已经做过1层子域爆破 当前实际递归层数是layer+1
+                    logger.log('INFOR', f'开始递归爆破{self.domain}的第{layer_num+1}层子域')
+                    for subdomain in self.subdomains.copy():
+                        if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域爆破的限制条件
+                            loop.run_until_complete(self.main(subdomain, rx_queue))
+
+
+            while not rx_queue.empty():  # 队列不空就一直取数据存数据库
+                database.save_db(db_conn, table_name, rx_queue.get())  # 将结果存入数据库中
+            database.copy_table(db_conn, table_name)
+            database.deduplicate_subdomain(db_conn, table_name)
+            database.remove_invalid(db_conn, table_name)
+
+            end = time.time()
+            self.elapsed = round(end - start, 1)
+            logger.log('INFOR', f'结束执行{self.source}模块爆破域名{self.domain}')
+            logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒发现{self.domain}的域名{len(self.subdomains)}个')
+            logger.log('DEBUG', f'{self.source}模块发现{self.domain}的的域名 {self.subdomains}')
+
+
+def do(domain, result):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param result: 结果集队列
+    """
+    brute = AIOBrute(domain)
+    brute.run(result)
+
+
+if __name__ == '__main__':
+    # fire.Fire(AIOBrute)
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/collect.py

@@ -0,0 +1,92 @@
+# coding=utf-8
+"""
+被动收集类
+"""
+import time
+import queue
+import threading
+import importlib
+import config
+import dbexport
+from common import database
+
+
+class Collect(object):
+    """
+    收集子域名类
+    """
+    def __init__(self, domain, export=True):
+        self.domain = domain
+        self.elapsed = 0.0
+        self.modules = []
+        self.collect_func = []
+        self.path = None
+        self.export = export
+        self.format = 'xlsx'
+
+    def get_mod(self):
+        """
+        获取要运行的模块
+        :return: None
+        """
+        if config.enable_all_module:
+            # modules = ['brute', 'certificates', 'crawl', 'datasets', 'intelligence', 'search']
+            modules = ['certificates', 'check', 'datasets', 'dnsquery', 'intelligence', 'search']  # crawl模块还有点问题
+            # modules = ['intelligence']  # crawl模块还有点问题
+            for module in modules:
+                module_path = config.oneforall_module_path.joinpath(module)
+                for path in module_path.rglob('*.py'):
+                    import_module = ('modules.' + module, path.stem)  # 需要导入的类
+                    self.modules.append(import_module)
+        else:
+            self.modules = config.enable_partial_module
+
+    def import_func(self):
+        """
+        导入脚本的do函数
+        """
+        for package, name in self.modules:
+            import_object = importlib.import_module('.'+name, package)
+            self.collect_func.append(getattr(import_object, 'do'))
+
+    def run(self, rx_queue=None):
+        """
+        类运行入口
+        """
+        start = time.time()
+        self.get_mod()
+        self.import_func()
+
+        if not rx_queue:
+            rx_queue = queue.Queue(maxsize=len(self.collect_func))  # 结果集队列
+        threads = []
+        # 创建多个子域收集线程
+        for collect_func in self.collect_func:
+            thread = threading.Thread(target=collect_func, args=(self.domain, rx_queue), daemon=True)
+            threads.append(thread)
+        # 启动所有线程
+        for thread in threads:
+            thread.start()
+        # 等待所有线程完成
+        for thread in threads:
+            thread.join()
+
+        db_conn = database.connect_db()
+        table_name = self.domain.replace('.', '_')
+        database.create_table(db_conn, table_name)
+        database.copy_table(db_conn, table_name)
+        database.deduplicate_subdomain(db_conn, table_name)
+        database.remove_invalid(db_conn, table_name)
+        db_conn.close()
+        # 数据库导出
+        if self.export:
+            if not self.path:
+                self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
+            dbexport.export(table_name, path=self.path, format=self.format)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+
+
+if __name__ == '__main__':
+    a = Collect('example.com')
+    a.run()

oneforall/common/__init__.py

@@ -0,0 +1 @@
+# coding=utf-8

oneforall/common/crawl.py

@@ -0,0 +1,10 @@
+# coding=utf-8
+from .module import Module
+
+
+class Crawl(Module):
+    """
+
+    """
+    def __init__(self):
+        Module.__init__(self)

oneforall/common/database.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+"""
+SQLite数据库初始化和操作
+"""
+
+import records
+import config
+from records import Connection
+from config import logger
+
+
+def connect_db(db_path=None):
+    """
+    获取数据库对象
+
+    :param db_path: 数据库连接或路径
+    :return: SQLite数据库
+    """
+    logger.log('DEBUG', f'正在获取数据库连接')
+    if isinstance(db_path, Connection):
+        return db_path
+    protocol = 'sqlite:///'
+    if not db_path:  # 数据库路径为空连接默认数据库
+        db_path = f'{protocol}{config.result_save_path}/result.sqlite3'
+    else:
+        db_path = protocol + db_path
+    db = records.Database(db_path)  # 不存在数据库时会新建一个数据库
+    logger.log('DEBUG', f'使用数据库: {db_path}')
+    return db.get_connection()
+
+
+def create_table(db_conn, table_name):
+    """
+    初始化数据库
+
+    :param db_conn: 数据库连接
+    :param str table_name: 要创建的表名
+    """
+    logger.log('DEBUG', f'正在创建{table_name}表')
+    try:
+        db_conn.query(f'create table if not exists {table_name} ('
+                      f'id integer primary key,'
+                      f'url text,'
+                      f'subdomain text,'
+                      f'port int,'
+                      f'ips text,'
+                      f'status int,'
+                      f'reason text,'
+                      f'valid int,'
+                      f'title text,'
+                      f'banner text,'
+                      f'module text,'
+                      f'source text,'
+                      f'elapsed float,'
+                      f'count int)')
+    except Exception as e:
+        logger.log('ERROR', e)
+
+
+def save_db(db_conn, table_name, results, module_name=None):
+    """
+    将各模块结果存入数据库
+
+    :param db_conn: 数据库连接
+    :param str table_name: 表名
+    :param list results: 结果列表
+    :param str module_name: 模块名
+    """
+    logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域结果存入数据库')
+    if results:
+        try:
+            db_conn.bulk_query(f'insert into {table_name} (id, url, subdomain, port, ips, status,'
+                               f'reason, valid, title, banner, module, source, elapsed, count)'
+                               f'values (:id, :url, :subdomain, :port, :ips, :status, :reason, :valid,'
+                               f':title, :banner, :module, :source, :elapsed, :count)', results)
+        except Exception as e:
+            logger.log('ERROR', e)
+
+
+def copy_table(db_conn, table_name):
+    """
+    复制表创建备份
+
+    :param db_conn: 数据库连接
+    :param str table_name: 表名
+    """
+    new_table_name = table_name + '_bak'
+    logger.log('DEBUG', f'正在将{table_name}表复制到{new_table_name}新表')
+    try:
+        db_conn.query(f'drop table if exists {new_table_name}')
+        db_conn.query(f'create table {new_table_name} as select * from {table_name}')
+    except Exception as e:
+        logger.log('ERROR', e)
+
+
+def clear_table(db_conn, table_name):
+    """
+    清空表中数据
+
+    :param db_conn: 数据库连接
+    :param str table_name: 表名
+    """
+    logger.log('DEBUG', f'正在清空{table_name}表中的数据')
+    try:
+        db_conn.query(f'delete from {table_name}')
+    except Exception as e:
+        logger.log('ERROR', e)
+
+
+def deduplicate_subdomain(db_conn, table_name):
+    """
+    去重表中的子域并删除空值和无效值
+
+    :param db_conn: 数据库连接
+    :param str table_name: 表名
+    """
+    logger.log('DEBUG', f'正在去重{table_name}表中的子域')
+    try:
+        db_conn.query(f'delete from {table_name} where id not in (select min(id) from {table_name} group by subdomain)')
+    except Exception as e:
+        logger.log('ERROR', e)
+
+
+def remove_invalid(db_conn, table_name):
+    """
+        去除表中的空值或无效子域
+
+        :param db_conn: 数据库连接
+        :param str table_name: 表名
+        """
+    logger.log('DEBUG', f'正在去除{table_name}表中的无效子域')
+    try:
+        db_conn.query(f'delete from {table_name} where subdomain is null or valid == 0')
+    except Exception as e:
+        logger.log('ERROR', e)
+
+
+def get_data(db_conn, table_name):
+    """
+    获取表中的所有数据
+
+    :param db_conn: 数据库连接
+    :param str table_name: 表名
+    """
+    logger.log('DEBUG', f'获取{table_name}表中的所有数据')
+    try:
+        rows = db_conn.query(f'select * from {table_name}')
+    except Exception as e:
+        logger.log('ERROR', e)
+    else:
+        return rows
+
+
+def get_subdomain(db_conn, table_name, valid):
+    """
+    获取表中的子域数据
+
+    :param db_conn: 数据库连接
+    :param str table_name: 表名
+    :param int valid: 是否有效
+    """
+    logger.log('DEBUG', f'获取{table_name}表中的所有数据')
+    try:
+        rows = db_conn.query(f'select * from {table_name} where valid = {valid}')
+    except Exception as e:
+        logger.log('ERROR', e)
+    else:
+        return rows

oneforall/common/domain.py

@@ -0,0 +1,64 @@
+# coding=utf-8
+import re
+import tldextract
+import config
+
+
+class Domain(object):
+    """
+    域名处理类
+
+    :param str string: 传入的字符串
+    """
+    def __init__(self, string):
+        self.string = str(string)
+        self.regexp = r'\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b'
+        self.domain = None
+
+    def match(self):
+        """
+        域名匹配
+
+        :return: 匹配结果
+        """
+        result = re.search(self.regexp, self.string, re.I)
+        if result:
+            return result.group()
+        else:
+            return None
+
+    def extract(self):
+        """
+        域名导出
+
+        >>>d = Domain('www.example.com')
+        <domain.Domain object>
+        >>>d.extract()
+        ExtractResult(subdomain='www', domain='example', suffix='com')
+
+        :return: 导出结果
+        """
+        extract_cache_file = config.data_storage_path.joinpath('public_suffix_list.dat')
+        tldext = tldextract.TLDExtract(extract_cache_file)
+        result = self.match()
+        if result:
+            return tldext(result)
+        else:
+            return None
+
+    def registered(self):
+        """
+        获取注册域名
+
+        >>>d = Domain('www.example.com')
+        <domain.Domain object>
+        >>>d.registered()
+        example.com
+
+        :return: 注册域名
+        """
+        result = self.extract()
+        if result:
+            return result.registered_domain
+        else:
+            return None

oneforall/common/module.py

@@ -0,0 +1,191 @@
+# coding=utf-8
+"""
+模块基类
+"""
+
+import json
+import re
+import threading
+
+import requests
+import config
+from config import logger
+from . import utils
+from .domain import Domain
+from common import database
+
+
+lock = threading.Lock()
+
+
+class Module(object):
+    def __init__(self):
+        self.module = 'Module'
+        self.source = 'BaseModule'
+        self.cookie = None
+        self.header = dict()
+        self.proxy = None
+        self.delay = config.request_delay  # 请求睡眠时延
+        self.timeout = config.request_timeout  # 请求超时时间
+        self.verify = config.request_verify  # 请求SSL验证
+        self.domain = ''  # 要进行子域名收集的域名
+        self.subdomains = set()  # 存放发现的子域
+        self.records = dict()  # 存放子域解析记录
+        self.results = list()  # 存放模块结果
+        self.elapsed = 0.0  # 模块执行耗时
+
+    def get(self, url, params=None, **kwargs):
+        """
+        自定义get请求
+
+        :param str url: 请求地址
+        :param dict params: 请求参数
+        :param kwargs: 其他参数
+        :return: requests响应对象
+        """
+        try:
+            resp = requests.get(url, params=params, cookies=self.cookie, headers=self.header,
+                                proxies=self.proxy, timeout=self.timeout, verify=self.verify, **kwargs)
+        except Exception as e:
+            logger.log('ERROR', e)
+            return None
+        if resp.status_code != 200:
+            logger.log('ALERT', f'GET {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
+            content_type = resp.headers.get('Content-Type')
+            if content_type:
+                if 'json' in content_type:
+                    logger.log('ALERT', resp.json())
+            return None
+        logger.log('DEBUG', f'GET {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
+        return resp
+
+    def post(self, url, data=None, **kwargs):
+        """
+        自定义post请求
+
+        :param str url: 请求地址
+        :param dict data: 请求数据
+        :param kwargs: 其他参数
+        :return: requests响应对象
+        """
+        try:
+            resp = requests.post(url, data=data, cookies=self.cookie, headers=self.header,
+                                 proxies=self.proxy, timeout=self.timeout, verify=self.verify, **kwargs)
+        except Exception as e:
+            logger.log('ERROR', e)
+            return None
+        if resp.status_code != 200:
+            content_type = resp.headers.get('Content-Type')
+            if content_type:
+                if 'json' in content_type:
+                    logger.log('ALERT', resp.json())
+            return None
+        logger.log('DEBUG', f'POST {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
+        return resp
+
+    def get_header(self):
+        """
+        获取请求头
+
+        :return: 请求头
+        """
+        # logger.log('DEBUG', f'获取请求头')
+        if config.enable_fake_header:
+            return utils.gen_fake_header()
+        else:
+            return self.header
+
+    def get_proxy(self, module):
+        """
+        获取代理
+
+        :param str module: 模块名
+        :return: 代理字典
+        """
+        if not config.enable_proxy:
+            logger.log('DEBUG', f'所有模块不使用代理')
+            return self.proxy
+        if config.proxy_all_module:
+            logger.log('DEBUG', f'{module}模块使用代理')
+            return utils.get_random_proxy()
+        if module in config.proxy_partial_module:
+            logger.log('DEBUG', f'{module}模块使用代理')
+            return utils.get_random_proxy()
+        else:
+            logger.log('DEBUG', f'{module}模块不使用代理')
+            return self.proxy
+
+    @staticmethod
+    def match(domain, html, distinct=True):
+        """
+        正则匹配出子域
+
+        :param str domain: 域名
+        :param str html: 要匹配的html响应体
+        :param bool distinct: 匹配结果去除
+        :return: 匹配出的子域集合或列表
+        :rtype: set or list
+        """
+        logger.log('DEBUG', f'正则匹配响应体中的子域')
+        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+                 + domain.replace('.', r'\.')
+        result = re.findall(regexp, html, re.I)
+        if not result:
+            return set()
+        deal = map(lambda s: re.sub(r'(?:http://|https://)', '', s[1:].lower(), flags=re.I), result)
+        if distinct:
+            return set(deal)
+        else:
+            return list(deal)
+
+    @staticmethod
+    def register(domain):
+        """
+        获取注册域名
+
+        :param str domain: 域名
+        :return: 注册域名
+        """
+        return Domain(domain).registered()
+
+    def save_json(self):
+        """
+        将各模块结果保存为json文件
+        """
+        logger.log('DEBUG', f'将{self.source}模块发现的子域结果保存为json文件')
+        if config.save_module_result:
+            dirpath = config.result_save_path.joinpath(self.domain, self.module)
+            dirpath.mkdir(parents=True, exist_ok=True)
+            name = self.source + '.json'
+            path = dirpath.joinpath(name)
+            with open(path, mode='w', encoding='utf-8') as file:
+                result = {'domain': self.domain, 'name': self.module, 'source': self.source, 'elapsed': self.elapsed,
+                          'count': len(self.subdomains), 'subdomains': list(self.subdomains), 'records': self.records}
+                json.dump(result, file, ensure_ascii=False, indent=4)
+
+    def gen_result(self):
+        results = list()
+        if not len(self.subdomains):  # 一个子域都没有发现的情况
+            result = {'id': None, 'url': None, 'subdomain': None, 'port': None, 'ips': None, 'status': None,
+                      'reason': None, 'valid': 1, 'title': None, 'banner': None, 'module': self.module,
+                      'source': self.source, 'elapsed': self.elapsed, 'count': 0}
+            results.append(result)
+            self.results = (self.source, results)
+        else:
+            for subdomain in self.subdomains:
+                url = 'http://' + subdomain
+                ips = self.records.get(subdomain)
+                result = {'id': None, 'url': url, 'subdomain': subdomain, 'port': None, 'ips': ips, 'status': None,
+                          'reason': None, 'valid': 1, 'title': None, 'banner': None, 'module': self.module,
+                          'source': self.source, 'elapsed': self.elapsed, 'count': len(self.subdomains)}
+                results.append(result)
+            self.results = (self.source, results)
+
+    def save_db(self):
+        lock.acquire()
+        db_conn = database.connect_db()
+        table_name = self.domain.replace('.', '_')
+        database.create_table(db_conn, table_name)
+        source, results = self.results
+        database.save_db(db_conn, table_name, results, source)  # 将结果存入数据库中
+        lock.release()

oneforall/common/query.py

@@ -0,0 +1,10 @@
+# coding=utf-8
+from .module import Module
+
+
+class Query(Module):
+    """
+    查询基类
+    """
+    def __init__(self):
+        Module.__init__(self)

oneforall/common/request.py

@@ -0,0 +1,115 @@
+# coding=utf-8
+
+import asyncio
+import functools
+import aiohttp
+from aiohttp import ClientSession
+from aiohttp.resolver import AsyncResolver
+from bs4 import BeautifulSoup
+import config
+from common import utils
+from config import logger
+
+
+def get_ports(port):
+    logger.log('INFOR', f'正在获取请求端口范围')
+    ports = set()
+    if isinstance(port, set):
+        ports = port
+    elif isinstance(port, str):
+        if port not in {'small', 'medium', 'large', 'xlarge'}:
+            logger.log('ERROR', f'不存在{port}等端口范围')
+            port = 'medium'
+        ports = config.ports.get(port)
+        logger.log('INFOR', f'使用{port}等端口范围')
+    if not ports:  # 意外情况 ports_range为空使用使用中等端口范围
+        logger.log('ALERT', f'使用medium等端口范围')
+        ports = config.ports.get('medium')
+    return ports
+
+
+def gen_new_datas(datas, ports):
+    logger.log('INFOR', f'正在生成请求地址')
+    new_datas = []
+    protocols = ['http://', 'https://']
+    for data in datas:
+        if data.get('valid'):  # 有效的子域才进行http请求探测
+            subdomain = data.get('subdomain')
+            for port in ports:
+                for protocol in protocols:
+                    url = f'{protocol}{subdomain}:{port}'
+                    data['id'] = None
+                    data['url'] = url
+                    data['port'] = port
+                    new_datas.append(data)
+                    data = dict(data)  # 需要生成一个新的字典对象
+    return new_datas
+
+
+async def fetch(session, url, semaphore):
+    """
+    请求
+
+    :param session: session对象
+    :param url: url地址
+    :param semaphore: 同步对象（控制并发量）
+    :return: 响应对象和响应文本
+    """
+    timeout = aiohttp.ClientTimeout(total=config.get_timeout)
+    async with semaphore:
+        async with session.get(url, allow_redirects=config.get_redirects,
+                               timeout=timeout, proxy=config.get_proxy) as resp:
+            text = await resp.text()
+            return resp, text
+
+
+def request_callback(future, index, datas):
+    try:
+        resp, text = future.result()
+    except Exception as e:
+        logger.log('DEBUG', e.args)
+        datas[index]['reason'] = str(e.args)
+        datas[index]['valid'] = 0
+    else:
+        datas[index]['reason'] = resp.reason
+        datas[index]['status'] = resp.status
+        if resp.status == 400 or resp.status >= 500:
+            datas[index]['valid'] = 0
+        else:
+            headers = resp.headers
+            banner = str({'Server': headers.get('Server'), 'Via': headers.get('Via'),
+                          'X-Powered-By': headers.get('X-Powered-By')})
+            datas[index]['banner'] = banner
+            soup = BeautifulSoup(text, 'lxml')
+            title = soup.title
+            head = soup.head
+            if title:
+                datas[index]['title'] = title.text
+            elif head:
+                datas[index]['title'] = head.text
+            else:
+                datas[index]['title'] = text
+
+
+async def bulk_get_request(datas, port):
+    logger.log('INFOR', f'正在异步进行子域的GET请求')
+    ports = get_ports(port)
+    new_datas = gen_new_datas(datas, ports)
+    header = None
+    if config.fake_header:
+        header = utils.gen_fake_header()
+    resolver = AsyncResolver(nameservers=config.resolver_nameservers)  # 使用异步域名解析器 自定义域名服务器
+    conn = aiohttp.TCPConnector(verify_ssl=config.verify_ssl, limit=config.limit_open_conn,
+                                limit_per_host=config.limit_per_host, resolver=resolver)
+    semaphore = asyncio.Semaphore(utils.get_semaphore())
+    async with ClientSession(connector=conn, headers=header) as session:
+        tasks = []
+        for i, data in enumerate(new_datas):
+            url = data.get('url')
+            task = asyncio.ensure_future(fetch(session, url, semaphore))
+            task.add_done_callback(functools.partial(request_callback, index=i, datas=new_datas))
+            tasks.append(task)
+        if tasks:  # 任务列表里有任务不空时才进行解析
+            await asyncio.wait(tasks)  # 等待所有task完成
+    logger.log('INFOR', f'完成异步进行子域的GET请求')
+    return new_datas

oneforall/common/resolve.py

@@ -0,0 +1,88 @@
+# coding=utf-8
+import asyncio
+import functools
+
+import dns.resolver
+import aiodns
+import config
+from common import utils
+from config import logger
+
+
+def dns_resolver():
+    """
+    dns解析器
+    """
+    resolver = dns.resolver.Resolver()
+    resolver.nameservers = config.resolver_nameservers
+    resolver.timeout = config.resolver_timeout
+    resolver.lifetime = config.resolver_lifetime
+    return resolver
+
+
+def dns_query_a(hostname):
+    """
+    查询A记录
+
+    :param str hostname: 主机名
+    :return: 查询结果
+    """
+    resolver = dns_resolver()
+    return resolver.query(hostname, 'A')
+
+
+def aiodns_resolver():
+    """
+    异步dns解析器
+    """
+    return aiodns.DNSResolver(nameservers=config.resolver_nameservers,
+                              timeout=config.resolver_timeout)
+
+
+async def aiodns_query_a(hostname, semaphore):
+    """
+    异步查询A记录
+
+    :param str hostname: 主机名
+    :param semaphore: 并发查询数量
+    :return: 主机名或查询结果或查询异常
+    """
+    async with semaphore:
+        resolver = aiodns_resolver()
+        answers = await resolver.query(hostname, 'A')
+        return hostname, answers
+
+
+def resolve_callback(future, index, datas):
+    try:
+        result = future.result()
+    except aiodns.error.DNSError as e:
+        datas[index]['ips'] = str(e.args)
+        datas[index]['valid'] = 0
+    else:
+        if isinstance(result, tuple):
+            _, answers = result
+            ips = {record.host for record in answers}
+            datas[index]['ips'] = str(ips)
+
+
+async def bulk_query_a(datas):
+    """
+    批量查询A记录
+
+    :param datas: 待查的数据集
+    :return: 查询过得到的结果集
+    """
+    logger.log('INFOR', '正在异步查询子域的A记录')
+    tasks = []
+    semaphore = asyncio.Semaphore(utils.get_semaphore())
+    for i, data in enumerate(datas):
+        if not data.get('ips'):
+            subdomain = data.get('subdomain')
+            task = asyncio.ensure_future(aiodns_query_a(subdomain, semaphore))
+            task.add_done_callback(functools.partial(resolve_callback, index=i, datas=datas))  # 回调
+            tasks.append(task)
+    if tasks:  # 任务列表里有任务不空时才进行解析
+        await asyncio.wait(tasks)  # 等待所有task完成
+    logger.log('INFOR', '完成异步查询子域的A记录')
+    return datas

oneforall/common/search.py

@@ -0,0 +1,50 @@
+# coding=utf-8
+import requests
+import config
+from .module import Module
+from . import utils
+
+
+class Search(Module):
+    """
+    搜索基类
+    """
+    def __init__(self):
+        Module.__init__(self)
+        self.page_num = 0  # 要显示搜索起始条数
+        self.per_page_num = 50  # 每页显示搜索条数
+        self.recursive_search = config.enable_recursive_search
+        self.recursive_times = config.search_recursive_times
+
+    @staticmethod
+    def filter(domain, subdomain):
+        """
+        生成搜索过滤语句
+        使用搜索引擎支持的-site:语法过滤掉搜索页面较多的子域以发现新域
+
+        :param str domain: 域名
+        :param set subdomain: 子域名集合
+        :return: 过滤语句
+        :rtype: str
+        """
+        statements_list = []
+        subdomains_temp = set(map(lambda x: x + '.' + domain, config.subdomains_common))
+        subdomains_temp = list(subdomain.intersection(subdomains_temp))
+        for i in range(0, len(subdomains_temp), 2):  # 同时排除2个子域
+            statements_list.append(''.join(set(map(lambda s: ' -site:' + s, subdomains_temp[i:i + 2]))))
+        return statements_list
+
+    def match_location(self, domain, url):
+        """
+        匹配跳转之后的url
+        针对部分搜索引擎（如百度搜索）搜索展示url时有显示不全的情况
+        此函数会向每条结果的链接发送head请求获取响应头的location值并做子域匹配
+
+        :param str domain: 域名
+        :param str url: 展示结果的url链接
+        :return: 匹配的子域
+        :rtype set
+        """
+        resp = requests.head(url, headers=self.header, proxies=self.proxy, timeout=self.timeout, allow_redirects=False)
+        location = resp.headers.get('location')
+        return set(utils.match_subdomain(domain, location))

oneforall/common/utils.py

@@ -0,0 +1,130 @@
+# coding=utf-8
+import re
+import pathlib
+import random
+import ipaddress
+import platform
+import config
+from fake_useragent import UserAgent
+from common.domain import Domain
+from config import logger
+
+
+def match_subdomain(domain, text, distinct=True):
+    """
+    匹配text中domain的子域名
+
+    :param str domain: 域名
+    :param str text: 响应文本
+    :param bool distinct: 结果去重
+    :return: 匹配结果
+    :rtype: set or list
+    """
+    regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' + domain.replace('.', r'\.')
+    result = re.findall(regexp, text, re.I)
+    if not result:
+        return set()
+    deal = map(lambda s: s.lower(), result)
+    if distinct:
+        return set(deal)
+    else:
+        return list(deal)
+
+
+def gen_random_ip():
+    """
+    生成随机的点分十进制的IP字符串
+    """
+    while True:
+        ip = ipaddress.IPv4Address(random.randint(0, 2 ** 32 - 1))
+        if ip.is_global:
+            return ip.exploded
+
+
+def gen_fake_header():
+    """
+    生成伪造请求头
+    """
+    ua = UserAgent()
+    ip = gen_random_ip()
+    headers = {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+        'Cache-Control': 'max-age=0',
+        'Connection': 'keep-alive',
+        'DNT': '1',
+        'Referer': 'https://www.google.com/',
+        'Upgrade-Insecure-Requests': '1',
+        'User-Agent': ua.random,
+        'X-Forwarded-For': ip,
+        'X-Real-IP': ip
+    }
+    return headers
+
+
+def get_random_proxy():
+    """
+    获取随机代理
+    """
+    try:
+        return random.choice(config.proxy_pool)
+    except IndexError:
+        return None
+
+
+def split_list(ls, size):
+    """
+    将ls列表按size大小划分并返回新的划分结果列表
+
+    :param list ls: 要划分的列表
+    :param int size: 划分大小
+    :return 划分结果
+
+    >>> split_list([1, 2, 3, 4], 3)
+    [[1, 2, 3], [4]]
+    """
+    if size == 0:
+        return ls
+    return [ls[i:i+size] for i in range(0, len(ls), size)]
+
+
+def get_domains(target):
+    """
+    获取域名
+
+    :param set or str target:
+    :return: 域名集合
+    """
+    domains = set()
+    logger.log('INFOR', f'正在获取域名')
+    if isinstance(target, set):
+        domains = target
+    elif isinstance(target, str):
+        path = pathlib.Path(target)
+        if path.is_file():
+            with open(target) as file:
+                for line in file:
+                    domain = Domain(line.strip()).match()
+                    if domain:
+                        domains.add(domain)
+        if Domain(target).match():
+            domains = {target}
+    logger.log('INFOR', f'获取到{len(domains)}个域名')
+    return domains
+
+
+def get_semaphore():
+    """
+    获取查询并发值
+
+    :return: 并发整型值
+    """
+    system = platform.system()
+    if system == 'Windows':
+        return 500
+    elif system == 'Linux':
+        return 1000
+    elif system == 'Darwin':
+        return 1000
+

oneforall/config.py

@@ -0,0 +1,181 @@
+# coding=utf-8
+"""
+OneForAll配置
+"""
+
+import sys
+import pathlib
+from loguru import logger
+
+# 路径设置
+oneforall_relpath = pathlib.Path(__file__).parent  # oneforall代码相对路径
+oneforall_abspath = oneforall_relpath.resolve()  # oneforall代码绝对路径
+oneforall_module_path = oneforall_relpath.joinpath('modules')  # oneforall模块目录
+data_storage_path = oneforall_relpath.joinpath('data')  # 数据存放目录
+result_save_path = oneforall_relpath.joinpath('results')  # 结果保存目录
+
+# 模块设置
+save_module_result = True  # 保存模块中各脚本结果 默认保存
+enable_all_module = True  # 启用所有模块 默认启用
+enable_partial_module = []  # 启用部分模块 必须禁用enable_all_module才能生效
+# enable_partial_module = [('modules.search', 'google')]  # 启用部分模块 必须禁用enable_all_module才能生效
+
+
+# 爆破模块设置
+enable_brute_module = False  # 使用爆破模块（默认禁用）
+enable_wildcard_check = True  # 开启泛解析检测 会去掉泛解析的子域
+brute_processes_num = None  # 爆破时使用的进程数（根据系统中CPU数量情况设置 不宜大于CPU数量 默认None为系统中的CPU数量）
+brute_coroutine_num = 128  # 爆破时每个进程下的协程数（不宜大于1000）
+brute_wordlist_path = None  # 爆破所使用的字典路径 默认data/subdomains.dict
+use_general_wordlist = False  # 是否使用通用字典（默认禁用）
+enable_recursive_brute = False  # 是否使用递归爆破（默认禁用）
+brute_recursive_depth = 2  # 递归爆破深度（默认2层）
+recursive_namelist_path = None  # 爆破下一层子域所使用的字典路径 默认data/next_subdomains.dict
+enable_fuzz = False  # 是否使用fuzz模式枚举域名
+fuzz_rule = ''  # fuzz域名的正则 示例：[a-z][0-9] 第一位是字母 第二位是数字
+ips_appear_maximum = 10  # 同一IP集合出现次数超过10认为是泛解析
+
+# 代理设置
+enable_proxy = True  # 是否使用代理 全局开关
+proxy_all_module = False  # 代理所有模块
+proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch', 'GoogleAPISearch',
+                        'GoogleSearch', 'YahooSearch', 'YandexSearch']  # 代理自定义的模块
+proxy_pool = [{'http': 'http://127.0.0.1:1080', 'https': 'https://127.0.0.1:1080'}]  # 代理池
+# proxy_pool = [{'http': 'socks5://127.0.0.1:10808', 'https': 'socks5://127.0.0.1:10808'}]  # 代理池
+
+
+# 网络请求设置
+enable_fake_header = True  # 启用伪造请求头
+request_delay = 1  # 请求时延
+request_timeout = 30  # 请求超时(AskSearch和YahooSearch比较慢)
+request_verify = True  # 请求SSL验证
+
+# 搜索模块设置
+enable_recursive_search = False  # 递归搜索子域
+search_recursive_times = 2  # 递归搜索层数
+
+# DNS解析设置
+resolver_nameservers = [
+    '119.29.29.29', '182.254.116.116',  # DNSPod
+    '180.76.76.76',  # Baidu DNS
+    '223.5.5.5', '223.6.6.6',  # AliDNS
+    '114.114.114.114', '114.114.115.115'  # 114DNS
+    # '8.8.8.8', '8.8.4.4',  # Google DNS
+    # '1.0.0.1', '1.1.1.1'  # CloudFlare DNS
+    # '208.67.222.222', '208.67.220.220'  # OpenDNS
+]  # 指定查询的DNS域名服务器
+resolver_timeout = 5.0  # 解析超时时间
+resolver_lifetime = 30.0  # 解析存活时间
+
+# http探测设置
+small_ports = {80, 443}
+medium_ports = {80, 443, 8000, 8080, 8443}  # 默认使用
+large_ports = {80, 81, 443, 591, 2082, 2087, 2095, 2096, 3000, 8000, 8001,
+               8008, 8080, 8083, 8443, 8834, 8888}
+xlarge_ports = {80, 81, 300, 443, 591, 593, 832, 981, 1010, 1311, 2082,
+                2087, 2095, 2096, 2480, 3000, 3128, 3333, 4243, 4567, 4711,
+                4712, 4993, 5000, 5104, 5108, 5800, 6543, 7000, 7396, 7474,
+                8000, 8001, 8008, 8014, 8042, 8069, 8080, 8081, 8088, 8090,
+                8091, 8118, 8123, 8172, 8222, 8243, 8280, 8281, 8333, 8443,
+                8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080, 9090,
+                9091, 9200, 9443, 9800, 9981, 12443, 16080, 18091, 18092,
+                20720, 28017}
+ports = {'small': small_ports, 'medium': medium_ports, 'large': large_ports, 'xlarge': xlarge_ports}
+verify_ssl = False
+get_proxy = None  # aiohttp 支持 HTTP/HTTPS形式的代理 proxy="http://user:pass@some.proxy.com"
+get_timeout = 10  # http请求探测总超时时间 None或者0则表示不检测超时
+get_redirects = True  # 允许请求跳转
+fake_header = True  # 使用伪造请求头
+limit_open_conn = 100  # 限制同一时间打开的连接数（默认100），0表示不限制
+limit_per_host = 0  # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数（默认0表示不限制）
+
+
+# 模块API配置
+# Censys可以免费注册获取API：https://censys.io/api
+censys_api_id = ''
+censys_api_secret = ''
+
+# Binaryedge可以免费注册获取API：https://app.binaryedge.io/account/api
+# 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
+binaryedge_api = ''
+
+# Binaryedge可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
+chinaz_api = ''
+
+# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/cognitive-services/bing-web-search-api/#web-json
+bing_api_id = ''
+bing_api_key = ''
+
+# SecurityTrails可以免费注册获取API：https://securitytrails.com/corp/api
+securitytrails_api = ''
+
+# https://fofa.so/api
+fofa_api_email = ''  # fofa用户邮箱
+fofa_api_key = ''  # fofa用户key
+
+# Google可以免费注册获取API: https://developers.google.com/custom-search/v1/overview
+# 免费的API只能查询前100条结果
+google_api_key = ''  # Google API搜索key
+google_api_cx = ''  # Google API搜索cx
+
+# https://api.passivetotal.org/api/docs/
+riskiq_api_username = ''
+riskiq_api_key = ''
+
+# Shodan可以免费注册获取API: https://account.shodan.io/register
+# 免费的API限速1秒查询1次
+shodan_api_key = ''
+# ThreatBook API 查询子域名需要收费 https://x.threatbook.cn/nodev4/vb4/myAPI
+threatbook_api_key = ''
+
+# VirusTotal可以免费注册获取API: https://developers.virustotal.com/reference
+virustotal_api_key = ''
+
+# https://www.zoomeye.org/doc?channel=api
+zoomeye_api_username = ''
+zoomeye_api_password = ''
+
+# Certdb可以免费注册获取API: https://spyse.com/
+certdb_api_token = ''
+
+# https://www.circl.lu/services/passive-dns/
+circl_api_username = ''
+circl_api_password = ''
+
+# https://www.dnsdb.info/
+dnsdb_api_key = ''
+
+# ipv4info可以免费注册获取API: http://ipv4info.com/tools/api/
+# 免费的API有效期只有2天，到期之后可以再次生成，每天可以查询50次。
+ipv4info_api_key = ''
+
+subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs', 'web', 'www', 'job', 'docs', 'news', 'blog',
+                     'data', 'help', 'live', 'mall', 'blogs', 'files', 'forum', 'store', 'mobile'}
+
+# 日志配置
+log_fmt = '<light-green>{time:HH:mm:ss,SSS}</light-green> ' \
+          '[<level>{level: <5}</level>] ' \
+          '<cyan>{process.name}</cyan>:<cyan>{thread.name: <10}</cyan> | ' \
+          '<blue>{module}</blue>.<blue>{function}</blue>:<blue>{line}</blue> - ' \
+          '<level>{message}</level>'
+
+log_path = result_save_path.joinpath('oneforall.log')
+
+logger.remove()
+logger.level(name='TRACE', no=5, color='<cyan><bold>', icon='✏️')
+logger.level(name='DEBUG', no=10, color='<blue><bold>', icon='🐞 ')
+logger.level(name='INFOR', no=20, color='<green><bold>', icon='ℹ️')
+logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️')
+logger.level(name='ERROR', no=40, color='<red><bold>', icon='❌️')
+logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')
+
+logger.add(sys.stdout, level='INFOR', format=log_fmt, enqueue=True)
+logger.add(log_path, level='TRACE', format=log_fmt, enqueue=True, encoding='utf-8')
+
+# 调试模式
+# import urllib3
+# urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+# request_proxy = [{'http': 'http://127.0.0.1:8080', 'https': 'https://127.0.0.1:8080'}]
+# request_verify = False
+# enable_all_module = False  # 启用所有模块 默认启用
+# enable_partial_module = [('modules.certificates', 'censys_api')]  # 启用部分模块 必须禁用enable_all_module才能生效

oneforall/data/next_subdomains.txt

@@ -0,0 +1,163 @@
+test
+test2
+t
+dev
+1
+2
+3
+s1
+s2
+s3
+admin
+adm
+a
+b
+c
+m
+ht
+adminht
+webht
+web
+gm
+sys
+system
+manage
+manager
+mgr
+passport
+bata
+wei
+weixin
+wechat
+wx
+wiki
+upload
+ftp
+pic
+jira
+zabbix
+nagios
+bug
+bugzilla
+sql
+mysql
+db
+stmp
+pop
+imap
+mail
+zimbra
+exchange
+forum
+bbs
+list
+count
+counter
+img
+img01
+img02
+img03
+img04
+api
+cache
+js
+css
+app
+apps
+wap
+sms
+zip
+monitor
+proxy
+update
+upgrade
+stat
+stats
+data
+portal
+blog
+autodiscover
+en
+search
+so
+oa
+database
+home
+sso
+help
+vip
+s
+w
+down
+download
+downloads
+dl
+svn
+git
+log
+staff
+vpn
+sslvpn
+ssh
+scanner
+sandbox
+ldap
+lab
+go
+demo
+console
+cms
+auth
+crm
+erp
+res
+static
+old
+new
+beta
+image
+service
+login
+3g
+docs
+it
+e
+live
+library
+files
+i
+d
+cp
+connect
+gateway
+lib
+preview
+backup
+share
+status
+assets
+user
+vote
+bugs
+cas
+feedback
+id
+edm
+survey
+union
+ceshi
+dev1
+updates
+phpmyadmin
+pma
+edit
+master
+xml
+control
+profile
+zhidao
+tool
+toolbox
+boss
+activity
+www

oneforall/data/srv_names.json

@@ -0,0 +1,102 @@
+[
+"_afpovertcp._tcp.",
+"_aix._tcp.",
+"_autodiscover._tcp.",
+"_caldav._tcp.",
+"_certificates._tcp.",
+"_client._smtp.",
+"_cmp._tcp.",
+"_crls._tcp.",
+"_crl._tcp.",
+"_finger._tcp.",
+"_ftp._tcp.",
+"_gc._tcp.",
+"_h323be._tcp.",
+"_h323be._udp.",
+"_h323cs._tcp.",
+"_h323cs._udp.",
+"_h323ls._tcp.",
+"_h323ls._udp.",
+"_h323rs._tcp.",
+"_hkps._tcp.",
+"_hkp._tcp.",
+"_http._tcp.",
+"_iax.udp.",
+"_imaps._tcp.",
+"_imap._tcp.",
+"_jabber-client._tcp.",
+"_jabber-client._udp.",
+"_jabber._tcp.",
+"_jabber._udp.",
+"_kerberos-adm._tcp.",
+"_kerberos._tcp.",
+"_kerberos._tcp.dc._msdcs.",
+"_kerberos._udp.",
+"_kpasswd._tcp.",
+"_kpasswd._udp.",
+"_ldap._tcp.",
+"_ldap._tcp.dc._msdcs.",
+"_ldap._tcp.gc._msdcs.",
+"_ldap._tcp.pdc._msdcs.",
+"_msdcs.",
+"_mysqlsrv._tcp.",
+"_nntp._tcp.",
+"_ntp._udp.",
+"_ocsp._tcp.",
+"_pgpkeys._tcp.",
+"_pgprevokations._tcp.",
+"_PKIXREP._tcp.",
+"_pop3s._tcp.",
+"_pop3._tcp.",
+"_sipfederationtls._tcp.",
+"_sipinternal._tcp.",
+"_sipinternaltls._tcp.",
+"_sips._tcp.",
+"_sip._tcp.",
+"_sip._tls.",
+"_sip._udp.",
+"_smtp._tcp.",
+"_ssh._tcp.",
+"_stun._tcp.",
+"_stun._udp.",
+"_svcp._tcp.",
+"_tcp.",
+"_telnet._tcp.",
+"_test._tcp.",
+"_tls.",
+"_udp.",
+"_vlmcs._tcp.",
+"_vlmcs._udp.",
+"_whois._tcp.",
+"_wpad._tcp.",
+"_xmpp-client._tcp.",
+"_xmpp-client._udp.",
+"_xmpp-server._tcp.",
+"_xmpp-server._udp.",
+"_https._tcp.",
+"_imap.tcp.",
+"_kerberos.tcp.dc._msdcs.",
+"_ldap._tcp.ForestDNSZones.",
+"_submission._tcp.",
+"_caldavs._tcp.",
+"_carddav._tcp.",
+"_carddavs._tcp.",
+"_x-puppet._tcp.",
+"_x-puppet-ca._tcp.",
+"_domainkey.",
+"_pkixrep._tcp.",
+"_cisco-phone-http.",
+"_cisco-phone-tftp.",
+"_cisco-uds._tcp.",
+"_ciscowtp._tcp.",
+"_collab-edge._tls.",
+"_cuplogin._tcp.",
+"_client._smtp._tcp.",
+"_sftp._tcp.",
+"_h323rs._udp.",
+"_sql._tcp.",
+"_sip._tcp.internal.",
+"_snmp._udp.",
+"_rdp._tcp.",
+"_xmpp-server._udp."
+]

oneforall/dbexport.py

@@ -0,0 +1,53 @@
+# coding=utf-8
+
+import fire
+from common import database
+from config import logger
+
+
+def export(table, db=None, valid=None, path=None, format='xlsx', output=False):
+    """
+    将数据库导出为指定格式文件
+
+    :param str table:   要导出的表
+    :param str db:      要导出的数据库路径（默认为results/result.sqlite3）
+    :param int valid:   导出子域的有效性（默认None）
+    :param str format:  导出格式（默认xlsx）
+    :param str path:    导出路径(默认None)
+    :param bool output: 是否将导出数据输出到终端（默认False）
+
+    Note:
+        参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+        参数format可选格式：'csv','tsv','json','yaml','html','xls','xlsx','dbf','latex','ods'
+        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+
+    Example:
+        python dbexport.py --db result.db --table name --format csv --output False
+        python dbexport.py --db result.db --table name --format csv --path= ./result.csv
+    """
+    db_conn = database.connect_db(db)
+    if valid is None:
+        rows = database.get_data(db_conn, table)
+    elif isinstance(valid, int):
+        rows = database.get_subdomain(db_conn, table, valid)
+    else:
+        rows = database.get_data(db_conn, table)  # 意外情况导出全部子域
+    if output:
+        print(rows.dataset)
+    if not path:
+        path = 'export.' + format
+    logger.log('INFOR', f'正在将数据库中{table}表导出到{path}')
+    try:
+        with open(path, 'w') as file:
+            file.write(rows.export(format))
+            logger.log('INFOR', '成功完成导出')
+    except TypeError:
+        with open(path, 'wb') as file:
+            file.write(rows.export(format))
+            logger.log('INFOR', '成功完成导出')
+    except Exception as e:
+        logger.log('ERROR', e)
+
+
+if __name__ == '__main__':
+    fire.Fire(export)

oneforall/domains.txt

@@ -0,0 +1 @@
+example.com

oneforall/modules/__init__.py

@@ -0,0 +1 @@
+# coding=utf-8

oneforall/modules/certificates/censys_api.py

@@ -0,0 +1,87 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.query import Query
+from config import logger
+
+
+class CensysAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Certificate'
+        self.source = "CensysAPIQuery"
+        self.addr = 'https://www.censys.io/api/v1/search/certificates'
+        self.id = config.censys_api_id
+        self.secret = config.censys_api_secret
+        self.delay = 3.0  # Censys 接口查询速率限制 最快2.5秒查1次
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        data = {
+            'query': 'parsed.names: owasp.org',
+            'page': 1,
+            'fields': ['parsed.subject_dn'],
+            'flatten': True}
+
+        resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
+        if not resp:
+            return
+        resp_json = resp.json()
+        status = resp_json.get('status')
+        if status != 'ok':
+            logger.log('ALERT', status)
+            return
+        subdomains_find = self.match(self.domain, str(resp_json))
+        self.subdomains = self.subdomains.union(subdomains_find)
+        pages = resp_json.get('metadata').get('pages')
+        for page in range(2, pages+1):
+            time.sleep(self.delay)
+            data['page'] = page
+            resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
+            if not resp:
+                return
+            subdomains_find = self.match(self.domain, str(resp.json()))
+            self.subdomains = self.subdomains.union(subdomains_find)
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        if not (self.id and self.secret):
+            logger.log('ERROR', f'{self.source}模块API配置错误')
+            logger.log('ALERT', f'不执行{self.source}模块')
+            return
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = CensysAPI(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/certificates/certdb_api.py

@@ -0,0 +1,74 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.query import Query
+from common import utils
+from config import logger
+
+
+class CertDBAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Certificate'
+        self.source = 'CertDBQuery'
+        self.addr = 'https://api.spyse.com/v1/subdomains'
+        self.token = config.certdb_api_token
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        page_num = 1
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            params = {'domain': self.domain, 'api_token': self.token, 'page': page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            resp_json = resp.json()
+            subdomains_find = utils.match_subdomain(self.domain, str(resp_json))
+            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            page_num += 1
+            if resp_json.get('count') < 30:  # 默认每次查询最多返回30条 当前条数小于30条说明已经查完
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        if not self.token:
+            logger.log('ERROR', f'{self.source}模块API配置错误')
+            logger.log('ALERT', f'不执行{self.source}模块')
+            return
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):   # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = CertDBAPI(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/certificates/certspotter.py

@@ -0,0 +1,62 @@
+# coding=utf-8
+import time
+import queue
+from common.query import Query
+from common import utils
+from config import logger
+
+
+class CertSpotter(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Certificate'
+        self.source = 'CertSpotterQuery'
+        self.addr = 'https://api.certspotter.com/v1/issuances'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'domain': self.domain, 'include_subdomains': 'true', 'expand': 'dns_names'}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = CertSpotter(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/certificates/crtsh.py

@@ -0,0 +1,62 @@
+# coding=utf-8
+import time
+import queue
+from common.query import Query
+from common import utils
+from config import logger
+
+
+class Crtsh(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Certificate'
+        self.source = 'CrtshQuery'
+        self.addr = 'https://crt.sh/'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'q': f'%.{self.domain}', 'output': 'json'}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains_find)
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = Crtsh(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/certificates/entrust.py

@@ -0,0 +1,62 @@
+# coding=utf-8
+import time
+import queue
+from common.query import Query
+from common import utils
+from config import logger
+
+
+class Entrust(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Certificate'
+        self.source = 'EntrustQuery'
+        self.addr = 'https://ctsearch.entrust.com/api/v1/certificates'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'fields': 'subjectDN', 'domain': self.domain, 'includeExpired': 'true'}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains_find)
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = Entrust(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/certificates/google.py

@@ -0,0 +1,62 @@
+# coding=utf-8
+import time
+import queue
+from common.query import Query
+from common import utils
+from config import logger
+
+
+class Google(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Certificate'
+        self.source = 'GoogleQuery'
+        self.addr = 'https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'include_expired': 'true', 'include_subdomains': 'true', 'domain': self.domain}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        subdomains_find = utils.match_subdomain(self.domain, resp.text)
+        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = Google(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/check/axfr.py

@@ -0,0 +1,98 @@
+# coding=utf-8
+"""
+查询域名的NS记录（域名服务器记录，记录该域名由哪台域名服务器解析），
+检查查出的域名服务器是否开启DNS域传送，如果开启且没做访问控制和身份验证便加以利用获取域名的所有记录
+
+DNS域传送（DNS zone transfer）指的是一台备用域名服务器使用来自主域名服务器的数据刷新自己的域数据库，
+目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
+当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获取某个域的所有记录。
+"""
+import time
+import queue
+import dns.resolver
+import dns.zone
+from config import logger
+from common import utils, resolve
+from common.module import Module
+
+
+class CheckAXFR(Module):
+    """
+    DNS域传送漏洞检查类
+    """
+    def __init__(self, domain: str):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Check'
+        self.source = 'AXFRCheck'
+        self.nsservers = []
+        self.results = []
+
+    def check(self):
+        """
+        正则匹配响应头中的内容安全策略字段以发现子域名
+        :return: None
+        """
+        resolver = resolve.dns_resolver()
+        try:
+            answers = resolver.query(self.domain, "NS")
+        except Exception as e:
+            logger.log('ERROR', e)
+            return
+        self.nsservers = [str(answer) for answer in answers]
+        if not len(self.nsservers):
+            logger.log('ALERT', f'没有找到{self.domain}的NS域名服务器记录')
+            return
+        for nsserver in self.nsservers:
+            logger.log('DEBUG', f'正在尝试对{self.domain}的域名服务器{nsserver}进行域传送')
+            try:
+                xfr = dns.query.xfr(nsserver, self.domain)
+                zone = dns.zone.from_xfr(xfr)
+            except Exception as e:
+                logger.log('DEBUG', str(e))
+                logger.log('INFOR', f'对{self.domain}的域名服务器{nsserver}进行域传送失败')
+                continue
+            else:
+                names = zone.nodes.keys()
+                for name in names:
+                    subdomain = utils.match_subdomain(self.domain, str(name)+'.'+self.domain)
+                    self.subdomains = self.subdomains.union(subdomain)
+                    record = zone[name].to_text(name)
+                    self.results.append(record)
+            if self.results:
+                logger.log('INFOR', f'发现{self.domain}在{nsserver}上的域传送记录')
+                logger.log('DEBUG', '\n'.join(self.results))
+                self.results = []
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}的域传送漏洞')
+        start = time.time()
+        self.check()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}的域传送漏洞')
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    check = CheckAXFR(domain)
+    check.run(rx_queue)
+    logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
+    logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
+
+
+if __name__ == '__main__':
+    # do('ZoneTransfer.me')
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/check/cdx.py

@@ -0,0 +1,64 @@
+# coding=utf-8
+"""
+检查crossdomain.xml文件收集子域名
+"""
+import time
+import queue
+from config import logger
+from common.utils import match_subdomain
+from common.module import Module
+
+
+class CheckCDX(Module):
+    """
+    检查crossdomain.xml文件收集子域名
+    """
+    def __init__(self, domain: str):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Check'
+        self.source = "CrossDomainXml"
+
+    def check(self):
+        """
+        检查crossdomain.xml收集子域名
+        :return:
+        """
+        url = f'http://{self.domain}/crossdomain.xml'
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        resp = self.get(url)
+        if not resp:
+            return
+        self.subdomains = match_subdomain(self.domain, resp.text)
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的crossdomain.xml')
+        start = time.time()
+        self.check()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的crossdomain.xml')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+    :param domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    check = CheckCDX(domain)
+    check.run(rx_queue)
+    logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
+    logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
+
+
+if __name__ == '__main__':
+    do('163.com')

oneforall/modules/check/cert.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+"""
+检查域名证书收集子域名
+"""
+import ssl
+import time
+import queue
+import socket
+from config import logger
+from common import utils
+from common.module import Module
+
+
+class CheckCert(Module):
+    def __init__(self, domain):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.port = 443  # ssl port
+        self.module = 'Check'
+        self.source = 'CertInfo'
+
+    def check(self):
+        """
+        获取域名证书并匹配证书中的子域名
+        """
+        ctx = ssl.create_default_context()
+        sock = ctx.wrap_socket(socket.socket(), server_hostname=self.domain)
+        try:
+            sock.connect((self.domain, self.port))
+            cert_dict = sock.getpeercert()
+        except Exception as e:
+            logger.log('ERROR', e)
+            return
+        subdomains_find = utils.match_subdomain(self.domain, str(cert_dict))
+        self.subdomains = self.subdomains.union(subdomains_find)
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的证书中的子域')
+        start = time.time()
+        self.check()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的证书中的子域')
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    check = CheckCert(domain)
+    check.run(rx_queue)
+    logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
+    logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/check/csp.py

@@ -0,0 +1,75 @@
+# coding=utf-8
+"""
+检查内容安全策略收集子域名收集子域名
+"""
+import time
+import queue
+from config import logger
+from common import utils
+from common.module import Module
+
+
+class CheckCSP(Module):
+    """
+    检查内容安全策略收集子域名
+    """
+    def __init__(self, domain, header):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Check'
+        self.source = 'Content-Security-Policy'
+        self.header = header
+
+    def check(self):
+        """
+        正则匹配响应头中的内容安全策略字段以发现子域名
+        """
+        if not self.header:
+            url = f'http://www.{self.domain}'
+            resp = self.get(url)
+            if not resp:
+                return
+            self.header = resp.headers
+        csp = self.header.get('Content-Security-Policy')
+        if not csp:
+            logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段')
+            return
+        logger.log('DEBUG', f'{self.domain}域的响应头存在内容安全策略字段')
+        self.subdomains = utils.match_subdomain(self.domain, csp)
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
+        start = time.time()
+        self.check()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+
+
+def do(domain, rx_queue, header=None):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    :param dict or None header: 响应头
+    """
+    check = CheckCSP(domain, header)
+    check.run(rx_queue)
+    logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
+    logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
+
+
+if __name__ == '__main__':
+    import requests
+    # resp = requests.get('https://content-security-policy.com/')
+    result_queue = queue.Queue()
+    resp = requests.get('https://www.baidu.com/')
+    do('google-analytics.com', result_queue, resp.headers)

oneforall/modules/crawl/archivecrawl.py

@@ -0,0 +1,71 @@
+# coding=utf-8
+import time
+import queue
+import cdx_toolkit
+from common.crawl import Crawl
+from config import logger
+
+
+class ArchiveCrawl(Crawl):
+    def __init__(self, domain):
+        Crawl.__init__(self)
+        self.domain = domain
+        self.module = 'Crawl'
+        self.source = 'ArchiveCrawl'
+
+    def crawl(self, domain, limit):
+        """
+
+        :param domain:
+        :param limit:
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        cdx = cdx_toolkit.CDXFetcher(source='ia')
+        url = f'*.{domain}/*'
+        size = cdx.get_size_estimate(url)
+        logger.log('DEBUG', f'{url} ArchiveCrawl size estimate {size}')
+
+        for resp in cdx.iter(url, limit=limit):
+            if resp.data.get('status') not in ['301', '302']:
+                url = resp.data.get('url')
+                subdomains_find = self.match(self.register(domain), url + resp.text)
+                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.crawl(self.domain, 50)
+
+        # 爬取已发现的子域以发现新的子域
+        for subdomain in self.subdomains:
+            if subdomain != self.domain:
+                self.crawl(subdomain, 10)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    crawl = ArchiveCrawl(domain)
+    crawl.run(result)
+    logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
+    logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/crawl/commoncrawl.py

@@ -0,0 +1,71 @@
+# coding=utf-8
+import time
+import queue
+from tqdm import tqdm
+import cdx_toolkit
+from common.crawl import Crawl
+from config import logger
+
+
+class CommonCrawl(Crawl):
+    def __init__(self, domain):
+        Crawl.__init__(self)
+        self.domain = domain
+        self.module = 'Crawl'
+        self.source = 'CommonCrawl'
+
+    def crawl(self, domain, limit):
+        """
+
+        :param domain:
+        :param limit:
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        cdx = cdx_toolkit.CDXFetcher()
+        url = f'*.{domain}/*'
+        size = cdx.get_size_estimate(url)
+        print(url, 'CommonCrawl size estimate', size)
+
+        for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
+            if resp.data.get('status') not in ['301', '302']:
+                subdomains_find = self.match(self.register(domain), resp.text)
+                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.crawl(self.domain, 50)
+
+        # 爬取已发现的子域以发现新的子域
+        for subdomain in self.subdomains:
+            if subdomain != self.domain:
+                self.crawl(subdomain, 10)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    crawl = CommonCrawl(domain)
+    crawl.run(result)
+    logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
+    logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/binaryedge_api.py

@@ -0,0 +1,68 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.query import Query
+from config import logger
+
+
+class BinaryEdgeAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'BinaryEdgeAPIQuery'
+        self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/'
+        self.api = config.binaryedge_api
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.header.update({'X-Key': self.api})
+        self.proxy = self.get_proxy(self.source)
+        url = self.addr + self.domain
+        resp = self.get(url)
+        if not resp:
+            return
+        subdomains_find = self.match(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains_find)
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        if not self.api:
+            logger.log('ERROR', f'{self.source}模块API配置错误')
+            logger.log('ALERT', f'不执行{self.source}模块')
+            return
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = BinaryEdgeAPI(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/bufferover.py

@@ -0,0 +1,61 @@
+# coding=utf-8
+import time
+import queue
+from common.query import Query
+from config import logger
+
+
+class BufferOver(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'BufferOverQuery'
+        self.addr = 'https://dns.bufferover.run/dns'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'q': self.domain}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        subdomains_find = self.match(self.domain, resp.text)
+        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = BufferOver(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/chinaz.py

@@ -0,0 +1,61 @@
+# coding=utf-8
+import time
+import queue
+from common.query import Query
+from config import logger
+
+
+class Chinaz(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'ChinazQuery'
+        self.addr = 'https://alexa.chinaz.com/'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        self.addr = self.addr + self.domain
+        resp = self.get(self.addr)
+        if not resp:
+            return
+        subdomains_find = self.match(self.domain, resp.text)
+        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = Chinaz(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/chinaz_api.py

@@ -0,0 +1,67 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.query import Query
+from config import logger
+
+
+class ChinazAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'ChinazAPIQuery'
+        self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa'
+        self.api = config.chinaz_api
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'key': self.api, 'domainName': self.domain}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        subdomains_find = self.match(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        if not self.api:
+            logger.log('ERROR', f'{self.source}模块API配置错误')
+            logger.log('ALERT', f'不执行{self.source}模块')
+            return
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = ChinazAPI(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/circl_api.py

@@ -0,0 +1,64 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.query import Query
+from config import logger
+
+
+class CirclAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'CirclAPIQuery'
+        self.addr = 'https://www.circl.lu/pdns/query/'
+        self.user = config.circl_api_username
+        self.pwd = config.circl_api_password
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
+        if not resp:
+            return
+        subdomains_find = self.match(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        if self.user and self.pwd:
+            start = time.time()
+            self.query()
+            end = time.time()
+            self.elapsed = round(end - start, 1)
+            self.save_json()
+            self.gen_result()
+            self.save_db()
+            rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = CirclAPI(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/dnsdb.py

@@ -0,0 +1,76 @@
+# coding=utf-8
+import random
+import time
+import queue
+from bs4 import BeautifulSoup
+from common.query import Query
+from config import logger
+
+
+class DNSdb(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'DNSdbQuery'
+        self.addr = 'https://www.dnsdb.org/'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({'Referer': self.addr})
+        self.proxy = self.get_proxy(self.source)
+        url = self.addr + self.domain + '/'
+        resp = self.get(url)
+        if not resp:
+            return
+        if resp.status_code == 200:
+            if 'index' in resp.text:
+                soup = BeautifulSoup(resp.text, features='lxml')
+                index_urls = set(map(lambda x: self.addr + self.domain + x.text, soup.find_all('a')))
+                for url in index_urls:
+                    self.delay = random.randint(2, 5)  # 休眠绕过CloudFlare的DDoS保护
+                    time.sleep(self.delay)
+                    resp = self.get(url)
+                    if not resp:
+                        return
+                    subdomains_find = self.match(self.domain, resp.text)
+                    self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            else:
+                subdomains_find = self.match(self.domain, resp.text)
+                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = DNSdb(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/dnsdb_api.py

@@ -0,0 +1,70 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.query import Query
+from common import utils
+from config import logger
+
+
+class DNSdbAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'DNSdbAPIQuery'
+        self.addr = 'https://api.dnsdb.info/lookup/rrset/name/'
+        self.api = config.dnsdb_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.header.update({'X-API-Key': self.api})
+        self.proxy = self.get_proxy(self.source)
+        url = f'{self.addr}*.{self.domain}'
+        resp = self.get(url)
+        if not resp:
+            return
+        if resp.status_code == 200:
+            subdomains_find = utils.match_subdomain(self.domain, resp.text)
+            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        if not self.api:
+            logger.log('ERROR', f'{self.source}模块API配置错误')
+            logger.log('ALERT', f'不执行{self.source}模块')
+            return
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = DNSdbAPI(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/dnsdumpster.py

@@ -0,0 +1,71 @@
+# coding=utf-8
+import time
+import queue
+from common.query import Query
+from common import utils
+from config import logger
+
+
+class DNSdumpster(Query):
+    """
+
+    """
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = "DNSdumpsterQuery"
+        self.addr = 'https://dnsdumpster.com/'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.header.update({'Referer': 'https://dnsdumpster.com'})
+        self.proxy = self.get_proxy(self.source)
+        resp = self.get(self.addr)
+        if not resp:
+            return
+        self.cookie = resp.cookies
+        data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'), 'targetip': self.domain}
+        resp = self.post(self.addr, data)
+        if not resp:
+            return
+        subdomains_find = utils.match_subdomain(self.domain, resp.text)
+        if subdomains_find:
+            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = DNSdumpster(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/hackertarget.py

@@ -0,0 +1,63 @@
+# coding=utf-8
+import time
+import queue
+from common.query import Query
+from common import utils
+from config import logger
+
+
+class HackerTarget(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = "HackerTargetQuery"
+        self.addr = 'https://api.hackertarget.com/hostsearch/'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'q': self.domain}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        if resp.status_code == 200:
+            subdomains_find = utils.match_subdomain(self.domain, resp.text)
+            if subdomains_find:
+                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = HackerTarget(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/ipv4info.py

@@ -0,0 +1,75 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.query import Query
+from config import logger
+
+
+class IPv4InfoAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'IPv4InfoAPIQuery'
+        self.addr = ' http://ipv4info.com/api_v1/'
+        self.api = config.ipv4info_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        page = 0
+        while True:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            params = {'type': 'SUBDOMAINS', 'key': self.api, 'value': self.domain, 'page': page}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            if resp.status_code != 200:
+                break  # 请求不正常通常网络是有问题，不再继续请求下去
+            resp_json = resp.json()
+            subdomains_find = self.match(self.domain, str(resp_json))
+            if not subdomains_find:
+                break
+            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            subdomains = resp_json.get('Subdomains')  # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
+            if len(subdomains) < 300:  # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
+                break
+            page += 1
+            if page >= 50:  # ipv4info子域查询接口最多允许查询50页
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = IPv4InfoAPI(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/netcraft.py

@@ -0,0 +1,86 @@
+# coding=utf-8
+import re
+import time
+import queue
+import hashlib
+from urllib import parse
+from common.query import Query
+from config import logger
+
+
+class NetCraft(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'NetCraftQuery'
+        self.init = 'https://searchdns.netcraft.com/'
+        self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains'
+        self.page_num = 1
+        self.per_page_num = 20
+
+    def bypass_verification(self):
+        """
+        绕过NetCraft的JS验证
+        """
+        self.header = self.get_header()  # Netcraft会检查User-Agent
+        self.cookie = self.get(self.init).cookies
+        cookie_value = self.cookie['netcraft_js_verification_challenge']
+        verify_taken = hashlib.sha1(parse.unquote(cookie_value).encode('utf-8')).hexdigest()
+        self.cookie['netcraft_js_verification_response'] = verify_taken
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.bypass_verification()
+        last = ''
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            params = {'host': '*.' + self.domain, 'from': self.page_num}
+            resp = self.get(self.addr + last, params)
+            if not resp:
+                return
+            subdomains_find = self.match(self.domain, resp.text)
+            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            if 'Next page' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+                break
+            last = re.search(r'&last=.*' + self.domain, resp.text).group(0)
+            self.page_num += self.per_page_num
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = NetCraft(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/ptrarchive.py

@@ -0,0 +1,65 @@
+# coding=utf-8
+import time
+import queue
+import random
+from common.query import Query
+from common import utils
+from config import logger
+
+
+class PTRArchive(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = "PTRArchiveQuery"
+        self.addr = 'http://ptrarchive.com/tools/search3.htm'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        self.cookie = {'pa_id': str(random.randint(0, 1000000000))}  # 绕过主页前端JS验证
+        params = {'label': self.domain, 'date': 'ALL'}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        if resp.status_code == 200:
+            subdomains_find = utils.match_subdomain(self.domain, resp.text)
+            if subdomains_find:
+                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = PTRArchive(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/riddler.py

@@ -0,0 +1,61 @@
+# coding=utf-8
+import time
+import queue
+from common.query import Query
+from config import logger
+
+
+class Riddler(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'RiddlerQuery'
+        self.addr = 'https://riddler.io/search'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'q': 'pld:' + self.domain}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        subdomains_find = self.match(self.domain, resp.text)
+        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = Riddler(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/robtex.py

@@ -0,0 +1,72 @@
+# coding=utf-8
+import time
+import queue
+import json
+from common.query import Query
+from config import logger
+
+
+class Robtex(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = "RobtexQuery"
+        self.addr = 'https://freeapi.robtex.com/pdns/'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        url = self.addr + 'forward/' + self.domain
+        resp = self.get(url)
+        if not resp:
+            return
+        text_list = resp.text.splitlines()
+        text_json = list(map(lambda x: json.loads(x), text_list))
+        for record in text_json:
+            if record.get('rrtype') in ['A', 'AAAA']:
+                time.sleep(self.delay)  # Robtex有查询频率限制
+                ip = record.get('rrdata')
+                url = self.addr + 'reverse/' + ip
+                resp = self.get(url)
+                if not resp:
+                    return
+                subdomains_find = self.match(self.domain, resp.text)
+                if subdomains_find:
+                    self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = Robtex(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/securitytrails_api.py

@@ -0,0 +1,71 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.query import Query
+from config import logger
+
+
+class SecurityTrailsAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'SecurityTrailsQuery'
+        self.addr = 'https://api.securitytrails.com/v1/domain/'
+        self.api = config.securitytrails_api
+        self.delay = 2  # SecurityTrails查询时延至少2秒
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'apikey': self.api}
+        url = f'{self.addr}{self.domain}/subdomains'
+        resp = self.get(url, params)
+        if not resp:
+            return
+        subdomains_prefix = resp.json()['subdomains']
+        subdomains_find = [f'{prefix}.{self.domain}' for prefix in subdomains_prefix]
+        if subdomains_find:
+            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        if not self.api:
+            logger.log('ERROR', f'{self.source}模块API配置错误')
+            logger.log('ALERT', f'不执行{self.source}模块')
+            return
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = SecurityTrailsAPI(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/datasets/sitedossier.py

@@ -0,0 +1,70 @@
+# coding=utf-8
+import time
+import queue
+from common.query import Query
+from config import logger
+
+
+class SiteDossier(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'SiteDossierQuery'
+        self.addr = 'http://www.sitedossier.com/parentdomain/'
+        self.delay = 2
+        self.page_num = 1
+        self.per_page_num = 100
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            url = f'{self.addr}{self.domain}/{self.page_num}'
+            resp = self.get(url)
+            if not resp:
+                return
+            subdomains_find = self.match(self.domain, resp.text)
+            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            if 'Show next 100 items' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+                break
+            self.page_num += self.per_page_num
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = SiteDossier(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/dnsquery/srv.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+"""
+通过枚举域名常见的SRV记录并做查询来发现子域
+"""
+
+import time
+import queue
+import json
+import asyncio
+import aiodns
+from common.module import Module
+from common import utils
+from config import logger, data_storage_path, resolver_nameservers
+
+
+class BruteSRV(Module):
+    def __init__(self, domain: str):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'dnsquery'
+        self.source = "BruteSRV"
+        self.loop = asyncio.new_event_loop()
+        self.nameservers = resolver_nameservers
+        self.resolver = aiodns.DNSResolver(self.nameservers, self.loop)
+
+    async def query(self, name):
+        """
+        查询域名的SRV记录
+        :param str name: SRV记录
+        :return: 查询结果
+        """
+        logger.log('TRACE', f'尝试查询{name}的SRV记录')
+        try:
+            answers = await self.resolver.query(name, 'SRV')
+        except Exception as e:
+            logger.log('TRACE', e)
+            logger.log('TRACE', f'查询{name}的SRV记录失败')
+            return None
+        else:
+            logger.log('TRACE', f'查询{name}的SRV记录成功')
+            return answers
+
+    def brute(self):
+        """
+        枚举域名的SRV记录
+        """
+        names_path = data_storage_path.joinpath('srv_names.json')
+        with open(names_path) as fp:
+            names_dict = json.load(fp)
+        query_map = map(lambda name: name + self.domain, names_dict)
+
+        tasks = []
+        for query in query_map:
+            tasks.append(self.query(query))
+        task_group = asyncio.gather(*tasks, loop=self.loop)
+        self.loop.run_until_complete(asyncio.gather(task_group))
+        self.loop.close()
+        results = task_group.result()
+        for result in results:
+            if result:
+                for answer in result:
+                    subdomain = utils.match_subdomain(self.domain, answer.host)
+                    if subdomain:
+                        self.subdomains = self.subdomains.union(subdomain)
+                    else:
+                        logger.log('DEBUG', f'{answer.host}不是{self.domain}的子域')
+        if not len(self.subdomains):
+            logger.log('DEBUG', f'没有找到{self.domain}的SRV记录')
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始枚举{self.domain}域的SRV记录')
+        start = time.time()
+        self.brute()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束枚举{self.domain}域的SRV记录')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    brute = BruteSRV(domain)
+    brute.run(rx_queue)
+    logger.log('INFOR', f'{brute.source}模块耗时{brute.elapsed}秒发现{brute.domain}的子域{len(brute.subdomains)}个')
+    logger.log('DEBUG', f'{brute.source}模块发现{brute.domain}的子域 {brute.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/intelligence/riskiq.py

@@ -0,0 +1,66 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.query import Query
+from config import logger
+
+
+class RiskIQ(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Intelligence'
+        self.source = 'RiskIQQuery'
+        self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
+        self.username = config.riskiq_api_username
+        self.key = config.riskiq_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'query': self.domain}
+        resp = self.get(url=self.addr, params=params, auth=(self.username, self.key))
+        if not resp:
+            return
+        resp_json = resp.json()
+        subdomains_find = resp_json.get('subdomains')
+        if subdomains_find:
+            self.subdomains = set(map(lambda x: x + '.' + self.domain, subdomains_find))
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    return
+    query = RiskIQ(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/intelligence/threatbook.py

@@ -0,0 +1,66 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.query import Query
+from config import logger
+
+
+class ThreatBookAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Intelligence'
+        self.source = 'ThreatBookAPIQuery'
+        self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
+        self.key = config.threatbook_api_key
+
+    def query(self, domain):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'apikey': self.key, 'domain': domain, 'field': 'sub_domains'}
+        resp = self.post(self.addr, params)
+        if not resp:
+            return
+        subdomain_find = self.match(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomain_find)
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        if not self.key:
+            logger.log('ERROR', f'{self.source}模块API配置错误')
+            logger.log('ALERT', f'不执行{self.source}模块')
+            return
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query(self.domain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = ThreatBookAPI(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/intelligence/threatminer.py

@@ -0,0 +1,61 @@
+# coding=utf-8
+import time
+import queue
+from common.query import Query
+from config import logger
+
+
+class ThreatMiner(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Intelligence'
+        self.source = 'ThreatMinerQuery'
+        self.addr = 'https://www.threatminer.org/getData.php'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'e': 'subdomains_container', 'q': self.domain, 't': 0, 'rt': 10}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        subdomains_find = self.match(self.domain, resp.text)
+        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = ThreatMiner(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/intelligence/virustotal.py

@@ -0,0 +1,85 @@
+# coding=utf-8
+import time
+import queue
+from common.query import Query
+from config import logger
+
+
+'''
+最多查询100条
+'''
+
+
+class VirusTotal(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.source = 'VirusTotalQuery'
+        self.module = 'Intelligence'
+        self.addr = 'https://www.virustotal.com/ui/domains/{}/subdomains'
+        self.domain = self.register(domain)
+
+    def query(self, domain):
+        """
+        向接口查询子域并做子域匹配
+        """
+        next_cursor = ''
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.header.update({'Referer': 'https://www.virustotal.com/',
+                                'TE': 'Trailers'})
+            self.proxy = self.get_proxy(self.source)
+            params = {'limit': '40', 'cursor': next_cursor}
+            resp = self.get(url=self.addr.format(domain), params=params)
+            if not resp:
+                return
+            resp_json = resp.json()
+            subdomain_find = set()
+            datas = resp_json.get('data')
+
+            if datas:
+                for data in datas:
+                    subdomain = data.get('id')
+                    if subdomain:
+                        subdomain_find.add(subdomain)
+            else:
+                break
+            self.subdomains = self.subdomains.union(subdomain_find)
+            meta = resp_json.get('meta')
+            if meta:
+                next_cursor = meta.get('cursor')
+            else:
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query(self.domain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    query = VirusTotal(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/intelligence/virustotal_api.py

@@ -0,0 +1,68 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.query import Query
+from config import logger
+
+
+class VirusTotalAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Intelligence'
+        self.source = 'VirusTotalAPIQuery'
+        self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'
+        self.key = config.virustotal_api_key
+
+    def query(self, domain):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'apikey': self.key, 'domain': domain}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        resp_json = resp.json()
+        subdomain_find = set(resp_json.get('subdomains'))
+        self.subdomains = self.subdomains.union(subdomain_find)
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        if not self.key:
+            logger.log('ERROR', f'{self.source}模块API配置错误')
+            logger.log('ALERT', f'不执行{self.source}模块')
+            return
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        start = time.time()
+        self.query(self.domain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    return
+    query = VirusTotalAPI(domain)
+    query.run(rx_queue)
+    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
+    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/ask.py

@@ -0,0 +1,89 @@
+# coding=utf-8
+import time
+import queue
+from common.search import Search
+from config import logger
+
+
+class Ask(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'AskSearch'
+        self.addr = 'https://www.search.ask.com/web'
+        self.limit_num = 200  # 限制搜索条数
+        self.per_page_num = 10  # 默认每页显示10页
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 1
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:' + domain + filtered_subdomain
+            params = {'q': query, 'page': self.page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomain_find = self.match(domain, resp.text)
+            if not subdomain_find:
+                break
+            if not full_search:
+                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                    break
+            self.subdomains = self.subdomains.union(subdomain_find)
+            self.page_num += 1
+            if '>Next<' not in resp.text:
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+                for subdomain in self.subdomains:
+                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                        self.search(subdomain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = Ask(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/baidu.py

@@ -0,0 +1,110 @@
+# coding=utf-8
+import time
+import queue
+from common.search import Search
+from bs4 import BeautifulSoup
+from config import logger
+
+
+class Baidu(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.module = 'Search'
+        self.source = 'BaiduSearch'
+        self.init = 'https://www.baidu.com/'
+        self.addr = 'https://www.baidu.com/s'
+        self.domain = domain
+        self.limit_num = 750  # 限制搜索条数
+
+    def redirect_match(self, domain, html):
+        """
+
+        :param domain:
+        :param html:
+        :return:
+        """
+        bs = BeautifulSoup(html, features='lxml')
+        subdomains_all = set()
+        for find_res in bs.find_all('a', {'class': 'c-showurl'}):  # 获取搜索结果中所有的跳转URL地址
+            url = find_res.get('href')
+            subdomain = self.match_location(domain, url)
+            subdomains_all = subdomains_all.union(subdomain)
+        return subdomains_all
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 0  # 二次搜索重新置0
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:' + domain + filtered_subdomain
+            params = {'wd': query, 'pn': self.page_num, 'rn': self.per_page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            if len(domain) > 12:  # 解决百度搜索结果中域名过长会显示不全的问题
+                subdomains_find = self.redirect_match(domain, resp.text)  # 获取百度跳转URL响应头的Location字段获取直链
+            else:
+                subdomains_find = self.match(domain, resp.text)
+            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+                break
+            if not full_search:
+                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                    break
+            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            self.page_num += self.per_page_num
+            if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+                break
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+                for subdomain in self.subdomains:
+                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                        self.search(subdomain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = Baidu(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/bing.py

@@ -0,0 +1,96 @@
+# coding=utf-8
+import time
+import queue
+from common.search import Search
+from config import logger
+
+
+class Bing(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'BingSearch'
+        self.init = 'https://www.bing.com/'
+        self.addr = 'https://www.bing.com/search'
+        self.limit_num = 1000  # 限制搜索条数
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 0  # 二次搜索重新置0
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        resp = self.get(self.init)
+        if not resp:
+            return
+        self.cookie = resp.cookies  # 获取cookie bing在搜索时需要带上cookie
+        while True:
+            time.sleep(self.delay)
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:' + domain + filtered_subdomain
+            params = {'q': query, 'first': self.page_num, 'count': self.per_page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomains_find = self.match(domain, resp.text)
+            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+                break
+            if not full_search:
+                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                    break
+            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            if '<div class="sw_next>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+                break
+            self.page_num += self.per_page_num
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+                for subdomain in self.subdomains:
+                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                        self.search(subdomain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = Bing(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/bing_api.py

@@ -0,0 +1,98 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.search import Search
+from config import logger
+
+
+class BingAPI(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'BingCustomSearch'
+        self.addr = 'https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search'
+        self.id = config.bing_api_id
+        self.key = config.bing_api_key
+        self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数
+        self.delay = 1  # 必应自定义搜索限制时延1秒
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 0  # 二次搜索重新置0
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.header = {'Ocp-Apim-Subscription-Key': self.key}
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:' + domain + filtered_subdomain
+            params = {'q': query, 'customconfig': self.id, 'safesearch': 'Off',
+                      'count': self.per_page_num, 'offset': self.page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomains_find = self.match(domain, str(resp.json()))
+            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+                break
+            if not full_search:
+                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                    break
+            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            self.page_num += self.per_page_num
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        if not (self.id and self.key):
+            logger.log('ERROR', f'{self.source}模块API配置错误')
+            logger.log('ALERT', f'不执行{self.source}模块')
+            return
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+                for subdomain in self.subdomains:
+                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                        self.search(subdomain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = BingAPI(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/duckduckgo.py

@@ -0,0 +1,93 @@
+# coding=utf-8
+import re
+import time
+import queue
+from common.search import Search
+from config import logger
+
+
+class DuckDuckGO(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'DuckDuckGoSearch'
+        self.addr = 'https://duckduckgo.com/html/'
+        self.header = self.get_header()
+        self.delay = 2
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        query = 'site:' + domain + filtered_subdomain
+        data = {'q': query, 'kl': 'us-en', 'v': 'l'}
+        while True:
+            time.sleep(self.delay)
+            self.proxy = self.get_proxy(self.source)
+            resp = self.post(self.addr, data)
+            if not resp:
+                return
+            subdomain_find = self.match(domain, resp.text)
+            if not subdomain_find:
+                break
+            if not full_search:
+                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                    break
+            self.subdomains = self.subdomains.union(subdomain_find)
+            try:
+                s = re.findall(r'name="s" value="(\d.*)"', resp.text)[-1]
+                dc = re.findall(r'name="dc" value="(\d.*)"', resp.text)
+            except Exception as e:
+                logger.error(e)
+                break
+            data.update({'s': s, 'nextParams': '', 'o': 'json', 'dc': dc, 'api': '/d.js'})
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+                for subdomain in self.subdomains:
+                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                        self.search(subdomain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    # return  # 暂时还有点问题
+    search = DuckDuckGO(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/exalead.py

@@ -0,0 +1,93 @@
+# coding=utf-8
+import time
+import queue
+import random
+from common.search import Search
+from config import logger
+
+
+class Exalead(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = "ExaleadSearch"
+        self.addr = "http://www.exalead.com/search/web/results/"
+        self.per_page_num = 30
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 0
+        while True:
+            self.delay = random.randint(1, 5)
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:' + domain + filtered_subdomain
+            params = {'q': query, 'elements_per_page': '30', "start_index": self.page_num}
+            resp = self.get(url=self.addr, params=params)
+            if not resp:
+                return
+            subdomain_find = self.match(domain, resp.text)
+            if not subdomain_find:
+                break
+            if not full_search:
+                if subdomain_find.issubset(self.subdomains):
+                    break
+            self.subdomains = self.subdomains.union(subdomain_find)
+            self.page_num += self.per_page_num
+            if self.page_num > 1999:
+                break
+            if 'title="Go to the next page"' not in resp.text:
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            statement = statement.replace('-site', 'and -site')
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+                for subdomain in self.subdomains:
+                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                        self.search(subdomain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = Exalead(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/fofa.py

@@ -0,0 +1,71 @@
+# coding=utf-8
+import time
+import queue
+import json
+import base64
+import config
+from common.search import Search
+from config import logger
+
+
+class FoFa(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'FoFaSearch'
+        self.addr = 'https://fofa.so/api/v1/search/all'
+        self.delay = 1
+        self.email = config.fofa_api_email
+        self.key = config.fofa_api_key
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        self.page_num = 1
+        query_base64 = base64.b64encode(f'domain={self.domain}'.encode('utf-8'))
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = {'email': self.email, 'key': self.key, 'qbase64': query_base64, 'page': self.page_num}
+            resp = self.get(self.addr, query)
+            if not resp:
+                return
+            subdomain_find = self.match(self.domain, resp.text)
+            self.subdomains = self.subdomains.union(subdomain_find)
+            self.page_num += 1
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = FoFa(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/google.py

@@ -0,0 +1,101 @@
+# coding=utf-8
+import time
+import queue
+import random
+from common.search import Search
+from config import logger
+
+
+class Google(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'GoogleSearch'
+        self.init = 'https://www.google.com/'
+        self.addr = 'https://www.google.com/search'
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        page_num = 1
+        per_page_num = 50
+        self.header = self.get_header()
+        self.header.update({'User-Agent': 'Googlebot',
+                            'Referer': 'https://www.google.com'})
+        self.proxy = self.get_proxy(self.source)
+        resp = self.get(self.init)
+        if not resp:
+            return
+        self.cookie = resp.cookies
+        while True:
+            self.delay = random.randint(1, 5)
+            time.sleep(self.delay)
+            self.proxy = self.get_proxy(self.source)
+            word = 'site:' + domain + filtered_subdomain
+            payload = {'q': word, 'start': page_num, 'num': per_page_num,
+                       'filter': '0', 'btnG': 'Search', 'gbv': '1', 'hl': 'en'}
+            resp = self.get(url=self.addr, params=payload)
+            if not resp:
+                return
+            subdomain_find = self.match(domain, resp.text)
+            if not subdomain_find:
+                break
+            if not full_search:
+                if subdomain_find.issubset(self.subdomains):
+                    break
+            self.subdomains = self.subdomains.union(subdomain_find)
+            page_num += per_page_num
+            if 'start='+str(page_num) not in resp.text:
+                break
+            if '302 Moved' in resp.text:
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+                for subdomain in self.subdomains:
+                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                        self.search(subdomain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = Google(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/google_api.py

@@ -0,0 +1,97 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.search import Search
+from config import logger
+
+
+class GoogleAPI(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'GoogleAPISearch'
+        self.addr = 'https://www.googleapis.com/customsearch/v1'
+        self.delay = 1
+        self.key = config.google_api_key
+        self.cx = config.google_api_cx
+        self.per_page_num = 10  # 每次只能请求10个结果
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 1
+        while True:
+            word = 'site:' + domain + filtered_subdomain
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            params = {'key': self.key, 'cx': self.cx, 'q': word, 'fields': 'items/link',
+                      'start': self.page_num, 'num': self.per_page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomain_find = self.match(domain, str(resp.json()))
+            if not subdomain_find:
+                break
+            if not full_search:
+                if subdomain_find.issubset(self.subdomains):
+                    break
+            self.subdomains = self.subdomains.union(subdomain_find)
+            self.page_num += self.per_page_num
+            if self.page_num > 100:  # 免费的API只能查询前100条结果
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        if not (self.cx and self.key):
+            logger.log('ERROR', f'{self.source}模块API配置错误')
+            logger.log('ALERT', f'不执行{self.source}模块')
+            return
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+                for subdomain in self.subdomains:
+                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                        self.search(subdomain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = GoogleAPI(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/shodan_api.py

@@ -0,0 +1,73 @@
+# coding=utf-8
+import time
+import queue
+import queue
+import config
+# from shodan import Shodan
+from common.search import Search
+from config import logger
+
+
+class ShodanAPI(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Search'
+        self.source = 'ShodanSearch'
+        self.addr = 'https://api.shodan.io/shodan/host/search'
+        self.key = config.shodan_api_key
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        query = 'hostname:.' + self.domain
+        page = 1
+        while True:
+            params = {'key': self.key, 'page': page, 'query': query, 'minify': True, 'facets': {'hostnames'}}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomain_find = self.match(self.domain, resp.text)
+            if subdomain_find:
+                self.subdomains = self.subdomains.union(subdomain_find)
+            page += 1
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        if not self.key:
+            logger.log('ERROR', f'{self.source}模块API配置错误')
+            logger.log('ALERT', f'不执行{self.source}模块')
+            return
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = ShodanAPI(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    results = queue.Queue()
+    do('qq.com', results)

oneforall/modules/search/so.py

@@ -0,0 +1,91 @@
+# coding=utf-8
+import time
+import queue
+from common.search import Search
+from config import logger
+
+
+class So(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'SoSearch'
+        self.addr = 'http://www.so.com/s'
+        self.limit_num = 640  # 限制搜索条数
+        self.per_page_num = 10  # 默认每页显示10页
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        page_num = 1
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            word = 'site:' + domain + filtered_subdomain
+            payload = {'q': word, 'pn': page_num}
+            resp = self.get(url=self.addr, params=payload)
+            if not resp:
+                return
+            subdomain_find = self.match(domain, resp.text)
+            if not subdomain_find:
+                break
+            if not full_search:
+                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                    break
+            self.subdomains = self.subdomains.union(subdomain_find)
+            page_num += 1
+            if '<a id="snext"' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+                break
+            if self.page_num * self.per_page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+                for subdomain in self.subdomains:
+                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                        self.search(subdomain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = So(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/sogou.py

@@ -0,0 +1,89 @@
+# coding=utf-8
+import time
+import queue
+from common.search import Search
+from config import logger
+
+
+class Sogou(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'SogouSearch'
+        self.addr = 'https://www.sogou.com/web'
+        self.limit_num = 1000  # 限制搜索条数
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 1
+        while True:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            word = 'site:' + domain + filtered_subdomain
+            payload = {'query': word, 'page': self.page_num, "num": self.per_page_num}
+            resp = self.get(self.addr, payload)
+            if not resp:
+                return
+            subdomain_find = self.match(domain, resp.text)
+            if not subdomain_find:
+                break
+            if not full_search:
+                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                    break
+            self.subdomains = self.subdomains.union(subdomain_find)
+            self.page_num += 1
+            if '<a id="sogou_next"' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+                break
+            if self.page_num * self.per_page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+                for subdomain in self.subdomains:
+                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                        self.search(subdomain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = Sogou(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/yahoo.py

@@ -0,0 +1,97 @@
+# coding=utf-8
+import time
+import queue
+from common.search import Search
+from config import logger
+
+
+class Yahoo(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'YahooSearch'
+        self.init = 'https://hk.search.yahoo.com/'
+        self.addr = 'https://hk.search.yahoo.com/search'
+        self.limit_num = 1000  # 限制搜索条数
+        self.delay = 5
+        self.per_page_num = 40
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 0
+        resp = self.get(self.init)
+        if not resp:
+            return
+        self.cookie = resp.cookies  # 获取cookie Yahoo在搜索时需要带上cookie
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:' + domain + filtered_subdomain
+            params = {'q': query, 'b': self.page_num, 'n': self.per_page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomains_find = self.match(domain, resp.text)
+            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+                break
+            if not full_search:
+                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                    break
+            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            if '>Next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+                break
+            self.page_num += self.per_page_num
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+                for subdomain in self.subdomains:
+                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                        self.search(subdomain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = Yahoo(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/yandex.py

@@ -0,0 +1,93 @@
+# coding=utf-8
+import time
+import queue
+from common.search import Search
+from config import logger
+
+
+class Yandex(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'YandexSearch'
+        self.init = 'https://yandex.com/'
+        self.addr = 'https://yandex.com/search'
+        self.limit_num = 1000  # 限制搜索条数
+        self.delay = 5
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 0  # 二次搜索重新置0
+        self.cookie = self.get(self.init).cookies  # 获取cookie bing在搜索时需要带上cookie
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:' + domain + filtered_subdomain
+            params = {'text': query, 'p': self.page_num, 'numdoc': self.per_page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomains_find = self.match(domain, resp.text)
+            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+                break
+            if not full_search:
+                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                    break
+            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            if '>next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+                break
+            self.page_num += 1
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+                for subdomain in self.subdomains:
+                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                        self.search(subdomain)
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = Yandex(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/modules/search/zoomeye_api.py

@@ -0,0 +1,97 @@
+# coding=utf-8
+import time
+import queue
+import config
+from common.search import Search
+from config import logger
+
+
+class ZoomEyeAPI(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'ZoomEyeAPISearch'
+        self.addr = 'https://api.zoomeye.org/web/search'
+        self.delay = 2
+        self.user = config.zoomeye_api_username
+        self.pwd = config.zoomeye_api_password
+
+    def login(self):
+        """
+        登陆获取查询taken
+        :return:
+        """
+        url = 'https://api.zoomeye.org/user/login'
+        data = {'username': self.user, 'password': self.pwd}
+        resp = self.post(url=url, json=data)
+        if not resp:
+            logger.log('FETAL', f'登录失败无法获取{self.source}的访问token')
+            return
+        resp_json = resp.json()
+        if resp.status_code == 200:
+            # print('登陆成功')
+            return resp_json.get('access_token')
+        else:
+            logger.log('ALERT', resp_json.get('message'))
+            exit(1)
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        page_num = 1
+        access_token = self.login()
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            self.header.update({'Authorization': 'JWT ' + access_token})
+            params = {'query': 'hostname:' + self.domain, 'page': page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomain_find = self.match(self.domain, resp.text)
+            self.subdomains = self.subdomains.union(subdomain_find)
+            page_num += 1
+            if page_num > 500:
+                break
+            if resp.status_code == 403:
+                break
+
+    def run(self, rx_queue):
+        """
+        类执行入口
+        """
+        if not (self.user and self.pwd):
+            logger.log('ERROR', f'{self.source}模块API配置错误')
+            logger.log('ALERT', f'不执行{self.source}模块')
+            return
+        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        start = time.time()
+        self.search()
+        end = time.time()
+        self.elapsed = round(end - start, 1)
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+        rx_queue.put(self.results)
+        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
+
+
+def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param rx_queue: 结果集队列
+    """
+    search = ZoomEyeAPI(domain)
+    search.run(rx_queue)
+    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
+    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+
+
+if __name__ == '__main__':
+    result_queue = queue.Queue()
+    do('owasp.org', result_queue)

oneforall/oneforall.py

@@ -0,0 +1,98 @@
+#!/usr/bin/python3
+# coding=utf-8
+
+"""
+OneForAll是一款强大的子域收集神器
+"""
+
+import asyncio
+import fire
+import config
+import dbexport
+from config import logger
+from collect import Collect
+from aiobrute import AIOBrute
+from common import utils, database, resolve, request
+
+__author__ = 'Jing Ling, Black Star'
+__contact__ = 'admin@hackfun.org'
+__copyright__ = 'Copyright (c) 2019, Jing Ling. All rights reserved.'
+__license__ = 'GNU General Public License v3.0'
+__version__ = '0.0.1'
+
+
+class OneForAll(object):
+    """
+    OneForAll是一款强大的子域收集神器
+
+    Version: 0.0.1
+    Project: https://github.com/shmilylty/OneForAll/
+
+    :param str target:  单个域名或者每行一个域名的文件路径
+    :param bool brute:  是否使用爆破模块（默认禁用）
+    :param str port:    HTTP请求验证的端口范围（默认medium）
+    :param int valid:   导出子域的有效性（默认1）
+    :param str format:  导出格式（默认xlsx）
+    :param str path:    导出路径(默认None)
+    :param bool output: 是否将导出数据输出到终端（默认False）
+
+    Note:
+        参数valid可选值有1，0，None，分别表示导出有效，无效，全部子域
+        参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+        参数format可选格式有'csv','tsv','json','yaml','html','xls','xlsx','dbf','latex','ods'
+        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+
+    Example:
+    python oneforall.py --target example.com run
+    python oneforall.py --target example.com --brute True --port medium valid 1 run
+    python oneforall.py --target ./domains.txt --format csv --path= ./result.csv  --output True run
+    """
+    def __init__(self, target, brute=False, port='medium', valid=1, path=None, format='xlsx', output=False):
+        self.target = target
+        self.port = port
+        self.domains = set()
+        self.domain = ''
+        self.datas = list()
+        self.brute = brute or config.enable_brute_module
+        self.valid = valid
+        self.path = path
+        self.format = format
+        self.output = output
+
+    def run(self):
+        logger.log('INFOR', f'开始运行OneForAll')
+        self.domains = utils.get_domains(self.target)
+        if self.domains:
+            for self.domain in self.domains:
+                collect = Collect(self.domain, export=False)
+                collect.run()
+                if self.brute:
+                    # 由于爆破会有大量dns解析请求 并发常常会导致其他任务中的网络请求超时
+                    brute = AIOBrute(self.domain)
+                    brute.run()
+                table_name = self.domain.replace('.', '_')
+                db_conn = database.connect_db()
+                self.datas = database.get_data(db_conn, table_name).as_dict()
+                loop = asyncio.get_event_loop()
+                asyncio.set_event_loop(loop)
+                self.datas = loop.run_until_complete(resolve.bulk_query_a(self.datas))
+                self.datas = loop.run_until_complete(request.bulk_get_request(self.datas, self.port))
+                loop.run_until_complete(asyncio.sleep(0.25))  # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
+                loop.close()
+                database.clear_table(db_conn, table_name)
+                database.save_db(db_conn, table_name, self.datas)
+                # database.deduplicate_subdomain(db_conn, table_name)
+                # database.remove_invalid(db_conn, table_name)
+                # 数据库导出
+                if not self.path:
+                    self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
+                dbexport.export(table_name, db_conn, self.valid, self.path, self.format, self.output)
+                db_conn.close()
+        else:
+            logger.log('FATAL', f'获取域名失败')
+        logger.log('INFOR', f'结束运行OneForAll')
+
+
+if __name__ == '__main__':
+    fire.Fire(OneForAll)
+    # OneForAll('owasp.org').run()