重构项目目录结构

common/__init__.py

@@ -0,0 +1 @@
+# coding=utf-8

common/crawl.py

@@ -0,0 +1,9 @@
+from .module import Module
+
+
+class Crawl(Module):
+    """
+    爬虫基类
+    """
+    def __init__(self):
+        Module.__init__(self)

common/database.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+"""
+SQLite数据库初始化和操作
+"""
+
+import records
+
+from records import Connection
+from config.log import logger
+from config import setting
+
+
+class Database(object):
+    def __init__(self, db_path=None):
+        self.conn = self.get_conn(db_path)
+
+    @staticmethod
+    def get_conn(db_path):
+        """
+        获取数据库对象
+
+        :param db_path: 数据库连接或路径
+        :return: SQLite数据库
+        """
+        logger.log('TRACE', f'正在获取数据库连接')
+        if isinstance(db_path, Connection):
+            return db_path
+        protocol = 'sqlite:///'
+        if not db_path:  # 数据库路径为空连接默认数据库
+            db_path = f'{protocol}{setting.result_save_dir}/result.sqlite3'
+        else:
+            db_path = protocol + db_path
+        db = records.Database(db_path)  # 不存在数据库时会新建一个数据库
+        logger.log('TRACE', f'使用数据库: {db_path}')
+        return db.get_connection()
+
+    def query(self, sql):
+        try:
+            results = self.conn.query(sql)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+        else:
+            return results
+
+    def create_table(self, table_name):
+        """
+        创建表结构
+
+        :param str table_name: 要创建的表名
+        """
+        table_name = table_name.replace('.', '_')
+        if self.exist_table(table_name):
+            logger.log('TRACE', f'已经存在{table_name}表')
+            return
+        logger.log('TRACE', f'正在创建{table_name}表')
+        self.query(f'create table "{table_name}" ('
+                   f'id integer primary key,'
+                   f'type text,'
+                   f'alive int,'
+                   f'request int,'
+                   f'resolve int,'
+                   f'new int,'
+                   f'url text,'
+                   f'subdomain text,'
+                   f'port int,'
+                   f'level int,'
+                   f'cname text,'
+                   f'content text,'
+                   f'public int,'
+                   f'status int,'
+                   f'reason text,'
+                   f'title text,'
+                   f'banner text,'
+                   f'header text,'
+                   f'response text,'
+                   f'times text,'
+                   f'ttl text,'
+                   f'resolver text,'
+                   f'module text,'
+                   f'source text,'
+                   f'elapse float,'
+                   f'find int,'
+                   f'brute int,'
+                   f'valid int)')
+
+    def save_db(self, table_name, results, module_name=None):
+        """
+        将各模块结果存入数据库
+
+        :param str table_name: 表名
+        :param list results: 结果列表
+        :param str module_name: 模块名
+        """
+        logger.log('TRACE', f'正在将{module_name}模块发现{table_name}的子域'
+                            '结果存入数据库')
+        table_name = table_name.replace('.', '_')
+        if results:
+            try:
+                self.conn.bulk_query(
+                    f'insert into "{table_name}" ('
+                    f'id, type, alive, resolve, request, new, url, subdomain,'
+                    f'port, level, cname, content, public, status, reason,'
+                    f'title, banner, header, response, times, ttl, resolver,'
+                    f'module, source, elapse, find, brute, valid) '
+                    f'values (:id, :type, :alive, :resolve, :request, :new,'
+                    f':url, :subdomain, :port, :level, :cname, :content,'
+                    f':public, :status, :reason, :title, :banner, :header,'
+                    f':response, :times, :ttl, :resolver, :module, :source,'
+                    f':elapse, :find, :brute, :valid)', results)
+            except Exception as e:
+                logger.log('ERROR', e)
+
+    def exist_table(self, table_name):
+        """
+        判断是否存在某表
+
+        :param str table_name: 表名
+        :return: 是否存在某表
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('TRACE', f'正在查询是否存在{table_name}表')
+        results = self.query(f'select count() from sqlite_master '
+                             f'where type = "table" and '
+                             f'name = "{table_name}"')
+        if results.scalar() == 0:
+            return False
+        else:
+            return True
+
+    def copy_table(self, table_name, bak_table_name):
+        """
+        复制表创建备份
+
+        :param str table_name: 表名
+        :param str bak_table_name: 新表名
+        """
+        table_name = table_name.replace('.', '_')
+        bak_table_name = bak_table_name.replace('.', '_')
+        logger.log('TRACE', f'正在将{table_name}表复制到{bak_table_name}新表')
+        self.query(f'drop table if exists "{bak_table_name}"')
+        self.query(f'create table "{bak_table_name}" '
+                   f'as select * from "{table_name}"')
+
+    def clear_table(self, table_name):
+        """
+        清空表中数据
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('TRACE', f'正在清空{table_name}表中的数据')
+        self.query(f'delete from "{table_name}"')
+
+    def drop_table(self, table_name):
+        """
+        删除表
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('TRACE', f'正在删除{table_name}表')
+        self.query(f'drop table if exists "{table_name}"')
+
+    def rename_table(self, table_name, new_table_name):
+        """
+        重命名表名
+
+        :param str table_name: 表名
+        :param str new_table_name: 新表名
+        """
+        table_name = table_name.replace('.', '_')
+        new_table_name = new_table_name.replace('.', '_')
+        logger.log('TRACE', f'正在将{table_name}表重命名为{table_name}表')
+        self.query(f'alter table "{table_name}" '
+                   f'rename to "{new_table_name}"')
+
+    def deduplicate_subdomain(self, table_name):
+        """
+        去重表中的子域
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('TRACE', f'正在去重{table_name}表中的子域')
+        self.query(f'delete from "{table_name}" where '
+                   f'id not in (select min(id) '
+                   f'from "{table_name}" group by subdomain)')
+
+    def remove_invalid(self, table_name):
+        """
+        去除表中的空值或无效子域
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('TRACE', f'正在去除{table_name}表中的无效子域')
+        self.query(f'delete from "{table_name}" where '
+                   f'subdomain is null or resolve == 0')
+
+    def deal_table(self, deal_table_name, backup_table_name):
+        """
+        收集任务完成时对表进行处理
+
+        :param str deal_table_name: 待处理的表名
+        :param str backup_table_name: 备份的表名
+        """
+        self.copy_table(deal_table_name, backup_table_name)
+        self.remove_invalid(deal_table_name)
+        self.deduplicate_subdomain(deal_table_name)
+
+    def get_data(self, table_name):
+        """
+        获取表中的所有数据
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('TRACE', f'获取{table_name}表中的所有数据')
+        return self.query(f'select * from "{table_name}"')
+
+    def export_data(self, table_name, alive, limit):
+        """
+        获取表中的部分数据
+
+        :param str table_name: 表名
+        :param any alive: 存活
+        :param str limit: 限制字段
+        """
+        table_name = table_name.replace('.', '_')
+        query = f'select id, type, new, alive, request, resolve, url, ' \
+                f'subdomain, level, cname, content, public, port, status, ' \
+                f'reason, title, banner, times, ttl, resolver, module, ' \
+                f'source, elapse, find, brute, valid from "{table_name}"'
+        if alive and limit:
+            if limit in ['resolve', 'request']:
+                where = f' where {limit} = 1'
+                query += where
+        elif alive:
+            where = f' where alive = 1'
+            query += where
+        logger.log('TRACE', f'获取{table_name}表中的数据')
+        return self.query(query)
+
+    def close(self):
+        """
+        关闭数据库连接
+        """
+        self.conn.close()

common/domain.py

@@ -0,0 +1,64 @@
+import re
+import tldextract
+from config import setting
+
+
+class Domain(object):
+    """
+    域名处理类
+
+    :param str string: 传入的字符串
+    """
+    def __init__(self, string):
+        self.string = str(string)
+        self.regexp = r'\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b'
+        self.domain = None
+
+    def match(self):
+        """
+        域名匹配
+
+        :return: 匹配结果
+        """
+        result = re.search(self.regexp, self.string, re.I)
+        if result:
+            return result.group()
+        else:
+            return None
+
+    def extract(self):
+        """
+        域名导出
+
+        >>> d = Domain('www.example.com')
+        <domain.Domain object>
+        >>> d.extract()
+        ExtractResult(subdomain='www', domain='example', suffix='com')
+
+        :return: 导出结果
+        """
+        data_storage_dir = setting.data_storage_dir
+        extract_cache_file = data_storage_dir.joinpath('public_suffix_list.dat')
+        tldext = tldextract.TLDExtract(extract_cache_file)
+        result = self.match()
+        if result:
+            return tldext(result)
+        else:
+            return None
+
+    def registered(self):
+        """
+        获取注册域名
+
+        >>> d = Domain('www.example.com')
+        <domain.Domain object>
+        >>> d.registered()
+        example.com
+
+        :return: 注册域名
+        """
+        result = self.extract()
+        if result:
+            return result.registered_domain
+        else:
+            return None

common/lookup.py

@@ -0,0 +1,25 @@
+from .module import Module
+from common import utils
+
+
+class Lookup(Module):
+    """
+    DNS查询基类
+    """
+    def __init__(self):
+        Module.__init__(self)
+
+    def query(self):
+        """
+        查询域名的TXT记录
+        :return: 查询结果
+        """
+        answer = utils.dns_query(self.domain, self.type)
+        if answer is None:
+            return None
+        for item in answer:
+            record = item.to_text()
+            subdomains = utils.match_subdomain(self.domain, record)
+            self.subdomains = self.subdomains.union(subdomains)
+            self.gen_record(subdomains, record)
+        return self.subdomains

common/module.py

@@ -0,0 +1,359 @@
+# coding=utf-8
+"""
+模块基类
+"""
+
+import json
+import re
+import threading
+import time
+
+import requests
+from config.log import logger
+from config import setting
+from . import utils
+from .domain import Domain
+from common.database import Database
+
+lock = threading.Lock()
+
+
+class Module(object):
+    def __init__(self):
+        self.module = 'Module'
+        self.source = 'BaseModule'
+        self.cookie = None
+        self.header = dict()
+        self.proxy = None
+        self.delay = setting.request_delay  # 请求睡眠时延
+        self.timeout = setting.request_timeout  # 请求超时时间
+        self.verify = setting.request_verify  # 请求SSL验证
+        self.domain = str()  # 当前进行子域名收集的主域
+        self.type = 'A'  # 对主域进行子域收集时利用的DNS记录查询类型(默认利用A记录)
+        self.subdomains = set()  # 存放发现的子域
+        self.records = dict()  # 存放子域解析记录
+        self.results = list()  # 存放模块结果
+        self.start = time.time()  # 模块开始执行时间
+        self.end = None  # 模块结束执行时间
+        self.elapse = None  # 模块执行耗时
+
+    def check(self, *apis):
+        """
+        简单检查是否配置了api信息
+
+        :param apis: api信息元组
+        :return: 检查结果
+        """
+        if not all(apis):
+            logger.log('ALERT', f'{self.source}模块没有配置API跳过执行')
+            return False
+        return True
+
+    def begin(self):
+        """
+        输出模块开始信息
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块收集{self.domain}的子域')
+
+    def finish(self):
+        """
+        输出模块结束信息
+        """
+        self.end = time.time()
+        self.elapse = round(self.end - self.start, 1)
+        logger.log('DEBUG', f'结束执行{self.source}模块收集{self.domain}的子域')
+        logger.log('INFOR', f'{self.source}模块耗时{self.elapse}秒发现子域'
+                            f'{len(self.subdomains)}个')
+        logger.log('DEBUG', f'{self.source}模块发现{self.domain}的子域\n'
+                            f'{self.subdomains}')
+
+    def head(self, url, params=None, check=True, **kwargs):
+        """
+        自定义head请求
+
+        :param str url: 请求地址
+        :param dict params: 请求参数
+        :param bool check: 检查响应
+        :param kwargs: 其他参数
+        :return: requests响应对象
+        """
+        try:
+            resp = requests.head(url,
+                                 params=params,
+                                 cookies=self.cookie,
+                                 headers=self.header,
+                                 proxies=self.proxy,
+                                 timeout=self.timeout,
+                                 verify=self.verify,
+                                 **kwargs)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return None
+        if not check:
+            return resp
+        if utils.check_response('HEAD', resp):
+            return resp
+        return None
+
+    def get(self, url, params=None, check=True, **kwargs):
+        """
+        自定义get请求
+
+        :param str url: 请求地址
+        :param dict params: 请求参数
+        :param bool check: 检查响应
+        :param kwargs: 其他参数
+        :return: requests响应对象
+        """
+        try:
+            resp = requests.get(url,
+                                params=params,
+                                cookies=self.cookie,
+                                headers=self.header,
+                                proxies=self.proxy,
+                                timeout=self.timeout,
+                                verify=self.verify,
+                                **kwargs)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return None
+        if not check:
+            return resp
+        if utils.check_response('GET', resp):
+            return resp
+        return None
+
+    def post(self, url, data=None, check=True, **kwargs):
+        """
+        自定义post请求
+
+        :param str url: 请求地址
+        :param dict data: 请求数据
+        :param bool check: 检查响应
+        :param kwargs: 其他参数
+        :return: requests响应对象
+        """
+        try:
+            resp = requests.post(url,
+                                 data=data,
+                                 cookies=self.cookie,
+                                 headers=self.header,
+                                 proxies=self.proxy,
+                                 timeout=self.timeout,
+                                 verify=self.verify,
+                                 **kwargs)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return None
+        if not check:
+            return resp
+        if utils.check_response('POST', resp):
+            return resp
+        return None
+
+    def get_header(self):
+        """
+        获取请求头
+
+        :return: 请求头
+        """
+        # logger.log('DEBUG', f'获取请求头')
+        if setting.enable_fake_header:
+            return utils.gen_fake_header()
+        else:
+            return self.header
+
+    def get_proxy(self, module):
+        """
+        获取代理
+
+        :param str module: 模块名
+        :return: 代理字典
+        """
+        if not setting.enable_proxy:
+            logger.log('TRACE', f'所有模块不使用代理')
+            return self.proxy
+        if setting.proxy_all_module:
+            logger.log('TRACE', f'{module}模块使用代理')
+            return utils.get_random_proxy()
+        if module in setting.proxy_partial_module:
+            logger.log('TRACE', f'{module}模块使用代理')
+            return utils.get_random_proxy()
+        else:
+            logger.log('TRACE', f'{module}模块不使用代理')
+            return self.proxy
+
+    @staticmethod
+    def match(domain, html, distinct=True):
+        """
+        正则匹配出子域
+
+        :param str domain: 域名
+        :param str html: 要匹配的html响应体
+        :param bool distinct: 匹配结果去除
+        :return: 匹配出的子域集合或列表
+        :rtype: set or list
+        """
+        logger.log('TRACE', f'正则匹配响应体中的子域')
+        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
+                 r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+                 + domain.replace('.', r'\.')
+        result = re.findall(regexp, html, re.I)
+        if not result:
+            return set()
+        regexp = r'(?:http://|https://)'
+        deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
+        if distinct:
+            return set(deal)
+        else:
+            return list(deal)
+
+    @staticmethod
+    def register(domain):
+        """
+        获取注册域名
+
+        :param str domain: 域名
+        :return: 注册域名
+        """
+        return Domain(domain).registered()
+
+    def save_json(self):
+        """
+        将各模块结果保存为json文件
+
+        :return： 是否保存成功
+        """
+        if not setting.save_module_result:
+            return False
+        logger.log('TRACE', f'将{self.source}模块发现的子域结果保存为json文件')
+        path = setting.result_save_dir.joinpath(self.domain, self.module)
+        path.mkdir(parents=True, exist_ok=True)
+        name = self.source + '.json'
+        path = path.joinpath(name)
+        with open(path, mode='w', encoding='utf-8', errors='ignore') as file:
+            result = {'domain': self.domain,
+                      'name': self.module,
+                      'source': self.source,
+                      'elapse': self.elapse,
+                      'find': len(self.subdomains),
+                      'subdomains': list(self.subdomains),
+                      'records': self.records}
+            json.dump(result, file, ensure_ascii=False, indent=4)
+        return True
+
+    def gen_record(self, subdomains, record):
+        """
+        生成记录字典
+        """
+        item = dict()
+        item['content'] = record
+        for subdomain in subdomains:
+            self.records[subdomain] = item
+
+    def gen_result(self, find=0, brute=None, valid=0):
+        """
+        生成结果
+        """
+        logger.log('DEBUG', f'正在生成最终结果')
+        if not len(self.subdomains):  # 该模块一个子域都没有发现的情况
+            logger.log('DEBUG', f'{self.source}模块收集结果为空')
+            result = {'id': None,
+                      'type': self.type,
+                      'alive': None,
+                      'request': None,
+                      'resolve': None,
+                      'new': None,
+                      'url': None,
+                      'subdomain': None,
+                      'level': None,
+                      'cname': None,
+                      'content': None,
+                      'public': None,
+                      'port': None,
+                      'status': None,
+                      'reason': None,
+                      'title': None,
+                      'banner': None,
+                      'header': None,
+                      'response': None,
+                      'times': None,
+                      'ttl': None,
+                      'resolver': None,
+                      'module': self.module,
+                      'source': self.source,
+                      'elapse': self.elapse,
+                      'find': find,
+                      'brute': brute,
+                      'valid': valid}
+            self.results.append(result)
+        else:
+            for subdomain in self.subdomains:
+                url = 'http://' + subdomain
+                level = subdomain.count('.') - self.domain.count('.')
+                record = self.records.get(subdomain)
+                if record is None:
+                    record = dict()
+                resolve = record.get('resolve')
+                request = record.get('request')
+                alive = record.get('alive')
+                if self.type != 'A':  # 不是利用的DNS记录的A记录查询子域默认都有效
+                    resolve = 1
+                    request = 1
+                    alive = 1
+                reason = record.get('reason')
+                resolver = record.get('resolver')
+                cname = record.get('cname')
+                content = record.get('content')
+                times = record.get('times')
+                ttl = record.get('ttl')
+                public = record.get('public')
+                if isinstance(cname, list):
+                    cname = ','.join(cname)
+                    content = ','.join(content)
+                    times = ','.join([str(num) for num in times])
+                    ttl = ','.join([str(num) for num in ttl])
+                    public = ','.join([str(num) for num in public])
+                result = {'id': None,
+                          'type': self.type,
+                          'alive': alive,
+                          'request': request,
+                          'resolve': resolve,
+                          'new': None,
+                          'url': url,
+                          'subdomain': subdomain,
+                          'level': level,
+                          'cname': cname,
+                          'content': content,
+                          'public': public,
+                          'port': 80,
+                          'status': None,
+                          'reason': reason,
+                          'title': None,
+                          'banner': None,
+                          'header': None,
+                          'response': None,
+                          'times': times,
+                          'ttl': ttl,
+                          'resolver': resolver,
+                          'module': self.module,
+                          'source': self.source,
+                          'elapse': self.elapse,
+                          'find': find,
+                          'brute': brute,
+                          'valid': valid,
+                          }
+                self.results.append(result)
+
+    def save_db(self):
+        """
+        将模块结果存入数据库中
+
+        """
+        logger.log('DEBUG', f'正在将结果存入到数据库')
+        lock.acquire()
+        db = Database()
+        db.create_table(self.domain)
+        db.save_db(self.domain, self.results, self.source)
+        db.close()
+        lock.release()

common/query.py

@@ -0,0 +1,9 @@
+from .module import Module
+
+
+class Query(Module):
+    """
+    查询基类
+    """
+    def __init__(self):
+        Module.__init__(self)

common/request.py

@@ -0,0 +1,274 @@
+import asyncio
+import functools
+
+import aiohttp
+import tqdm
+from aiohttp import ClientSession
+from bs4 import BeautifulSoup
+
+from common import utils
+from config.log import logger
+from config import setting
+from common.database import Database
+
+
+def get_limit_conn():
+    limit_open_conn = setting.limit_open_conn
+    if limit_open_conn is None:  # 默认情况
+        limit_open_conn = utils.get_semaphore()
+    elif not isinstance(limit_open_conn, int):  # 如果传入不是数字的情况
+        limit_open_conn = utils.get_semaphore()
+    return limit_open_conn
+
+
+def get_ports(port):
+    logger.log('DEBUG', f'正在获取请求端口范围')
+    ports = set()
+    if isinstance(port, (set, list, tuple)):
+        ports = port
+    elif isinstance(port, int):
+        if 0 <= port <= 65535:
+            ports = {port}
+    elif port in {'default', 'small', 'large'}:
+        logger.log('DEBUG', f'请求{port}等端口范围')
+        ports = setting.ports.get(port)
+    if not ports:  # 意外情况
+        logger.log('ERROR', f'指定请求端口范围有误')
+        ports = {80}
+    logger.log('INFOR', f'请求端口范围：{ports}')
+    return set(ports)
+
+
+def gen_req_data(data, ports):
+    logger.log('INFOR', f'正在生成请求地址')
+    new_data = []
+    for data in data:
+        resolve = data.get('resolve')
+        # 解析失败(0)的子域不进行http请求探测
+        if resolve == 0:
+            continue
+        subdomain = data.get('subdomain')
+        for port in ports:
+            if str(port).endswith('443'):
+                url = f'https://{subdomain}:{port}'
+                if port == 443:
+                    url = f'https://{subdomain}'
+                data['id'] = None
+                data['url'] = url
+                data['port'] = port
+                new_data.append(data)
+                data = dict(data)  # 需要生成一个新的字典对象
+            else:
+                url = f'http://{subdomain}:{port}'
+                if port == 80:
+                    url = f'http://{subdomain}'
+                data['id'] = None
+                data['url'] = url
+                data['port'] = port
+                new_data.append(data)
+                data = dict(data)  # 需要生成一个新的字典对象
+    return new_data
+
+
+async def fetch(session, url):
+    """
+    请求
+
+    :param session: session对象
+    :param str url: url地址
+    :return: 响应对象和响应文本
+    """
+    method = setting.request_method.upper()
+    timeout = aiohttp.ClientTimeout(total=None,
+                                    connect=None,
+                                    sock_read=setting.sockread_timeout,
+                                    sock_connect=setting.sockconn_timeout)
+    try:
+        if method == 'HEAD':
+            async with session.head(url,
+                                    ssl=setting.verify_ssl,
+                                    allow_redirects=setting.allow_redirects,
+                                    timeout=timeout,
+                                    proxy=setting.aiohttp_proxy) as resp:
+                text = await resp.text()
+        else:
+            async with session.get(url,
+                                   ssl=setting.verify_ssl,
+                                   allow_redirects=setting.allow_redirects,
+                                   timeout=timeout,
+                                   proxy=setting.aiohttp_proxy) as resp:
+
+                try:
+                    # 先尝试用utf-8解码
+                    text = await resp.text(encoding='utf-8', errors='strict')
+                except UnicodeError:
+                    try:
+                        # 再尝试用gb18030解码
+                        text = await resp.text(encoding='gb18030',
+                                               errors='strict')
+                    except UnicodeError:
+                        # 最后尝试自动解码
+                        text = await resp.text(encoding=None,
+                                               errors='ignore')
+        return resp, text
+    except Exception as e:
+        return e
+
+
+def get_title(markup):
+    """
+    获取标题
+
+    :param markup: html标签
+    :return: 标题
+    """
+    soup = BeautifulSoup(markup, 'html.parser')
+
+    title = soup.title
+    if title:
+        return title.text
+
+    h1 = soup.h1
+    if h1:
+        return h1.text
+
+    h2 = soup.h2
+    if h2:
+        return h2.text
+
+    h3 = soup.h3
+    if h2:
+        return h3.text
+
+    desc = soup.find('meta', attrs={'name': 'description'})
+    if desc:
+        return desc['content']
+
+    word = soup.find('meta', attrs={'name': 'keywords'})
+    if word:
+        return word['content']
+
+    text = soup.text
+    if len(text) <= 200:
+        return text
+
+    return 'None'
+
+
+def request_callback(future, index, datas):
+    result = future.result()
+    if isinstance(result, BaseException):
+        logger.log('TRACE', result.args)
+        name = utils.get_classname(result)
+        datas[index]['reason'] = name + ' ' + str(result)
+        datas[index]['request'] = 0
+        datas[index]['alive'] = 0
+    elif isinstance(result, tuple):
+        resp, text = result
+        datas[index]['reason'] = resp.reason
+        datas[index]['status'] = resp.status
+        if resp.status == 400 or resp.status >= 500:
+            datas[index]['request'] = 0
+            datas[index]['alive'] = 0
+        else:
+            datas[index]['request'] = 1
+            datas[index]['alive'] = 1
+            headers = resp.headers
+            datas[index]['banner'] = utils.get_sample_banner(headers)
+            datas[index]['header'] = str(dict(headers))[1:-1]
+            if isinstance(text, str):
+                title = get_title(text).strip()
+                datas[index]['title'] = utils.remove_invalid_string(title)
+                datas[index]['response'] = utils.remove_invalid_string(text)
+
+
+def get_connector():
+    limit_open_conn = get_limit_conn()
+    return aiohttp.TCPConnector(ttl_dns_cache=300,
+                                ssl=setting.verify_ssl,
+                                limit=limit_open_conn,
+                                limit_per_host=setting.limit_per_host)
+
+
+def get_header():
+    header = None
+    if setting.fake_header:
+        header = utils.gen_fake_header()
+    return header
+
+
+async def bulk_request(data, port):
+    ports = get_ports(port)
+    no_req_data = utils.get_filtered_data(data)
+    to_req_data = gen_req_data(data, ports)
+    method = setting.request_method
+    logger.log('INFOR', f'请求使用{method}方法')
+    logger.log('INFOR', f'正在进行异步子域请求')
+    connector = get_connector()
+    header = get_header()
+    async with ClientSession(connector=connector, headers=header) as session:
+        tasks = []
+        for i, data in enumerate(to_req_data):
+            url = data.get('url')
+            task = asyncio.ensure_future(fetch(session, url))
+            task.add_done_callback(functools.partial(request_callback,
+                                                     index=i,
+                                                     datas=to_req_data))
+            tasks.append(task)
+        # 任务列表里有任务不空时才进行解析
+        if tasks:
+            # 等待所有task完成 错误聚合到结果列表里
+            futures = asyncio.as_completed(tasks)
+            for future in tqdm.tqdm(futures,
+                                    total=len(tasks),
+                                    desc='Request Progress',
+                                    ncols=80):
+                await future
+    return to_req_data + no_req_data
+
+
+def set_loop_policy():
+    try:
+        import uvloop
+    except ImportError:
+        pass
+    else:
+        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+
+
+def run_request(domain, data, port):
+    """
+    调用子域请求入口函数
+
+    :param str domain: 待请求的主域
+    :param list data: 待请求的子域数据
+    :param str port: 待请求的端口范围
+    :return: 请求后得到的结果列表
+    :rtype: list
+    """
+    logger.log('INFOR', f'开始执行子域请求模块')
+    set_loop_policy()
+    loop = asyncio.get_event_loop()
+    asyncio.set_event_loop(loop)
+    data = utils.set_id_none(data)
+    request_coroutine = bulk_request(data, port)
+    data = loop.run_until_complete(request_coroutine)
+    # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
+    loop.run_until_complete(asyncio.sleep(0.25))
+    count = utils.count_alive(data)
+    logger.log('INFOR', f'经验证{domain}存活子域{count}个')
+    return data
+
+
+def save_data(name, data):
+    """
+    保存请求结果到数据库
+
+    :param str name: 保存表名
+    :param list data: 待保存的数据
+    """
+    db = Database()
+    db.drop_table(name)
+    db.create_table(name)
+    db.save_db(name, data, 'request')
+    db.close()

common/resolve.py

@@ -0,0 +1,169 @@
+import gc
+import json
+
+from config.log import logger
+from config import setting
+from common import utils
+from common.database import Database
+
+
+def filter_subdomain(data):
+    """
+    过滤出无解析内容的子域到新的子域列表
+
+    :param list data: 待过滤的数据列表
+    :return: 符合条件的子域列表
+    """
+    logger.log('DEBUG', f'正在过滤出待解析的子域')
+    subdomains = []
+    for data in data:
+        if not data.get('content'):
+            subdomain = data.get('subdomain')
+            subdomains.append(subdomain)
+    return subdomains
+
+
+def update_data(data, records):
+    """
+    更新解析结果
+
+    :param list data: 待更新的数据列表
+    :param dict records: 解析结果字典
+    :return: 更新后的数据列表
+    """
+    logger.log('DEBUG', f'正在更新解析结果')
+    if not records:
+        logger.log('ERROR', f'无有效解析结果')
+        return data
+    for index, items in enumerate(data):
+        if not items.get('content'):
+            subdomain = items.get('subdomain')
+            record = records.get(subdomain)
+            items.update(record)
+            data[index] = items
+    return data
+
+
+def save_data(name, data):
+    """
+    保存解析结果到数据库
+
+    :param str name: 保存表名
+    :param list data: 待保存的数据
+    """
+    logger.log('INFOR', f'正在保存解析结果')
+    db = Database()
+    db.drop_table(name)
+    db.create_table(name)
+    db.save_db(name, data, 'resolve')
+    db.close()
+
+
+def save_subdomains(save_path, subdomain_list):
+    logger.log('DEBUG', f'正在保存待解析的子域')
+    subdomain_data = '\n'.join(subdomain_list)
+    if not utils.save_data(save_path, subdomain_data):
+        logger.log('FATAL', '保存待解析的子域出错')
+        exit(1)
+
+
+def deal_output(output_path):
+    logger.log('INFOR', f'正在处理解析结果')
+    records = dict()  # 用来记录所有域名解析数据
+    with open(output_path) as fd:
+        for line in fd:
+            line = line.strip()
+            try:
+                items = json.loads(line)
+            except Exception as e:
+                logger.log('ERROR', e.args)
+                logger.log('ERROR', f'解析行{line}出错跳过解析该行')
+                continue
+            record = dict()
+            record['resolver'] = items.get('resolver')
+            qname = items.get('name')[:-1]  # 去出最右边的`.`点号
+            status = items.get('status')
+            if status != 'NOERROR':
+                record['alive'] = 0
+                record['resolve'] = 0
+                record['reason'] = status
+                records[qname] = record
+                continue
+            data = items.get('data')
+            if 'answers' not in data:
+                record['alive'] = 0
+                record['resolve'] = 0
+                record['reason'] = 'NOANSWER'
+                records[qname] = record
+                continue
+            flag = False
+            cname = list()
+            ips = list()
+            public = list()
+            ttls = list()
+            answers = data.get('answers')
+            for answer in answers:
+                if answer.get('type') == 'A':
+                    flag = True
+                    cname.append(answer.get('name')[:-1])  # 去出最右边的`.`点号
+                    ip = answer.get('data')
+                    ips.append(ip)
+                    ttl = answer.get('ttl')
+                    ttls.append(str(ttl))
+                    is_public = utils.ip_is_public(ip)
+                    public.append(str(is_public))
+                    record['resolve'] = 1
+                    record['reason'] = status
+                    record['cname'] = ','.join(cname)
+                    record['content'] = ','.join(ips)
+                    record['public'] = ','.join(public)
+                    record['ttl'] = ','.join(ttls)
+                    records[qname] = record
+            if not flag:
+                record['alive'] = 0
+                record['resolve'] = 0
+                record['reason'] = 'NOARECORD'
+                records[qname] = record
+    return records
+
+
+def run_resolve(domain, data):
+    """
+    调用子域解析入口函数
+
+    :param str domain: 待解析的主域
+    :param list data: 待解析的子域数据列表
+    :return: 解析得到的结果列表
+    :rtype: list
+    """
+    logger.log('INFOR', f'开始解析{domain}的子域')
+    subdomains = filter_subdomain(data)
+    if not subdomains:
+        return data
+
+    massdns_dir = setting.third_party_dir.joinpath('massdns')
+    result_dir = setting.result_save_dir
+    temp_dir = result_dir.joinpath('temp')
+    utils.check_dir(temp_dir)
+    massdns_path = utils.get_massdns_path(massdns_dir)
+    timestring = utils.get_timestring()
+
+    save_name = f'collected_subdomains_{domain}_{timestring}.txt'
+    save_path = temp_dir.joinpath(save_name)
+    save_subdomains(save_path, subdomains)
+    del subdomains
+    gc.collect()
+
+    output_name = f'resolved_result_{domain}_{timestring}.json'
+    output_path = temp_dir.joinpath(output_name)
+    log_path = result_dir.joinpath('massdns.log')
+
+    ns_path = setting.brute_nameservers_path
+
+    utils.call_massdns(massdns_path, save_path, ns_path,
+                       output_path, log_path, quiet_mode=True)
+
+    records = deal_output(output_path)
+    data = update_data(data, records)
+    logger.log('INFOR', f'结束解析{domain}的子域')
+    return data

common/search.py

@@ -0,0 +1,54 @@
+from config import setting
+from .module import Module
+from . import utils
+
+
+class Search(Module):
+    """
+    搜索基类
+    """
+    def __init__(self):
+        Module.__init__(self)
+        self.page_num = 0  # 要显示搜索起始条数
+        self.per_page_num = 50  # 每页显示搜索条数
+        self.recursive_search = setting.enable_recursive_search
+        self.recursive_times = setting.search_recursive_times
+
+    @staticmethod
+    def filter(domain, subdomain):
+        """
+        生成搜索过滤语句
+        使用搜索引擎支持的-site:语法过滤掉搜索页面较多的子域以发现新域
+
+        :param str domain: 域名
+        :param set subdomain: 子域名集合
+        :return: 过滤语句
+        :rtype: str
+        """
+        statements_list = []
+        subdomains_temp = set(map(lambda x: x + '.' + domain,
+                                  setting.subdomains_common))
+        subdomains_temp = list(subdomain.intersection(subdomains_temp))
+        for i in range(0, len(subdomains_temp), 2):  # 同时排除2个子域
+            statements_list.append(''.join(set(map(lambda s: ' -site:' + s,
+                                                   subdomains_temp[i:i + 2]))))
+        return statements_list
+
+    def match_location(self, domain, url):
+        """
+        匹配跳转之后的url
+        针对部分搜索引擎(如百度搜索)搜索展示url时有显示不全的情况
+        此函数会向每条结果的链接发送head请求获取响应头的location值并做子域匹配
+
+        :param str domain: 域名
+        :param str url: 展示结果的url链接
+        :return: 匹配的子域
+        :rtype set
+        """
+        resp = self.head(url, check=False, allow_redirects=False)
+        if not resp:
+            return set()
+        location = resp.headers.get('location')
+        if not location:
+            return set()
+        return set(utils.match_subdomain(domain, location))

common/utils.py

@@ -0,0 +1,588 @@
+import os
+import re
+import sys
+import time
+import random
+import platform
+import subprocess
+from ipaddress import IPv4Address, ip_address
+from stat import S_IXUSR
+
+import psutil
+import tenacity
+import requests
+from pathlib import Path
+from records import Record, RecordCollection
+from dns.resolver import Resolver
+
+from common.domain import Domain
+from config import setting
+from config.log import logger
+
+user_agents = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/68.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:61.0) '
+    'Gecko/20100101 Firefox/68.0',
+    'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0']
+
+
+def match_subdomain(domain, text, distinct=True):
+    """
+    匹配text中domain的子域名
+
+    :param str domain: 域名
+    :param str text: 响应文本
+    :param bool distinct: 结果去重
+    :return: 匹配结果
+    :rtype: set or list
+    """
+    regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+             + domain.replace('.', r'\.')
+    result = re.findall(regexp, text, re.I)
+    if not result:
+        return set()
+    deal = map(lambda s: s.lower(), result)
+    if distinct:
+        return set(deal)
+    else:
+        return list(deal)
+
+
+def gen_random_ip():
+    """
+    生成随机的点分十进制的IP字符串
+    """
+    while True:
+        ip = IPv4Address(random.randint(0, 2 ** 32 - 1))
+        if ip.is_global:
+            return ip.exploded
+
+
+def gen_fake_header():
+    """
+    生成伪造请求头
+    """
+    ua = random.choice(user_agents)
+    ip = gen_random_ip()
+    headers = {
+        'Accept': 'text/html,application/xhtml+xml,'
+                  'application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+        'Cache-Control': 'max-age=0',
+        'Connection': 'close',
+        'DNT': '1',
+        'Referer': 'https://www.google.com/',
+        'Upgrade-Insecure-Requests': '1',
+        'User-Agent': ua,
+        'X-Forwarded-For': ip,
+        'X-Real-IP': ip
+    }
+    return headers
+
+
+def get_random_proxy():
+    """
+    获取随机代理
+    """
+    try:
+        return random.choice(setting.proxy_pool)
+    except IndexError:
+        return None
+
+
+def split_list(ls, size):
+    """
+    将ls列表按size大小划分并返回新的划分结果列表
+
+    :param list ls: 要划分的列表
+    :param int size: 划分大小
+    :return 划分结果
+
+    >>> split_list([1, 2, 3, 4], 3)
+    [[1, 2, 3], [4]]
+    """
+    if size == 0:
+        return ls
+    return [ls[i:i + size] for i in range(0, len(ls), size)]
+
+
+def get_domains(target):
+    """
+    获取域名
+
+    :param set or str target:
+    :return: 域名集合
+    """
+    domains = list()
+    logger.log('DEBUG', f'正在获取域名')
+    if isinstance(target, (set, tuple)):
+        domains = list(target)
+    elif isinstance(target, list):
+        domains = target
+    elif isinstance(target, str):
+        path = Path(target)
+        if path.exists() and path.is_file():
+            with open(target, encoding='utf-8', errors='ignore') as file:
+                for line in file:
+                    line = line.lower().strip()
+                    domain = Domain(line).match()
+                    if domain:
+                        domains.append(domain)
+        else:
+            target = target.lower().strip()
+            domain = Domain(target).match()
+            if domain:
+                domains.append(domain)
+    count = len(domains)
+    if count == 0:
+        logger.log('FATAL', f'获取到{count}个域名')
+        exit(1)
+    logger.log('INFOR', f'获取到{count}个域名')
+    return domains
+
+
+def get_semaphore():
+    """
+    获取查询并发值
+
+    :return: 并发整型值
+    """
+    system = platform.system()
+    if system == 'Windows':
+        return 800
+    elif system == 'Linux':
+        return 800
+    elif system == 'Darwin':
+        return 800
+
+
+def check_dir(dir_path):
+    if not dir_path.exists():
+        logger.log('INFOR', f'不存在{dir_path}目录将会新建')
+        dir_path.mkdir(parents=True, exist_ok=True)
+
+
+def check_path(path, name, format):
+    """
+    检查结果输出目录路径
+
+    :param path: 保存路径
+    :param name: 导出名字
+    :param format: 保存格式
+    :return: 保存路径
+    """
+    filename = f'{name}.{format}'
+    default_path = setting.result_save_dir.joinpath(filename)
+    if isinstance(path, str):
+        path = repr(path).replace('\\', '/')  # 将路径中的反斜杠替换为正斜杠
+        path = path.replace('\'', '')  # 去除多余的转义
+    else:
+        path = default_path
+    path = Path(path)
+    if not path.suffix:  # 输入是目录的情况
+        path = path.joinpath(filename)
+    parent_dir = path.parent
+    if not parent_dir.exists():
+        logger.log('ALERT', f'不存在{parent_dir}目录将会新建')
+        parent_dir.mkdir(parents=True, exist_ok=True)
+    if path.exists():
+        logger.log('ALERT', f'存在{path}文件将会覆盖')
+    return path
+
+
+def check_format(format, count):
+    """
+    检查导出格式
+
+    :param format: 传入的导出格式
+    :param count: 数量
+    :return: 导出格式
+    """
+    formats = ['rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+               'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods']
+    if format == 'xls' and count > 65000:
+        logger.log('ALERT', 'xls文件限制为最多65000行')
+        logger.log('ALERT', '使用xlsx格式导出')
+        return 'xlsx'
+    if format in formats:
+        return format
+    else:
+        logger.log('ALERT', f'不支持{format}格式导出')
+        logger.log('ALERT', '默认使用csv格式导出')
+        return 'csv'
+
+
+def save_data(path, data):
+    """
+    保存数据到文件
+
+    :param path: 保存路径
+    :param data: 待存数据
+    :return: 保存成功与否
+    """
+    try:
+        with open(path, 'w', encoding="utf-8",
+                  errors='ignore', newline='') as file:
+            file.write(data)
+            return True
+    except TypeError:
+        with open(path, 'wb') as file:
+            file.write(data)
+            return True
+    except Exception as e:
+        logger.log('ERROR', e.args)
+        return False
+
+
+def check_response(method, resp):
+    """
+    检查响应 输出非正常响应返回json的信息
+
+    :param method: 请求方法
+    :param resp: 响应体
+    :return: 是否正常响应
+    """
+    if resp.status_code == 200 and resp.content:
+        return True
+    logger.log('ALERT', f'{method} {resp.url} {resp.status_code} - '
+                        f'{resp.reason} {len(resp.content)}')
+    content_type = resp.headers.get('Content-Type')
+    if content_type and 'json' in content_type and resp.content:
+        try:
+            msg = resp.json()
+        except Exception as e:
+            logger.log('DEBUG', e.args)
+        else:
+            logger.log('ALERT', msg)
+    return False
+
+
+def mark_subdomain(old_data, now_data):
+    """
+    标记新增子域并返回新的数据集
+
+    :param list old_data: 之前子域数据
+    :param list now_data: 现在子域数据
+    :return: 标记后的的子域数据
+    :rtype: list
+    """
+    # 第一次收集子域的情况
+    mark_data = now_data.copy()
+    if not old_data:
+        for index, item in enumerate(mark_data):
+            item['new'] = 1
+            mark_data[index] = item
+        return mark_data
+    # 非第一次收集子域的情况
+    old_subdomains = {item.get('subdomain') for item in old_data}
+    for index, item in enumerate(mark_data):
+        subdomain = item.get('subdomain')
+        if subdomain in old_subdomains:
+            item['new'] = 0
+        else:
+            item['new'] = 1
+        mark_data[index] = item
+    return mark_data
+
+
+def remove_invalid_string(string):
+    # Excel文件中单元格值不能直接存储以下非法字符
+    return re.sub(r'[\000-\010]|[\013-\014]|[\016-\037]', r'', string)
+
+
+def check_value(values):
+    if not isinstance(values, dict):
+        return values
+    for key, value in values.items():
+        if value is None:
+            continue
+        if isinstance(value, str) and len(value) > 32767:
+            # Excel文件中单元格值长度不能超过32767
+            values[key] = value[:32767]
+    return values
+
+
+def export_all(format, path, datas):
+    """
+    将所有结果数据导出到一个文件
+
+    :param str format: 导出文件格式
+    :param str path: 导出文件路径
+    :param list datas: 待导出的结果数据
+    """
+    format = check_format(format, len(datas))
+    timestamp = get_timestring()
+    name = f'all_subdomain_result_{timestamp}'
+    path = check_path(path, name, format)
+    logger.log('INFOR', f'所有主域的子域结果 {path}')
+    row_list = list()
+    for row in datas:
+        if 'header' in row:
+            row.pop('header')
+        if 'response' in row:
+            row.pop('response')
+        keys = row.keys()
+        values = row.values()
+        if format in {'xls', 'xlsx'}:
+            values = check_value(values)
+        row_list.append(Record(keys, values))
+    rows = RecordCollection(iter(row_list))
+    content = rows.export(format)
+    save_data(path, content)
+
+
+def dns_resolver():
+    """
+    dns解析器
+    """
+    resolver = Resolver()
+    resolver.nameservers = setting.resolver_nameservers
+    resolver.timeout = setting.resolver_timeout
+    resolver.lifetime = setting.resolver_lifetime
+    return resolver
+
+
+def dns_query(qname, qtype):
+    """
+    查询域名DNS记录
+
+    :param str qname: 待查域名
+    :param str qtype: 查询类型
+    :return: 查询结果
+    """
+    logger.log('TRACE', f'尝试查询{qname}的{qtype}记录')
+    resolver = dns_resolver()
+    try:
+        answer = resolver.query(qname, qtype)
+    except Exception as e:
+        logger.log('TRACE', e.args)
+        logger.log('TRACE', f'查询{qname}的{qtype}记录失败')
+        return None
+    else:
+        logger.log('TRACE', f'查询{qname}的{qtype}记录成功')
+        return answer
+
+
+def get_timestamp():
+    return int(time.time())
+
+
+def get_timestring():
+    return time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time()))
+
+
+def get_classname(classobj):
+    return classobj.__class__.__name__
+
+
+def python_version():
+    return sys.version
+
+
+def count_alive(data):
+    return len(list(filter(lambda item: item.get('alive') == 1, data)))
+
+
+def get_subdomains(data):
+    return set(map(lambda item: item.get('subdomain'), data))
+
+
+def set_id_none(data):
+    new_data = []
+    for item in data:
+        item['id'] = None
+        new_data.append(item)
+    return new_data
+
+
+def get_filtered_data(data):
+    filtered_data = []
+    for item in data:
+        valid = item.get('resolve')
+        if valid == 0:
+            filtered_data.append(item)
+    return filtered_data
+
+
+def get_sample_banner(headers):
+    temp_list = []
+    server = headers.get('Server')
+    if server:
+        temp_list.append(server)
+    via = headers.get('Via')
+    if via:
+        temp_list.append(via)
+    power = headers.get('X-Powered-By')
+    if power:
+        temp_list.append(power)
+    banner = ','.join(temp_list)
+    return banner
+
+
+def check_ip_public(ip_list):
+    for ip_str in ip_list:
+        ip = ip_address(ip_str)
+        if not ip.is_global:
+            return 0
+    return 1
+
+
+def ip_is_public(ip_str):
+    ip = ip_address(ip_str)
+    if not ip.is_global:
+        return 0
+    return 1
+
+
+def get_process_num():
+    process_num = setting.brute_process_num
+    if isinstance(process_num, int):
+        return min(os.cpu_count(), process_num)
+    else:
+        return 1
+
+
+def get_coroutine_num():
+    coroutine_num = setting.resolve_coroutine_num
+    if isinstance(coroutine_num, int):
+        return max(64, coroutine_num)
+    elif coroutine_num is None:
+        mem = psutil.virtual_memory()
+        total_mem = mem.total
+        g_size = 1024 * 1024 * 1024
+        if total_mem <= 1 * g_size:
+            return 64
+        elif total_mem <= 2 * g_size:
+            return 128
+        elif total_mem <= 4 * g_size:
+            return 256
+        elif total_mem <= 8 * g_size:
+            return 512
+        elif total_mem <= 16 * g_size:
+            return 1024
+        else:
+            return 2048
+    else:
+        return 64
+
+
+def uniq_dict_list(dict_list):
+    return list(filter(lambda name: dict_list.count(name) == 1, dict_list))
+
+
+def delete_file(*paths):
+    for path in paths:
+        try:
+            path.unlink()
+        except Exception as e:
+            logger.log('ERROR', e.args)
+
+
+@tenacity.retry(stop=tenacity.stop_after_attempt(3))
+def check_net():
+    logger.log('INFOR', '正在检查网络环境')
+    urls = ['http://www.example.com', 'http://www.baidu.com',
+            'http://www.bing.com', 'http://www.taobao.com',
+            'http://www.linkedin.com', 'http://www.msn.com',
+            'http://www.apple.com', 'http://microsoft.com']
+    url = random.choice(urls)
+    logger.log('INFOR', f'正在尝试访问 {url}')
+    try:
+        rsp = requests.get(url)
+    except Exception as e:
+        logger.log('ERROR', e.args)
+        logger.log('ALERT', '访问外网出错 重新检查中')
+        raise tenacity.TryAgain
+    if rsp.status_code != 200:
+        logger.log('ALERT', f'{rsp.request.method} {rsp.request.url} '
+                            f'{rsp.status_code} {rsp.reason}')
+        logger.log('ALERT', '不能正常访问外网 重新检查中')
+        raise tenacity.TryAgain
+    logger.log('INFOR', '能正常访问外网')
+
+
+def check_pre():
+    logger.log('INFOR', '正在检查依赖环境')
+    system = platform.system()
+    implementation = platform.python_implementation()
+    version = platform.python_version()
+    if implementation != 'CPython':
+        logger.log('FATAL', f'OneForAll只在CPython下测试通过')
+        exit(1)
+    if version < '3.6':
+        logger.log('FATAL', 'OneForAll需要Python 3.6以上版本')
+        exit(1)
+    if system == 'Windows' and implementation == 'CPython':
+        if version < '3.8':
+            logger.log('FATAL', 'OneForAll在Windows系统运行时需要Python 3.8以上版本')
+            exit(1)
+    if system in {"Linux", "Darwin"}:
+        try:
+            import uvloop
+        except ImportError:
+            logger.log('ALERT', f'请手动安装uvloop的Python库加速子域请求')
+
+
+def check_env():
+    logger.log('INFOR', '正在检查运行环境')
+    try:
+        check_net()
+    except Exception as e:
+        logger.log('DEBUG', e.args)
+        logger.log('FATAL', '不能正常访问外网')
+        exit(1)
+    check_pre()
+
+
+def get_maindomain(domain):
+    return Domain(domain).registered()
+
+
+def call_massdns(massdns_path, dict_path, ns_path, output_path, log_path,
+                 query_type='A', process_num=1, concurrent_num=10000,
+                 quiet_mode=False):
+    logger.log('INFOR', f'开始执行massdns')
+    quiet = ''
+    if quiet_mode:
+        quiet = '--quiet'
+    status_format = setting.brute_status_format
+    socket_num = setting.brute_socket_num
+    resolve_num = setting.brute_resolve_num
+    cmd = f'{massdns_path} {quiet} --status-format {status_format} ' \
+          f'--processes {process_num} --socket-count {socket_num} ' \
+          f'--hashmap-size {concurrent_num} --resolvers {ns_path} ' \
+          f'--resolve-count {resolve_num} --type {query_type} ' \
+          f'--flush --output J --outfile {output_path} ' \
+          f'--error-log {log_path} {dict_path}'
+    logger.log('INFOR', f'执行命令 {cmd}')
+    subprocess.run(args=cmd, shell=True)
+    logger.log('INFOR', f'结束执行massdns')
+
+
+def get_massdns_path(massdns_dir):
+    path = setting.brute_massdns_path
+    if path:
+        return path
+    system = platform.system().lower()
+    machine = platform.machine().lower()
+    name = f'massdns_{system}_{machine}'
+    if system == 'windows':
+        name = name + '.exe'
+        if machine == 'amd64':
+            massdns_dir = massdns_dir.joinpath('windows', 'x64')
+        else:
+            massdns_dir = massdns_dir.joinpath('windows', 'x84')
+    path = massdns_dir.joinpath(name)
+    path.chmod(S_IXUSR)
+    if not path.exists():
+        logger.log('FATAL', '暂无该系统平台及架构的massdns')
+        logger.log('INFOR', '请尝试自行编译massdns并在配置里指定路径')
+        exit(0)
+    return path