1.重构请求，边请求边存入数据库，解决内存占用过大问题。

2.format参数改为fmt。
3.添加信息富化模块。
4.优化iscdn模块代码。
5.移除数据库new字段。

README.md

@@ -170,7 +170,7 @@ DESCRIPTION
         python3 oneforall.py --target example.com --valid None run
         python3 oneforall.py --target example.com --brute True run
         python3 oneforall.py --target example.com --port small run
-        python3 oneforall.py --target example.com --format csv run
+        python3 oneforall.py --target example.com --fmt csv run
         python3 oneforall.py --target example.com --dns False run
         python3 oneforall.py --target example.com --req False run
         python3 oneforall.py --target example.com --takeover False run
@@ -179,7 +179,7 @@ DESCRIPTION
     Note:
         参数alive可选值True，False分别表示导出存活，全部子域结果
         参数port可选值有'default', 'small', 'large', 详见config.py配置
-        参数format可选格式有 'csv','json'
+        参数fmt可选格式有 'csv','json'
         参数path默认None使用OneForAll结果目录生成路径
 
 ARGUMENTS
@@ -199,7 +199,7 @@ FLAGS
         请求验证子域的端口范围(默认只探测80端口)
     --valid=VALID
         只导出存活的子域结果(默认False)
-    --format=FORMAT
+    --fmt=FMT
         结果保存格式(默认csv)
     --path=PATH
         结果保存路径(默认None)
@@ -269,7 +269,6 @@ FLAGS
 ## ⌛后续计划
 
 - [ ] 各模块持续优化和完善
-- [x] 子域监控（标记每次新发现的子域）
 - [x] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
 - [ ] 操作强大交互人性的前端界面实现
 

brute.py

@@ -384,7 +384,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
     info = dict()
     cnames = list()
     ips = list()
-    public = list()
     times = list()
     ttls = list()
     is_valid_flags = list()
@@ -401,7 +400,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
         cnames.append(cname)  # 去除最右边的`.`点号
         ip = answer.get('data')
         ips.append(ip)
-        public.append(utils.ip_is_public(ip))
         num = ip_times.get(ip)
         times.append(num)
         isvalid, reason = is_valid_subdomain(ip, ttl, num, wc_ips, wc_ttl, cname)
@@ -416,7 +414,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
         info['ttl'] = ttls
         info['cname'] = cnames
         info['ip'] = ips
-        info['public'] = public
         info['times'] = times
         info['resolver'] = resolver
         infos[qname] = info
@@ -559,7 +556,7 @@ class Brute(Module):
         brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
 
     Note:
-        --format csv/json (result format)
+        --fmt csv/json (result fmt)
         --path   Result path (default None, automatically generated)
 
 
@@ -578,13 +575,13 @@ class Brute(Module):
     :param str  rule:       Specify the regexp rules used in fuzz mode (required if use fuzz mode)
     :param str  fuzzlist:   Dictionary path used in fuzz mode (default use ./config/default.py)
     :param bool export:     Export the results (default True)
-    :param str  format:     Result format (default csv)
+    :param str  fmt:        Result format (default csv)
     :param str  path:       Result directory (default None)
     """
     def __init__(self, target=None, targets=None, process=None, concurrent=None,
                  word=False, wordlist=None, recursive=False, depth=None, nextlist=None,
                  fuzz=False, place=None, rule=None, fuzzlist=None, export=True,
-                 alive=True, format='csv', path=None):
+                 alive=True, fmt='csv', path=None):
         Module.__init__(self)
         self.module = 'Brute'
         self.source = 'Brute'
@@ -603,7 +600,7 @@ class Brute(Module):
         self.fuzzlist = fuzzlist or settings.fuzz_list
         self.export = export
         self.alive = alive
-        self.format = format
+        self.fmt = fmt
         self.path = path
         self.bulk = False  # 是否是批量爆破场景
         self.domains = list()  # 待爆破的所有域名集合
@@ -761,7 +758,7 @@ class Brute(Module):
 
             logger.log('INFOR', f'Finished {self.source} module to brute {self.domain}')
             if not self.path:
-                name = f'{self.domain}_brute_result.{self.format}'
+                name = f'{self.domain}_brute_result.{self.fmt}'
                 self.path = settings.result_save_dir.joinpath(name)
             # 数据库导出
             if self.export:
@@ -770,7 +767,7 @@ class Brute(Module):
                                 alive=self.alive,
                                 limit='resolve',
                                 path=self.path,
-                                format=self.format)
+                                fmt=self.fmt)
 
 
 if __name__ == '__main__':

common/database.py

@@ -57,7 +57,6 @@ class Database(object):
                    f'alive int,'
                    f'request int,'
                    f'resolve int,'
-                   f'new int,'
                    f'url text,'
                    f'subdomain text,'
                    f'port int,'
@@ -86,6 +85,20 @@ class Database(object):
                    f'elapse float,'
                    f'find int)')
 
+    def insert_table(self, table_name, result):
+        table_name = table_name.replace('.', '_')
+        self.conn.query(
+            f'insert into "{table_name}" '
+            f'(id, alive, resolve, request, url, subdomain, port, level,'
+            f'cname, ip, public, cdn, status, reason, title, banner, header,'
+            f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,'
+            f'module, source, elapse, find) '
+            f'values (:id, :alive, :resolve, :request, :url,'
+            f':subdomain, :port, :level, :cname, :ip, :public, :cdn,'
+            f':status, :reason, :title, :banner, :header, :history, :response,'
+            f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,'
+            f':source, :elapse, :find)', **result)
+
     def save_db(self, table_name, results, module_name=None):
         """
         Save the results of each module in the database
@@ -101,11 +114,11 @@ class Database(object):
             try:
                 self.conn.bulk_query(
                     f'insert into "{table_name}" '
-                    f'(id, alive, resolve, request, new, url, subdomain, port, level,'
+                    f'(id, alive, resolve, request, url, subdomain, port, level,'
                     f'cname, ip, public, cdn, status, reason, title, banner, header,'
                     f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,'
                     f'module, source, elapse, find) '
-                    f'values (:id, :alive, :resolve, :request, :new, :url,'
+                    f'values (:id, :alive, :resolve, :request, :url,'
                     f':subdomain, :port, :level, :cname, :ip, :public, :cdn,'
                     f':status, :reason, :title, :banner, :header, :history, :response,'
                     f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,'
@@ -178,7 +191,7 @@ class Database(object):
 
     def deduplicate_subdomain(self, table_name):
         """
-        Deduplicates of subdomains in the table
+        Deduplicate subdomains in the table
 
         :param str table_name: table name
         """
@@ -199,17 +212,6 @@ class Database(object):
         self.query(f'delete from "{table_name}" where '
                    f'subdomain is null or resolve == 0')
 
-    def deal_table(self, deal_table_name, backup_table_name):
-        """
-        Process the table when the collection task is complete
-
-        :param str deal_table_name: Pending table name
-        :param str backup_table_name: Table name for backup
-        """
-        self.copy_table(deal_table_name, backup_table_name)
-        self.remove_invalid(deal_table_name)
-        self.deduplicate_subdomain(deal_table_name)
-
     def get_data(self, table_name):
         """
         Get all the data in the table
@@ -218,7 +220,7 @@ class Database(object):
         """
         table_name = table_name.replace('.', '_')
         logger.log('TRACE', f'Get all the data from {table_name} table')
-        return self.query(f'select * from "{table_name}"')
+        return self.query(f'select * from {table_name}')
 
     def export_data(self, table_name, alive, limit):
         """
@@ -229,18 +231,42 @@ class Database(object):
         :param str limit: limit value
         """
         table_name = table_name.replace('.', '_')
-        query = f'select id, new, alive, request, resolve, url, subdomain, level,' \
+        sql = f'select id, alive, request, resolve, url, subdomain, level,' \
-                f'cname, ip, public, cdn, port, status, reason, title, banner,' \
+              f'cname, ip, public, cdn, port, status, reason, title, banner,' \
-                f'cidr, asn, org, addr, isp, source from "{table_name}"'
+              f'cidr, asn, org, addr, isp, source from {table_name} order by subdomain'
         if alive and limit:
             if limit in ['resolve', 'request']:
                 where = f' where {limit} = 1'
-                query += where
+                sql += where
         elif alive:
             where = f' where alive = 1'
-            query += where
+            sql += where
         logger.log('TRACE', f'Get the data from {table_name} table')
-        return self.query(query)
+        return self.query(sql)
+
+    def count_alive(self, table_name):
+        table_name = table_name.replace('.', '_')
+        sql = f'select count() from {table_name} where alive = 1'
+        return self.query(sql)
+
+    def get_resp_by_url(self, table_name, url):
+        table_name = table_name.replace('.', '_')
+        sql = f'select response from {table_name} where url = "{url}"'
+        logger.log('TRACE', f'Get response data from {url}')
+        return self.query(sql).scalar()
+
+    def get_data_by_fields(self, table_name, fields):
+        table_name = table_name.replace('.', '_')
+        field_str = ', '.join(fields)
+        sql = f"select {field_str} from {table_name}"
+        logger.log('TRACE', f'Get specified field data {fields} from {table_name} table')
+        return self.query(sql)
+
+    def update_data_by_url(self, table_name, info, url):
+        table_name = table_name.replace('.', '_')
+        field_str = ', '.join(map(lambda kv: f'{kv[0]} = "{kv[1]}"', info.items()))
+        sql = f'update {table_name} set {field_str} where url = "{url}"'
+        return self.query(sql)
 
     def close(self):
         """

common/module.py

@@ -275,7 +275,6 @@ class Module(object):
                       'alive': None,
                       'request': None,
                       'resolve': None,
-                      'new': None,
                       'url': None,
                       'subdomain': None,
                       'port': None,
@@ -315,25 +314,22 @@ class Module(object):
                 ip = info.get('ip')
                 times = info.get('times')
                 ttl = info.get('ttl')
-                public = info.get('public')
                 if isinstance(cname, list):
                     cname = ','.join(cname)
                     ip = ','.join(ip)
                     times = ','.join([str(num) for num in times])
                     ttl = ','.join([str(num) for num in ttl])
-                    public = ','.join([str(num) for num in public])
                 result = {'id': None,
                           'alive': info.get('alive'),
                           'request': info.get('request'),
                           'resolve': info.get('resolve'),
-                          'new': None,
                           'url': url,
                           'subdomain': subdomain,
                           'port': 80,
                           'level': level,
                           'cname': cname,
                           'ip': ip,
-                          'public': public,
+                          'public': info.get('public'),
                           'cdn': info.get('cdn'),
                           'status': None,
                           'reason': info.get('reason'),

common/request.py

@@ -1,7 +1,6 @@
 import json
 from threading import Thread
 from queue import Queue
-from operator import attrgetter
 
 import tqdm
 import requests
@@ -9,6 +8,7 @@ from bs4 import BeautifulSoup
 
 from common import utils
 from config.log import logger
+from common.database import Database
 from config import settings
 
 
@@ -127,7 +127,7 @@ def get_progress_bar(total):
     return bar
 
 
-def get(url, resp_list, session):
+def get_resp(url, session):
     timeout = settings.request_timeout_second
     redirect = settings.request_allow_redirect
     proxy = utils.get_proxy()
@@ -136,13 +136,14 @@ def get(url, resp_list, session):
     except Exception as e:
         logger.log('DEBUG', e.args)
         resp = e
-    resp_list.append((url, resp))
+    return resp
 
 
-def request(urls_queue, resp_list, session):
+def request(urls_queue, resp_queue, session):
     while not urls_queue.empty():
-        url = urls_queue.get()
+        index, url = urls_queue.get()
-        get(url, resp_list, session)
+        resp = get_resp(url, session)
+        resp_queue.put((index, resp))
         urls_queue.task_done()
 
 
@@ -168,31 +169,6 @@ def get_session():
     return session
 
 
-def bulk_request(urls):
-    logger.log('INFOR', 'Requesting urls in bulk')
-    resp_list = list()
-    urls_queue = Queue()
-    for url in urls:
-        urls_queue.put(url)
-    total = len(urls)
-    session = get_session()
-    thread_count = req_thread_count()
-    bar = get_progress_bar(total)
-
-    progress_thread = Thread(target=progress, name='ProgressThread',
-                             args=(bar, total, urls_queue), daemon=True)
-    progress_thread.start()
-
-    for i in range(thread_count):
-        request_thread = Thread(target=request, name=f'RequestThread-{i}',
-                                args=(urls_queue, resp_list, session), daemon=True)
-        request_thread.start()
-
-    urls_queue.join()
-
-    return resp_list
-
-
 def gen_new_info(info, resp):
     if isinstance(resp, Exception):
         info['reason'] = str(resp.args)
@@ -220,13 +196,54 @@ def gen_new_info(info, resp):
     return info
 
 
-def gen_new_data(data, resp_list):
+def save(name, total, req_data, resp_queue):
-    new_data = list()
+    db = Database()
-    for url, resp in resp_list:
+    db.create_table(name)
-        for info in data:
+    i = 0
-            if info.get('url') == url:
+    while True:
-                new_data.append(gen_new_info(info, resp))
+        if not resp_queue.empty():
-    return new_data
+            i += 1
+            index, resp = resp_queue.get()
+            old_info = req_data[index]
+            new_info = gen_new_info(old_info, resp)
+            db.insert_table(name, new_info)
+            resp_queue.task_done()
+        if i >= total:
+            break
+    db.close()
+
+
+def bulk_request(domain, req_data, ret=False):
+    logger.log('INFOR', 'Requesting urls in bulk')
+    resp_queue = Queue()
+    urls_queue = Queue()
+    total = len(req_data)
+    for index, info in enumerate(req_data):
+        try:
+            url = info.get('url')
+        except Exception:
+            pass
+        urls_queue.put((index, url))
+    session = get_session()
+    thread_count = req_thread_count()
+    bar = get_progress_bar(total)
+
+    progress_thread = Thread(target=progress, name='ProgressThread',
+                             args=(bar, total, urls_queue), daemon=True)
+    progress_thread.start()
+
+    for i in range(thread_count):
+        request_thread = Thread(target=request, name=f'RequestThread-{i}',
+                                args=(urls_queue, resp_queue, session), daemon=True)
+        request_thread.start()
+    if ret:
+        urls_queue.join()
+        return resp_queue
+    save_thread = Thread(target=save, name=f'SaveThread',
+                         args=(domain, total, req_data, resp_queue), daemon=True)
+    save_thread.start()
+    urls_queue.join()
+    save_thread.join()
 
 
 def run_request(domain, data, port):
@@ -242,20 +259,6 @@ def run_request(domain, data, port):
     data = utils.set_id_none(data)
     ports = get_port_seq(port)
     req_data, req_urls = gen_req_data(data, ports)
-    resp_list = bulk_request(req_urls)
+    bulk_request(domain, req_data)
-    new_data = gen_new_data(req_data, resp_list)
+    count = utils.count_alive(domain)
-    count = utils.count_alive(new_data)
     logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
-    sorted_data = utils.sort_by_subdomain(new_data)
-    return sorted_data
-
-
-def save_db(name, data):
-    """
-    Save request results to database
-
-    :param str  name: table name
-    :param list data: data to be saved
-    """
-    logger.log('INFOR', f'Saving requested results')
-    utils.save_db(name, data, 'request')

common/resolve.py

@@ -4,12 +4,6 @@ import json
 from config.log import logger
 from config import settings
 from common import utils
-from common.ipasn import IPAsnInfo
-from common.ipreg import IpRegData
-
-
-ip_asn = IPAsnInfo()
-ip_reg = IpRegData()
 
 
 def filter_subdomain(data):
@@ -79,13 +73,7 @@ def gen_infos(data, qname, info, infos):
     flag = False
     cname = list()
     ips = list()
-    public = list()
     ttl = list()
-    cidr = list()
-    asn = list()
-    org = list()
-    addr = list()
-    isp = list()
     answers = data.get('answers')
     for answer in answers:
         if answer.get('type') == 'A':
@@ -94,25 +82,11 @@ def gen_infos(data, qname, info, infos):
             ip = answer.get('data')
             ips.append(ip)
             ttl.append(str(answer.get('ttl')))
-            public.append(str(utils.ip_is_public(ip)))
-            asn_info = ip_asn.find(ip)
-            cidr.append(asn_info.get('cidr'))
-            asn.append(asn_info.get('asn'))
-            org.append(asn_info.get('org'))
-            ip_info = ip_reg.query(ip)
-            addr.append(ip_info.get('addr'))
-            isp.append(ip_info.get('isp'))
             info['resolve'] = 1
             info['reason'] = 'OK'
             info['cname'] = ','.join(cname)
             info['ip'] = ','.join(ips)
-            info['public'] = ','.join(public)
             info['ttl'] = ','.join(ttl)
-            info['cidr'] = ','.join(cidr)
-            info['asn'] = ','.join(asn)
-            info['org'] = ','.join(org)
-            info['addr'] = ','.join(addr)
-            info['isp'] = ','.join(isp)
             infos[qname] = info
     if not flag:
         info['alive'] = 0

common/utils.py

@@ -176,16 +176,16 @@ def check_dir(dir_path):
         dir_path.mkdir(parents=True, exist_ok=True)
 
 
-def check_path(path, name, format):
+def check_path(path, name, fmt):
     """
     检查结果输出目录路径
 
     :param path: 保存路径
     :param name: 导出名字
-    :param format: 保存格式
+    :param fmt: 保存格式
     :return: 保存路径
     """
-    filename = f'{name}.{format}'
+    filename = f'{name}.{fmt}'
     default_path = settings.result_save_dir.joinpath(filename)
     if isinstance(path, str):
         path = repr(path).replace('\\', '/')  # 将路径中的反斜杠替换为正斜杠
@@ -204,19 +204,18 @@ def check_path(path, name, format):
     return path
 
 
-def check_format(format, count):
+def check_format(fmt):
     """
     检查导出格式
 
-    :param format: 传入的导出格式
+    :param fmt: 传入的导出格式
-    :param count: 数量
     :return: 导出格式
     """
     formats = ['csv', 'json', ]
-    if format in formats:
+    if fmt in formats:
-        return format
+        return fmt
     else:
-        logger.log('ALERT', f'Does not support {format} format')
+        logger.log('ALERT', f'Does not support {fmt} format')
         logger.log('ALERT', 'So use csv format by default')
         return 'csv'
 
@@ -333,8 +332,8 @@ def remove_invalid_string(string):
     return re.sub(r'[\000-\010]|[\013-\014]|[\016-\037]', r'', string)
 
 
-def export_all_results(path, name, format, datas):
+def export_all_results(path, name, fmt, datas):
-    path = check_path(path, name, format)
+    path = check_path(path, name, fmt)
     logger.log('ALERT', f'The subdomain result for all main domains: {path}')
     row_list = list()
     for row in datas:
@@ -346,7 +345,7 @@ def export_all_results(path, name, format, datas):
         values = row.values()
         row_list.append(Record(keys, values))
     rows = RecordCollection(iter(row_list))
-    content = rows.export(format)
+    content = rows.export(fmt)
     save_data(path, content)
 
 
@@ -366,19 +365,19 @@ def export_all_subdomains(alive, path, name, datas):
     save_data(path, data)
 
 
-def export_all(alive, format, path, datas):
+def export_all(alive, fmt, path, datas):
     """
     将所有结果数据导出
 
     :param bool alive: 只导出存活子域结果
-    :param str format: 导出文件格式
+    :param str fmt: 导出文件格式
     :param str path: 导出文件路径
     :param list datas: 待导出的结果数据
     """
-    format = check_format(format, len(datas))
+    fmt = check_format(fmt)
     timestamp = get_timestring()
     name = f'all_subdomain_result_{timestamp}'
-    export_all_results(path, name, format, datas)
+    export_all_results(path, name, fmt, datas)
     export_all_subdomains(alive, path, name, datas)
 
 
@@ -430,10 +429,18 @@ def python_version():
     return sys.version
 
 
-def count_alive(data):
+def calc_alive(data):
     return len(list(filter(lambda item: item.get('alive') == 1, data)))
 
 
+def count_alive(name):
+    db = Database()
+    result = db.count_alive(name)
+    count = result.scalar()
+    db.close()
+    return count
+
+
 def get_subdomains(data):
     return set(map(lambda item: item.get('subdomain'), data))
 
@@ -833,3 +840,23 @@ def looks_like_ip(maybe_ip):
             return True
     except socket.error:
         return False
+
+
+def deal_data(domain):
+    db = Database()
+    db.remove_invalid(domain)
+    db.deduplicate_subdomain(domain)
+    db.close()
+
+
+def get_data(domain):
+    db = Database()
+    data = db.get_data(domain).as_dict()
+    db.close()
+    return data
+
+
+def clear_data(domain):
+    db = Database()
+    db.drop_table(domain)
+    db.close()

config/default.py

@@ -24,7 +24,7 @@ enable_dns_resolve = True  # 使用DNS解析子域(默认True)
 enable_http_request = True  # 使用HTTP请求子域(默认True)
 enable_finder_module = True  # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True)
 enable_altdns_module = False  # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
-enable_cdn_check = True  # 开启cdn检查模块(默认True)
+enable_enrich_module = True  # 开启enrich模块，开启会富化出信息，如ip的cdn，cidr，asn，org，addr和isp等信息
 enable_banner_identify = True  # 开启WEB指纹识别模块(默认True)
 enable_takeover_check = False  # 开启子域接管风险检查(默认False)
 # 参数可选值有'small', 'medium', 'large'

data/cdn_asn_list.json

@@ -1,134 +1,134 @@
 [
-    "10576",
+    "AS10576",
-    "10762",
+    "AS10762",
-    "11748",
+    "AS11748",
-    "131099",
+    "AS131099",
-    "132601",
+    "AS132601",
-    "133496",
+    "AS133496",
-    "134409",
+    "AS134409",
-    "135295",
+    "AS135295",
-    "136764",
+    "AS136764",
-    "137187",
+    "AS137187",
-    "13777",
+    "AS13777",
-    "13890",
+    "AS13890",
-    "14103",
+    "AS14103",
-    "14520",
+    "AS14520",
-    "17132",
+    "AS17132",
-    "199251",
+    "AS199251",
-    "200013",
+    "AS200013",
-    "200325",
+    "AS200325",
-    "200856",
+    "AS200856",
-    "201263",
+    "AS201263",
-    "202294",
+    "AS202294",
-    "203075",
+    "AS203075",
-    "203139",
+    "AS203139",
-    "204248",
+    "AS204248",
-    "204286",
+    "AS204286",
-    "204545",
+    "AS204545",
-    "206227",
+    "AS206227",
-    "206734",
+    "AS206734",
-    "206848",
+    "AS206848",
-    "206986",
+    "AS206986",
-    "207158",
+    "AS207158",
-    "208559",
+    "AS208559",
-    "209403",
+    "AS209403",
-    "21030",
+    "AS21030",
-    "21257",
+    "AS21257",
-    "23327",
+    "AS23327",
-    "23393",
+    "AS23393",
-    "23637",
+    "AS23637",
-    "23794",
+    "AS23794",
-    "24997",
+    "AS24997",
-    "26492",
+    "AS26492",
-    "268843",
+    "AS268843",
-    "28709",
+    "AS28709",
-    "29264",
+    "AS29264",
-    "30282",
+    "AS30282",
-    "30637",
+    "AS30637",
-    "328126",
+    "AS328126",
-    "36408",
+    "AS36408",
-    "38107",
+    "AS38107",
-    "397192",
+    "AS397192",
-    "40366",
+    "AS40366",
-    "43303",
+    "AS43303",
-    "44907",
+    "AS44907",
-    "46071",
+    "AS46071",
-    "46177",
+    "AS46177",
-    "47542",
+    "AS47542",
-    "49287",
+    "AS49287",
-    "49689",
+    "AS49689",
-    "51286",
+    "AS51286",
-    "55082",
+    "AS55082",
-    "55254",
+    "AS55254",
-    "56636",
+    "AS56636",
-    "57363",
+    "AS57363",
-    "58127",
+    "AS58127",
-    "59730",
+    "AS59730",
-    "59776",
+    "AS59776",
-    "60068",
+    "AS60068",
-    "60626",
+    "AS60626",
-    "60922",
+    "AS60922",
-    "61107",
+    "AS61107",
-    "61159",
+    "AS61159",
-    "62026",
+    "AS62026",
-    "62229",
+    "AS62229",
-    "63062",
+    "AS63062",
-    "64232",
+    "AS64232",
-    "8868",
+    "AS8868",
-    "9053",
+    "AS9053",
-    "55770",
+    "AS55770",
-    "49846",
+    "AS49846",
-    "49249",
+    "AS49249",
-    "48163",
+    "AS48163",
-    "45700",
+    "AS45700",
-    "43639",
+    "AS43639",
-    "39836",
+    "AS39836",
-    "393560",
+    "AS393560",
-    "393234",
+    "AS393234",
-    "36183",
+    "AS36183",
-    "35994",
+    "AS35994",
-    "35993",
+    "AS35993",
-    "35204",
+    "AS35204",
-    "34850",
+    "AS34850",
-    "34164",
+    "AS34164",
-    "33905",
+    "AS33905",
-    "32787",
+    "AS32787",
-    "31377",
+    "AS31377",
-    "31110",
+    "AS31110",
-    "31109",
+    "AS31109",
-    "31108",
+    "AS31108",
-    "31107",
+    "AS31107",
-    "30675",
+    "AS30675",
-    "24319",
+    "AS24319",
-    "23903",
+    "AS23903",
-    "23455",
+    "AS23455",
-    "23454",
+    "AS23454",
-    "22207",
+    "AS22207",
-    "21399",
+    "AS21399",
-    "21357",
+    "AS21357",
-    "21342",
+    "AS21342",
-    "20940",
+    "AS20940",
-    "20189",
+    "AS20189",
-    "18717",
+    "AS18717",
-    "18680",
+    "AS18680",
-    "17334",
+    "AS17334",
-    "16702",
+    "AS16702",
-    "16625",
+    "AS16625",
-    "12222",
+    "AS12222",
-    "209101",
+    "AS209101",
-    "201585",
+    "AS201585",
-    "135429",
+    "AS135429",
-    "395747",
+    "AS395747",
-    "394536",
+    "AS394536",
-    "209242",
+    "AS209242",
-    "203898",
+    "AS203898",
-    "202623",
+    "AS202623",
-    "14789",
+    "AS14789",
-    "133877",
+    "AS133877",
-    "13335",
+    "AS13335",
-    "132892",
+    "AS132892",
-    "21859",
+    "AS21859",
-    "6185",
+    "AS6185",
-    "47823",
+    "AS47823",
-    "4134"
+    "AS4134"
 ]

dbexport.py

@@ -15,17 +15,17 @@ from common.database import Database
 from config.log import logger
 
 
-def export(target, type='target', db=None, alive=False, limit=None, path=None, format='csv', show=False):
+def export(target, type='target', db=None, alive=False, limit=None, path=None, fmt='csv', show=False):
     """
     OneForAll export from database module
 
     Example:
-        python3 dbexport.py --target name --format csv --dir= ./result.csv
+        python3 dbexport.py --target name --fmt csv --dir= ./result.csv
         python3 dbexport.py --db result.db --target name --show False
         python3 dbexport.py --target table_name --tb True --show False
 
     Note:
-        --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
+        --fmt csv/json (result format)
         --path   Result directory (default directory is ./results)
 
     :param str  target:  Table to be exported
@@ -33,7 +33,7 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f
     :param str  db:      Database path to be exported (default ./results/result.sqlite3)
     :param bool alive:   Only export the results of alive subdomains (default False)
     :param str  limit:   Export limit (default None)
-    :param str  format:  Result format (default csv)
+    :param str  fmt:     Result format (default csv)
     :param str  path:    Result directory (default None)
     :param bool show:    Displays the exported data in terminal (default False)
     """
@@ -48,29 +48,29 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f
                 rows = database.export_data(table_name, alive, limit)
                 if rows is None:
                     continue
-                data = export_data(format, path, rows, show, table_name, target)
+                data = export_data(fmt, path, rows, show, table_name, target)
                 datas.extend(data)
         database.close()
         if len(domains) > 1:
-            utils.export_all(alive, format, path, datas)
+            utils.export_all(alive, fmt, path, datas)
     elif type == 'table':
         database = Database(db)
         rows = database.export_data(target, alive, limit)
-        data = export_data(format, path, rows, show, target, target)
+        data, _, _ = export_data(fmt, path, rows, show, target, target)
         database.close()
         return data
 
 
-def export_data(format, path, rows, show, table_name, target):
+def export_data(fmt, path, rows, show, table_name, target):
-    format = utils.check_format(format, len(rows))
+    fmt = utils.check_format(fmt)
-    path = utils.check_path(path, target, format)
+    path = utils.check_path(path, target, fmt)
     if show:
         print(rows.dataset)
-    data = rows.export(format)
+    data = rows.export(fmt)
     utils.save_data(path, data)
     logger.log('ALERT', f'The subdomain result for {table_name}: {path}')
     data = rows.as_dict()
-    return data, format, path
+    return data, fmt, path
 
 
 def domain_to_table(table):

docs/en-us/README.md

@@ -164,7 +164,7 @@ DESCRIPTION
         python3 oneforall.py --target example.com --alive False run
         python3 oneforall.py --target example.com --brute True run
         python3 oneforall.py --target example.com --port medium run
-        python3 oneforall.py --target example.com --format csv run
+        python3 oneforall.py --target example.com --fmt csv run
         python3 oneforall.py --target example.com --dns False run
         python3 oneforall.py --target example.com --req False run
         python3 oneforall.py --target example.com --takeover False run
@@ -173,7 +173,7 @@ DESCRIPTION
     Note:
         --alive  True/False           Only export alive subdomains or not (default False)
         --port   default/small/large  See details in ./config/setting.py(default port 80)
-        --format csv/json (result format)
+        --fmt csv/json (result format)
         --path   Result directory (default directory is ./results)
 
 ARGUMENTS
@@ -193,7 +193,7 @@ FLAGS
         The port range request to the subdomains (default port 80)
     --alive=ALIVE
         Only export alive subdomains (default False)
-    --format=FORMAT
+    --fmt=FMT
         Result format (default csv)
     --path=PATH
         Result directory (default None)
@@ -270,7 +270,6 @@ The project uses [SemVer](https://semver.org/) for version management, and you c
 ## ⌛Follow-up plan
 
 - [ ] Continuous optimize and improve of each module
-- [x] Subdomain monitoring (mark newly discovered subdomain)
 - [x] Subdomain collection crawler (collect subdomains from static files such as JS)
 - [ ] Implementation of front-end interface for powerful interaction
 

modules/altdns.py

@@ -193,9 +193,7 @@ class Altdns(Module):
         self.end = time.time()
         self.elapse = round(self.end - self.start, 1)
         self.gen_result()
-        temp_data = resolve.run_resolve(self.domain, self.results)
+        resolved_data = resolve.run_resolve(self.domain, self.results)
-        fina_data = request.run_request(self.domain, temp_data, port)
+        request.run_request(self.domain, resolved_data, port)
-        data = data + fina_data
         logger.log('INFOR', f'Saving altdns results')
         utils.save_db(self.domain, data, 'altdns')
-        return data

modules/enrich.py

@@ -0,0 +1,72 @@
+from modules import iscdn
+from common import utils
+from common.database import Database
+from common.ipasn import IPAsnInfo
+from common.ipreg import IpRegData
+
+
+def get_ips(info):
+    ip = info.get('ip')
+    if not ip:
+        return None
+    ips = ip.split(',')
+    return ips
+
+
+def enrich_info(data):
+    ip_asn = IPAsnInfo()
+    ip_reg = IpRegData()
+    for index, info in enumerate(data):
+        ips = get_ips(info)
+        if not ips:
+            continue
+        public = list()
+        cidr = list()
+        asn = list()
+        org = list()
+        addr = list()
+        isp = list()
+        for ip in ips:
+            public.append(str(utils.ip_is_public(ip)))
+            asn_info = ip_asn.find(ip)
+            cidr.append(asn_info.get('cidr'))
+            asn.append(asn_info.get('asn'))
+            org.append(asn_info.get('org'))
+            ip_info = ip_reg.query(ip)
+            addr.append(ip_info.get('addr'))
+            isp.append(ip_info.get('isp'))
+        data[index]['public'] = ','.join(public)
+        data[index]['cidr'] = ','.join(cidr)
+        data[index]['asn'] = ','.join(asn)
+        data[index]['org'] = ','.join(org)
+        data[index]['addr'] = ','.join(addr)
+        data[index]['isp'] = ','.join(isp)
+        return data
+
+
+class Enrich(object):
+    def __init__(self, domain):
+        self.domain = domain
+
+    def get_data(self):
+        db = Database()
+        fields = ['url', 'cname', 'ip', 'public', 'cdn', 'header',
+                  'cidr', 'asn', 'org', 'addr', 'isp']
+        results = db.get_data_by_fields(self.domain, fields)
+        return results.as_dict()
+
+    def save_db(self, data):
+        db = Database()
+        for info in data:
+            url = info.pop('url')
+            info.pop('cname')
+            info.pop('ip')
+            info.pop('header')
+            db.update_data_by_url(self.domain, info, url)
+        db.close()
+
+    def run(self):
+        data = self.get_data()
+        data = enrich_info(data)
+        data = iscdn.do_check(data)
+        self.save_db(data)

modules/finder.py

@@ -7,6 +7,7 @@ from common import utils
 from common import resolve
 from common import request
 from common.module import Module
+from common.database import Database
 from config import settings
 from config.log import logger
 
@@ -25,16 +26,11 @@ class Finder(Module):
         new_subdomains = found_subdomains - existing_subdomains
         if not len(new_subdomains):
             self.finish()  # 未发现新的子域就直接返回
-            return data
         self.subdomains = new_subdomains
         self.finish()
         self.gen_result()
-        temp_data = resolve.run_resolve(domain, self.results)
+        resolved_data = resolve.run_resolve(domain, self.results)
-        fina_data = request.run_request(domain, temp_data, port)
+        request.run_request(domain, resolved_data, port)
-        data = data + fina_data
-        logger.log('INFOR', f'Saving finder results')
-        utils.save_db(domain, data, 'finder')
-        return data
 
 
 file_path = settings.data_storage_dir.joinpath('common_js_library.json')
@@ -177,22 +173,31 @@ def find_js_urls(domain, req_url, rsp_html):
     return js_urls
 
 
+def convert_to_dict(url_list):
+    url_dict = []
+    for url in url_list:
+        url_dict.append({'url': url})
+    return url_dict
+
 def find_subdomains(domain, data):
     subdomains = set()
     js_urls = set()
+    db = Database()
     for infos in data:
         jump_history = infos.get('history')
         req_url = infos.get('url')
         subdomains.update(find_in_history(domain, req_url, jump_history))
-        rsp_html = infos.get('response')
+        rsp_html = db.get_resp_by_url(domain, req_url)
         if not rsp_html:
             logger.log('DEBUG', f'an abnormal response occurred in the request {req_url}')
             continue
         subdomains.update(find_in_resp(domain, req_url, rsp_html))
         js_urls.update(find_js_urls(domain, req_url, rsp_html))
 
-    resp_data = request.bulk_request(js_urls)
+    req_data = convert_to_dict(js_urls)
-    for _, resp in resp_data:
+    resp_data = request.bulk_request(domain, req_data, ret=True)
+    while not resp_data.empty():
+        _, resp = resp_data.get()
         if not isinstance(resp, Response):
             continue
         text = utils.decode_resp_text(resp)

modules/iscdn.py

@@ -3,6 +3,7 @@ import ipaddress
 
 from config import settings
 from common import utils
+from common.database import Database
 from config.log import logger
 
 data_dir = settings.data_storage_dir
@@ -18,6 +19,8 @@ cdn_header_key = utils.load_json(data_dir.joinpath('cdn_header_keys.json'))
 
 
 def check_cname_keyword(cname):
+    if not cname:
+        return False
     names = cname.lower().split(',')
     for name in names:
         for keyword in cdn_cname_keyword.keys():
@@ -26,14 +29,22 @@ def check_cname_keyword(cname):
 
 
 def check_header_key(header):
-    header = set(map(lambda x: x.lower(), header.keys()))
+    if isinstance(header, str):
-    for key in cdn_header_key:
+        header = json.loads(header)
-        if key in header:
+    if isinstance(header, dict):
-            return True
+        header = set(map(lambda x: x.lower(), header.keys()))
+        for key in cdn_header_key:
+            if key in header:
+                return True
+    else:
+        return False
 
 
-def check_cdn_cidr(content):
+def check_cdn_cidr(ips):
-    ips = set(content.split(','))
+    if isinstance(ips, str):
+        ips = set(ips.split(','))
+    else:
+        return False
     for ip in ips:
         try:
             ip = ipaddress.ip_address(ip)
@@ -46,39 +57,30 @@ def check_cdn_cidr(content):
 
 
 def check_cdn_asn(asn):
-    if str(asn) in cdn_asn_list:
+    if isinstance(asn, str):
-        return True
+        if asn in cdn_asn_list:
+            return True
+    return False
 
 
-def check_cdn(data):
+def do_check(data):
-    logger.log('DEBUG', f'Start cdn check module')
+    logger.log('DEBUG', f'Checking cdn')
     for index, item in enumerate(data):
         cname = item.get('cname')
-        if cname:
+        if check_cname_keyword(cname):
-            if check_cname_keyword(cname):
+            data[index]['cdn'] = 1
-                data[index]['cdn'] = 1
+            continue
-                continue
         header = item.get('header')
-        if header:
+        if check_header_key(header):
-            header = json.loads(header)
+            data[index]['cdn'] = 1
-            if check_header_key(header):
+            continue
-                data[index]['cdn'] = 1
-                continue
         ip = item.get('ip')
-        if ip:
+        if check_cdn_cidr(ip):
-            if check_cdn_cidr(ip):
+            data[index]['cdn'] = 1
-                data[index]['cdn'] = 1
+            continue
-                continue
         asn = item.get('asn')
-        if asn:
+        if check_cdn_asn(asn):
-            asn = asn[2:]  # 去除AS
+            data[index]['cdn'] = 1
-            if check_cdn_asn(asn):
+            continue
-                data[index]['cdn'] = 1
-                continue
         data[index]['cdn'] = 0
     return data
-
-
-def save_db(name, data):
-    logger.log('DEBUG', f'Saving cdn check results')
-    utils.save_db(name, data, 'cdn')

oneforall.py

@@ -15,12 +15,11 @@ from datetime import datetime
 import dbexport
 from brute import Brute
 from common import utils, resolve, request
-from common.database import Database
 from modules.collect import Collect
 from modules.srv import BruteSRV
 from modules.finder import Finder
 from modules.altdns import Altdns
-from modules import iscdn
+from modules.enrich import Enrich
 from config import settings
 from config.log import logger
 from takeover import Takeover
@@ -60,7 +59,7 @@ class OneForAll(object):
         python3 oneforall.py --target example.com --alive False run
         python3 oneforall.py --target example.com --brute False run
         python3 oneforall.py --target example.com --port medium run
-        python3 oneforall.py --target example.com --format csv run
+        python3 oneforall.py --target example.com --fmt csv run
         python3 oneforall.py --target example.com --dns False run
         python3 oneforall.py --target example.com --req False run
         python3 oneforall.py --target example.com --takeover False run
@@ -68,22 +67,22 @@ class OneForAll(object):
 
     Note:
         --port   small/medium/large  See details in ./config/setting.py(default small)
-        --format csv/json (result format)
+        --fmt csv/json (result format)
         --path   Result path (default None, automatically generated)
 
     :param str  target:     One domain (target or targets must be provided)
     :param str  targets:    File path of one domain per line
-    :param bool brute:     Use brute module (default True)
+    :param bool brute:      Use brute module (default True)
-    :param bool dns:       Use DNS resolution (default True)
+    :param bool dns:        Use DNS resolution (default True)
-    :param bool req:       HTTP request subdomains (default True)
+    :param bool req:        HTTP request subdomains (default True)
     :param str  port:       The port range to request (default small port is 80,443)
-    :param bool alive:     Only export alive subdomains (default False)
+    :param bool alive:      Only export alive subdomains (default False)
-    :param str  format:     Result format (default csv)
+    :param str  fmt:        Result format (default csv)
     :param str  path:       Result path (default None, automatically generated)
-    :param bool takeover:  Scan subdomain takeover (default False)
+    :param bool takeover:   Scan subdomain takeover (default False)
     """
     def __init__(self, target=None, targets=None, brute=None, dns=None, req=None,
-                 port=None, alive=None, format=None, path=None, takeover=None):
+                 port=None, alive=None, fmt=None, path=None, takeover=None):
         self.target = target
         self.targets = targets
         self.brute = brute
@@ -91,17 +90,13 @@ class OneForAll(object):
         self.req = req
         self.port = port
         self.alive = alive
-        self.format = format
+        self.fmt = fmt
         self.path = path
         self.takeover = takeover
         self.domain = str()  # The domain currently being collected
         self.domains = set()  # All domains that are to be collected
         self.data = list()  # The subdomain results of the current domain
         self.datas = list()  # All subdomain results of the domain
-        self.old_table = str()  # The table name of the last result
-        self.new_table = str()  # The table name of the current result
-        self.origin_table = str()  # The table name of the origin result
-        self.resolve_table = str()  # The table name of the resolute result
 
     def config_param(self):
         """
@@ -119,8 +114,8 @@ class OneForAll(object):
             self.port = settings.http_request_port
         if self.alive is None:
             self.alive = bool(settings.result_export_alive)
-        if self.format is None:
+        if self.fmt is None:
-            self.format = settings.result_save_format
+            self.fmt = settings.result_save_format
         if self.path is None:
             self.path = settings.result_save_path
 
@@ -134,47 +129,13 @@ class OneForAll(object):
 
     def export(self, table):
         """
-        Export data from the database and do some follow-up processing
+        Export data from the database
 
         :param table: table name
         :return: export data
         :rtype: list
         """
-        db = Database()
+        return dbexport.export(table, type='table', alive=self.alive, fmt=self.fmt)
-        data = dbexport.export(table, type='table', alive=self.alive, format=self.format)
-        db.drop_table(self.new_table)
-        db.rename_table(self.domain, self.new_table)
-        db.close()
-        return data
-
-    def deal_db(self):
-        """
-        Process the data when the collection task is completed
-        """
-        db = Database()
-        db.deal_table(self.domain, self.origin_table)
-        db.close()
-
-    def mark(self):
-        """
-        Mark the new discovered subdomain
-
-        :return: marked data
-        :rtype: list
-        """
-        db = Database()
-        old_data = list()
-        now_data = db.get_data(self.domain).as_dict()
-        # Database pre-processing when it is not the first time to collect this subdomain
-        if db.exist_table(self.new_table):
-            # If there is the last collection result table, delete it first
-            db.drop_table(self.old_table)
-            # Rename the new table to the old table
-            db.rename_table(self.new_table, self.old_table)
-            old_data = db.get_data(self.old_table).as_dict()
-            db.close()
-        marked_data = utils.mark_subdomain(old_data, now_data)
-        return marked_data
 
     def main(self):
         """
@@ -183,11 +144,6 @@ class OneForAll(object):
         :return: subdomain results
         :rtype: list
         """
-        self.old_table = self.domain + '_old_result'
-        self.new_table = self.domain + '_now_result'
-        self.origin_table = self.domain + '_origin_result'
-        self.resolve_table = self.domain + '_resolve_result'
-
         collect = Collect(self.domain)
         collect.run()
 
@@ -202,49 +158,44 @@ class OneForAll(object):
             brute.quite = True
             brute.run()
 
-        # Database processing
+        utils.deal_data(self.domain)
-        self.deal_db()
-        # Mark the new discovered subdomain
-        self.data = self.mark()
-
         # Export results without resolve
         if not self.dns:
             return self.export(self.domain)
 
+        self.data = utils.get_data(self.domain)
+
         # Resolve subdomains
+        utils.clear_data(self.domain)
         self.data = resolve.run_resolve(self.domain, self.data)
         # Save resolve results
-        resolve.save_db(self.resolve_table, self.data)
+        resolve.save_db(self.domain, self.data)
 
         # Export results without HTTP request
         if not self.req:
-            return self.export(self.resolve_table)
+            return self.export(self.domain)
 
         # HTTP request
-        self.data = request.run_request(self.domain, self.data, self.port)
+        utils.clear_data(self.domain)
-        # Save HTTP request result
+        request.run_request(self.domain, self.data, self.port)
-        request.save_db(self.domain, self.data)
 
         # Finder module
         if settings.enable_finder_module:
             finder = Finder()
-            self.data = finder.run(self.domain, self.data, self.port)
+            finder.run(self.domain, self.data, self.port)
 
         # altdns module
         if settings.enable_altdns_module:
-            finder = Altdns(self.domain)
+            altdns = Altdns(self.domain)
-            self.data = finder.run(self.data, self.port)
+            altdns.run(self.data, self.port)
 
-        # check cdn module
+        # Information enrichment module
-        if settings.enable_cdn_check:
+        if settings.enable_enrich_module:
-            self.data = iscdn.check_cdn(self.data)
+            enrich = Enrich(self.domain)
-            iscdn.save_db(self.domain, self.data)
+            enrich.run()
-
-        # Add the final result list to the total data list
-        self.datas.extend(self.data)
 
         # Export
-        self.export(self.domain)
+        self.datas.extend(self.export(self.domain))
 
         # Scan subdomain takeover
         if self.takeover:
@@ -277,7 +228,7 @@ class OneForAll(object):
             for domain in self.domains:
                 self.domain = utils.get_main_domain(domain)
                 self.main()
-            utils.export_all(self.alive, self.format, self.path, self.datas)
+            utils.export_all(self.alive, self.fmt, self.path, self.datas)
         else:
             logger.log('FATAL', 'Failed to obtain domain')
         logger.log('INFOR', 'Finished OneForAll')

takeover.py

@@ -45,21 +45,21 @@ class Takeover(Module):
     OneForAll subdomain takeover module
 
     Example:
-        python3 takeover.py --target www.example.com  --format csv run
+        python3 takeover.py --target www.example.com  --fmt csv run
         python3 takeover.py --targets ./subdomains.txt --thread 10 run
 
     Note:
-        --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
+        --fmt txt/csv/json (result format)
         --path   Result directory (default directory is ./results)
 
     :param str target:   One domain (target or targets must be provided)
     :param str targets:  File path of one domain per line
     :param int thread:   threads number (default 20)
-    :param str format:   Result format (default csv)
+    :param str fmt:      Result format (default csv)
     :param str path:     Result directory (default None)
     """
 
-    def __init__(self, target=None, targets=None, thread=20, path=None, format='csv'):
+    def __init__(self, target=None, targets=None, thread=20, path=None, fmt='csv'):
         Module.__init__(self)
         self.subdomains = set()
         self.module = 'Check'
@@ -68,7 +68,7 @@ class Takeover(Module):
         self.targets = targets
         self.thread = thread
         self.path = path
-        self.format = format
+        self.fmt = fmt
         self.fingerprints = None
         self.subdomainq = Queue()
         self.cnames = list()
@@ -76,10 +76,10 @@ class Takeover(Module):
 
     def save(self):
         logger.log('DEBUG', 'Saving results')
-        if self.format == 'txt':
+        if self.fmt == 'txt':
             data = str(self.results)
         else:
-            data = self.results.export(self.format)
+            data = self.results.export(self.fmt)
         utils.save_data(self.path, data)
 
     def compare(self, subdomain, cname, responses):
@@ -131,10 +131,10 @@ class Takeover(Module):
             self.subdomains = self.targets
         else:
             self.subdomains = utils.get_domains(self.target, self.targets)
-        self.format = utils.check_format(self.format, len(self.subdomains))
+        self.fmt = utils.check_format(self.fmt)
         timestamp = utils.get_timestamp()
         name = f'takeover_check_result_{timestamp}'
-        self.path = utils.check_path(self.path, name, self.format)
+        self.path = utils.check_path(self.path, name, self.fmt)
         if self.subdomains:
             logger.log('INFOR', f'Checking subdomain takeover')
             self.fingerprints = get_fingerprint()