修改日志输出格式，降低代码重复率。

oneforall/aiobrute.py

@@ -8,21 +8,22 @@ OneForAll多进程多协程异步子域爆破模块
 :license: GNU General Public License v3.0, see LICENSE for more details.
 """
 
-import os
-import time
-import queue
-import signal
-import pathlib
 import asyncio
+import os
+import queue
+import secrets
+import signal
+import time
+
+import aiomultiprocess
+import exrex
 import fire
 import tqdm
-import exrex
+
-import secrets
-import aiomultiprocess
 import config
-from config import logger
+from common import database, resolve, utils
-from common import utils, database, resolve
 from common.module import Module
+from config import logger
 
 
 def init_worker():
@@ -253,7 +254,6 @@ class AIOBrute(Module):
                         if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域爆破的限制条件
                             loop.run_until_complete(self.main(subdomain, rx_queue))
 
-
             while not rx_queue.empty():  # 队列不空就一直取数据存数据库
                 database.save_db(db_conn, table_name, rx_queue.get())  # 将结果存入数据库中
             database.copy_table(db_conn, table_name)
@@ -281,4 +281,4 @@ def do(domain, result):  # 统一入口名字 方便多线程调用
 if __name__ == '__main__':
     fire.Fire(AIOBrute)
     # result_queue = queue.Queue()
-    # do('owasp.org', result_queue)
+    # do('example.com', result_queue)

oneforall/collect.py

@@ -9,6 +9,7 @@ import importlib
 import config
 import dbexport
 from common import database
+from config import logger
 
 
 class Collect(object):
@@ -54,6 +55,7 @@ class Collect(object):
         类运行入口
         """
         start = time.time()
+        logger.log('INFOR', f'开始收集{self.domain}的子域')
         self.get_mod()
         self.import_func()
 

oneforall/common/module.py

@@ -6,6 +6,7 @@
 import json
 import re
 import threading
+import time
 
 import requests
 import config
@@ -32,7 +33,25 @@ class Module(object):
         self.subdomains = set()  # 存放发现的子域
         self.records = dict()  # 存放子域解析记录
         self.results = list()  # 存放模块结果
-        self.elapsed = 0.0  # 模块执行耗时
+        self.start = time.time()  # 模块开始执行时间
+        self.end = None
+        self.elapsed = time.time() - self.start  # 模块执行耗时
+
+    def begin(self):
+        """
+        输出模块开始信息
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+
+    def finish(self):
+        """
+        输出模块结束信息
+        """
+        self.end = time.time()
+        self.elapsed = round(self.end - self.start, 1)
+        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒发现子域{len(self.subdomains)}个')
+        logger.log('DEBUG', f'{self.source}模块发现{self.domain}的子域 {self.subdomains}')
 
     def get(self, url, params=None, **kwargs):
         """

oneforall/config.py

@@ -152,7 +152,11 @@ subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs', 'web',
                      'data', 'help', 'live', 'mall', 'blogs', 'files', 'forum', 'store', 'mobile'}
 
 # 日志配置
-log_fmt = '<light-green>{time:HH:mm:ss,SSS}</light-green> ' \
+stdout_fmt = '<light-green>{time:HH:mm:ss,SSS}</light-green> ' \
+          '[<level>{level: <5}</level>] ' \
+          '<blue>{module}</blue>:<cyan>{line}</cyan> - ' \
+          '<level>{message}</level>'
+logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
           '[<level>{level: <5}</level>] ' \
           '<cyan>{process.name}</cyan>:<cyan>{thread.name: <10}</cyan> | ' \
           '<blue>{module}</blue>.<blue>{function}</blue>:<blue>{line}</blue> - ' \
@@ -168,8 +172,8 @@ logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️')
 logger.level(name='ERROR', no=40, color='<red><bold>', icon='❌️')
 logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')
 
-logger.add(sys.stdout, level='INFOR', format=log_fmt, enqueue=True)
+logger.add(sys.stdout, level='INFOR', format=stdout_fmt, enqueue=True)
-logger.add(log_path, level='TRACE', format=log_fmt, enqueue=True, encoding='utf-8')
+logger.add(log_path, level='TRACE', format=logfile_fmt, enqueue=True, encoding='utf-8')
 
 # 调试模式
 # import urllib3

oneforall/dbexport.py

@@ -44,15 +44,17 @@ def export(table, db=None, valid=None, path=None, format='xlsx', output=False):
         print(rows.dataset)
     if not path:
         path = 'export.' + format
-    logger.log('INFOR', f'正在将数据库中{table}表导出到{path}')
+    logger.log('INFOR', f'正在将数据库中{table}表导出')
     try:
         with open(path, 'w') as file:
             file.write(rows.export(format))
             logger.log('INFOR', '成功完成导出')
+            logger.log('INFOR', path)
     except TypeError:
         with open(path, 'wb') as file:
             file.write(rows.export(format))
             logger.log('INFOR', '成功完成导出')
+            logger.log('INFOR', path)
     except Exception as e:
         logger.log('ERROR', e)
 

oneforall/modules/certificates/censys_api.py

@@ -24,11 +24,10 @@ class CensysAPI(Query):
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         data = {
-            'query': 'parsed.names: owasp.org',
+            'query': 'parsed.names: example.com',
             'page': 1,
             'fields': ['parsed.subject_dn'],
             'flatten': True}
-
         resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
         if not resp:
             return
@@ -57,16 +56,13 @@ class CensysAPI(Query):
             logger.log('ERROR', f'{self.source}模块API配置错误')
             logger.log('ALERT', f'不执行{self.source}模块')
             return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -78,10 +74,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = CensysAPI(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/certificates/certdb_api.py

@@ -44,16 +44,13 @@ class CertDBAPI(Query):
             logger.log('ERROR', f'{self.source}模块API配置错误')
             logger.log('ALERT', f'不执行{self.source}模块')
             return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -65,10 +62,8 @@ def do(domain, rx_queue):   # 统一入口名字 方便多线程调用
     """
     query = CertDBAPI(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/certificates/certspotter.py

@@ -1,9 +1,9 @@
 # coding=utf-8
-import time
 import queue
-from common.query import Query
+import time
+
 from common import utils
-from config import logger
+from common.query import Query
 
 
 class CertSpotter(Query):
@@ -32,16 +32,13 @@ class CertSpotter(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -53,10 +50,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = CertSpotter(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/certificates/crtsh.py

@@ -1,9 +1,9 @@
 # coding=utf-8
-import time
 import queue
-from common.query import Query
+import time
+
 from common import utils
-from config import logger
+from common.query import Query
 
 
 class Crtsh(Query):
@@ -32,16 +32,13 @@ class Crtsh(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -53,10 +50,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = Crtsh(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/certificates/entrust.py

@@ -1,9 +1,9 @@
 # coding=utf-8
-import time
 import queue
-from common.query import Query
+import time
+
 from common import utils
-from config import logger
+from common.query import Query
 
 
 class Entrust(Query):
@@ -32,16 +32,13 @@ class Entrust(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -53,10 +50,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = Entrust(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/certificates/google.py

@@ -1,9 +1,9 @@
 # coding=utf-8
-import time
 import queue
-from common.query import Query
+import time
+
 from common import utils
-from config import logger
+from common.query import Query
 
 
 class Google(Query):
@@ -32,16 +32,13 @@ class Google(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -53,10 +50,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = Google(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/check/axfr.py

@@ -7,19 +7,20 @@ DNS域传送(DNS zone transfer)指的是一台备用域名服务器使用来自
 目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
 当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获取某个域的所有记录。
 """
-import time
 import queue
 import dns.resolver
 import dns.zone
-from config import logger
+
-from common import utils, resolve
+from common import resolve, utils
 from common.module import Module
+from config import logger
 
 
 class CheckAXFR(Module):
     """
     DNS域传送漏洞检查类
     """
+
     def __init__(self, domain: str):
         Module.__init__(self)
         self.domain = self.register(domain)
@@ -69,10 +70,9 @@ class CheckAXFR(Module):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}的域传送漏洞')
-        start = time.time()
+
         self.check()
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}的域传送漏洞')
         self.save_json()
         self.gen_result()
@@ -95,4 +95,4 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
 if __name__ == '__main__':
     # do('ZoneTransfer.me')
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/check/cdx.py

@@ -2,17 +2,18 @@
 """
 检查crossdomain.xml文件收集子域名
 """
-import time
 import queue
-from config import logger
+
-from common.utils import match_subdomain
 from common.module import Module
+from common.utils import match_subdomain
+from config import logger
 
 
 class CheckCDX(Module):
     """
     检查crossdomain.xml文件收集子域名
     """
+
     def __init__(self, domain: str):
         Module.__init__(self)
         self.domain = self.register(domain)
@@ -37,10 +38,9 @@ class CheckCDX(Module):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的crossdomain.xml')
-        start = time.time()
+
         self.check()
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -61,4 +61,5 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
 
 
 if __name__ == '__main__':
-    do('163.com')
+    result_queue = queue.Queue()
+    do('163.com', result_queue)

oneforall/modules/check/cert.py

@@ -4,13 +4,13 @@
 """
 检查域名证书收集子域名
 """
-import ssl
-import time
 import queue
 import socket
-from config import logger
+import ssl
+
 from common import utils
 from common.module import Module
+from config import logger
 
 
 class CheckCert(Module):
@@ -41,10 +41,9 @@ class CheckCert(Module):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的证书中的子域')
-        start = time.time()
+
         self.check()
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的证书中的子域')
         self.save_json()
         self.gen_result()
@@ -67,4 +66,4 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/check/csp.py

@@ -2,17 +2,18 @@
 """
 检查内容安全策略收集子域名收集子域名
 """
-import time
 import queue
-from config import logger
+
 from common import utils
 from common.module import Module
+from config import logger
 
 
 class CheckCSP(Module):
     """
     检查内容安全策略收集子域名
     """
+
     def __init__(self, domain, header):
         Module.__init__(self)
         self.domain = self.register(domain)
@@ -42,10 +43,9 @@ class CheckCSP(Module):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
-        start = time.time()
+
         self.check()
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
         self.save_json()
         self.gen_result()
@@ -69,6 +69,7 @@ def do(domain, rx_queue, header=None):  # 统一入口名字 方便多线程调
 
 if __name__ == '__main__':
     import requests
+
     # resp = requests.get('https://content-security-policy.com/')
     result_queue = queue.Queue()
     resp = requests.get('https://www.baidu.com/')

oneforall/modules/crawl/archivecrawl.py

@@ -1,5 +1,4 @@
 # coding=utf-8
-import time
 import queue
 import cdx_toolkit
 from common.crawl import Crawl
@@ -36,21 +35,17 @@ class ArchiveCrawl(Crawl):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.crawl(self.domain, 50)
-
         # 爬取已发现的子域以发现新的子域
         for subdomain in self.subdomains:
             if subdomain != self.domain:
                 self.crawl(subdomain, 10)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -61,11 +56,11 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     :param rx_queue: 结果集队列
     """
     crawl = ArchiveCrawl(domain)
-    crawl.run(result)
+    crawl.run(rx_queue)
     logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
     logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/crawl/commoncrawl.py

@@ -1,8 +1,9 @@
 # coding=utf-8
-import time
 import queue
-from tqdm import tqdm
+
 import cdx_toolkit
+from tqdm import tqdm
+
 from common.crawl import Crawl
 from config import logger
 
@@ -36,21 +37,17 @@ class CommonCrawl(Crawl):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.crawl(self.domain, 50)
-
         # 爬取已发现的子域以发现新的子域
         for subdomain in self.subdomains:
             if subdomain != self.domain:
                 self.crawl(subdomain, 10)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -61,11 +58,11 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     :param rx_queue: 结果集队列
     """
     crawl = CommonCrawl(domain)
-    crawl.run(result)
+    crawl.run(rx_queue)
     logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
     logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/binaryedge_api.py

@@ -38,16 +38,13 @@ class BinaryEdgeAPI(Query):
             logger.log('ERROR', f'{self.source}模块API配置错误')
             logger.log('ALERT', f'不执行{self.source}模块')
             return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -59,10 +56,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = BinaryEdgeAPI(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/bufferover.py

@@ -2,7 +2,6 @@
 import time
 import queue
 from common.query import Query
-from config import logger
 
 
 class BufferOver(Query):
@@ -31,16 +30,13 @@ class BufferOver(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -52,10 +48,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = BufferOver(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/chinaz.py

@@ -1,8 +1,8 @@
 # coding=utf-8
-import time
 import queue
+import time
+
 from common.query import Query
-from config import logger
 
 
 class Chinaz(Query):
@@ -31,16 +31,13 @@ class Chinaz(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -52,10 +49,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = Chinaz(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/chinaz_api.py

@@ -37,16 +37,13 @@ class ChinazAPI(Query):
             logger.log('ERROR', f'{self.source}模块API配置错误')
             logger.log('ALERT', f'不执行{self.source}模块')
             return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -58,10 +55,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = ChinazAPI(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/circl_api.py

@@ -3,7 +3,6 @@ import time
 import queue
 import config
 from common.query import Query
-from config import logger
 
 
 class CirclAPI(Query):
@@ -33,17 +32,14 @@ class CirclAPI(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
         if self.user and self.pwd:
-            start = time.time()
             self.query()
-            end = time.time()
-            self.elapsed = round(end - start, 1)
             self.save_json()
             self.gen_result()
             self.save_db()
             rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -55,10 +51,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = CirclAPI(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/dnsdb.py

@@ -1,10 +1,11 @@
 # coding=utf-8
+import queue
 import random
 import time
-import queue
+
 from bs4 import BeautifulSoup
+
 from common.query import Query
-from config import logger
 
 
 class DNSdb(Query):
@@ -46,16 +47,13 @@ class DNSdb(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -67,10 +65,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = DNSdb(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/dnsdb_api.py

@@ -40,16 +40,13 @@ class DNSdbAPI(Query):
             logger.log('ERROR', f'{self.source}模块API配置错误')
             logger.log('ALERT', f'不执行{self.source}模块')
             return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -61,10 +58,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = DNSdbAPI(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/dnsdumpster.py

@@ -1,15 +1,16 @@
 # coding=utf-8
-import time
 import queue
-from common.query import Query
+import time
+
 from common import utils
-from config import logger
+from common.query import Query
 
 
 class DNSdumpster(Query):
     """
 
     """
+
     def __init__(self, domain):
         Query.__init__(self)
         self.domain = self.register(domain)
@@ -41,16 +42,13 @@ class DNSdumpster(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -62,10 +60,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = DNSdumpster(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/hackertarget.py

@@ -1,9 +1,8 @@
 # coding=utf-8
-import time
 import queue
-from common.query import Query
+
 from common import utils
-from config import logger
+from common.query import Query
 
 
 class HackerTarget(Query):
@@ -33,16 +32,13 @@ class HackerTarget(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -54,10 +50,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = HackerTarget(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/ipv4info.py

@@ -1,9 +1,8 @@
 # coding=utf-8
-import time
 import queue
+
 import config
 from common.query import Query
-from config import logger
 
 
 class IPv4InfoAPI(Query):
@@ -45,16 +44,13 @@ class IPv4InfoAPI(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -66,10 +62,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = IPv4InfoAPI(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/netcraft.py

@@ -1,11 +1,11 @@
 # coding=utf-8
+import hashlib
+import queue
 import re
 import time
-import queue
-import hashlib
 from urllib import parse
+
 from common.query import Query
-from config import logger
 
 
 class NetCraft(Query):
@@ -56,16 +56,13 @@ class NetCraft(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -77,10 +74,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = NetCraft(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/ptrarchive.py

@@ -1,10 +1,9 @@
 # coding=utf-8
-import time
 import queue
 import random
-from common.query import Query
+
 from common import utils
-from config import logger
+from common.query import Query
 
 
 class PTRArchive(Query):
@@ -35,16 +34,13 @@ class PTRArchive(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -56,10 +52,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = PTRArchive(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/riddler.py

@@ -1,8 +1,8 @@
 # coding=utf-8
-import time
 import queue
+import time
+
 from common.query import Query
-from config import logger
 
 
 class Riddler(Query):
@@ -31,16 +31,13 @@ class Riddler(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -52,10 +49,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = Riddler(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/robtex.py

@@ -1,9 +1,9 @@
 # coding=utf-8
-import time
-import queue
 import json
+import queue
+import time
+
 from common.query import Query
-from config import logger
 
 
 class Robtex(Query):
@@ -42,16 +42,13 @@ class Robtex(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -63,10 +60,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = Robtex(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/securitytrails_api.py

@@ -41,16 +41,13 @@ class SecurityTrailsAPI(Query):
             logger.log('ERROR', f'{self.source}模块API配置错误')
             logger.log('ALERT', f'不执行{self.source}模块')
             return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -62,10 +59,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = SecurityTrailsAPI(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/datasets/sitedossier.py

@@ -1,8 +1,8 @@
 # coding=utf-8
-import time
 import queue
+import time
+
 from common.query import Query
-from config import logger
 
 
 class SiteDossier(Query):
@@ -40,16 +40,13 @@ class SiteDossier(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -61,10 +58,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = SiteDossier(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/dnsquery/srv.py

@@ -5,14 +5,15 @@
 通过枚举域名常见的SRV记录并做查询来发现子域
 """
 
-import time
-import queue
-import json
 import asyncio
+import json
+import queue
+
 import aiodns
-from common.module import Module
+
 from common import utils
-from config import logger, data_storage_path, resolver_nameservers
+from common.module import Module
+from config import data_storage_path, logger, resolver_nameservers
 
 
 class BruteSRV(Module):
@@ -74,10 +75,9 @@ class BruteSRV(Module):
         类执行入口
         """
         logger.log('DEBUG', f'开始枚举{self.domain}域的SRV记录')
-        start = time.time()
+
         self.brute()
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -99,4 +99,4 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/intelligence/riskiq.py

@@ -1,9 +1,8 @@
 # coding=utf-8
-import time
 import queue
+
 import config
 from common.query import Query
-from config import logger
 
 
 class RiskIQ(Query):
@@ -35,16 +34,13 @@ class RiskIQ(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -54,13 +50,10 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     :param str domain: 域名
     :param rx_queue: 结果集队列
     """
-    return
     query = RiskIQ(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/intelligence/threatbook.py

@@ -1,6 +1,6 @@
 # coding=utf-8
-import time
 import queue
+
 import config
 from common.query import Query
 from config import logger
@@ -36,16 +36,13 @@ class ThreatBookAPI(Query):
             logger.log('ERROR', f'{self.source}模块API配置错误')
             logger.log('ALERT', f'不执行{self.source}模块')
             return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query(self.domain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -57,10 +54,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = ThreatBookAPI(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/intelligence/threatminer.py

@@ -1,8 +1,8 @@
 # coding=utf-8
-import time
 import queue
+import time
+
 from common.query import Query
-from config import logger
 
 
 class ThreatMiner(Query):
@@ -31,16 +31,13 @@ class ThreatMiner(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -52,10 +49,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = ThreatMiner(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/intelligence/virustotal.py

@@ -1,9 +1,8 @@
 # coding=utf-8
-import time
 import queue
-from common.query import Query
+import time
-from config import logger
 
+from common.query import Query
 
 '''
 最多查询100条
@@ -55,16 +54,13 @@ class VirusTotal(Query):
         """
         类执行入口
         """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query(self.domain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -76,10 +72,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     query = VirusTotal(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/intelligence/virustotal_api.py

@@ -1,6 +1,6 @@
 # coding=utf-8
-import time
 import queue
+
 import config
 from common.query import Query
 from config import logger
@@ -37,16 +37,13 @@ class VirusTotalAPI(Query):
             logger.log('ERROR', f'{self.source}模块API配置错误')
             logger.log('ALERT', f'不执行{self.source}模块')
             return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
+        self.begin()
-        start = time.time()
         self.query(self.domain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
         rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        self.finish()
 
 
 def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
@@ -56,13 +53,10 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     :param str domain: 域名
     :param rx_queue: 结果集队列
     """
-    return
     query = VirusTotalAPI(domain)
     query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/ask.py

@@ -49,7 +49,7 @@ class Ask(Search):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search(self.domain, full_search=True)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -62,8 +62,7 @@ class Ask(Search):
                 for subdomain in self.subdomains:
                     if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
                         self.search(subdomain)
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -80,10 +79,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = Ask(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/baidu.py

@@ -70,7 +70,7 @@ class Baidu(Search):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search(self.domain, full_search=True)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -83,8 +83,7 @@ class Baidu(Search):
                 for subdomain in self.subdomains:
                     if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
                         self.search(subdomain)
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -101,10 +100,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = Baidu(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/bing.py

@@ -56,7 +56,7 @@ class Bing(Search):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search(self.domain, full_search=True)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -69,8 +69,7 @@ class Bing(Search):
                 for subdomain in self.subdomains:
                     if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
                         self.search(subdomain)
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -87,10 +86,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = Bing(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/bing_api.py

@@ -58,7 +58,7 @@ class BingAPI(Search):
             logger.log('ALERT', f'不执行{self.source}模块')
             return
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search(self.domain, full_search=True)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -71,8 +71,7 @@ class BingAPI(Search):
                 for subdomain in self.subdomains:
                     if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
                         self.search(subdomain)
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -89,10 +88,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = BingAPI(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/duckduckgo.py

@@ -52,7 +52,7 @@ class DuckDuckGO(Search):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search(self.domain, full_search=True)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -65,8 +65,7 @@ class DuckDuckGO(Search):
                 for subdomain in self.subdomains:
                     if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
                         self.search(subdomain)
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -84,10 +83,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     # return  # 暂时还有点问题
     search = DuckDuckGO(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/exalead.py

@@ -52,7 +52,7 @@ class Exalead(Search):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search(self.domain, full_search=True)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -66,8 +66,7 @@ class Exalead(Search):
                 for subdomain in self.subdomains:
                     if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
                         self.search(subdomain)
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -84,10 +83,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = Exalead(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/fofa.py

@@ -1,8 +1,8 @@
 # coding=utf-8
-import time
-import queue
-import json
 import base64
+import queue
+import time
+
 import config
 from common.search import Search
 from config import logger
@@ -42,10 +42,9 @@ class FoFa(Search):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search()
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -62,10 +61,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = FoFa(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/google.py

@@ -61,7 +61,7 @@ class Google(Search):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search(self.domain, full_search=True)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -74,8 +74,7 @@ class Google(Search):
                 for subdomain in self.subdomains:
                     if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
                         self.search(subdomain)
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -92,10 +91,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = Google(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/google_api.py

@@ -57,7 +57,7 @@ class GoogleAPI(Search):
             logger.log('ALERT', f'不执行{self.source}模块')
             return
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search(self.domain, full_search=True)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -70,8 +70,7 @@ class GoogleAPI(Search):
                 for subdomain in self.subdomains:
                     if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
                         self.search(subdomain)
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -88,10 +87,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = GoogleAPI(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/shodan_api.py

@@ -1,7 +1,6 @@
 # coding=utf-8
-import time
-import queue
 import queue
+
 import config
 # from shodan import Shodan
 from common.search import Search
@@ -44,10 +43,9 @@ class ShodanAPI(Search):
             logger.log('ALERT', f'不执行{self.source}模块')
             return
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search()
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -64,8 +62,6 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = ShodanAPI(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':

oneforall/modules/search/so.py

@@ -51,7 +51,7 @@ class So(Search):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search(self.domain, full_search=True)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -64,8 +64,7 @@ class So(Search):
                 for subdomain in self.subdomains:
                     if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
                         self.search(subdomain)
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -82,10 +81,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = So(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/sogou.py

@@ -1,6 +1,6 @@
 # coding=utf-8
-import time
 import queue
+
 from common.search import Search
 from config import logger
 
@@ -49,7 +49,7 @@ class Sogou(Search):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search(self.domain, full_search=True)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -62,8 +62,7 @@ class Sogou(Search):
                 for subdomain in self.subdomains:
                     if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
                         self.search(subdomain)
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -80,10 +79,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = Sogou(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/yahoo.py

@@ -57,7 +57,7 @@ class Yahoo(Search):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search(self.domain, full_search=True)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -70,8 +70,7 @@ class Yahoo(Search):
                 for subdomain in self.subdomains:
                     if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
                         self.search(subdomain)
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -88,10 +87,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = Yahoo(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/yandex.py

@@ -53,7 +53,7 @@ class Yandex(Search):
         类执行入口
         """
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search(self.domain, full_search=True)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -66,8 +66,7 @@ class Yandex(Search):
                 for subdomain in self.subdomains:
                     if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
                         self.search(subdomain)
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -84,10 +83,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = Yandex(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/modules/search/zoomeye_api.py

@@ -68,10 +68,9 @@ class ZoomEyeAPI(Search):
             logger.log('ALERT', f'不执行{self.source}模块')
             return
         logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+
         self.search()
-        end = time.time()
+
-        self.elapsed = round(end - start, 1)
         self.save_json()
         self.gen_result()
         self.save_db()
@@ -88,10 +87,8 @@ def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
     """
     search = ZoomEyeAPI(domain)
     search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
 
 
 if __name__ == '__main__':
     result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com', result_queue)

oneforall/oneforall.py

@@ -78,8 +78,6 @@ class OneForAll(object):
                 loop.close()
                 database.clear_table(db_conn, table_name)
                 database.save_db(db_conn, table_name, self.datas)
-                # database.deduplicate_subdomain(db_conn, table_name)
-                # database.remove_invalid(db_conn, table_name)
                 # 数据库导出
                 if not self.path:
                     self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
@@ -91,5 +89,5 @@ class OneForAll(object):
 
 
 if __name__ == '__main__':
-    fire.Fire(OneForAll)
+    # fire.Fire(OneForAll)
-    # OneForAll('owasp.org').run()
+    OneForAll('example.com').run()