调整执行finish函数位置

oneforall/common/module.py

@@ -35,7 +35,7 @@ class Module(object):
         self.results = list()  # 存放模块结果
         self.start = time.time()  # 模块开始执行时间
         self.end = None
-        self.elapsed = time.time() - self.start  # 模块执行耗时
+        self.elapsed = None  # 模块执行耗时
 
     def check(self, *apis):
         """

oneforall/modules/certificates/censys_api.py

@@ -54,10 +54,10 @@ class CensysAPI(Query):
             return
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/certificates/certdb_api.py

@@ -45,10 +45,10 @@ class CertDBAPI(Query):
             return
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/certificates/certspotter.py

@@ -33,10 +33,10 @@ class CertSpotter(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/certificates/crtsh.py

@@ -31,10 +31,10 @@ class Crtsh(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/certificates/entrust.py

@@ -32,10 +32,10 @@ class Entrust(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/certificates/google.py

@@ -34,10 +34,10 @@ class Google(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/check/axfr.py

@@ -75,10 +75,10 @@ class CheckAXFR(Module):
         """
         self.begin()
         self.check()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/check/cdx.py

@@ -4,7 +4,6 @@
 
 from common.module import Module
 from common import utils
-from config import logger
 
 
 class CheckCDX(Module):
@@ -21,7 +20,6 @@ class CheckCDX(Module):
     def check(self):
         """
         检查crossdomain.xml收集子域名
-        :return:
         """
         urls = [f'http://{self.domain}/crossdomain.xml',
                 f'https://{self.domain}/crossdomain.xml',
@@ -44,10 +42,10 @@ class CheckCDX(Module):
         """
         self.begin()
         self.check()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/check/cert.py

@@ -40,10 +40,10 @@ class CheckCert(Module):
         """
         self.begin()
         self.check()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/check/csp.py

@@ -50,10 +50,10 @@ class CheckCSP(Module):
         """
         self.begin()
         self.check()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain, header=None):  # 统一入口名字 方便多线程调用

oneforall/modules/check/robots.py

@@ -44,10 +44,10 @@ class CheckRobots(Module):
         """
         self.begin()
         self.check()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/check/sitemap.py

@@ -57,10 +57,10 @@ class CheckRobots(Module):
         """
         self.begin()
         self.check()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/crawl/archivecrawl.py

@@ -26,7 +26,8 @@ class ArchiveCrawl(Crawl):
         for resp in cdx.iter(url, limit=limit):
             if resp.data.get('status') not in ['301', '302']:
                 url = resp.data.get('url')
-                subdomains_find = self.match(self.register(domain), url + resp.text)
+                subdomains_find = self.match(self.register(domain),
+                                             url + resp.text)
                 # 合并搜索子域名搜索结果
                 self.subdomains = self.subdomains.union(subdomains_find)
 
@@ -40,10 +41,10 @@ class ArchiveCrawl(Crawl):
         for subdomain in self.subdomains:
             if subdomain != self.domain:
                 self.crawl(subdomain, 10)
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/crawl/commoncrawl.py

@@ -41,10 +41,10 @@ class CommonCrawl(Crawl):
         for subdomain in self.subdomains:
             if subdomain != self.domain:
                 self.crawl(subdomain, 10)
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/binaryedge_api.py

@@ -35,10 +35,10 @@ class BinaryEdgeAPI(Query):
             return
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/bufferover.py

@@ -31,10 +31,10 @@ class BufferOver(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/chinaz.py

@@ -31,10 +31,10 @@ class Chinaz(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/chinaz_api.py

@@ -35,10 +35,10 @@ class ChinazAPI(Query):
             return
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/circl_api.py

@@ -35,10 +35,10 @@ class CirclAPI(Query):
             return
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/dnsdb.py

@@ -26,8 +26,9 @@ class DNSdb(Query):
         if resp.status_code == 200:
             if 'index' in resp.text:
                 soup = BeautifulSoup(resp.text, features='lxml')
-                index_urls = set(map(lambda x: self.addr + self.domain + x.text, soup.find_all('a')))
-                for url in index_urls:
+                urls = set(map(lambda x: self.addr + self.domain + x.text,
+                               soup.find_all('a')))
+                for url in urls:
                     # 休眠绕过CloudFlare的DDoS保护
                     self.delay = random.randint(2, 5)
                     time.sleep(self.delay)
@@ -48,10 +49,10 @@ class DNSdb(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用
@@ -59,12 +60,10 @@ def do(domain):  # 统一入口名字 方便多线程调用
     类统一调用入口
 
     :param str domain: 域名
-
     """
     query = DNSdb(domain)
     query.run()
 
 
 if __name__ == '__main__':
-
     do('example.com')

oneforall/modules/datasets/dnsdb_api.py

@@ -38,10 +38,10 @@ class DNSdbAPI(Query):
             return
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/dnsdumpster.py

@@ -23,7 +23,8 @@ class DNSdumpster(Query):
         if not resp:
             return
         self.cookie = resp.cookies
-        data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'), 'targetip': self.domain}
+        data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'),
+                'targetip': self.domain}
         resp = self.post(self.addr, data)
         if not resp:
             return
@@ -38,10 +39,10 @@ class DNSdumpster(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/hackertarget.py

@@ -32,10 +32,10 @@ class HackerTarget(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/ipv4info.py

@@ -47,10 +47,10 @@ class IPv4InfoAPI(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/netcraft.py

@@ -62,10 +62,10 @@ class NetCraft(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/ptrarchive.py

@@ -36,10 +36,10 @@ class PTRArchive(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/riddler.py

@@ -31,10 +31,10 @@ class Riddler(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/robtex.py

@@ -43,10 +43,10 @@ class Robtex(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/securitytrails_api.py

@@ -39,10 +39,10 @@ class SecurityTrailsAPI(Query):
             return
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/datasets/sitedossier.py

@@ -42,10 +42,10 @@ class SiteDossier(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/dnsquery/srv.py

@@ -73,13 +73,11 @@ class BruteSRV(Module):
         类执行入口
         """
         self.begin()
-        logger.log('DEBUG', f'开始枚举{self.domain}域的SRV记录')
         self.brute()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        logger.log('DEBUG', f'结束枚举{self.domain}域的SRV记录')
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/intelligence/riskiq_api.py

@@ -19,13 +19,16 @@ class RiskIQ(Query):
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         params = {'query': self.domain}
-        resp = self.get(url=self.addr, params=params, auth=(self.user, self.key))
+        resp = self.get(url=self.addr,
+                        params=params,
+                        auth=(self.user, self.key))
         if not resp:
             return
         resp_json = resp.json()
-        subdomains_find = resp_json.get('subdomains')
-        if subdomains_find:
-            self.subdomains = set(map(lambda x: x + '.' + self.domain, subdomains_find))
+        subnames = resp_json.get('subdomains')
+        if subnames:
+            self.subdomains = set(map(lambda sub: f'{sub}.{self.domain}',
+                                      subnames))
 
     def run(self):
         """
@@ -35,10 +38,10 @@ class RiskIQ(Query):
             return
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/intelligence/threatbook_api.py

@@ -11,13 +11,15 @@ class ThreatBookAPI(Query):
         self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
         self.key = config.threatbook_api_key
 
-    def query(self, domain):
+    def query(self):
         """
         向接口查询子域并做子域匹配
         """
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
-        params = {'apikey': self.key, 'domain': domain, 'field': 'sub_domains'}
+        params = {'apikey': self.key,
+                  'domain': self.domain,
+                  'field': 'sub_domains'}
         resp = self.post(self.addr, params)
         if not resp:
             return
@@ -31,11 +33,11 @@ class ThreatBookAPI(Query):
         if not self.check(self.key):
             return
         self.begin()
-        self.query(self.domain)
+        self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/intelligence/threatminer.py

@@ -33,10 +33,10 @@ class ThreatMiner(Query):
         """
         self.begin()
         self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/intelligence/virustotal.py

@@ -15,7 +15,7 @@ class VirusTotal(Query):
         self.addr = 'https://www.virustotal.com/ui/domains/{}/subdomains'
         self.domain = self.register(domain)
 
-    def query(self, domain):
+    def query(self):
         """
         向接口查询子域并做子域匹配
         """
@@ -27,7 +27,7 @@ class VirusTotal(Query):
                                 'TE': 'Trailers'})
             self.proxy = self.get_proxy(self.source)
             params = {'limit': '40', 'cursor': next_cursor}
-            resp = self.get(url=self.addr.format(domain), params=params)
+            resp = self.get(url=self.addr.format(self.domain), params=params)
             if not resp:
                 return
             resp_json = resp.json()
@@ -53,11 +53,11 @@ class VirusTotal(Query):
         类执行入口
         """
         self.begin()
-        self.query(self.domain)
+        self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/intelligence/virustotal_api.py

@@ -11,13 +11,13 @@ class VirusTotalAPI(Query):
         self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'
         self.key = config.virustotal_api_key
 
-    def query(self, domain):
+    def query(self):
         """
         向接口查询子域并做子域匹配
         """
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
-        params = {'apikey': self.key, 'domain': domain}
+        params = {'apikey': self.key, 'domain': self.domain}
         resp = self.get(self.addr, params)
         if not resp:
             return
@@ -32,11 +32,11 @@ class VirusTotalAPI(Query):
         if not self.check(self.key):
             return
         self.begin()
-        self.query(self.domain)
+        self.query()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/ask.py

@@ -61,10 +61,10 @@ class Ask(Search):
                     count = subdomain.count('.') - self.domain.count('.')
                     if count == layer_num:
                         self.search(subdomain)
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/baidu.py

@@ -89,11 +89,10 @@ class Baidu(Search):
                     count = subdomain.count('.') - self.domain.count('.')
                     if count == layer_num:
                         self.search(subdomain)
-
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/bing.py

@@ -72,11 +72,10 @@ class Bing(Search):
                     count = subdomain.count('.') - self.domain.count('.')
                     if count == layer_num:
                         self.search(subdomain)
-
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/bing_api.py

@@ -9,7 +9,8 @@ class BingAPI(Search):
         self.domain = domain
         self.module = 'Search'
         self.source = 'BingAPISearch'
-        self.addr = 'https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search'
+        self.addr = 'https://api.cognitive.microsoft.com/' \
+                    'bingcustomsearch/v7.0/search'
         self.id = config.bing_api_id
         self.key = config.bing_api_key
         self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数
@@ -70,11 +71,10 @@ class BingAPI(Search):
                     count = subdomain.count('.') - self.domain.count('.')
                     if count == layer_num:
                         self.search(subdomain)
-
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/duckduckgo.py

@@ -68,11 +68,10 @@ class DuckDuckGO(Search):
                     count = subdomain.count('.') - self.domain.count('.')
                     if count == layer_num:
                         self.search(subdomain)
-
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/exalead.py

@@ -67,11 +67,10 @@ class Exalead(Search):
                     count = subdomain.count('.') - self.domain.count('.')
                     if count == layer_num:
                         self.search(subdomain)
-
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/fofa_api.py

@@ -44,10 +44,10 @@ class FoFa(Search):
             return
         self.begin()
         self.search()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/google.py

@@ -74,11 +74,10 @@ class Google(Search):
                     count = subdomain.count('.') - self.domain.count('.')
                     if count == layer_num:
                         self.search(subdomain)
-
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/google_api.py

@@ -68,11 +68,10 @@ class GoogleAPI(Search):
                     count = subdomain.count('.') - self.domain.count('.')
                     if count == layer_num:
                         self.search(subdomain)
-
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/shodan_api.py

@@ -38,10 +38,10 @@ class ShodanAPI(Search):
             return
         self.begin()
         self.search()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/so.py

@@ -67,11 +67,10 @@ class So(Search):
                     count = subdomain.count('.') - self.domain.count('.')
                     if count == layer_num:
                         self.search(subdomain)
-
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/sogou.py

@@ -65,11 +65,10 @@ class Sogou(Search):
                     count = subdomain.count('.') - self.domain.count('.')
                     if count == layer_num:
                         self.search(subdomain)
-
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/yahoo.py

@@ -72,11 +72,10 @@ class Yahoo(Search):
                     count = subdomain.count('.') - self.domain.count('.')
                     if count == layer_num:
                         self.search(subdomain)
-
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/yandex.py

@@ -72,11 +72,10 @@ class Yandex(Search):
                     count = subdomain.count('.') - self.domain.count('.')
                     if count == layer_num:
                         self.search(subdomain)
-
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用

oneforall/modules/search/zoomeye_api.py

@@ -64,10 +64,10 @@ class ZoomEyeAPI(Search):
             return
         self.begin()
         self.search()
+        self.finish()
         self.save_json()
         self.gen_result()
         self.save_db()
-        self.finish()
 
 
 def do(domain):  # 统一入口名字 方便多线程调用