模块优化

common/module.py

@@ -208,8 +208,19 @@ class Module(object):
             logger.log('TRACE', f'{module} module does not use proxy')
             return self.proxy
 
-    def match_subdomains(self, html, distinct=True, fuzzy=True):
+    def match_subdomains(self, resp, distinct=True, fuzzy=True):
-        return utils.match_subdomains(self.domain, html, distinct, fuzzy)
+        if not resp:
+            return set()
+        elif isinstance(resp, str):
+            return utils.match_subdomains(self.domain, resp, distinct, fuzzy)
+        elif hasattr(resp, 'text'):
+            return utils.match_subdomains(self.domain, resp.text, distinct, fuzzy)
+        else:
+            return set()
+
+    def collect_subdomains(self, resp):
+        subdomains = self.match_subdomains(resp)
+        return self.subdomains.union(subdomains)
 
     def save_json(self):
         """

modules/certificates/censys_api.py

@@ -39,10 +39,7 @@ class CensysAPI(Query):
         for page in range(2, pages + 1):
             data['page'] = page
             resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
-            if not resp:
+            self.subdomains = self.collect_subdomains(resp)
-                return
-            subdomains = self.match_subdomains(resp.text)
-            self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/certificates/certspotter.py

@@ -19,11 +19,7 @@ class CertSpotter(Query):
                   'include_subdomains': 'true',
                   'expand': 'dns_names'}
         resp = self.get(self.addr, params)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/certificates/google.py

@@ -20,11 +20,7 @@ class Google(Query):
                   'include_subdomains': 'true',
                   'domain': self.domain}
         resp = self.get(self.addr, params)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/crawl/archivecrawl.py

@@ -28,7 +28,6 @@ class ArchiveCrawl(Crawl):
                 url = resp.data.get('url')
                 subdomains = self.match_subdomains(self.get_maindomain(domain),
                                                    url + resp.text)
-                # 合并搜索子域名搜索结果
                 self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/crawl/commoncrawl.py

@@ -27,7 +27,6 @@ class CommonCrawl(Crawl):
         for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
             if resp.data.get('status') not in ['301', '302']:
                 subdomains = self.match_subdomains(self.get_maindomain(domain), resp.text)
-                # 合并搜索子域名搜索结果
                 self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/datasets/binaryedge_api.py

@@ -20,10 +20,7 @@ class BinaryEdgeAPI(Query):
         self.proxy = self.get_proxy(self.source)
         url = self.addr + self.domain
         resp = self.get(url)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/bufferover.py

@@ -26,9 +26,7 @@ class BufferOver(Query):
             return
         if resp.status_code != 200:
             return
-        subdomains = self.match_subdomains(resp.text)
+        self.subdomains = self.collect_subdomains(resp)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/cebaidu.py

@@ -17,11 +17,7 @@ class CeBaidu(Query):
         self.proxy = self.get_proxy(self.source)
         params = {'site_address': self.domain}
         resp = self.get(self.addr, params)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/chinaz.py

@@ -17,11 +17,7 @@ class Chinaz(Query):
         self.proxy = self.get_proxy(self.source)
         self.addr = self.addr + self.domain
         resp = self.get(self.addr)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/chinaz_api.py

@@ -19,11 +19,7 @@ class ChinazAPI(Query):
         self.proxy = self.get_proxy(self.source)
         params = {'key': self.api, 'domainName': self.domain}
         resp = self.get(self.addr, params)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/circl_api.py

@@ -19,11 +19,7 @@ class CirclAPI(Query):
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/cloudflare_api.py

@@ -48,7 +48,8 @@ class CloudFlareAPI(Query):
                     return
             elif zones_resp.status_code == 403:
                 logger.log('DEBUG',
-                           f'{self.domain} is banned or not a registered domain, so cannot be added to Cloudflare.')
+                           f'{self.domain} is banned or not a registered domain, '
+                           f'so cannot be added to Cloudflare.')
                 return
             else:
                 logger.log('DEBUG',
@@ -67,12 +68,14 @@ class CloudFlareAPI(Query):
         if create_zone_resp.json()['success']:
             return create_zone_resp.json()['result']['id']
         else:
-            logger.log('DEBUG', f'{self.domain} is temporarily banned and cannot be added to Cloudflare')
+            logger.log('DEBUG', f'{self.domain} is temporarily banned '
+                                f'and cannot be added to Cloudflare')
             return False
 
     def list_dns(self, zone_id):
         page = 1
-        list_dns_resp = self.get(self.addr + f'zones/{zone_id}/dns_records', params={'page': page, 'per_page': 10})
+        list_dns_resp = self.get(self.addr + f'zones/{zone_id}/dns_records',
+                                 params={'page': page, 'per_page': 10})
         if not list_dns_resp:
             logger.log('DEBUG',
                        f'{list_dns_resp.status_code} {list_dns_resp.text}')

modules/datasets/dnsdb_api.py

@@ -20,11 +20,7 @@ class DNSdbAPI(Query):
         self.proxy = self.get_proxy(self.source)
         url = f'{self.addr}*.{self.domain}'
         resp = self.get(url)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/dnsdumpster.py

@@ -23,12 +23,7 @@ class DNSdumpster(Query):
         data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'),
                 'targetip': self.domain}
         resp = self.post(self.addr, data)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        if subdomains:
-            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/hackertarget.py

@@ -17,13 +17,7 @@ class HackerTarget(Query):
         self.proxy = self.get_proxy(self.source)
         params = {'q': self.domain}
         resp = self.get(self.addr, params)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        if resp.status_code == 200:
-            subdomains = self.match_subdomains(resp.text)
-            if subdomains:
-                # 合并搜索子域名搜索结果
-                self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/ip138.py

@@ -17,11 +17,7 @@ class IP138(Query):
         self.proxy = self.get_proxy(self.source)
         self.addr = self.addr.format(domain=self.domain)
         resp = self.get(self.addr)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/ipv4info_api.py

@@ -35,7 +35,6 @@ class IPv4InfoAPI(Query):
             subdomains = self.match_subdomains(str(json))
             if not subdomains:
                 break
-            # 合并搜索子域名搜索结果
             self.subdomains = self.subdomains.union(subdomains)
             # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
             subdomains = json.get('Subdomains')

modules/datasets/netcraft.py

@@ -47,12 +47,9 @@ class NetCraft(Query):
                       'host': '.' + self.domain,
                       'from': self.page_num}
             resp = self.get(self.addr + last, params)
-            if not resp:
+            subdomains = self.match_subdomains(resp)
-                return
-            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
-            # 合并搜索子域名搜索结果
             self.subdomains = self.subdomains.union(subdomains)
             if 'Next Page' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                 break

modules/datasets/passivedns_api.py

@@ -20,11 +20,7 @@ class PassiveDnsAPI(Query):
         self.proxy = self.get_proxy(self.source)
         url = self.addr + '/flint/rrset/*.' + self.domain
         resp = self.get(url)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/phonebook.py

@@ -33,10 +33,7 @@ class PhoneBook(Query):
             return
         url = f'{addr}/result?k={key}&id={ids}&limit=10000'
         resp = self.get(url)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/qianxun.py

@@ -25,10 +25,7 @@ class QianXun(Query):
             url = f'https://www.dnsscan.cn/dns.html?' \
                   f'keywords={self.domain}&page={num}'
             resp = self.post(url, data)
-            if not resp:
+            self.subdomains = self.collect_subdomains(resp)
-                break
-            subdomains = self.match_subdomains(resp.text)
-            self.subdomains = self.subdomains.union(subdomains)
             if '<div id="page" class="pagelist">' not in resp.text:
                 break
             if '<li class="disabled"><span>&raquo;</span></li>' in resp.text:

modules/datasets/rapiddns.py

@@ -17,11 +17,7 @@ class RapidDNS(Query):
         url = f'http://rapiddns.io/subdomain/{self.domain}'
         params = {'full': '1'}
         resp = self.get(url, params)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/riddler.py

@@ -17,11 +17,7 @@ class Riddler(Query):
         self.proxy = self.get_proxy(self.source)
         params = {'q': 'pld:' + self.domain}
         resp = self.get(self.addr, params)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/robtex.py

@@ -30,12 +30,7 @@ class Robtex(Query):
                 ip = record.get('rrdata')
                 url = self.addr + 'reverse/' + ip
                 resp = self.get(url)
-                if not resp:
+                self.subdomains = self.collect_subdomains(resp)
-                    return
-                subdomains = self.match_subdomains(resp.text)
-                if subdomains:
-                    # 合并搜索子域名搜索结果
-                    self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/securitytrails_api.py

@@ -26,7 +26,6 @@ class SecurityTrailsAPI(Query):
         prefixs = resp.json()['subdomains']
         subdomains = [f'{prefix}.{self.domain}' for prefix in prefixs]
         if subdomains:
-            # 合并搜索子域名搜索结果
             self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/datasets/sitedossier.py

@@ -22,12 +22,9 @@ class SiteDossier(Query):
             self.proxy = self.get_proxy(self.source)
             url = f'{self.addr}{self.domain}/{self.page_num}'
             resp = self.get(url)
-            if not resp:
+            subdomains = self.match_subdomains(resp)
-                return
-            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
-            # 合并搜索子域名搜索结果
             self.subdomains = self.subdomains.union(subdomains)
             # 搜索页面没有出现下一页时停止搜索
             if 'Show next 100 items' not in resp.text:

modules/datasets/spyse_api.py

@@ -29,7 +29,6 @@ class SpyseAPI(Query):
             subdomains = self.match_subdomains(str(json))
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
-            # 合并搜索子域名搜索结果
             self.subdomains = self.subdomains.union(subdomains)
             offset += limit
             if len(json.get('data').get('items')) < limit:

modules/datasets/sublist3r.py

@@ -17,10 +17,7 @@ class Sublist3r(Query):
         addr = 'https://api.sublist3r.com/search.php'
         param = {'domain': self.domain}
         resp = self.get(addr, param)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/datasets/threatcrowd.py

@@ -26,7 +26,6 @@ class ThreatCrowd(Query):
         if resp.status_code != 200:
             return
         subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/datasets/ximcx.py

@@ -21,7 +21,6 @@ class Ximcx(Query):
             return
         json = resp.json()
         subdomains = self.match_subdomains(str(json))
-        # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/intelligence/alienvault.py

@@ -18,19 +18,11 @@ class AlienVault(Query):
         base = 'https://otx.alienvault.com/api/v1/indicators/domain'
         dns = f'{base}/{self.domain}/passive_dns'
         resp = self.get(dns)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        json = resp.json()
-        subdomains = self.match_subdomains(str(json))
-        self.subdomains = self.subdomains.union(subdomains)
 
         url = f'{base}/{self.domain}/url_list'
         resp = self.get(url)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        json = resp.json()
-        subdomains = self.match_subdomains(str(json))
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/intelligence/threatbook_api.py

@@ -20,10 +20,7 @@ class ThreatBookAPI(Query):
         params = {'apikey': self.key,
                   'resource': self.domain}
         resp = self.post(self.addr, params)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/intelligence/threatminer.py

@@ -18,11 +18,7 @@ class ThreatMiner(Query):
         params = {'e': 'subdomains_container',
                   'q': self.domain, 't': 0, 'rt': 10}
         resp = self.get(self.addr, params)
-        if not resp:
+        self.subdomains = self.collect_subdomains(resp)
-            return
-        subdomains = self.match_subdomains(resp.text)
-        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/search/baidu.py

@@ -58,7 +58,6 @@ class Baidu(Search):
             if not full_search and subdomains.issubset(self.subdomains):
                 # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
                 break
-            # 合并搜索子域名搜索结果
             self.subdomains = self.subdomains.union(subdomains)
             self.page_num += self.per_page_num
             # 搜索页面没有出现下一页时停止搜索

modules/search/bing.py

@@ -42,7 +42,6 @@ class Bing(Search):
             if not full_search and subdomains.issubset(self.subdomains):
                 # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
                 break
-            # 合并搜索子域名搜索结果
             self.subdomains = self.subdomains.union(subdomains)
             # 搜索页面没有出现下一页时停止搜索
             if '<div class="sw_next">' not in resp.text:

modules/search/bing_api.py

@@ -34,15 +34,12 @@ class BingAPI(Search):
                       'count': self.per_page_num,
                       'offset': self.page_num}
             resp = self.get(self.addr, params)
-            if not resp:
+            subdomains = self.match_subdomains(resp)
-                return
-            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             if not full_search and subdomains.issubset(self.subdomains):
                 # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
                 break
-            # 合并搜索子域名搜索结果
             self.subdomains = self.subdomains.union(subdomains)
             self.page_num += self.per_page_num
             if self.page_num >= self.limit_num:  # 搜索条数限制

modules/search/google_api.py

@@ -33,9 +33,7 @@ class GoogleAPI(Search):
                       'q': word, 'fields': 'items/link',
                       'start': self.page_num, 'num': self.per_page_num}
             resp = self.get(self.addr, params)
-            if not resp:
+            subdomains = self.match_subdomains(resp)
-                return
-            subdomains = self.match_subdomains(resp.text)
             if not subdomains:
                 break
             if not full_search and subdomains.issubset(self.subdomains):

modules/search/shodan_api.py

@@ -23,9 +23,7 @@ class ShodanAPI(Search):
             params = {'key': self.key, 'page': page, 'query': query,
                       'minify': True, 'facets': {'hostnames'}}
             resp = self.get(self.addr, params)
-            if not resp:
+            subdomains = self.match_subdomains(resp)
-                return
-            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             if subdomains:

modules/search/yahoo.py

@@ -42,7 +42,6 @@ class Yahoo(Search):
             if not full_search and subdomains.issubset(self.subdomains):
                 # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
                 break
-            # 合并搜索子域名搜索结果
             self.subdomains = self.subdomains.union(subdomains)
             if '>Next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                 break

modules/search/yandex.py

@@ -42,7 +42,6 @@ class Yandex(Search):
             if not full_search and subdomains.issubset(self.subdomains):
                 # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
                 break
-            # 合并搜索子域名搜索结果
             self.subdomains = self.subdomains.union(subdomains)
             if '>next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                 break

modules/search/zoomeye_api.py

@@ -46,9 +46,7 @@ class ZoomEyeAPI(Search):
             self.header.update({'Authorization': 'JWT ' + access_token})
             params = {'query': 'hostname:' + self.domain, 'page': page_num}
             resp = self.get(self.addr, params)
-            if not resp:
+            subdomains = self.match_subdomains(resp)
-                return
-            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             self.subdomains = self.subdomains.union(subdomains)