模块优化

modules/certificates/censys_api.py

@@ -41,7 +41,7 @@ class CensysAPI(Query):
             resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
             if not resp:
                 return
-            subdomains = self.match_subdomains(self.domain, str(resp.json()))
+            subdomains = self.match_subdomains(self.domain, resp.text)
             self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/certificates/certspotter.py

@@ -22,7 +22,7 @@ class CertSpotter(Query):
         resp = self.get(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, str(resp.json()))
+        subdomains = self.match_subdomains(self.domain, resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/check/cdx.py

@@ -27,12 +27,11 @@ class CheckCDX(Module):
         for url in urls:
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
-            response = self.get(url, check=False)
+            resp = self.get(url, check=False)
-            if not response:
+            if not resp:
                 return
-            if response and len(response.content):
+            if resp and len(resp.content):
-                self.subdomains = self.match_subdomains(self.domain,
+                self.subdomains = self.match_subdomains(self.domain, resp.text)
-                                                        response.text)
 
     def run(self):
         """

modules/check/robots.py

@@ -26,12 +26,11 @@ class CheckRobots(Module):
         for url in urls:
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
-            response = self.get(url, check=False, allow_redirects=False)
+            resp = self.get(url, check=False, allow_redirects=False)
-            if not response:
+            if not resp:
                 return
-            if response and len(response.content):
+            if resp and len(resp.content):
-                self.subdomains = self.match_subdomains(self.domain,
+                self.subdomains = self.match_subdomains(self.domain, resp.text)
-                                                        response.text)
 
     def run(self):
         """

modules/check/sitemap.py

@@ -20,32 +20,22 @@ class CheckRobots(Module):
         正则匹配域名的sitemap文件中的子域
         """
         urls = [f'http://{self.domain}/sitemap.xml',
-                f'https://{self.domain}/sitemap.xml',
                 f'http://www.{self.domain}/sitemap.xml',
-                f'https://www.{self.domain}/sitemap.xml',
                 f'http://{self.domain}/sitemap.txt',
-                f'https://{self.domain}/sitemap.txt',
                 f'http://www.{self.domain}/sitemap.txt',
-                f'https://www.{self.domain}/sitemap.txt',
                 f'http://{self.domain}/sitemap.html',
-                f'https://{self.domain}/sitemap.html',
                 f'http://www.{self.domain}/sitemap.html',
-                f'https://www.{self.domain}/sitemap.html',
                 f'http://{self.domain}/sitemap_index.xml',
-                f'https://{self.domain}/sitemap_index.xml',
+                f'http://www.{self.domain}/sitemap_index.xml']
-                f'http://www.{self.domain}/sitemap_index.xml',
-                f'https://www.{self.domain}/sitemap_index.xml']
         for url in urls:
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             self.timeout = 10
-            response = self.get(url, check=False, allow_redirects=False)
+            resp = self.get(url, check=False)
-            if not response:
+            if not resp:
                 return
-            if response and len(response.content):
+            if resp and len(resp.content):
-                self.subdomains = self.match_subdomains(self.domain,
+                self.subdomains = self.match_subdomains(self.domain, resp.text)
-                                                        response.text)
-
 
     def run(self):
         """

modules/datasets/binaryedge_api.py

@@ -22,7 +22,7 @@ class BinaryEdgeAPI(Query):
         resp = self.get(url)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, str(resp.json()))
+        subdomains = self.match_subdomains(self.domain, resp.text)
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/datasets/bufferover.py

@@ -26,7 +26,7 @@ class BufferOver(Query):
             return
         if resp.status_code != 200:
             return
-        subdomains = self.match_subdomains(self.domain, str(resp.json()))
+        subdomains = self.match_subdomains(self.domain, resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/cebaidu.py

@@ -19,7 +19,7 @@ class CeBaidu(Query):
         resp = self.get(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, str(resp.json()))
+        subdomains = self.match_subdomains(self.domain, resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/chinaz_api.py

@@ -21,7 +21,7 @@ class ChinazAPI(Query):
         resp = self.get(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, str(resp.json()))
+        subdomains = self.match_subdomains(self.domain, resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/circl_api.py

@@ -21,7 +21,7 @@ class CirclAPI(Query):
         resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, str(resp.json()))
+        subdomains = self.match_subdomains(self.domain, resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/passivedns_api.py

@@ -22,7 +22,7 @@ class PassiveDnsAPI(Query):
         resp = self.get(url)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, str(resp.json()))
+        subdomains = self.match_subdomains(self.domain, resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/phonebook.py

@@ -15,11 +15,13 @@ class PhoneBook(Query):
         """
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
-        self.header.update({'Referer': 'https://phonebook.cz/', 'Origin': 'https://phonebook.cz'})
+        self.header.update({'Referer': 'https://phonebook.cz/',
+                            'Origin': 'https://phonebook.cz'})
         addr = 'https://public.intelx.io/phonebook/search'
-        key = 'ac572eea-3902-4e9a-972d-f5996d76174c'
+        key = 'd7d1ed06-f0c5-49d4-a9ca-a167e6d2ffab'
         url = f'{addr}?k={key}'
-        data = {"term": self.domain, "maxresults": 1000000, "media": 0, "target": 1,
+        data = {"term": self.domain, "maxresults": 10000,
+                "media": 0, "target": 1,
                 "terminate": [], "timeout": 20}
         resp = self.post(url, json=data)
         if not resp:
@@ -29,8 +31,10 @@ class PhoneBook(Query):
         if not ids:
             logger.log('ALERT', f'Get PhoneBook id fail')
             return
-        url = f'{addr}/result?k={key}&id={ids}&limit=1000000'
+        url = f'{addr}/result?k={key}&id={ids}&limit=10000'
         resp = self.get(url)
+        if not resp:
+            return
         subdomains = self.match_subdomains(self.domain, resp.text)
         self.subdomains = self.subdomains.union(subdomains)
 
@@ -57,4 +61,4 @@ def do(domain):  # 统一入口名字 方便多线程调用
 
 
 if __name__ == '__main__':
-    do('example.com')
+    do('freebuf.com')

modules/datasets/threatcrowd.py

@@ -25,7 +25,7 @@ class ThreatCrowd(Query):
             return
         if resp.status_code != 200:
             return
-        subdomains = self.match_subdomains(self.domain, str(resp.json()))
+        subdomains = self.match_subdomains(self.domain, resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/intelligence/threatbook_api.py

@@ -22,7 +22,7 @@ class ThreatBookAPI(Query):
         resp = self.post(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, str(resp.json()))
+        subdomains = self.match_subdomains(self.domain, resp.text)
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/search/bing_api.py

@@ -37,7 +37,7 @@ class BingAPI(Search):
             resp = self.get(self.addr, params)
             if not resp:
                 return
-            subdomains = self.match_subdomains(domain, str(resp.json()))
+            subdomains = self.match_subdomains(domain, resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             if not full_search:

modules/search/google_api.py

@@ -35,7 +35,7 @@ class GoogleAPI(Search):
             resp = self.get(self.addr, params)
             if not resp:
                 return
-            subdomains = self.match_subdomains(domain, str(resp.json()))
+            subdomains = self.match_subdomains(domain, resp.text)
             if not subdomains:
                 break
             if not full_search: