From dda116b9e45de2aab838cd679fe9efb16073833e Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Wed, 8 Jul 2020 15:16:26 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A8=A1=E5=9D=97=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/certificates/censys_api.py | 2 +- modules/certificates/certspotter.py | 2 +- modules/check/cdx.py | 9 ++++----- modules/check/robots.py | 9 ++++----- modules/check/sitemap.py | 20 +++++--------------- modules/datasets/binaryedge_api.py | 2 +- modules/datasets/bufferover.py | 2 +- modules/datasets/cebaidu.py | 2 +- modules/datasets/chinaz_api.py | 2 +- modules/datasets/circl_api.py | 2 +- modules/datasets/passivedns_api.py | 2 +- modules/datasets/phonebook.py | 14 +++++++++----- modules/datasets/threatcrowd.py | 2 +- modules/intelligence/threatbook_api.py | 2 +- modules/search/bing_api.py | 2 +- modules/search/google_api.py | 2 +- 16 files changed, 34 insertions(+), 42 deletions(-) diff --git a/modules/certificates/censys_api.py b/modules/certificates/censys_api.py index 1c0c3a3..263cebc 100644 --- a/modules/certificates/censys_api.py +++ b/modules/certificates/censys_api.py @@ -41,7 +41,7 @@ class CensysAPI(Query): resp = self.post(self.addr, json=data, auth=(self.id, self.secret)) if not resp: return - subdomains = self.match_subdomains(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, resp.text) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/certificates/certspotter.py b/modules/certificates/certspotter.py index b6b32cf..76906c6 100644 --- a/modules/certificates/certspotter.py +++ b/modules/certificates/certspotter.py @@ -22,7 +22,7 @@ class CertSpotter(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/check/cdx.py b/modules/check/cdx.py index bdc3156..80bd9a0 100644 --- a/modules/check/cdx.py +++ b/modules/check/cdx.py @@ -27,12 +27,11 @@ class CheckCDX(Module): for url in urls: self.header = self.get_header() self.proxy = self.get_proxy(self.source) - response = self.get(url, check=False) - if not response: + resp = self.get(url, check=False) + if not resp: return - if response and len(response.content): - self.subdomains = self.match_subdomains(self.domain, - response.text) + if resp and len(resp.content): + self.subdomains = self.match_subdomains(self.domain, resp.text) def run(self): """ diff --git a/modules/check/robots.py b/modules/check/robots.py index 8116c7a..a67eb5d 100644 --- a/modules/check/robots.py +++ b/modules/check/robots.py @@ -26,12 +26,11 @@ class CheckRobots(Module): for url in urls: self.header = self.get_header() self.proxy = self.get_proxy(self.source) - response = self.get(url, check=False, allow_redirects=False) - if not response: + resp = self.get(url, check=False, allow_redirects=False) + if not resp: return - if response and len(response.content): - self.subdomains = self.match_subdomains(self.domain, - response.text) + if resp and len(resp.content): + self.subdomains = self.match_subdomains(self.domain, resp.text) def run(self): """ diff --git a/modules/check/sitemap.py b/modules/check/sitemap.py index c03122a..066de7d 100644 --- a/modules/check/sitemap.py +++ b/modules/check/sitemap.py @@ -20,32 +20,22 @@ class CheckRobots(Module): 正则匹配域名的sitemap文件中的子域 """ urls = [f'http://{self.domain}/sitemap.xml', - f'https://{self.domain}/sitemap.xml', f'http://www.{self.domain}/sitemap.xml', - f'https://www.{self.domain}/sitemap.xml', f'http://{self.domain}/sitemap.txt', - f'https://{self.domain}/sitemap.txt', f'http://www.{self.domain}/sitemap.txt', - f'https://www.{self.domain}/sitemap.txt', f'http://{self.domain}/sitemap.html', - f'https://{self.domain}/sitemap.html', f'http://www.{self.domain}/sitemap.html', - f'https://www.{self.domain}/sitemap.html', f'http://{self.domain}/sitemap_index.xml', - f'https://{self.domain}/sitemap_index.xml', - f'http://www.{self.domain}/sitemap_index.xml', - f'https://www.{self.domain}/sitemap_index.xml'] + f'http://www.{self.domain}/sitemap_index.xml'] for url in urls: self.header = self.get_header() self.proxy = self.get_proxy(self.source) self.timeout = 10 - response = self.get(url, check=False, allow_redirects=False) - if not response: + resp = self.get(url, check=False) + if not resp: return - if response and len(response.content): - self.subdomains = self.match_subdomains(self.domain, - response.text) - + if resp and len(resp.content): + self.subdomains = self.match_subdomains(self.domain, resp.text) def run(self): """ diff --git a/modules/datasets/binaryedge_api.py b/modules/datasets/binaryedge_api.py index 6a63e1f..59ffdc2 100644 --- a/modules/datasets/binaryedge_api.py +++ b/modules/datasets/binaryedge_api.py @@ -22,7 +22,7 @@ class BinaryEdgeAPI(Query): resp = self.get(url) if not resp: return - subdomains = self.match_subdomains(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, resp.text) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/datasets/bufferover.py b/modules/datasets/bufferover.py index b5eedb6..e9c9401 100644 --- a/modules/datasets/bufferover.py +++ b/modules/datasets/bufferover.py @@ -26,7 +26,7 @@ class BufferOver(Query): return if resp.status_code != 200: return - subdomains = self.match_subdomains(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/cebaidu.py b/modules/datasets/cebaidu.py index 5311246..f8363dd 100644 --- a/modules/datasets/cebaidu.py +++ b/modules/datasets/cebaidu.py @@ -19,7 +19,7 @@ class CeBaidu(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/chinaz_api.py b/modules/datasets/chinaz_api.py index 69ba907..f878702 100644 --- a/modules/datasets/chinaz_api.py +++ b/modules/datasets/chinaz_api.py @@ -21,7 +21,7 @@ class ChinazAPI(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/circl_api.py b/modules/datasets/circl_api.py index b2548f2..e4d6a36 100644 --- a/modules/datasets/circl_api.py +++ b/modules/datasets/circl_api.py @@ -21,7 +21,7 @@ class CirclAPI(Query): resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd)) if not resp: return - subdomains = self.match_subdomains(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/passivedns_api.py b/modules/datasets/passivedns_api.py index 30a9dfa..ff5805b 100644 --- a/modules/datasets/passivedns_api.py +++ b/modules/datasets/passivedns_api.py @@ -22,7 +22,7 @@ class PassiveDnsAPI(Query): resp = self.get(url) if not resp: return - subdomains = self.match_subdomains(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/phonebook.py b/modules/datasets/phonebook.py index 2a1be4b..9a63640 100644 --- a/modules/datasets/phonebook.py +++ b/modules/datasets/phonebook.py @@ -15,11 +15,13 @@ class PhoneBook(Query): """ self.header = self.get_header() self.proxy = self.get_proxy(self.source) - self.header.update({'Referer': 'https://phonebook.cz/', 'Origin': 'https://phonebook.cz'}) + self.header.update({'Referer': 'https://phonebook.cz/', + 'Origin': 'https://phonebook.cz'}) addr = 'https://public.intelx.io/phonebook/search' - key = 'ac572eea-3902-4e9a-972d-f5996d76174c' + key = 'd7d1ed06-f0c5-49d4-a9ca-a167e6d2ffab' url = f'{addr}?k={key}' - data = {"term": self.domain, "maxresults": 1000000, "media": 0, "target": 1, + data = {"term": self.domain, "maxresults": 10000, + "media": 0, "target": 1, "terminate": [], "timeout": 20} resp = self.post(url, json=data) if not resp: @@ -29,8 +31,10 @@ class PhoneBook(Query): if not ids: logger.log('ALERT', f'Get PhoneBook id fail') return - url = f'{addr}/result?k={key}&id={ids}&limit=1000000' + url = f'{addr}/result?k={key}&id={ids}&limit=10000' resp = self.get(url) + if not resp: + return subdomains = self.match_subdomains(self.domain, resp.text) self.subdomains = self.subdomains.union(subdomains) @@ -57,4 +61,4 @@ def do(domain): # 统一入口名字 方便多线程调用 if __name__ == '__main__': - do('example.com') + do('freebuf.com') diff --git a/modules/datasets/threatcrowd.py b/modules/datasets/threatcrowd.py index 03b118a..0971a43 100644 --- a/modules/datasets/threatcrowd.py +++ b/modules/datasets/threatcrowd.py @@ -25,7 +25,7 @@ class ThreatCrowd(Query): return if resp.status_code != 200: return - subdomains = self.match_subdomains(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/intelligence/threatbook_api.py b/modules/intelligence/threatbook_api.py index 29a6041..409c399 100644 --- a/modules/intelligence/threatbook_api.py +++ b/modules/intelligence/threatbook_api.py @@ -22,7 +22,7 @@ class ThreatBookAPI(Query): resp = self.post(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, resp.text) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/search/bing_api.py b/modules/search/bing_api.py index 2864811..8e46ee3 100644 --- a/modules/search/bing_api.py +++ b/modules/search/bing_api.py @@ -37,7 +37,7 @@ class BingAPI(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(domain, str(resp.json())) + subdomains = self.match_subdomains(domain, resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break if not full_search: diff --git a/modules/search/google_api.py b/modules/search/google_api.py index ae9613c..8647a2d 100644 --- a/modules/search/google_api.py +++ b/modules/search/google_api.py @@ -35,7 +35,7 @@ class GoogleAPI(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(domain, str(resp.json())) + subdomains = self.match_subdomains(domain, resp.text) if not subdomains: break if not full_search: