diff --git a/common/module.py b/common/module.py index 43a0140..3655ef5 100644 --- a/common/module.py +++ b/common/module.py @@ -208,8 +208,19 @@ class Module(object): logger.log('TRACE', f'{module} module does not use proxy') return self.proxy - def match_subdomains(self, html, distinct=True, fuzzy=True): - return utils.match_subdomains(self.domain, html, distinct, fuzzy) + def match_subdomains(self, resp, distinct=True, fuzzy=True): + if not resp: + return set() + elif isinstance(resp, str): + return utils.match_subdomains(self.domain, resp, distinct, fuzzy) + elif hasattr(resp, 'text'): + return utils.match_subdomains(self.domain, resp.text, distinct, fuzzy) + else: + return set() + + def collect_subdomains(self, resp): + subdomains = self.match_subdomains(resp) + return self.subdomains.union(subdomains) def save_json(self): """ diff --git a/modules/certificates/censys_api.py b/modules/certificates/censys_api.py index 6b74188..559bc9d 100644 --- a/modules/certificates/censys_api.py +++ b/modules/certificates/censys_api.py @@ -39,10 +39,7 @@ class CensysAPI(Query): for page in range(2, pages + 1): data['page'] = page resp = self.post(self.addr, json=data, auth=(self.id, self.secret)) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/certificates/certspotter.py b/modules/certificates/certspotter.py index c4fa32b..148d97a 100644 --- a/modules/certificates/certspotter.py +++ b/modules/certificates/certspotter.py @@ -19,11 +19,7 @@ class CertSpotter(Query): 'include_subdomains': 'true', 'expand': 'dns_names'} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/certificates/google.py b/modules/certificates/google.py index cc9b991..36b0de4 100644 --- a/modules/certificates/google.py +++ b/modules/certificates/google.py @@ -20,11 +20,7 @@ class Google(Query): 'include_subdomains': 'true', 'domain': self.domain} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/crawl/archivecrawl.py b/modules/crawl/archivecrawl.py index cd632f8..1b6445a 100644 --- a/modules/crawl/archivecrawl.py +++ b/modules/crawl/archivecrawl.py @@ -28,7 +28,6 @@ class ArchiveCrawl(Crawl): url = resp.data.get('url') subdomains = self.match_subdomains(self.get_maindomain(domain), url + resp.text) - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/crawl/commoncrawl.py b/modules/crawl/commoncrawl.py index 6d6b07f..d79e3c0 100644 --- a/modules/crawl/commoncrawl.py +++ b/modules/crawl/commoncrawl.py @@ -27,7 +27,6 @@ class CommonCrawl(Crawl): for resp in tqdm(cdx.iter(url, limit=limit), total=limit): if resp.data.get('status') not in ['301', '302']: subdomains = self.match_subdomains(self.get_maindomain(domain), resp.text) - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/datasets/binaryedge_api.py b/modules/datasets/binaryedge_api.py index b7e2477..f4e438f 100644 --- a/modules/datasets/binaryedge_api.py +++ b/modules/datasets/binaryedge_api.py @@ -20,10 +20,7 @@ class BinaryEdgeAPI(Query): self.proxy = self.get_proxy(self.source) url = self.addr + self.domain resp = self.get(url) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/bufferover.py b/modules/datasets/bufferover.py index 86cc8fc..56928a9 100644 --- a/modules/datasets/bufferover.py +++ b/modules/datasets/bufferover.py @@ -26,9 +26,7 @@ class BufferOver(Query): return if resp.status_code != 200: return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/cebaidu.py b/modules/datasets/cebaidu.py index 8fbb5e5..c074deb 100644 --- a/modules/datasets/cebaidu.py +++ b/modules/datasets/cebaidu.py @@ -17,11 +17,7 @@ class CeBaidu(Query): self.proxy = self.get_proxy(self.source) params = {'site_address': self.domain} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/chinaz.py b/modules/datasets/chinaz.py index 3598555..8f39355 100644 --- a/modules/datasets/chinaz.py +++ b/modules/datasets/chinaz.py @@ -17,11 +17,7 @@ class Chinaz(Query): self.proxy = self.get_proxy(self.source) self.addr = self.addr + self.domain resp = self.get(self.addr) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/chinaz_api.py b/modules/datasets/chinaz_api.py index 28afca5..4d6a746 100644 --- a/modules/datasets/chinaz_api.py +++ b/modules/datasets/chinaz_api.py @@ -19,11 +19,7 @@ class ChinazAPI(Query): self.proxy = self.get_proxy(self.source) params = {'key': self.api, 'domainName': self.domain} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/circl_api.py b/modules/datasets/circl_api.py index f160869..b1c00b2 100644 --- a/modules/datasets/circl_api.py +++ b/modules/datasets/circl_api.py @@ -19,11 +19,7 @@ class CirclAPI(Query): self.header = self.get_header() self.proxy = self.get_proxy(self.source) resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd)) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/cloudflare_api.py b/modules/datasets/cloudflare_api.py index 5dd1e01..157029e 100644 --- a/modules/datasets/cloudflare_api.py +++ b/modules/datasets/cloudflare_api.py @@ -48,7 +48,8 @@ class CloudFlareAPI(Query): return elif zones_resp.status_code == 403: logger.log('DEBUG', - f'{self.domain} is banned or not a registered domain, so cannot be added to Cloudflare.') + f'{self.domain} is banned or not a registered domain, ' + f'so cannot be added to Cloudflare.') return else: logger.log('DEBUG', @@ -67,12 +68,14 @@ class CloudFlareAPI(Query): if create_zone_resp.json()['success']: return create_zone_resp.json()['result']['id'] else: - logger.log('DEBUG', f'{self.domain} is temporarily banned and cannot be added to Cloudflare') + logger.log('DEBUG', f'{self.domain} is temporarily banned ' + f'and cannot be added to Cloudflare') return False def list_dns(self, zone_id): page = 1 - list_dns_resp = self.get(self.addr + f'zones/{zone_id}/dns_records', params={'page': page, 'per_page': 10}) + list_dns_resp = self.get(self.addr + f'zones/{zone_id}/dns_records', + params={'page': page, 'per_page': 10}) if not list_dns_resp: logger.log('DEBUG', f'{list_dns_resp.status_code} {list_dns_resp.text}') diff --git a/modules/datasets/dnsdb_api.py b/modules/datasets/dnsdb_api.py index f763cea..13e8031 100644 --- a/modules/datasets/dnsdb_api.py +++ b/modules/datasets/dnsdb_api.py @@ -20,11 +20,7 @@ class DNSdbAPI(Query): self.proxy = self.get_proxy(self.source) url = f'{self.addr}*.{self.domain}' resp = self.get(url) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/dnsdumpster.py b/modules/datasets/dnsdumpster.py index f809cf4..aabb8ad 100644 --- a/modules/datasets/dnsdumpster.py +++ b/modules/datasets/dnsdumpster.py @@ -23,12 +23,7 @@ class DNSdumpster(Query): data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'), 'targetip': self.domain} resp = self.post(self.addr, data) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - if subdomains: - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/hackertarget.py b/modules/datasets/hackertarget.py index de2e7ca..4fedf26 100644 --- a/modules/datasets/hackertarget.py +++ b/modules/datasets/hackertarget.py @@ -17,13 +17,7 @@ class HackerTarget(Query): self.proxy = self.get_proxy(self.source) params = {'q': self.domain} resp = self.get(self.addr, params) - if not resp: - return - if resp.status_code == 200: - subdomains = self.match_subdomains(resp.text) - if subdomains: - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/ip138.py b/modules/datasets/ip138.py index 1e49ad4..a27ff82 100644 --- a/modules/datasets/ip138.py +++ b/modules/datasets/ip138.py @@ -17,11 +17,7 @@ class IP138(Query): self.proxy = self.get_proxy(self.source) self.addr = self.addr.format(domain=self.domain) resp = self.get(self.addr) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/ipv4info_api.py b/modules/datasets/ipv4info_api.py index aed58a9..ff5a609 100644 --- a/modules/datasets/ipv4info_api.py +++ b/modules/datasets/ipv4info_api.py @@ -35,7 +35,6 @@ class IPv4InfoAPI(Query): subdomains = self.match_subdomains(str(json)) if not subdomains: break - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名 subdomains = json.get('Subdomains') diff --git a/modules/datasets/netcraft.py b/modules/datasets/netcraft.py index 76f472a..6cc37b8 100644 --- a/modules/datasets/netcraft.py +++ b/modules/datasets/netcraft.py @@ -47,12 +47,9 @@ class NetCraft(Query): 'host': '.' + self.domain, 'from': self.page_num} resp = self.get(self.addr + last, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) + subdomains = self.match_subdomains(resp) if not subdomains: # 搜索没有发现子域名则停止搜索 break - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) if 'Next Page' not in resp.text: # 搜索页面没有出现下一页时停止搜索 break diff --git a/modules/datasets/passivedns_api.py b/modules/datasets/passivedns_api.py index e8217d5..f8eb54e 100644 --- a/modules/datasets/passivedns_api.py +++ b/modules/datasets/passivedns_api.py @@ -20,11 +20,7 @@ class PassiveDnsAPI(Query): self.proxy = self.get_proxy(self.source) url = self.addr + '/flint/rrset/*.' + self.domain resp = self.get(url) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/phonebook.py b/modules/datasets/phonebook.py index c708974..ff2fdb6 100644 --- a/modules/datasets/phonebook.py +++ b/modules/datasets/phonebook.py @@ -33,10 +33,7 @@ class PhoneBook(Query): return url = f'{addr}/result?k={key}&id={ids}&limit=10000' resp = self.get(url) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/qianxun.py b/modules/datasets/qianxun.py index 66cbf51..df9149a 100644 --- a/modules/datasets/qianxun.py +++ b/modules/datasets/qianxun.py @@ -25,10 +25,7 @@ class QianXun(Query): url = f'https://www.dnsscan.cn/dns.html?' \ f'keywords={self.domain}&page={num}' resp = self.post(url, data) - if not resp: - break - subdomains = self.match_subdomains(resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) if '