From 8a035c76ca69766f67663d16da2b323048bfb080 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Mon, 10 Aug 2020 18:19:11 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A8=A1=E5=9D=97=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/module.py | 15 +++++++++++++-- modules/certificates/censys_api.py | 5 +---- modules/certificates/certspotter.py | 6 +----- modules/certificates/google.py | 6 +----- modules/crawl/archivecrawl.py | 1 - modules/crawl/commoncrawl.py | 1 - modules/datasets/binaryedge_api.py | 5 +---- modules/datasets/bufferover.py | 4 +--- modules/datasets/cebaidu.py | 6 +----- modules/datasets/chinaz.py | 6 +----- modules/datasets/chinaz_api.py | 6 +----- modules/datasets/circl_api.py | 6 +----- modules/datasets/cloudflare_api.py | 9 ++++++--- modules/datasets/dnsdb_api.py | 6 +----- modules/datasets/dnsdumpster.py | 7 +------ modules/datasets/hackertarget.py | 8 +------- modules/datasets/ip138.py | 6 +----- modules/datasets/ipv4info_api.py | 1 - modules/datasets/netcraft.py | 5 +---- modules/datasets/passivedns_api.py | 6 +----- modules/datasets/phonebook.py | 5 +---- modules/datasets/qianxun.py | 5 +---- modules/datasets/rapiddns.py | 6 +----- modules/datasets/riddler.py | 6 +----- modules/datasets/robtex.py | 7 +------ modules/datasets/securitytrails_api.py | 1 - modules/datasets/sitedossier.py | 5 +---- modules/datasets/spyse_api.py | 1 - modules/datasets/sublist3r.py | 5 +---- modules/datasets/threatcrowd.py | 1 - modules/datasets/ximcx.py | 1 - modules/intelligence/alienvault.py | 12 ++---------- modules/intelligence/threatbook_api.py | 5 +---- modules/intelligence/threatminer.py | 6 +----- modules/search/baidu.py | 1 - modules/search/bing.py | 1 - modules/search/bing_api.py | 5 +---- modules/search/google_api.py | 4 +--- modules/search/shodan_api.py | 4 +--- modules/search/yahoo.py | 1 - modules/search/yandex.py | 1 - modules/search/zoomeye_api.py | 4 +--- 42 files changed, 49 insertions(+), 153 deletions(-) diff --git a/common/module.py b/common/module.py index 43a0140..3655ef5 100644 --- a/common/module.py +++ b/common/module.py @@ -208,8 +208,19 @@ class Module(object): logger.log('TRACE', f'{module} module does not use proxy') return self.proxy - def match_subdomains(self, html, distinct=True, fuzzy=True): - return utils.match_subdomains(self.domain, html, distinct, fuzzy) + def match_subdomains(self, resp, distinct=True, fuzzy=True): + if not resp: + return set() + elif isinstance(resp, str): + return utils.match_subdomains(self.domain, resp, distinct, fuzzy) + elif hasattr(resp, 'text'): + return utils.match_subdomains(self.domain, resp.text, distinct, fuzzy) + else: + return set() + + def collect_subdomains(self, resp): + subdomains = self.match_subdomains(resp) + return self.subdomains.union(subdomains) def save_json(self): """ diff --git a/modules/certificates/censys_api.py b/modules/certificates/censys_api.py index 6b74188..559bc9d 100644 --- a/modules/certificates/censys_api.py +++ b/modules/certificates/censys_api.py @@ -39,10 +39,7 @@ class CensysAPI(Query): for page in range(2, pages + 1): data['page'] = page resp = self.post(self.addr, json=data, auth=(self.id, self.secret)) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/certificates/certspotter.py b/modules/certificates/certspotter.py index c4fa32b..148d97a 100644 --- a/modules/certificates/certspotter.py +++ b/modules/certificates/certspotter.py @@ -19,11 +19,7 @@ class CertSpotter(Query): 'include_subdomains': 'true', 'expand': 'dns_names'} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/certificates/google.py b/modules/certificates/google.py index cc9b991..36b0de4 100644 --- a/modules/certificates/google.py +++ b/modules/certificates/google.py @@ -20,11 +20,7 @@ class Google(Query): 'include_subdomains': 'true', 'domain': self.domain} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/crawl/archivecrawl.py b/modules/crawl/archivecrawl.py index cd632f8..1b6445a 100644 --- a/modules/crawl/archivecrawl.py +++ b/modules/crawl/archivecrawl.py @@ -28,7 +28,6 @@ class ArchiveCrawl(Crawl): url = resp.data.get('url') subdomains = self.match_subdomains(self.get_maindomain(domain), url + resp.text) - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/crawl/commoncrawl.py b/modules/crawl/commoncrawl.py index 6d6b07f..d79e3c0 100644 --- a/modules/crawl/commoncrawl.py +++ b/modules/crawl/commoncrawl.py @@ -27,7 +27,6 @@ class CommonCrawl(Crawl): for resp in tqdm(cdx.iter(url, limit=limit), total=limit): if resp.data.get('status') not in ['301', '302']: subdomains = self.match_subdomains(self.get_maindomain(domain), resp.text) - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/datasets/binaryedge_api.py b/modules/datasets/binaryedge_api.py index b7e2477..f4e438f 100644 --- a/modules/datasets/binaryedge_api.py +++ b/modules/datasets/binaryedge_api.py @@ -20,10 +20,7 @@ class BinaryEdgeAPI(Query): self.proxy = self.get_proxy(self.source) url = self.addr + self.domain resp = self.get(url) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/bufferover.py b/modules/datasets/bufferover.py index 86cc8fc..56928a9 100644 --- a/modules/datasets/bufferover.py +++ b/modules/datasets/bufferover.py @@ -26,9 +26,7 @@ class BufferOver(Query): return if resp.status_code != 200: return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/cebaidu.py b/modules/datasets/cebaidu.py index 8fbb5e5..c074deb 100644 --- a/modules/datasets/cebaidu.py +++ b/modules/datasets/cebaidu.py @@ -17,11 +17,7 @@ class CeBaidu(Query): self.proxy = self.get_proxy(self.source) params = {'site_address': self.domain} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/chinaz.py b/modules/datasets/chinaz.py index 3598555..8f39355 100644 --- a/modules/datasets/chinaz.py +++ b/modules/datasets/chinaz.py @@ -17,11 +17,7 @@ class Chinaz(Query): self.proxy = self.get_proxy(self.source) self.addr = self.addr + self.domain resp = self.get(self.addr) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/chinaz_api.py b/modules/datasets/chinaz_api.py index 28afca5..4d6a746 100644 --- a/modules/datasets/chinaz_api.py +++ b/modules/datasets/chinaz_api.py @@ -19,11 +19,7 @@ class ChinazAPI(Query): self.proxy = self.get_proxy(self.source) params = {'key': self.api, 'domainName': self.domain} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/circl_api.py b/modules/datasets/circl_api.py index f160869..b1c00b2 100644 --- a/modules/datasets/circl_api.py +++ b/modules/datasets/circl_api.py @@ -19,11 +19,7 @@ class CirclAPI(Query): self.header = self.get_header() self.proxy = self.get_proxy(self.source) resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd)) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/cloudflare_api.py b/modules/datasets/cloudflare_api.py index 5dd1e01..157029e 100644 --- a/modules/datasets/cloudflare_api.py +++ b/modules/datasets/cloudflare_api.py @@ -48,7 +48,8 @@ class CloudFlareAPI(Query): return elif zones_resp.status_code == 403: logger.log('DEBUG', - f'{self.domain} is banned or not a registered domain, so cannot be added to Cloudflare.') + f'{self.domain} is banned or not a registered domain, ' + f'so cannot be added to Cloudflare.') return else: logger.log('DEBUG', @@ -67,12 +68,14 @@ class CloudFlareAPI(Query): if create_zone_resp.json()['success']: return create_zone_resp.json()['result']['id'] else: - logger.log('DEBUG', f'{self.domain} is temporarily banned and cannot be added to Cloudflare') + logger.log('DEBUG', f'{self.domain} is temporarily banned ' + f'and cannot be added to Cloudflare') return False def list_dns(self, zone_id): page = 1 - list_dns_resp = self.get(self.addr + f'zones/{zone_id}/dns_records', params={'page': page, 'per_page': 10}) + list_dns_resp = self.get(self.addr + f'zones/{zone_id}/dns_records', + params={'page': page, 'per_page': 10}) if not list_dns_resp: logger.log('DEBUG', f'{list_dns_resp.status_code} {list_dns_resp.text}') diff --git a/modules/datasets/dnsdb_api.py b/modules/datasets/dnsdb_api.py index f763cea..13e8031 100644 --- a/modules/datasets/dnsdb_api.py +++ b/modules/datasets/dnsdb_api.py @@ -20,11 +20,7 @@ class DNSdbAPI(Query): self.proxy = self.get_proxy(self.source) url = f'{self.addr}*.{self.domain}' resp = self.get(url) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/dnsdumpster.py b/modules/datasets/dnsdumpster.py index f809cf4..aabb8ad 100644 --- a/modules/datasets/dnsdumpster.py +++ b/modules/datasets/dnsdumpster.py @@ -23,12 +23,7 @@ class DNSdumpster(Query): data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'), 'targetip': self.domain} resp = self.post(self.addr, data) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - if subdomains: - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/hackertarget.py b/modules/datasets/hackertarget.py index de2e7ca..4fedf26 100644 --- a/modules/datasets/hackertarget.py +++ b/modules/datasets/hackertarget.py @@ -17,13 +17,7 @@ class HackerTarget(Query): self.proxy = self.get_proxy(self.source) params = {'q': self.domain} resp = self.get(self.addr, params) - if not resp: - return - if resp.status_code == 200: - subdomains = self.match_subdomains(resp.text) - if subdomains: - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/ip138.py b/modules/datasets/ip138.py index 1e49ad4..a27ff82 100644 --- a/modules/datasets/ip138.py +++ b/modules/datasets/ip138.py @@ -17,11 +17,7 @@ class IP138(Query): self.proxy = self.get_proxy(self.source) self.addr = self.addr.format(domain=self.domain) resp = self.get(self.addr) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/ipv4info_api.py b/modules/datasets/ipv4info_api.py index aed58a9..ff5a609 100644 --- a/modules/datasets/ipv4info_api.py +++ b/modules/datasets/ipv4info_api.py @@ -35,7 +35,6 @@ class IPv4InfoAPI(Query): subdomains = self.match_subdomains(str(json)) if not subdomains: break - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名 subdomains = json.get('Subdomains') diff --git a/modules/datasets/netcraft.py b/modules/datasets/netcraft.py index 76f472a..6cc37b8 100644 --- a/modules/datasets/netcraft.py +++ b/modules/datasets/netcraft.py @@ -47,12 +47,9 @@ class NetCraft(Query): 'host': '.' + self.domain, 'from': self.page_num} resp = self.get(self.addr + last, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) + subdomains = self.match_subdomains(resp) if not subdomains: # 搜索没有发现子域名则停止搜索 break - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) if 'Next Page' not in resp.text: # 搜索页面没有出现下一页时停止搜索 break diff --git a/modules/datasets/passivedns_api.py b/modules/datasets/passivedns_api.py index e8217d5..f8eb54e 100644 --- a/modules/datasets/passivedns_api.py +++ b/modules/datasets/passivedns_api.py @@ -20,11 +20,7 @@ class PassiveDnsAPI(Query): self.proxy = self.get_proxy(self.source) url = self.addr + '/flint/rrset/*.' + self.domain resp = self.get(url) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/phonebook.py b/modules/datasets/phonebook.py index c708974..ff2fdb6 100644 --- a/modules/datasets/phonebook.py +++ b/modules/datasets/phonebook.py @@ -33,10 +33,7 @@ class PhoneBook(Query): return url = f'{addr}/result?k={key}&id={ids}&limit=10000' resp = self.get(url) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/qianxun.py b/modules/datasets/qianxun.py index 66cbf51..df9149a 100644 --- a/modules/datasets/qianxun.py +++ b/modules/datasets/qianxun.py @@ -25,10 +25,7 @@ class QianXun(Query): url = f'https://www.dnsscan.cn/dns.html?' \ f'keywords={self.domain}&page={num}' resp = self.post(url, data) - if not resp: - break - subdomains = self.match_subdomains(resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) if '
' not in resp.text: break if '
  • »
    • ' in resp.text: diff --git a/modules/datasets/rapiddns.py b/modules/datasets/rapiddns.py index eee5e49..3df56ed 100644 --- a/modules/datasets/rapiddns.py +++ b/modules/datasets/rapiddns.py @@ -17,11 +17,7 @@ class RapidDNS(Query): url = f'http://rapiddns.io/subdomain/{self.domain}' params = {'full': '1'} resp = self.get(url, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/riddler.py b/modules/datasets/riddler.py index 900947f..6470c34 100644 --- a/modules/datasets/riddler.py +++ b/modules/datasets/riddler.py @@ -17,11 +17,7 @@ class Riddler(Query): self.proxy = self.get_proxy(self.source) params = {'q': 'pld:' + self.domain} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/robtex.py b/modules/datasets/robtex.py index 4ecdb98..9e21bd2 100644 --- a/modules/datasets/robtex.py +++ b/modules/datasets/robtex.py @@ -30,12 +30,7 @@ class Robtex(Query): ip = record.get('rrdata') url = self.addr + 'reverse/' + ip resp = self.get(url) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - if subdomains: - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/securitytrails_api.py b/modules/datasets/securitytrails_api.py index a2cb388..7f5451c 100644 --- a/modules/datasets/securitytrails_api.py +++ b/modules/datasets/securitytrails_api.py @@ -26,7 +26,6 @@ class SecurityTrailsAPI(Query): prefixs = resp.json()['subdomains'] subdomains = [f'{prefix}.{self.domain}' for prefix in prefixs] if subdomains: - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/datasets/sitedossier.py b/modules/datasets/sitedossier.py index 6db05c0..fc2de44 100644 --- a/modules/datasets/sitedossier.py +++ b/modules/datasets/sitedossier.py @@ -22,12 +22,9 @@ class SiteDossier(Query): self.proxy = self.get_proxy(self.source) url = f'{self.addr}{self.domain}/{self.page_num}' resp = self.get(url) - if not resp: - return - subdomains = self.match_subdomains(resp.text) + subdomains = self.match_subdomains(resp) if not subdomains: # 搜索没有发现子域名则停止搜索 break - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) # 搜索页面没有出现下一页时停止搜索 if 'Show next 100 items' not in resp.text: diff --git a/modules/datasets/spyse_api.py b/modules/datasets/spyse_api.py index 96e253f..2c1730d 100644 --- a/modules/datasets/spyse_api.py +++ b/modules/datasets/spyse_api.py @@ -29,7 +29,6 @@ class SpyseAPI(Query): subdomains = self.match_subdomains(str(json)) if not subdomains: # 搜索没有发现子域名则停止搜索 break - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) offset += limit if len(json.get('data').get('items')) < limit: diff --git a/modules/datasets/sublist3r.py b/modules/datasets/sublist3r.py index 477586d..be0d244 100644 --- a/modules/datasets/sublist3r.py +++ b/modules/datasets/sublist3r.py @@ -17,10 +17,7 @@ class Sublist3r(Query): addr = 'https://api.sublist3r.com/search.php' param = {'domain': self.domain} resp = self.get(addr, param) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/datasets/threatcrowd.py b/modules/datasets/threatcrowd.py index 6256a92..5ea17c9 100644 --- a/modules/datasets/threatcrowd.py +++ b/modules/datasets/threatcrowd.py @@ -26,7 +26,6 @@ class ThreatCrowd(Query): if resp.status_code != 200: return subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/datasets/ximcx.py b/modules/datasets/ximcx.py index 2c83973..7ddf768 100644 --- a/modules/datasets/ximcx.py +++ b/modules/datasets/ximcx.py @@ -21,7 +21,6 @@ class Ximcx(Query): return json = resp.json() subdomains = self.match_subdomains(str(json)) - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/intelligence/alienvault.py b/modules/intelligence/alienvault.py index 75faf1e..f0c7d93 100644 --- a/modules/intelligence/alienvault.py +++ b/modules/intelligence/alienvault.py @@ -18,19 +18,11 @@ class AlienVault(Query): base = 'https://otx.alienvault.com/api/v1/indicators/domain' dns = f'{base}/{self.domain}/passive_dns' resp = self.get(dns) - if not resp: - return - json = resp.json() - subdomains = self.match_subdomains(str(json)) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) url = f'{base}/{self.domain}/url_list' resp = self.get(url) - if not resp: - return - json = resp.json() - subdomains = self.match_subdomains(str(json)) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/intelligence/threatbook_api.py b/modules/intelligence/threatbook_api.py index f721a06..ff17916 100644 --- a/modules/intelligence/threatbook_api.py +++ b/modules/intelligence/threatbook_api.py @@ -20,10 +20,7 @@ class ThreatBookAPI(Query): params = {'apikey': self.key, 'resource': self.domain} resp = self.post(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/intelligence/threatminer.py b/modules/intelligence/threatminer.py index 8fe58fd..0f79900 100644 --- a/modules/intelligence/threatminer.py +++ b/modules/intelligence/threatminer.py @@ -18,11 +18,7 @@ class ThreatMiner(Query): params = {'e': 'subdomains_container', 'q': self.domain, 't': 0, 'rt': 10} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains = self.collect_subdomains(resp) def run(self): """ diff --git a/modules/search/baidu.py b/modules/search/baidu.py index ad1c6d4..23666aa 100644 --- a/modules/search/baidu.py +++ b/modules/search/baidu.py @@ -58,7 +58,6 @@ class Baidu(Search): if not full_search and subdomains.issubset(self.subdomains): # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索 break - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) self.page_num += self.per_page_num # 搜索页面没有出现下一页时停止搜索 diff --git a/modules/search/bing.py b/modules/search/bing.py index ae2b56f..a2e8f9c 100644 --- a/modules/search/bing.py +++ b/modules/search/bing.py @@ -42,7 +42,6 @@ class Bing(Search): if not full_search and subdomains.issubset(self.subdomains): # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索 break - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) # 搜索页面没有出现下一页时停止搜索 if '
    ' not in resp.text: diff --git a/modules/search/bing_api.py b/modules/search/bing_api.py index 8c6018a..8e01110 100644 --- a/modules/search/bing_api.py +++ b/modules/search/bing_api.py @@ -34,15 +34,12 @@ class BingAPI(Search): 'count': self.per_page_num, 'offset': self.page_num} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) + subdomains = self.match_subdomains(resp) if not subdomains: # 搜索没有发现子域名则停止搜索 break if not full_search and subdomains.issubset(self.subdomains): # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索 break - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) self.page_num += self.per_page_num if self.page_num >= self.limit_num: # 搜索条数限制 diff --git a/modules/search/google_api.py b/modules/search/google_api.py index aae3e50..8511be8 100644 --- a/modules/search/google_api.py +++ b/modules/search/google_api.py @@ -33,9 +33,7 @@ class GoogleAPI(Search): 'q': word, 'fields': 'items/link', 'start': self.page_num, 'num': self.per_page_num} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) + subdomains = self.match_subdomains(resp) if not subdomains: break if not full_search and subdomains.issubset(self.subdomains): diff --git a/modules/search/shodan_api.py b/modules/search/shodan_api.py index 21b0814..2553730 100644 --- a/modules/search/shodan_api.py +++ b/modules/search/shodan_api.py @@ -23,9 +23,7 @@ class ShodanAPI(Search): params = {'key': self.key, 'page': page, 'query': query, 'minify': True, 'facets': {'hostnames'}} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) + subdomains = self.match_subdomains(resp) if not subdomains: # 搜索没有发现子域名则停止搜索 break if subdomains: diff --git a/modules/search/yahoo.py b/modules/search/yahoo.py index bc98214..ca2b716 100644 --- a/modules/search/yahoo.py +++ b/modules/search/yahoo.py @@ -42,7 +42,6 @@ class Yahoo(Search): if not full_search and subdomains.issubset(self.subdomains): # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索 break - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) if '>Next' not in resp.text: # 搜索页面没有出现下一页时停止搜索 break diff --git a/modules/search/yandex.py b/modules/search/yandex.py index e61d616..0eac9fe 100644 --- a/modules/search/yandex.py +++ b/modules/search/yandex.py @@ -42,7 +42,6 @@ class Yandex(Search): if not full_search and subdomains.issubset(self.subdomains): # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索 break - # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) if '>next' not in resp.text: # 搜索页面没有出现下一页时停止搜索 break diff --git a/modules/search/zoomeye_api.py b/modules/search/zoomeye_api.py index 39c540c..c2da394 100644 --- a/modules/search/zoomeye_api.py +++ b/modules/search/zoomeye_api.py @@ -46,9 +46,7 @@ class ZoomEyeAPI(Search): self.header.update({'Authorization': 'JWT ' + access_token}) params = {'query': 'hostname:' + self.domain, 'page': page_num} resp = self.get(self.addr, params) - if not resp: - return - subdomains = self.match_subdomains(resp.text) + subdomains = self.match_subdomains(resp) if not subdomains: # 搜索没有发现子域名则停止搜索 break self.subdomains = self.subdomains.union(subdomains)