diff --git a/common/module.py b/common/module.py index b04835b..b56be1d 100644 --- a/common/module.py +++ b/common/module.py @@ -87,7 +87,7 @@ class Module(object): verify=self.verify, **kwargs) except Exception as e: - logger.log('ERROR', e.args) + logger.log('ERROR', e.args[0]) return None if not check: return resp @@ -124,9 +124,9 @@ class Module(object): except Exception as e: if raise_error: if isinstance(e, requests.exceptions.ConnectTimeout): - logger.log(level, e.args) + logger.log(level, e.args[0]) raise e - logger.log(level, e.args) + logger.log(level, e.args[0]) return None if not check: return resp @@ -156,7 +156,7 @@ class Module(object): verify=self.verify, **kwargs) except Exception as e: - logger.log('ERROR', e.args) + logger.log('ERROR', e.args[0]) return None if not check: return resp @@ -184,7 +184,7 @@ class Module(object): verify=self.verify, **kwargs) except Exception as e: - logger.log('ERROR', e.args) + logger.log('ERROR', e.args[0]) return None if not check: return resp diff --git a/data/cdn_header_keys.json b/data/cdn_header_keys.json index 2f5e264..9c4fda9 100644 --- a/data/cdn_header_keys.json +++ b/data/cdn_header_keys.json @@ -2,6 +2,8 @@ "xcs", "via", "x-via", + "x-cdn", + "x-cdn-forward", "x-ser", "x-cf1", "cache", @@ -11,6 +13,7 @@ "x-hit-cache", "x-cache-status", "x-cache-hits", + "x-cache-lookup", "cc_cache", "webcache", "chinacache", @@ -20,14 +23,20 @@ "x-github-request-id", "x-sucuri-id", "x-amz-cf-id", + "x-airee-node", "x-cdn-provider", "x-fastly", + "x-iinfo", + "x-llid", + "sozu-id", + "x-cf-tsc", "x-ws-request-id", "fss-cache", "powered-by-chinacache", "verycdn", "yunjiasu", "skyparkcdn", + "x-beluga-cache-status", "x-content-type-options", "x-download-options", "x-proxy-node", @@ -36,5 +45,6 @@ "etag", "expires", "pragma", - "cache-control" + "cache-control", + "last-modified" ] diff --git a/modules/intelligence/riskiq_api.py b/modules/intelligence/riskiq_api.py index b388551..3a7d1d9 100644 --- a/modules/intelligence/riskiq_api.py +++ b/modules/intelligence/riskiq_api.py @@ -25,8 +25,8 @@ class RiskIQ(Query): if not resp: return data = resp.json() - subnames = data.get('subdomains') - subdomain_str = str(set(map(lambda subname: f'{subname}.{self.domain}', subnames))) + names = data.get('subdomains') + subdomain_str = str(set(map(lambda name: f'{name}.{self.domain}', names))) self.subdomains = self.collect_subdomains(subdomain_str) def run(self): diff --git a/modules/search/shodan_api.py b/modules/search/shodan_api.py index e80cf2a..3f505e2 100644 --- a/modules/search/shodan_api.py +++ b/modules/search/shodan_api.py @@ -8,7 +8,6 @@ class ShodanAPI(Search): self.domain = domain self.module = 'Search' self.source = 'ShodanAPISearch' - self.addr = 'https://api.shodan.io/shodan/host/search' self.key = settings.shodan_api_key def search(self): @@ -17,17 +16,14 @@ class ShodanAPI(Search): """ self.header = self.get_header() self.proxy = self.get_proxy(self.source) - query = 'hostname:.' + self.domain - page = 1 - while True: - params = {'key': self.key, 'page': page, 'query': query, - 'minify': True, 'facets': {'hostnames'}} - resp = self.get(self.addr, params) - subdomains = self.match_subdomains(resp) - if not subdomains: # 搜索没有发现子域名则停止搜索 - break - self.subdomains.update(subdomains) - page += 1 + url = f'https://api.shodan.io/dns/domain/{self.domain}?key={self.key}' + resp = self.get(url) + if not resp: + return + data = resp.json() + names = data.get('subdomains') + subdomain_str = str(set(map(lambda name: f'{name}.{self.domain}', names))) + self.subdomains = self.collect_subdomains(subdomain_str) def run(self): """ @@ -54,4 +50,4 @@ def run(domain): if __name__ == '__main__': - run('example.com') + run('freebuf.com')