From bb8a72903f8b091cadea812cdfffb5604b67fd64 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Fri, 4 Jun 2021 15:18:29 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dnetcraft=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/datasets/netcraft.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/modules/datasets/netcraft.py b/modules/datasets/netcraft.py index c729737..27d1235 100644 --- a/modules/datasets/netcraft.py +++ b/modules/datasets/netcraft.py @@ -12,39 +12,21 @@ class NetCraft(Query): self.domain = domain self.module = 'Dataset' self.source = 'NetCraftQuery' - self.init = 'https://searchdns.netcraft.com/' - self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains' + self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains&position=limited' self.page_num = 1 self.per_page_num = 20 - def bypass_verification(self): - """ - 绕过NetCraft的JS验证 - """ - resp = self.get(self.init) - if not resp: - return False - self.cookie = resp.cookies - cookie_value = self.cookie['netcraft_js_verification_challenge'] - cookie_encode = parse.unquote(cookie_value).encode('utf-8') - verify_taken = hashlib.sha1(cookie_encode).hexdigest() - self.cookie['netcraft_js_verification_response'] = verify_taken - return True - def query(self): """ 向接口查询子域并做子域匹配 """ self.header = self.get_header() # NetCraft会检查User-Agent self.proxy = self.get_proxy(self.source) - if not self.bypass_verification(): - return last = '' while True: time.sleep(self.delay) self.proxy = self.get_proxy(self.source) - params = {'restriction': 'site ends with', - 'host': '.' + self.domain, + params = {'host': '*.' + self.domain, 'from': self.page_num} resp = self.get(self.addr + last, params) subdomains = self.match_subdomains(resp)