This commit is contained in:
Jing Ling
2020-08-20 14:07:31 +08:00
parent e056834f1d
commit 0a770916cf
8 changed files with 18 additions and 15 deletions
+1 -1
View File
@@ -26,7 +26,7 @@ class CommonCrawl(Crawl):
for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
if resp.data.get('status') not in ['301', '302']:
subdomains = self.match_subdomains(self.get_maindomain(domain), resp.text)
subdomains = self.match_subdomains(domain, resp.text)
self.subdomains.update(subdomains)
def run(self):