diff --git a/oneforall/modules/datasets/dnsdb.py b/oneforall/modules/datasets/dnsdb.py index c4dbae1..60e275b 100644 --- a/oneforall/modules/datasets/dnsdb.py +++ b/oneforall/modules/datasets/dnsdb.py @@ -1,7 +1,9 @@ -import random import time +import random +import cloudscraper from bs4 import BeautifulSoup from common.query import Query +from config import logger class DNSdb(Query): @@ -10,38 +12,55 @@ class DNSdb(Query): self.domain = self.register(domain) self.module = 'Dataset' self.source = 'DNSdbQuery' - self.addr = 'https://www.dnsdb.org/' + self.addr = 'http://www.dnsdb.org/' + self.url = f'{self.addr}{self.domain}/' + + def get_tokens(self): + """ + 绕过cloudFlare验证并获取taken + + :return: 绕过失败返回None 成功返回tokens + """ + scraper = cloudscraper.create_scraper() + scraper.interpreter = 'js2py' + scraper.proxies = self.get_proxy(self.source) + try: + tokens = scraper.get_tokens(self.url) + except Exception as e: + logger.log('ERROR', e.args) + return None + if len(tokens) != 2: + return None + return tokens def query(self): """ 向接口查询子域并做子域匹配 """ - self.header = self.get_header() - self.header.update({'Referer': self.addr}) - self.proxy = self.get_proxy(self.source) - url = self.addr + self.domain + '/' - resp = self.get(url) + tokens = self.get_tokens() + if not tokens: + logger.log('ALERT', f'{self.source}模块绕过cloudFlare检查失败') + return False + self.cookie = tokens[0] + self.header = {'User-Agent': tokens[1]} + resp = self.get(self.url) if not resp: return - if resp.status_code == 200: - if 'index' in resp.text: - soup = BeautifulSoup(resp.text, features='lxml') - urls = set(map(lambda x: self.addr + self.domain + x.text, - soup.find_all('a'))) - for url in urls: - # 休眠绕过CloudFlare的DDoS保护 - self.delay = random.randint(2, 5) - time.sleep(self.delay) - resp = self.get(url) - if not resp: - return - subdomains_find = self.match(self.domain, resp.text) - # 合并搜索子域名搜索结果 - self.subdomains = self.subdomains.union(subdomains_find) - else: + if 'index' in resp.text: + soup = BeautifulSoup(resp.text, features='lxml') + urls = set(map(lambda x: self.addr + self.domain + x.text, + soup.find_all('a'))) + for url in urls: + resp = self.get(url) + if not resp: + return subdomains_find = self.match(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains_find) + else: + subdomains_find = self.match(self.domain, resp.text) + # 合并搜索子域名搜索结果 + self.subdomains = self.subdomains.union(subdomains_find) def run(self): """