添加绕过cloudFlare检查

oneforall/modules/datasets/dnsdb.py

@@ -1,7 +1,9 @@
-import random
 import time
+import random
+import cloudscraper
 from bs4 import BeautifulSoup
 from common.query import Query
+from config import logger
 
 
 class DNSdb(Query):
@@ -10,28 +12,45 @@ class DNSdb(Query):
         self.domain = self.register(domain)
         self.module = 'Dataset'
         self.source = 'DNSdbQuery'
-        self.addr = 'https://www.dnsdb.org/'
+        self.addr = 'http://www.dnsdb.org/'
+        self.url = f'{self.addr}{self.domain}/'
+
+    def get_tokens(self):
+        """
+        绕过cloudFlare验证并获取taken
+
+        :return: 绕过失败返回None 成功返回tokens
+        """
+        scraper = cloudscraper.create_scraper()
+        scraper.interpreter = 'js2py'
+        scraper.proxies = self.get_proxy(self.source)
+        try:
+            tokens = scraper.get_tokens(self.url)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return None
+        if len(tokens) != 2:
+            return None
+        return tokens
 
     def query(self):
         """
         向接口查询子域并做子域匹配
         """
-        self.header = self.get_header()
-        self.header.update({'Referer': self.addr})
-        self.proxy = self.get_proxy(self.source)
-        url = self.addr + self.domain + '/'
-        resp = self.get(url)
+        tokens = self.get_tokens()
+        if not tokens:
+            logger.log('ALERT', f'{self.source}模块绕过cloudFlare检查失败')
+            return False
+        self.cookie = tokens[0]
+        self.header = {'User-Agent': tokens[1]}
+        resp = self.get(self.url)
         if not resp:
             return
-        if resp.status_code == 200:
         if 'index' in resp.text:
             soup = BeautifulSoup(resp.text, features='lxml')
             urls = set(map(lambda x: self.addr + self.domain + x.text,
                            soup.find_all('a')))
             for url in urls:
-                    # 休眠绕过CloudFlare的DDoS保护
-                    self.delay = random.randint(2, 5)
-                    time.sleep(self.delay)
                 resp = self.get(url)
                 if not resp:
                     return