修复check模块下在解析出多个IP不通的情况下超时严重的问题

common/check.py

@@ -1,3 +1,5 @@
+import requests
+from config.log import logger
 from common.module import Module
 
 
@@ -12,15 +14,28 @@ class Check(Module):
 
     def to_check(self, filenames):
         urls = set()
+        urls_www = set()
         for filename in filenames:
-            urls.update((f'http://{self.domain}/{filename}',
-                         f'https://{self.domain}/{filename}',
-                         f'http://www.{self.domain}/{filename}',
-                         f'https://www.{self.domain}/{filename}'))
+            urls.update((
+                f'http://{self.domain}/{filename}',
+                f'https://{self.domain}/{filename}',
+            ))
+            urls_www.update((
+                f'http://www.{self.domain}/{filename}',
+                f'https://www.{self.domain}/{filename}'
+            ))
+        self.check_loop(urls)
+        self.check_loop(urls_www)
+
+    def check_loop(self, urls):
         for url in urls:
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
-            resp = self.get(url, check=False, ignore=True)
+            try:
+                resp = self.get(url, check=False, ignore=True, raise_error=True)
+            except requests.exceptions.ConnectTimeout:
+                logger.log('DEBUG', f'Connection to {url} timed out, so break check')
+                break
             self.subdomains = self.collect_subdomains(resp)
             if self.subdomains:
                 break

common/module.py

@@ -79,13 +79,13 @@ class Module(object):
         session.trust_env = False
         try:
             resp = session.head(url,
-                                 params=params,
-                                 cookies=self.cookie,
-                                 headers=self.header,
-                                 proxies=self.proxy,
-                                 timeout=self.timeout,
-                                 verify=self.verify,
-                                 **kwargs)
+                                params=params,
+                                cookies=self.cookie,
+                                headers=self.header,
+                                proxies=self.proxy,
+                                timeout=self.timeout,
+                                verify=self.verify,
+                                **kwargs)
         except Exception as e:
             logger.log('ERROR', e.args)
             return None
@@ -95,7 +95,7 @@ class Module(object):
             return resp
         return None
 
-    def get(self, url, params=None, check=True, ignore=False, **kwargs):
+    def get(self, url, params=None, check=True, ignore=False,raise_error=False, **kwargs):
         """
         Custom get request
 
@@ -103,6 +103,7 @@ class Module(object):
         :param dict params: request parameters
         :param bool check: check response
         :param bool ignore: ignore error
+        :param bool raise_error: raise error or not
         :param kwargs: other params
         :return: response object
         """
@@ -121,6 +122,10 @@ class Module(object):
                                verify=self.verify,
                                **kwargs)
         except Exception as e:
+            if raise_error:
+                if isinstance(e, requests.exceptions.ConnectTimeout):
+                    logger.log(level, e.args)
+                    raise e
             logger.log(level, e.args)
             return None
         if not check: