解决代理配置无效问题

modules/datasets/netcraft.py

@@ -21,7 +21,6 @@ class NetCraft(Query):
         """
         绕过NetCraft的JS验证
         """
-        self.header = self.get_header()  # Netcraft会检查User-Agent
         resp = self.get(self.init)
         if not resp:
             return False
@@ -36,12 +35,13 @@ class NetCraft(Query):
         """
         向接口查询子域并做子域匹配
         """
+        self.header = self.get_header()  # NetCraft会检查User-Agent
+        self.proxy = self.get_proxy(self.source)
         if not self.bypass_verification():
             return
         last = ''
         while True:
             time.sleep(self.delay)
-            self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             params = {'restriction': 'site ends with',
                       'host': '.' + self.domain,

modules/search/bing.py

@@ -19,9 +19,9 @@ class Bing(Search):
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
         """
-        self.page_num = 0  # 二次搜索重新置0
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
+        self.page_num = 0  # 二次搜索重新置0
         resp = self.get(self.init)
         if not resp:
             return

modules/search/gitee.py

@@ -11,7 +11,6 @@ class Gitee(Search):
         self.module = 'Search'
         self.addr = 'https://search.gitee.com/'
         self.domain = domain
-        self.header = self.get_header()
 
     def search(self):
         """
@@ -20,6 +19,8 @@ class Gitee(Search):
         page_num = 1
         while True:
             time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
             params = {'pageno': page_num, 'q': self.domain, 'type': 'code'}
             try:
                 resp = self.get(self.addr, params=params)

modules/search/yahoo.py

@@ -21,13 +21,14 @@ class Yahoo(Search):
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
         """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
         resp = self.get(self.init)
         if not resp:
             return
         self.cookie = resp.cookies  # 获取cookie Yahoo在搜索时需要带上cookie
         while True:
             time.sleep(self.delay)
-            self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             query = 'site:.' + domain + filtered_subdomain
             params = {'p': query, 'b': self.page_num, 'pz': self.per_page_num}

modules/search/yandex.py

@@ -20,6 +20,8 @@ class Yandex(Search):
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
         """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
         self.page_num = 0  # 二次搜索重新置0
         resp = self.get(self.init)
         if not resp:
@@ -27,7 +29,6 @@ class Yandex(Search):
         self.cookie = resp.cookies  # 获取cookie
         while True:
             time.sleep(self.delay)
-            self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             query = 'site:.' + domain + filtered_subdomain
             params = {'text': query, 'p': self.page_num,