添加自定义的请求头功能

common/module.py

@@ -182,9 +182,10 @@ class Module(object):
 
         :return: header
         """
-        # logger.log('DEBUG', f'Get request header')
+        headers = utils.gen_fake_header()
-        if settings.enable_fake_header:
+        if isinstance(headers, dict):
-            return utils.gen_fake_header()
+            self.header = headers
+            return headers
         else:
             return self.header
 

common/request.py

@@ -188,8 +188,8 @@ def get_connector():
 async def async_request(urls):
     results = list()
     connector = get_connector()
-    header = utils.get_random_header()
+    headers = utils.get_random_header()
-    async with ClientSession(connector=connector, headers=header) as session:
+    async with ClientSession(connector=connector, headers=headers) as session:
         tasks = []
         for i, url in enumerate(urls):
             task = asyncio.ensure_future(fetch(session, 'GET', url))
@@ -213,8 +213,8 @@ async def bulk_request(data, port):
     logger.log('INFOR', f'Use {method} method to request')
     logger.log('INFOR', 'Async subdomains request in progress')
     connector = get_connector()
-    header = utils.get_random_header()
+    headers = utils.get_random_header()
-    async with ClientSession(connector=connector, headers=header) as session:
+    async with ClientSession(connector=connector, headers=headers) as session:
         tasks = []
         for i, data in enumerate(to_req_data):
             url = data.get('url')

common/utils.py

@@ -49,21 +49,24 @@ def gen_fake_header():
     """
     Generate fake request headers
     """
-    ua = random.choice(user_agents)
+    headers = settings.headers
-    headers = dict()
+    if not isinstance(headers, dict):
-    headers['User-Agent'] = ua
+        headers = dict()
-    headers.update(settings.headers)
+    if settings.random_user_agent:
+        ua = random.choice(user_agents)
+        headers['User-Agent'] = ua
+    headers['Accept-Encoding'] = 'gzip, deflate'
     return headers
 
 
 def get_random_header():
     """
-    Get random proxy
+    Get random header
     """
-    header = None
+    headers = gen_fake_header()
-    if settings.fake_header:
+    if not isinstance(headers, dict):
-        header = gen_fake_header()
+        headers = None
-    return header
+    return headers
 
 
 def get_random_proxy():

config/default.py

@@ -95,7 +95,6 @@ proxy_pool = [{'http': 'http://127.0.0.1:1080',
 
 
 # 网络请求设置
-enable_fake_header = True  # 启用伪造请求头
 request_delay = 1  # 请求时延
 request_timeout = 60  # 请求超时
 request_verify = False  # 请求SSL验证
@@ -150,7 +149,6 @@ verify_ssl = False
 # aiohttp 支持 HTTP/HTTPS形式的代理
 aiohttp_proxy = None  # 示例 proxy="http://user:pass@some.proxy.com"
 allow_redirects = True  # 允许请求跳转
-fake_header = True  # 使用伪造请求头
 # 为了保证请求质量 请谨慎更改以下设置
 # request_method只能是HEAD或GET,HEAD请求方法更快，但是不能获取响应体并提取从中title及banner
 request_method = 'GET'  # 使用请求方法，默认GET
@@ -167,6 +165,7 @@ subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
                      'store', 'mobile'}
 
 # 请求头设置
+# 可以在headers里添加自定义请求头
 headers = {
     'Accept': 'text/html,application/xhtml+xml,'
               'application/xml;q=0.9,*/*;q=0.8',
@@ -175,6 +174,9 @@ headers = {
     'Cache-Control': 'max-age=0',
     'DNT': '1',
     'Referer': 'https://www.google.com/',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                  '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
     'Upgrade-Insecure-Requests': '1',
     'X-Forwarded-For': '127.0.0.1'
 }
+random_user_agent = True  # 使用随机UA(默认True，开启可以覆盖header的设置)

config/setting.py

@@ -83,7 +83,6 @@ proxy_pool = [{'http': 'http://127.0.0.1:1080',
 
 
 # 网络请求设置
-enable_fake_header = True  # 启用伪造请求头
 request_delay = 1  # 请求时延
 request_timeout = 60  # 请求超时
 request_verify = False  # 请求SSL验证
@@ -95,7 +94,6 @@ search_recursive_times = 2  # 递归搜索层数
 # aiohttp有关配置
 # aiohttp 支持 HTTP/HTTPS形式的代理
 aiohttp_proxy = None  # 示例 proxy="http://user:pass@some.proxy.com"
-fake_header = True  # 使用伪造请求头
 # 为了保证请求质量 请谨慎更改以下设置
 sockread_timeout = 6  # 每个请求socket读取超时时间，默认6秒
 sockconn_timeout = 3  # 每个请求socket连接超时时间，默认3秒