添加自定义的请求头功能

common/module.py

@@ -182,9 +182,10 @@ class Module(object):
 
         :return: header
         """
-        # logger.log('DEBUG', f'Get request header')
-        if settings.enable_fake_header:
-            return utils.gen_fake_header()
+        headers = utils.gen_fake_header()
+        if isinstance(headers, dict):
+            self.header = headers
+            return headers
         else:
             return self.header
 

common/request.py

@@ -188,8 +188,8 @@ def get_connector():
 async def async_request(urls):
     results = list()
     connector = get_connector()
-    header = utils.get_random_header()
-    async with ClientSession(connector=connector, headers=header) as session:
+    headers = utils.get_random_header()
+    async with ClientSession(connector=connector, headers=headers) as session:
         tasks = []
         for i, url in enumerate(urls):
             task = asyncio.ensure_future(fetch(session, 'GET', url))
@@ -213,8 +213,8 @@ async def bulk_request(data, port):
     logger.log('INFOR', f'Use {method} method to request')
     logger.log('INFOR', 'Async subdomains request in progress')
     connector = get_connector()
-    header = utils.get_random_header()
-    async with ClientSession(connector=connector, headers=header) as session:
+    headers = utils.get_random_header()
+    async with ClientSession(connector=connector, headers=headers) as session:
         tasks = []
         for i, data in enumerate(to_req_data):
             url = data.get('url')

common/utils.py

@@ -49,21 +49,24 @@ def gen_fake_header():
     """
     Generate fake request headers
     """
-    ua = random.choice(user_agents)
-    headers = dict()
-    headers['User-Agent'] = ua
-    headers.update(settings.headers)
+    headers = settings.headers
+    if not isinstance(headers, dict):
+        headers = dict()
+    if settings.random_user_agent:
+        ua = random.choice(user_agents)
+        headers['User-Agent'] = ua
+    headers['Accept-Encoding'] = 'gzip, deflate'
     return headers
 
 
 def get_random_header():
     """
-    Get random proxy
+    Get random header
     """
-    header = None
-    if settings.fake_header:
-        header = gen_fake_header()
-    return header
+    headers = gen_fake_header()
+    if not isinstance(headers, dict):
+        headers = None
+    return headers
 
 
 def get_random_proxy():