Merge branch 'master' of https://github.com/shmilylty/OneForAll into master

common/module.py

@@ -182,9 +182,10 @@ class Module(object):
 
         :return: header
         """
-        # logger.log('DEBUG', f'Get request header')
-        if settings.enable_fake_header:
-            return utils.gen_fake_header()
+        headers = utils.gen_fake_header()
+        if isinstance(headers, dict):
+            self.header = headers
+            return headers
         else:
             return self.header
 

common/request.py

@@ -188,8 +188,8 @@ def get_connector():
 async def async_request(urls):
     results = list()
     connector = get_connector()
-    header = utils.get_random_header()
-    async with ClientSession(connector=connector, headers=header) as session:
+    headers = utils.get_random_header()
+    async with ClientSession(connector=connector, headers=headers) as session:
         tasks = []
         for i, url in enumerate(urls):
             task = asyncio.ensure_future(fetch(session, 'GET', url))
@@ -213,8 +213,8 @@ async def bulk_request(data, port):
     logger.log('INFOR', f'Use {method} method to request')
     logger.log('INFOR', 'Async subdomains request in progress')
     connector = get_connector()
-    header = utils.get_random_header()
-    async with ClientSession(connector=connector, headers=header) as session:
+    headers = utils.get_random_header()
+    async with ClientSession(connector=connector, headers=headers) as session:
         tasks = []
         for i, data in enumerate(to_req_data):
             url = data.get('url')

common/utils.py

@@ -49,30 +49,24 @@ def gen_fake_header():
     """
     Generate fake request headers
     """
+    headers = settings.headers
+    if not isinstance(headers, dict):
+        headers = dict()
+    if settings.random_user_agent:
         ua = random.choice(user_agents)
-    headers = {
-        'Accept': 'text/html,application/xhtml+xml,'
-                  'application/xml;q=0.9,*/*;q=0.8',
-        'Accept-Encoding': 'gzip, deflate',
-        'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
-        'Cache-Control': 'max-age=0',
-        'DNT': '1',
-        'Referer': 'https://www.google.com/',
-        'Upgrade-Insecure-Requests': '1',
-        'User-Agent': ua,
-        'X-Forwarded-For': '127.0.0.1'
-    }
+        headers['User-Agent'] = ua
+    headers['Accept-Encoding'] = 'gzip, deflate'
     return headers
 
 
 def get_random_header():
     """
-    Get random proxy
+    Get random header
     """
-    header = None
-    if settings.fake_header:
-        header = gen_fake_header()
-    return header
+    headers = gen_fake_header()
+    if not isinstance(headers, dict):
+        headers = None
+    return headers
 
 
 def get_random_proxy():

config/default.py

@@ -95,7 +95,6 @@ proxy_pool = [{'http': 'http://127.0.0.1:1080',
 
 
 # 网络请求设置
-enable_fake_header = True  # 启用伪造请求头
 request_delay = 1  # 请求时延
 request_timeout = 60  # 请求超时
 request_verify = False  # 请求SSL验证
@@ -150,7 +149,6 @@ verify_ssl = False
 # aiohttp 支持 HTTP/HTTPS形式的代理
 aiohttp_proxy = None  # 示例 proxy="http://user:pass@some.proxy.com"
 allow_redirects = True  # 允许请求跳转
-fake_header = True  # 使用伪造请求头
 # 为了保证请求质量 请谨慎更改以下设置
 # request_method只能是HEAD或GET,HEAD请求方法更快，但是不能获取响应体并提取从中title及banner
 request_method = 'GET'  # 使用请求方法，默认GET
@@ -165,3 +163,20 @@ subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
                      'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
                      'help', 'live', 'mall', 'blogs', 'files', 'forum',
                      'store', 'mobile'}
+
+# 请求头设置
+# 可以在headers里添加自定义请求头
+headers = {
+    'Accept': 'text/html,application/xhtml+xml,'
+              'application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Encoding': 'gzip, deflate',
+    'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+    'Cache-Control': 'max-age=0',
+    'DNT': '1',
+    'Referer': 'https://www.google.com/',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                  '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Upgrade-Insecure-Requests': '1',
+    'X-Forwarded-For': '127.0.0.1'
+}
+random_user_agent = True  # 使用随机UA(默认True，开启可以覆盖header的设置)

config/setting.py

@@ -83,7 +83,6 @@ proxy_pool = [{'http': 'http://127.0.0.1:1080',
 
 
 # 网络请求设置
-enable_fake_header = True  # 启用伪造请求头
 request_delay = 1  # 请求时延
 request_timeout = 60  # 请求超时
 request_verify = False  # 请求SSL验证
@@ -95,7 +94,6 @@ search_recursive_times = 2  # 递归搜索层数
 # aiohttp有关配置
 # aiohttp 支持 HTTP/HTTPS形式的代理
 aiohttp_proxy = None  # 示例 proxy="http://user:pass@some.proxy.com"
-fake_header = True  # 使用伪造请求头
 # 为了保证请求质量 请谨慎更改以下设置
 sockread_timeout = 6  # 每个请求socket读取超时时间，默认6秒
 sockconn_timeout = 3  # 每个请求socket连接超时时间，默认3秒