From 5eba97ebcebc00284130e3c2c865bc6465117ded Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Wed, 12 Aug 2020 16:45:51 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=87=AA=E5=AE=9A=E4=B9=89?= =?UTF-8?q?=E7=9A=84=E8=AF=B7=E6=B1=82=E5=A4=B4=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/module.py | 7 ++++--- common/request.py | 8 ++++---- common/utils.py | 21 ++++++++++++--------- config/default.py | 6 ++++-- config/setting.py | 2 -- 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/common/module.py b/common/module.py index 3655ef5..9521231 100644 --- a/common/module.py +++ b/common/module.py @@ -182,9 +182,10 @@ class Module(object): :return: header """ - # logger.log('DEBUG', f'Get request header') - if settings.enable_fake_header: - return utils.gen_fake_header() + headers = utils.gen_fake_header() + if isinstance(headers, dict): + self.header = headers + return headers else: return self.header diff --git a/common/request.py b/common/request.py index 72e032d..3b774b9 100644 --- a/common/request.py +++ b/common/request.py @@ -188,8 +188,8 @@ def get_connector(): async def async_request(urls): results = list() connector = get_connector() - header = utils.get_random_header() - async with ClientSession(connector=connector, headers=header) as session: + headers = utils.get_random_header() + async with ClientSession(connector=connector, headers=headers) as session: tasks = [] for i, url in enumerate(urls): task = asyncio.ensure_future(fetch(session, 'GET', url)) @@ -213,8 +213,8 @@ async def bulk_request(data, port): logger.log('INFOR', f'Use {method} method to request') logger.log('INFOR', 'Async subdomains request in progress') connector = get_connector() - header = utils.get_random_header() - async with ClientSession(connector=connector, headers=header) as session: + headers = utils.get_random_header() + async with ClientSession(connector=connector, headers=headers) as session: tasks = [] for i, data in enumerate(to_req_data): url = data.get('url') diff --git a/common/utils.py b/common/utils.py index 998b03c..1ad3b5b 100644 --- a/common/utils.py +++ b/common/utils.py @@ -49,21 +49,24 @@ def gen_fake_header(): """ Generate fake request headers """ - ua = random.choice(user_agents) - headers = dict() - headers['User-Agent'] = ua - headers.update(settings.headers) + headers = settings.headers + if not isinstance(headers, dict): + headers = dict() + if settings.random_user_agent: + ua = random.choice(user_agents) + headers['User-Agent'] = ua + headers['Accept-Encoding'] = 'gzip, deflate' return headers def get_random_header(): """ - Get random proxy + Get random header """ - header = None - if settings.fake_header: - header = gen_fake_header() - return header + headers = gen_fake_header() + if not isinstance(headers, dict): + headers = None + return headers def get_random_proxy(): diff --git a/config/default.py b/config/default.py index d12dc73..93b921d 100644 --- a/config/default.py +++ b/config/default.py @@ -95,7 +95,6 @@ proxy_pool = [{'http': 'http://127.0.0.1:1080', # 网络请求设置 -enable_fake_header = True # 启用伪造请求头 request_delay = 1 # 请求时延 request_timeout = 60 # 请求超时 request_verify = False # 请求SSL验证 @@ -150,7 +149,6 @@ verify_ssl = False # aiohttp 支持 HTTP/HTTPS形式的代理 aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com" allow_redirects = True # 允许请求跳转 -fake_header = True # 使用伪造请求头 # 为了保证请求质量 请谨慎更改以下设置 # request_method只能是HEAD或GET,HEAD请求方法更快，但是不能获取响应体并提取从中title及banner request_method = 'GET' # 使用请求方法，默认GET @@ -167,6 +165,7 @@ subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs', 'store', 'mobile'} # 请求头设置 +# 可以在headers里添加自定义请求头 headers = { 'Accept': 'text/html,application/xhtml+xml,' 'application/xml;q=0.9,*/*;q=0.8', @@ -175,6 +174,9 @@ headers = { 'Cache-Control': 'max-age=0', 'DNT': '1', 'Referer': 'https://www.google.com/', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36', 'Upgrade-Insecure-Requests': '1', 'X-Forwarded-For': '127.0.0.1' } +random_user_agent = True # 使用随机UA(默认True，开启可以覆盖header的设置) diff --git a/config/setting.py b/config/setting.py index 5f85c4d..693815c 100644 --- a/config/setting.py +++ b/config/setting.py @@ -83,7 +83,6 @@ proxy_pool = [{'http': 'http://127.0.0.1:1080', # 网络请求设置 -enable_fake_header = True # 启用伪造请求头 request_delay = 1 # 请求时延 request_timeout = 60 # 请求超时 request_verify = False # 请求SSL验证 @@ -95,7 +94,6 @@ search_recursive_times = 2 # 递归搜索层数 # aiohttp有关配置 # aiohttp 支持 HTTP/HTTPS形式的代理 aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com" -fake_header = True # 使用伪造请求头 # 为了保证请求质量 请谨慎更改以下设置 sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒 sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒