diff --git a/common/module.py b/common/module.py index 3655ef5..9521231 100644 --- a/common/module.py +++ b/common/module.py @@ -182,9 +182,10 @@ class Module(object): :return: header """ - # logger.log('DEBUG', f'Get request header') - if settings.enable_fake_header: - return utils.gen_fake_header() + headers = utils.gen_fake_header() + if isinstance(headers, dict): + self.header = headers + return headers else: return self.header diff --git a/common/request.py b/common/request.py index 72e032d..3b774b9 100644 --- a/common/request.py +++ b/common/request.py @@ -188,8 +188,8 @@ def get_connector(): async def async_request(urls): results = list() connector = get_connector() - header = utils.get_random_header() - async with ClientSession(connector=connector, headers=header) as session: + headers = utils.get_random_header() + async with ClientSession(connector=connector, headers=headers) as session: tasks = [] for i, url in enumerate(urls): task = asyncio.ensure_future(fetch(session, 'GET', url)) @@ -213,8 +213,8 @@ async def bulk_request(data, port): logger.log('INFOR', f'Use {method} method to request') logger.log('INFOR', 'Async subdomains request in progress') connector = get_connector() - header = utils.get_random_header() - async with ClientSession(connector=connector, headers=header) as session: + headers = utils.get_random_header() + async with ClientSession(connector=connector, headers=headers) as session: tasks = [] for i, data in enumerate(to_req_data): url = data.get('url') diff --git a/common/utils.py b/common/utils.py index 7ee71c3..1ad3b5b 100644 --- a/common/utils.py +++ b/common/utils.py @@ -49,30 +49,24 @@ def gen_fake_header(): """ Generate fake request headers """ - ua = random.choice(user_agents) - headers = { - 'Accept': 'text/html,application/xhtml+xml,' - 'application/xml;q=0.9,*/*;q=0.8', - 'Accept-Encoding': 'gzip, deflate', - 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7', - 'Cache-Control': 'max-age=0', - 'DNT': '1', - 'Referer': 'https://www.google.com/', - 'Upgrade-Insecure-Requests': '1', - 'User-Agent': ua, - 'X-Forwarded-For': '127.0.0.1' - } + headers = settings.headers + if not isinstance(headers, dict): + headers = dict() + if settings.random_user_agent: + ua = random.choice(user_agents) + headers['User-Agent'] = ua + headers['Accept-Encoding'] = 'gzip, deflate' return headers def get_random_header(): """ - Get random proxy + Get random header """ - header = None - if settings.fake_header: - header = gen_fake_header() - return header + headers = gen_fake_header() + if not isinstance(headers, dict): + headers = None + return headers def get_random_proxy(): diff --git a/config/default.py b/config/default.py index 32e197f..93b921d 100644 --- a/config/default.py +++ b/config/default.py @@ -95,7 +95,6 @@ proxy_pool = [{'http': 'http://127.0.0.1:1080', # 网络请求设置 -enable_fake_header = True # 启用伪造请求头 request_delay = 1 # 请求时延 request_timeout = 60 # 请求超时 request_verify = False # 请求SSL验证 @@ -150,7 +149,6 @@ verify_ssl = False # aiohttp 支持 HTTP/HTTPS形式的代理 aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com" allow_redirects = True # 允许请求跳转 -fake_header = True # 使用伪造请求头 # 为了保证请求质量 请谨慎更改以下设置 # request_method只能是HEAD或GET,HEAD请求方法更快,但是不能获取响应体并提取从中title及banner request_method = 'GET' # 使用请求方法,默认GET @@ -165,3 +163,20 @@ subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs', 'web', 'www', 'job', 'docs', 'news', 'blog', 'data', 'help', 'live', 'mall', 'blogs', 'files', 'forum', 'store', 'mobile'} + +# 请求头设置 +# 可以在headers里添加自定义请求头 +headers = { + 'Accept': 'text/html,application/xhtml+xml,' + 'application/xml;q=0.9,*/*;q=0.8', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7', + 'Cache-Control': 'max-age=0', + 'DNT': '1', + 'Referer': 'https://www.google.com/', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36', + 'Upgrade-Insecure-Requests': '1', + 'X-Forwarded-For': '127.0.0.1' +} +random_user_agent = True # 使用随机UA(默认True,开启可以覆盖header的设置) diff --git a/config/setting.py b/config/setting.py index 5f85c4d..693815c 100644 --- a/config/setting.py +++ b/config/setting.py @@ -83,7 +83,6 @@ proxy_pool = [{'http': 'http://127.0.0.1:1080', # 网络请求设置 -enable_fake_header = True # 启用伪造请求头 request_delay = 1 # 请求时延 request_timeout = 60 # 请求超时 request_verify = False # 请求SSL验证 @@ -95,7 +94,6 @@ search_recursive_times = 2 # 递归搜索层数 # aiohttp有关配置 # aiohttp 支持 HTTP/HTTPS形式的代理 aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com" -fake_header = True # 使用伪造请求头 # 为了保证请求质量 请谨慎更改以下设置 sockread_timeout = 6 # 每个请求socket读取超时时间,默认6秒 sockconn_timeout = 3 # 每个请求socket连接超时时间,默认3秒