diff --git a/common/request.py b/common/request.py index 1c7c459..5ef3f25 100644 --- a/common/request.py +++ b/common/request.py @@ -208,7 +208,7 @@ def save(name, total, req_data, resp_queue): new_info = gen_new_info(old_info, resp) db.insert_table(name, new_info) resp_queue.task_done() - if i >= total: + if i >= total: # 得存入完所有请求结果才能结束 break db.close() diff --git a/common/utils.py b/common/utils.py index a673e25..c42e723 100644 --- a/common/utils.py +++ b/common/utils.py @@ -1,3 +1,4 @@ +import os import re import sys import time @@ -476,7 +477,7 @@ def ip_is_public(ip_str): def get_request_count(): - return 32 + return os.cpu_count() * 16 def uniq_dict_list(dict_list): diff --git a/config/default.py b/config/default.py index 3e3bc6b..c34caad 100644 --- a/config/default.py +++ b/config/default.py @@ -97,8 +97,8 @@ request_proxy_pool = [{'http': 'http://127.0.0.1:1080', # 请求设置 -request_thread_count = None # 请求线程数量(默认None,则根据内存大小设置) -request_timeout_second = (3.05, 27) # 请求超时秒数(默认connect timout推荐略大于3秒,read秒) +request_thread_count = None # 请求线程数量(默认None,则根据情况自动设置) +request_timeout_second = (3.05, 27) # 请求超时秒数(默认connect timout推荐略大于3秒) request_ssl_verify = False # 请求SSL验证(默认False) request_allow_redirect = True # 请求允许重定向(默认True) request_redirect_limit = 10 # 请求跳转限制(默认10次) diff --git a/config/setting.py b/config/setting.py index 6fc3f23..2d9602e 100644 --- a/config/setting.py +++ b/config/setting.py @@ -71,6 +71,30 @@ request_proxy_pool = [{'http': 'http://127.0.0.1:1080', # request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808', # 'https': 'socks5h://127.0.0.1:10808'}] # 代理池 + +# 请求设置 +request_thread_count = None # 请求线程数量(默认None,则根据情况自动设置) +request_timeout_second = (3.05, 27) # 请求超时秒数(默认connect timout推荐略大于3秒) +request_ssl_verify = False # 请求SSL验证(默认False) +request_allow_redirect = True # 请求允许重定向(默认True) +request_redirect_limit = 10 # 请求跳转限制(默认10次) +# 默认请求头 可以在headers里添加自定义请求头 +request_default_headers = { + 'Accept': 'text/html,application/xhtml+xml,' + 'application/xml;q=0.9,*/*;q=0.8', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7', + 'Cache-Control': 'max-age=0', + 'DNT': '1', + 'Referer': 'https://www.google.com/', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36', + 'Upgrade-Insecure-Requests': '1', + 'X-Forwarded-For': '127.0.0.1' +} +enable_random_ua = True # 使用随机UA(默认True,开启可以覆盖request_default_headers的UA) + + # 搜索模块设置 enable_recursive_search = False # 递归搜索子域 search_recursive_times = 2 # 递归搜索层数