From 56010d971284332f668dae9f400d6b64ea5e0445 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Fri, 29 Jan 2021 00:46:17 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=AF=B7=E6=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/request.py | 2 +- common/utils.py | 3 ++- config/default.py | 4 ++-- config/setting.py | 24 ++++++++++++++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/common/request.py b/common/request.py index 1c7c459..5ef3f25 100644 --- a/common/request.py +++ b/common/request.py @@ -208,7 +208,7 @@ def save(name, total, req_data, resp_queue): new_info = gen_new_info(old_info, resp) db.insert_table(name, new_info) resp_queue.task_done() - if i >= total: + if i >= total: # 得存入完所有请求结果才能结束 break db.close() diff --git a/common/utils.py b/common/utils.py index a673e25..c42e723 100644 --- a/common/utils.py +++ b/common/utils.py @@ -1,3 +1,4 @@ +import os import re import sys import time @@ -476,7 +477,7 @@ def ip_is_public(ip_str): def get_request_count(): - return 32 + return os.cpu_count() * 16 def uniq_dict_list(dict_list): diff --git a/config/default.py b/config/default.py index 3e3bc6b..c34caad 100644 --- a/config/default.py +++ b/config/default.py @@ -97,8 +97,8 @@ request_proxy_pool = [{'http': 'http://127.0.0.1:1080', # 请求设置 -request_thread_count = None # 请求线程数量(默认None，则根据内存大小设置) -request_timeout_second = (3.05, 27) # 请求超时秒数(默认connect timout推荐略大于3秒，read秒) +request_thread_count = None # 请求线程数量(默认None，则根据情况自动设置) +request_timeout_second = (3.05, 27) # 请求超时秒数(默认connect timout推荐略大于3秒) request_ssl_verify = False # 请求SSL验证(默认False) request_allow_redirect = True # 请求允许重定向(默认True) request_redirect_limit = 10 # 请求跳转限制(默认10次) diff --git a/config/setting.py b/config/setting.py index 6fc3f23..2d9602e 100644 --- a/config/setting.py +++ b/config/setting.py @@ -71,6 +71,30 @@ request_proxy_pool = [{'http': 'http://127.0.0.1:1080', # request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808', # 'https': 'socks5h://127.0.0.1:10808'}] # 代理池 + +# 请求设置 +request_thread_count = None # 请求线程数量(默认None，则根据情况自动设置) +request_timeout_second = (3.05, 27) # 请求超时秒数(默认connect timout推荐略大于3秒) +request_ssl_verify = False # 请求SSL验证(默认False) +request_allow_redirect = True # 请求允许重定向(默认True) +request_redirect_limit = 10 # 请求跳转限制(默认10次) +# 默认请求头 可以在headers里添加自定义请求头 +request_default_headers = { + 'Accept': 'text/html,application/xhtml+xml,' + 'application/xml;q=0.9,*/*;q=0.8', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7', + 'Cache-Control': 'max-age=0', + 'DNT': '1', + 'Referer': 'https://www.google.com/', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36', + 'Upgrade-Insecure-Requests': '1', + 'X-Forwarded-For': '127.0.0.1' +} +enable_random_ua = True # 使用随机UA(默认True，开启可以覆盖request_default_headers的UA) + + # 搜索模块设置 enable_recursive_search = False # 递归搜索子域 search_recursive_times = 2 # 递归搜索层数