From 814062b90880f8dc8d58beaff55db3ec7487b68d Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Fri, 14 Aug 2020 18:01:16 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/request.py | 22 ++++++++++++-------- common/utils.py | 53 ++++++++++++++--------------------------------- config/default.py | 5 ++--- config/setting.py | 3 +-- 4 files changed, 32 insertions(+), 51 deletions(-) diff --git a/common/request.py b/common/request.py index 3b774b9..00aad24 100644 --- a/common/request.py +++ b/common/request.py @@ -14,9 +14,10 @@ from config import settings def get_limit_conn(): limit_open_conn = settings.limit_open_conn - if limit_open_conn is None: # 默认情况 - limit_open_conn = utils.get_semaphore() - return limit_open_conn + if isinstance(limit_open_conn, int): + return max(32, limit_open_conn) + else: + return utils.get_coroutine_num() def get_ports(port): @@ -42,8 +43,8 @@ def gen_req_data(data, ports): new_data = [] for data in data: resolve = data.get('resolve') - # 解析失败(0)的子域不进行http请求探测 - if resolve == 0: + # 解析不成功的子域不进行http请求探测 + if resolve != 1: continue subdomain = data.get('subdomain') for port in ports: @@ -169,7 +170,8 @@ def request_callback(future, index, datas): else: datas[index]['alive'] = 1 headers = resp.headers - # datas[index]['banner'] = utils.get_sample_banner(headers) 采用webanalyzer的指纹识别 原banner识别弃用 + # 采用webanalyzer的指纹识别 原banner识别弃用 + # datas[index]['banner'] = utils.get_sample_banner(headers) datas[index]['header'] = json.dumps(dict(headers)) if isinstance(text, str): title = get_title(text).strip() @@ -216,11 +218,13 @@ async def bulk_request(data, port): headers = utils.get_random_header() async with ClientSession(connector=connector, headers=headers) as session: tasks = [] - for i, data in enumerate(to_req_data): + for num, data in enumerate(to_req_data): url = data.get('url') - task = asyncio.ensure_future(fetch(session, method, url)) + task = asyncio.create_task(fetch(session, method, url)) + task.set_name(f'RequestTask-{num}') + # logger.log('TRACE', f'RequestTask-{num} {url}') task.add_done_callback(functools.partial(request_callback, - index=i, + index=num, datas=to_req_data)) tasks.append(task) if tasks: diff --git a/common/utils.py b/common/utils.py index 9058ff0..28d6a6a 100644 --- a/common/utils.py +++ b/common/utils.py @@ -146,21 +146,6 @@ def get_domains(target): return domains -def get_semaphore(): - """ - 获取查询并发值 - - :return: 并发整型值 - """ - system = platform.system() - if system == 'Windows': - return 800 - elif system == 'Linux': - return 800 - elif system == 'Darwin': - return 800 - - def check_dir(dir_path): if not dir_path.exists(): logger.log('INFOR', f'{dir_path} does not exist, directory will be created') @@ -459,8 +444,8 @@ def set_id_none(data): def get_filtered_data(data): filtered_data = [] for item in data: - valid = item.get('resolve') - if valid == 0: + resolve = item.get('resolve') + if resolve != 1: filtered_data.append(item) return filtered_data @@ -504,27 +489,21 @@ def get_process_num(): def get_coroutine_num(): - coroutine_num = settings.resolve_coroutine_num - if isinstance(coroutine_num, int): - return max(64, coroutine_num) - elif coroutine_num is None: - mem = psutil.virtual_memory() - total_mem = mem.total - g_size = 1024 * 1024 * 1024 - if total_mem <= 1 * g_size: - return 64 - elif total_mem <= 2 * g_size: - return 128 - elif total_mem <= 4 * g_size: - return 256 - elif total_mem <= 8 * g_size: - return 512 - elif total_mem <= 16 * g_size: - return 1024 - else: - return 2048 - else: + mem = psutil.virtual_memory() + total_mem = mem.total + g_size = 1024 * 1024 * 1024 + if total_mem <= 1 * g_size: + return 32 + elif total_mem <= 2 * g_size: return 64 + elif total_mem <= 4 * g_size: + return 128 + elif total_mem <= 8 * g_size: + return 256 + elif total_mem <= 16 * g_size: + return 512 + else: + return 1024 def uniq_dict_list(dict_list): diff --git a/config/default.py b/config/default.py index d771a3e..b1327cb 100644 --- a/config/default.py +++ b/config/default.py @@ -105,7 +105,6 @@ enable_recursive_search = False # 递归搜索子域(默认False) search_recursive_times = 2 # 递归搜索层数(默认2) # DNS解析设置 -resolve_coroutine_num = 64 resolver_nameservers = [ '223.5.5.5', # AliDNS '119.29.29.29', # DNSPod @@ -151,8 +150,7 @@ allow_redirects = True # 允许请求跳转 request_method = 'GET' # 使用请求方法，默认GET sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒 sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒 -# 限制同一时间打开的连接总数 -limit_open_conn = 800 # 默认800 +limit_open_conn = None # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置 # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数 limit_per_host = 10 # 0表示不限制,默认10 @@ -169,6 +167,7 @@ headers = { 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7', 'Cache-Control': 'max-age=0', + 'Connection': 'close', 'DNT': '1', 'Referer': 'https://www.google.com/', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' diff --git a/config/setting.py b/config/setting.py index 693815c..08d3f89 100644 --- a/config/setting.py +++ b/config/setting.py @@ -97,7 +97,6 @@ aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com" # 为了保证请求质量 请谨慎更改以下设置 sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒 sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒 -# 限制同一时间打开的连接总数 -limit_open_conn = 800 # 默认800 +limit_open_conn = None # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置 # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数 limit_per_host = 10 # 0表示不限制,默认10