优化

common/request.py

@@ -14,9 +14,10 @@ from config import settings
 
 def get_limit_conn():
     limit_open_conn = settings.limit_open_conn
-    if limit_open_conn is None:  # 默认情况
-        limit_open_conn = utils.get_semaphore()
-    return limit_open_conn
+    if isinstance(limit_open_conn, int):
+        return max(32, limit_open_conn)
+    else:
+        return utils.get_coroutine_num()
 
 
 def get_ports(port):
@@ -42,8 +43,8 @@ def gen_req_data(data, ports):
     new_data = []
     for data in data:
         resolve = data.get('resolve')
-        # 解析失败(0)的子域不进行http请求探测
-        if resolve == 0:
+        # 解析不成功的子域不进行http请求探测
+        if resolve != 1:
             continue
         subdomain = data.get('subdomain')
         for port in ports:
@@ -169,7 +170,8 @@ def request_callback(future, index, datas):
         else:
             datas[index]['alive'] = 1
         headers = resp.headers
-        # datas[index]['banner'] = utils.get_sample_banner(headers) 采用webanalyzer的指纹识别 原banner识别弃用
+        # 采用webanalyzer的指纹识别 原banner识别弃用
+        # datas[index]['banner'] = utils.get_sample_banner(headers)
         datas[index]['header'] = json.dumps(dict(headers))
         if isinstance(text, str):
             title = get_title(text).strip()
@@ -216,11 +218,13 @@ async def bulk_request(data, port):
     headers = utils.get_random_header()
     async with ClientSession(connector=connector, headers=headers) as session:
         tasks = []
-        for i, data in enumerate(to_req_data):
+        for num, data in enumerate(to_req_data):
             url = data.get('url')
-            task = asyncio.ensure_future(fetch(session, method, url))
+            task = asyncio.create_task(fetch(session, method, url))
+            task.set_name(f'RequestTask-{num}')
+            # logger.log('TRACE', f'RequestTask-{num} {url}')
             task.add_done_callback(functools.partial(request_callback,
-                                                     index=i,
+                                                     index=num,
                                                      datas=to_req_data))
             tasks.append(task)
         if tasks:

common/utils.py

@@ -146,21 +146,6 @@ def get_domains(target):
     return domains
 
 
-def get_semaphore():
-    """
-    获取查询并发值
-
-    :return: 并发整型值
-    """
-    system = platform.system()
-    if system == 'Windows':
-        return 800
-    elif system == 'Linux':
-        return 800
-    elif system == 'Darwin':
-        return 800
-
-
 def check_dir(dir_path):
     if not dir_path.exists():
         logger.log('INFOR', f'{dir_path} does not exist, directory will be created')
@@ -459,8 +444,8 @@ def set_id_none(data):
 def get_filtered_data(data):
     filtered_data = []
     for item in data:
-        valid = item.get('resolve')
-        if valid == 0:
+        resolve = item.get('resolve')
+        if resolve != 1:
             filtered_data.append(item)
     return filtered_data
 
@@ -504,27 +489,21 @@ def get_process_num():
 
 
 def get_coroutine_num():
-    coroutine_num = settings.resolve_coroutine_num
-    if isinstance(coroutine_num, int):
-        return max(64, coroutine_num)
-    elif coroutine_num is None:
-        mem = psutil.virtual_memory()
-        total_mem = mem.total
-        g_size = 1024 * 1024 * 1024
-        if total_mem <= 1 * g_size:
-            return 64
-        elif total_mem <= 2 * g_size:
-            return 128
-        elif total_mem <= 4 * g_size:
-            return 256
-        elif total_mem <= 8 * g_size:
-            return 512
-        elif total_mem <= 16 * g_size:
-            return 1024
-        else:
-            return 2048
-    else:
+    mem = psutil.virtual_memory()
+    total_mem = mem.total
+    g_size = 1024 * 1024 * 1024
+    if total_mem <= 1 * g_size:
+        return 32
+    elif total_mem <= 2 * g_size:
         return 64
+    elif total_mem <= 4 * g_size:
+        return 128
+    elif total_mem <= 8 * g_size:
+        return 256
+    elif total_mem <= 16 * g_size:
+        return 512
+    else:
+        return 1024
 
 
 def uniq_dict_list(dict_list):

config/default.py

@@ -105,7 +105,6 @@ enable_recursive_search = False  # 递归搜索子域(默认False)
 search_recursive_times = 2  # 递归搜索层数(默认2)
 
 # DNS解析设置
-resolve_coroutine_num = 64
 resolver_nameservers = [
     '223.5.5.5',  # AliDNS
     '119.29.29.29',  # DNSPod
@@ -151,8 +150,7 @@ allow_redirects = True  # 允许请求跳转
 request_method = 'GET'  # 使用请求方法，默认GET
 sockread_timeout = 6  # 每个请求socket读取超时时间，默认6秒
 sockconn_timeout = 3  # 每个请求socket连接超时时间，默认3秒
-# 限制同一时间打开的连接总数
-limit_open_conn = 800  # 默认800
+limit_open_conn = None  # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置
 # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
 limit_per_host = 10  # 0表示不限制,默认10
 
@@ -169,6 +167,7 @@ headers = {
     'Accept-Encoding': 'gzip, deflate',
     'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
     'Cache-Control': 'max-age=0',
+    'Connection': 'close',
     'DNT': '1',
     'Referer': 'https://www.google.com/',
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '

config/setting.py

@@ -97,7 +97,6 @@ aiohttp_proxy = None  # 示例 proxy="http://user:pass@some.proxy.com"
 # 为了保证请求质量 请谨慎更改以下设置
 sockread_timeout = 6  # 每个请求socket读取超时时间，默认6秒
 sockconn_timeout = 3  # 每个请求socket连接超时时间，默认3秒
-# 限制同一时间打开的连接总数
-limit_open_conn = 800  # 默认800
+limit_open_conn = None  # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置
 # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
 limit_per_host = 10  # 0表示不限制,默认10