参数调优

oneforall/common/request.py

@@ -46,26 +46,24 @@ def gen_new_datas(datas, ports):
     return new_datas
 
 
-async def fetch(session, url):
+async def fetch(session, url, semaphore):
     """
     请求
 
     :param session: session对象
     :param url: url地址
+    :param semaphore: 并发信号量
     :return: 响应对象和响应文本
     """
-    header = None
-    if config.fake_header:
-        header = utils.gen_fake_header()
     timeout = aiohttp.ClientTimeout(total=config.get_timeout)
-    async with session.get(url,
-                           headers=header,
-                           ssl=config.verify_ssl,
-                           allow_redirects=config.get_redirects,
-                           timeout=timeout,
-                           proxy=config.get_proxy) as resp:
-        text = await resp.text()
-    return resp, text
+    async with semaphore:
+        async with session.get(url,
+                               ssl=config.verify_ssl,
+                               allow_redirects=config.get_redirects,
+                               timeout=timeout,
+                               proxy=config.get_proxy) as resp:
+            text = await resp.text()
+        return resp, text
 
 
 def deal_results(datas, results):
@@ -79,7 +77,7 @@ def deal_results(datas, results):
             resp, text = result
             datas[index]['reason'] = resp.reason
             datas[index]['status'] = resp.status
-            if resp.status == 400 or resp.status >= 500:
+            if resp.status >= 500:
                 datas[index]['valid'] = 0
             else:
                 datas[index]['valid'] = 1
@@ -95,7 +93,7 @@ def deal_results(datas, results):
                     datas[index]['title'] = title.text
                 elif head:
                     datas[index]['title'] = head.text
-                else:
+                elif len(text) <= 200:
                     datas[index]['title'] = text
     return datas
 
@@ -105,19 +103,25 @@ async def bulk_get_request(datas, port):
     new_datas = gen_new_datas(datas, ports)
     logger.log('INFOR', f'正在异步进行子域的GET请求')
 
+    limit_open_conn = config.limit_open_conn
+    if not limit_open_conn:
+        limit_open_conn = utils.get_semaphore()
     # 使用异步域名解析器 自定义域名服务器
     resolver = AsyncResolver(nameservers=config.resolver_nameservers)
     conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
-                                limit=config.limit_open_conn,
+                                limit=limit_open_conn,
                                 limit_per_host=config.limit_per_host,
                                 resolver=resolver)
-    # semaphore = asyncio.Semaphore(utils.get_semaphore())
-    async with ClientSession(connector=conn) as session:
+
+    semaphore = asyncio.Semaphore(limit_open_conn)
+    header = None
+    if config.fake_header:
+        header = utils.gen_fake_header()
+    async with ClientSession(connector=conn, headers=header) as session:
         tasks = []
         for i, data in enumerate(new_datas):
             url = data.get('url')
-            # task = asyncio.ensure_future(fetch(session, url, semaphore))
-            task = asyncio.ensure_future(fetch(session, url))
+            task = asyncio.ensure_future(fetch(session, url, semaphore))
             tasks.append(task)
         if tasks:  # 任务列表里有任务不空时才进行解析
             # 等待所有task完成 错误聚合到结果列表里

oneforall/config.py

@@ -44,7 +44,7 @@ fuzz_rule = ''  # fuzz域名的正则 示例：[a-z][0-9] 第一位是字母 第
 ips_appear_maximum = 10  # 同一IP集合出现次数超过10认为是泛解析
 
 # 代理设置
-enable_proxy = True  # 是否使用代理(全局开关)
+enable_proxy = False  # 是否使用代理(全局开关)
 proxy_all_module = False  # 代理所有模块
 proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
                         'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
@@ -101,9 +101,10 @@ get_proxy = None  # proxy="http://user:pass@some.proxy.com"
 get_timeout = 120  # http请求探测总超时时间 None或者0则表示不检测超时
 get_redirects = True  # 允许请求跳转
 fake_header = True  # 使用伪造请求头
-limit_open_conn = 100  # 限制同一时间打开的连接数(默认100)，0表示不限制
+# 限制同一时间打开的连接数(默认None，根据系统不同设置，Windows系统500 其他系统1000)
+limit_open_conn = None
 # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
-limit_per_host = 0  # 默认0表示不限制
+limit_per_host = 10  # 默认0表示不限制
 
 
 # 模块API配置
@@ -180,8 +181,8 @@ stdout_fmt = '<cyan>{time:HH:mm:ss,SSS}</cyan> ' \
 logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
           '[<level>{level: <5}</level>] ' \
           '<cyan>{process.name}</cyan>:<cyan>{thread.name: <10}</cyan> | ' \
-          '<blue>{module}</blue>.<blue>{function}</blue>:<blue>{line}</blue> - ' \
-          '<level>{message}</level>'
+          '<blue>{module}</blue>.<blue>{function}</blue>:' \
+          '<blue>{line}</blue> - <level>{message}</level>'
 
 log_path = result_save_path.joinpath('oneforall.log')