参数调优

oneforall/common/request.py

@@ -46,26 +46,24 @@ def gen_new_datas(datas, ports):
     return new_datas
 
 
-async def fetch(session, url):
+async def fetch(session, url, semaphore):
     """
     请求
 
     :param session: session对象
     :param url: url地址
+    :param semaphore: 并发信号量
     :return: 响应对象和响应文本
     """
-    header = None
-    if config.fake_header:
-        header = utils.gen_fake_header()
     timeout = aiohttp.ClientTimeout(total=config.get_timeout)
-    async with session.get(url,
-                           headers=header,
-                           ssl=config.verify_ssl,
-                           allow_redirects=config.get_redirects,
-                           timeout=timeout,
-                           proxy=config.get_proxy) as resp:
-        text = await resp.text()
-    return resp, text
+    async with semaphore:
+        async with session.get(url,
+                               ssl=config.verify_ssl,
+                               allow_redirects=config.get_redirects,
+                               timeout=timeout,
+                               proxy=config.get_proxy) as resp:
+            text = await resp.text()
+        return resp, text
 
 
 def deal_results(datas, results):
@@ -79,7 +77,7 @@ def deal_results(datas, results):
             resp, text = result
             datas[index]['reason'] = resp.reason
             datas[index]['status'] = resp.status
-            if resp.status == 400 or resp.status >= 500:
+            if resp.status >= 500:
                 datas[index]['valid'] = 0
             else:
                 datas[index]['valid'] = 1
@@ -95,7 +93,7 @@ def deal_results(datas, results):
                     datas[index]['title'] = title.text
                 elif head:
                     datas[index]['title'] = head.text
-                else:
+                elif len(text) <= 200:
                     datas[index]['title'] = text
     return datas
 
@@ -105,19 +103,25 @@ async def bulk_get_request(datas, port):
     new_datas = gen_new_datas(datas, ports)
     logger.log('INFOR', f'正在异步进行子域的GET请求')
 
+    limit_open_conn = config.limit_open_conn
+    if not limit_open_conn:
+        limit_open_conn = utils.get_semaphore()
     # 使用异步域名解析器 自定义域名服务器
     resolver = AsyncResolver(nameservers=config.resolver_nameservers)
     conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
-                                limit=config.limit_open_conn,
+                                limit=limit_open_conn,
                                 limit_per_host=config.limit_per_host,
                                 resolver=resolver)
-    # semaphore = asyncio.Semaphore(utils.get_semaphore())
-    async with ClientSession(connector=conn) as session:
+
+    semaphore = asyncio.Semaphore(limit_open_conn)
+    header = None
+    if config.fake_header:
+        header = utils.gen_fake_header()
+    async with ClientSession(connector=conn, headers=header) as session:
         tasks = []
         for i, data in enumerate(new_datas):
             url = data.get('url')
-            # task = asyncio.ensure_future(fetch(session, url, semaphore))
-            task = asyncio.ensure_future(fetch(session, url))
+            task = asyncio.ensure_future(fetch(session, url, semaphore))
             tasks.append(task)
         if tasks:  # 任务列表里有任务不空时才进行解析
             # 等待所有task完成 错误聚合到结果列表里