请求优化

oneforall/common/request.py

@@ -48,7 +48,8 @@ def gen_new_datas(datas, ports):
     protocols = ['http://', 'https://']
     for data in datas:
         valid = data.get('valid')
-        if valid is None:  # 子域有效性未知的才进行http请求探测
+        if valid == 0:  # 子域有效性未知的才进行http请求探测
+            continue
         subdomain = data.get('subdomain')
         for port in ports:
             for protocol in protocols:
@@ -75,6 +76,7 @@ async def fetch(session, url):
     :return: 响应对象和响应文本
     """
     timeout = aiohttp.ClientTimeout(total=config.get_timeout)
+    try:
         async with session.get(url,
                                ssl=config.verify_ssl,
                                allow_redirects=config.get_redirects,
@@ -86,6 +88,8 @@ async def fetch(session, url):
             except UnicodeDecodeError:
                 text = await resp.text(errors='ignore')
         return resp, text
+    except BaseException as exception:
+        return exception
 
 
 def get_title(markup):
@@ -135,6 +139,7 @@ def request_callback(future, index, datas):
         datas[index]['reason'] = str(e.args)
         datas[index]['valid'] = 0
     else:
+        if isinstance(result, tuple):
             resp, text = result
             datas[index]['reason'] = resp.reason
             datas[index]['status'] = resp.status
@@ -148,6 +153,9 @@ def request_callback(future, index, datas):
                               'X-Powered-By': headers.get('X-Powered-By')})
                 datas[index]['banner'] = banner
                 datas[index]['title'] = get_title(text)
+        else:
+            datas[index]['reason'] = 'Something error'
+            datas[index]['valid'] = 0
 
 
 async def bulk_get_request(datas, port):
@@ -156,17 +164,15 @@ async def bulk_get_request(datas, port):
     logger.log('INFOR', f'正在异步进行子域的GET请求')
 
     limit_open_conn = get_limit_conn()
-    # 使用异步域名解析器 自定义域名服务器
+    conn = aiohttp.TCPConnector(ttl_dns_cache=300,
-    conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
+                                ssl=config.verify_ssl,
                                 limit=limit_open_conn,
                                 limit_per_host=config.limit_per_host)
-
-    # semaphore = asyncio.Semaphore(limit_open_conn)
     header = None
     if config.fake_header:
         header = utils.gen_fake_header()
-    async with ClientSession(connector=conn, headers=header) as session:
     tasks = []
+    async with ClientSession(connector=conn, headers=header) as session:
         for i, data in enumerate(new_datas):
             url = data.get('url')
             task = asyncio.ensure_future(fetch(session, url))
@@ -174,7 +180,8 @@ async def bulk_get_request(datas, port):
                                                      index=i,
                                                      datas=new_datas))
             tasks.append(task)
-        if tasks:  # 任务列表里有任务不空时才进行解析
+        # 任务列表里有任务不空时才进行解析
+        if tasks:
             # 等待所有task完成 错误聚合到结果列表里
             futures = asyncio.as_completed(tasks)
             for future in tqdm.tqdm(futures,
@@ -182,10 +189,7 @@ async def bulk_get_request(datas, port):
                                     desc='Progress',
                                     smoothing=1.0,
                                     ncols=True):
-                try:
                 await future
-                except:
-                    pass
 
     logger.log('INFOR', f'完成异步进行子域的GET请求')
     return new_datas