不使用aiodns

oneforall/common/request.py

@@ -1,5 +1,3 @@
-# coding=utf-8
-
 import asyncio
 import functools
 
@@ -62,28 +60,26 @@ def gen_new_datas(datas, ports):
     return new_datas
 
 
-async def fetch(session, url, semaphore):
+async def fetch(session, url):
     """
     请求
 
     :param session: session对象
     :param url: url地址
-    :param semaphore: 并发信号量
     :return: 响应对象和响应文本
     """
     timeout = aiohttp.ClientTimeout(total=config.get_timeout)
-    async with semaphore:
-        async with session.get(url,
-                               ssl=config.verify_ssl,
-                               allow_redirects=config.get_redirects,
-                               timeout=timeout,
-                               proxy=config.get_proxy) as resp:
+    async with session.get(url,
+                           ssl=config.verify_ssl,
+                           allow_redirects=config.get_redirects,
+                           timeout=timeout,
+                           proxy=config.get_proxy) as resp:
 
-            try:
-                text = await resp.text(encoding='gb2312')  # 先尝试用fb2312解码
-            except UnicodeDecodeError:
-                text = await resp.text(errors='ignore')
-        return resp, text
+        try:
+            text = await resp.text(encoding='gb2312')  # 先尝试用fb2312解码
+        except UnicodeDecodeError:
+            text = await resp.text(errors='ignore')
+    return resp, text
 
 
 def get_title(markup):
@@ -159,7 +155,7 @@ async def bulk_get_request(datas, port):
                                 limit=limit_open_conn,
                                 limit_per_host=config.limit_per_host)
 
-    semaphore = asyncio.Semaphore(limit_open_conn)
+    # semaphore = asyncio.Semaphore(limit_open_conn)
     header = None
     if config.fake_header:
         header = utils.gen_fake_header()
@@ -167,7 +163,7 @@ async def bulk_get_request(datas, port):
         tasks = []
         for i, data in enumerate(new_datas):
             url = data.get('url')
-            task = asyncio.ensure_future(fetch(session, url, semaphore))
+            task = asyncio.ensure_future(fetch(session, url))
             task.add_done_callback(functools.partial(request_callback,
                                                      index=i,
                                                      datas=new_datas))

oneforall/common/resolve.py

@@ -22,7 +22,7 @@ def dns_resolver():
     return resolver
 
 
-def dns_query_a(hostname):
+async def dns_query_a(hostname):
     """
     查询A记录
 
@@ -30,7 +30,12 @@ def dns_query_a(hostname):
     :return: 查询结果
     """
     resolver = dns_resolver()
-    return resolver.query(hostname, 'A')
+    try:
+        answer = resolver.query(hostname, 'A')
+    except BaseException as e:
+        logger.log('DEBUG', e.args)
+        answer = None
+    return answer
 
 
 def aiodns_resolver():
@@ -76,18 +81,16 @@ def resolve_callback(future, index, datas):
     :param datas: 结果集
     """
     try:
-        result = future.result()
+        answer = future.result()
     except BaseException as e:
         datas[index]['ips'] = str(e.args)
         datas[index]['valid'] = 0
     else:
-        if isinstance(result, tuple):
-            _, answers = result
-            if answers:
-                ips = {record.host for record in answers}
-                datas[index]['ips'] = str(ips)
-            else:
-                datas[index]['ips'] = 'No answers'
+        if answer:
+            ips = {item.address for item in answer}
+            datas[index]['ips'] = str(ips)
+        else:
+            datas[index]['ips'] = 'No answers'
 
 
 async def bulk_query_a(datas):
@@ -99,11 +102,11 @@ async def bulk_query_a(datas):
     """
     logger.log('INFOR', '正在异步查询子域的A记录')
     tasks = []
-    semaphore = asyncio.Semaphore(config.limit_resolve_conn)
+    # semaphore = asyncio.Semaphore(config.limit_resolve_conn)
     for i, data in enumerate(datas):
         if not data.get('ips'):
             subdomain = data.get('subdomain')
-            task = asyncio.ensure_future(aiodns_query_a(subdomain, semaphore))
+            task = asyncio.ensure_future(dns_query_a(subdomain))
             task.add_done_callback(functools.partial(resolve_callback,
                                                      index=i,
                                                      datas=datas))  # 回调

oneforall/common/utils.py

@@ -137,7 +137,7 @@ def get_semaphore():
     """
     system = platform.system()
     if system == 'Windows':
-        return 300
+        return 800
     elif system == 'Linux':
         return 800
     elif system == 'Darwin':