不使用aiodns

oneforall/common/request.py

@@ -1,5 +1,3 @@
-# coding=utf-8
-
 import asyncio
 import functools
 
@@ -62,28 +60,26 @@ def gen_new_datas(datas, ports):
     return new_datas
 
 
-async def fetch(session, url, semaphore):
+async def fetch(session, url):
     """
     请求
 
     :param session: session对象
     :param url: url地址
-    :param semaphore: 并发信号量
     :return: 响应对象和响应文本
     """
     timeout = aiohttp.ClientTimeout(total=config.get_timeout)
-    async with semaphore:
-        async with session.get(url,
-                               ssl=config.verify_ssl,
-                               allow_redirects=config.get_redirects,
-                               timeout=timeout,
-                               proxy=config.get_proxy) as resp:
+    async with session.get(url,
+                           ssl=config.verify_ssl,
+                           allow_redirects=config.get_redirects,
+                           timeout=timeout,
+                           proxy=config.get_proxy) as resp:
 
-            try:
-                text = await resp.text(encoding='gb2312')  # 先尝试用fb2312解码
-            except UnicodeDecodeError:
-                text = await resp.text(errors='ignore')
-        return resp, text
+        try:
+            text = await resp.text(encoding='gb2312')  # 先尝试用fb2312解码
+        except UnicodeDecodeError:
+            text = await resp.text(errors='ignore')
+    return resp, text
 
 
 def get_title(markup):
@@ -159,7 +155,7 @@ async def bulk_get_request(datas, port):
                                 limit=limit_open_conn,
                                 limit_per_host=config.limit_per_host)
 
-    semaphore = asyncio.Semaphore(limit_open_conn)
+    # semaphore = asyncio.Semaphore(limit_open_conn)
     header = None
     if config.fake_header:
         header = utils.gen_fake_header()
@@ -167,7 +163,7 @@ async def bulk_get_request(datas, port):
         tasks = []
         for i, data in enumerate(new_datas):
             url = data.get('url')
-            task = asyncio.ensure_future(fetch(session, url, semaphore))
+            task = asyncio.ensure_future(fetch(session, url))
             task.add_done_callback(functools.partial(request_callback,
                                                      index=i,
                                                      datas=new_datas))

oneforall/common/resolve.py

@@ -22,7 +22,7 @@ def dns_resolver():
     return resolver
 
 
-def dns_query_a(hostname):
+async def dns_query_a(hostname):
     """
     查询A记录
 
@@ -30,7 +30,12 @@ def dns_query_a(hostname):
     :return: 查询结果
     """
     resolver = dns_resolver()
-    return resolver.query(hostname, 'A')
+    try:
+        answer = resolver.query(hostname, 'A')
+    except BaseException as e:
+        logger.log('DEBUG', e.args)
+        answer = None
+    return answer
 
 
 def aiodns_resolver():
@@ -76,18 +81,16 @@ def resolve_callback(future, index, datas):
     :param datas: 结果集
     """
     try:
-        result = future.result()
+        answer = future.result()
     except BaseException as e:
         datas[index]['ips'] = str(e.args)
         datas[index]['valid'] = 0
     else:
-        if isinstance(result, tuple):
-            _, answers = result
-            if answers:
-                ips = {record.host for record in answers}
-                datas[index]['ips'] = str(ips)
-            else:
-                datas[index]['ips'] = 'No answers'
+        if answer:
+            ips = {item.address for item in answer}
+            datas[index]['ips'] = str(ips)
+        else:
+            datas[index]['ips'] = 'No answers'
 
 
 async def bulk_query_a(datas):
@@ -99,11 +102,11 @@ async def bulk_query_a(datas):
     """
     logger.log('INFOR', '正在异步查询子域的A记录')
     tasks = []
-    semaphore = asyncio.Semaphore(config.limit_resolve_conn)
+    # semaphore = asyncio.Semaphore(config.limit_resolve_conn)
     for i, data in enumerate(datas):
         if not data.get('ips'):
             subdomain = data.get('subdomain')
-            task = asyncio.ensure_future(aiodns_query_a(subdomain, semaphore))
+            task = asyncio.ensure_future(dns_query_a(subdomain))
             task.add_done_callback(functools.partial(resolve_callback,
                                                      index=i,
                                                      datas=datas))  # 回调

oneforall/common/utils.py

@@ -137,7 +137,7 @@ def get_semaphore():
     """
     system = platform.system()
     if system == 'Windows':
-        return 300
+        return 800
     elif system == 'Linux':
         return 800
     elif system == 'Darwin':

oneforall/config.py

@@ -31,7 +31,7 @@ enable_verify_subdomain = True  # 验证子域有效性(默认True)
 enable_wildcard_check = True  # 开启泛解析检测 会去掉泛解析的子域
 # 爆破时使用的进程数(根据系统中CPU数量情况设置 不宜大于CPU数量 默认为系统中的CPU数量)
 brute_process_num = os.cpu_count()
-brute_coroutine_num = 128  # 爆破时每个进程下的协程数(不宜大于1000)
+brute_coroutine_num = 64  # 爆破时每个进程下的协程数(不宜大于500)
 # 爆破所使用的字典路径 默认data/subdomains.txt
 brute_wordlist_path = data_storage_path.joinpath('subnames.txt')
 brute_task_segment = 500
@@ -82,7 +82,7 @@ resolver_nameservers = [
 ]  # 指定查询的DNS域名服务器
 resolver_timeout = 5.0  # 解析超时时间
 resolver_lifetime = 30.0  # 解析存活时间
-limit_resolve_conn = 50  # 限制同一时间解析的数量(默认50)
+limit_resolve_conn = 500  # 限制同一时间解析的数量(默认500)
 
 # http探测设置
 small_ports = {80, 443}
@@ -106,7 +106,7 @@ get_timeout = 120  # http请求探测总超时时间 None或者0则表示不检
 get_redirects = True  # 允许请求跳转
 fake_header = True  # 使用伪造请求头
 # 限制同一时间打开的连接数(默认None，根据系统不同设置，Windows系统400 其他系统800)
-limit_open_conn = None
+limit_open_conn = 0
 # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
 limit_per_host = 0  # 默认0表示不限制
 

oneforall/modules/dnsquery/srv.py

@@ -50,14 +50,15 @@ class BruteSRV(Module):
         tasks = []
         for name in names:
             tasks.append(self.query(name))
-        loop = asyncio.get_event_loop()
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
         group = asyncio.gather(*tasks)
         results = loop.run_until_complete(group)
-        for result in results:
-            if result is None:
+        for answer in results:
+            if answer is None:
                 continue
-            for answer in result:
-                subdomains = utils.match_subdomain(self.domain, str(answer))
+            for item in answer:
+                subdomains = utils.match_subdomain(self.domain, str(item))
                 self.subdomains = self.subdomains.union(subdomains)
         if not len(self.subdomains):
             logger.log('DEBUG', f'没有找到{self.domain}的SRV记录')