参数调优

oneforall/common/request.py

@@ -46,28 +46,26 @@ def gen_new_datas(datas, ports):
     return new_datas
 
 
-async def fetch(session, url, semaphore):
+async def fetch(session, url):
     """
     请求
 
     :param session: session对象
     :param url: url地址
-    :param semaphore: 同步对象(控制并发量)
     :return: 响应对象和响应文本
     """
     header = None
     if config.fake_header:
         header = utils.gen_fake_header()
     timeout = aiohttp.ClientTimeout(total=config.get_timeout)
-    async with semaphore:
-        async with session.get(url,
-                               headers=header,
-                               ssl=config.verify_ssl,
-                               allow_redirects=config.get_redirects,
-                               timeout=timeout,
-                               proxy=config.get_proxy) as resp:
-            text = await resp.text()
-            return resp, text
+    async with session.get(url,
+                           headers=header,
+                           ssl=config.verify_ssl,
+                           allow_redirects=config.get_redirects,
+                           timeout=timeout,
+                           proxy=config.get_proxy) as resp:
+        text = await resp.text()
+    return resp, text
 
 
 def deal_results(datas, results):
@@ -103,9 +101,9 @@ def deal_results(datas, results):
 
 
 async def bulk_get_request(datas, port):
-    logger.log('INFOR', f'正在异步进行子域的GET请求')
     ports = get_ports(port)
     new_datas = gen_new_datas(datas, ports)
+    logger.log('INFOR', f'正在异步进行子域的GET请求')
 
     # 使用异步域名解析器 自定义域名服务器
     resolver = AsyncResolver(nameservers=config.resolver_nameservers)
@@ -113,12 +111,13 @@ async def bulk_get_request(datas, port):
                                 limit=config.limit_open_conn,
                                 limit_per_host=config.limit_per_host,
                                 resolver=resolver)
-    semaphore = asyncio.Semaphore(utils.get_semaphore())
+    # semaphore = asyncio.Semaphore(utils.get_semaphore())
     async with ClientSession(connector=conn) as session:
         tasks = []
         for i, data in enumerate(new_datas):
             url = data.get('url')
-            task = asyncio.ensure_future(fetch(session, url, semaphore))
+            # task = asyncio.ensure_future(fetch(session, url, semaphore))
+            task = asyncio.ensure_future(fetch(session, url))
             tasks.append(task)
         if tasks:  # 任务列表里有任务不空时才进行解析
             # 等待所有task完成 错误聚合到结果列表里

oneforall/common/resolve.py

@@ -89,7 +89,7 @@ async def bulk_query_a(datas):
     """
     logger.log('INFOR', '正在异步查询子域的A记录')
     tasks = []
-    semaphore = asyncio.Semaphore(utils.get_semaphore())
+    semaphore = asyncio.Semaphore(config.limit_resolve_conn)
     for i, data in enumerate(datas):
         if not data.get('ips'):
             subdomain = data.get('subdomain')

oneforall/config.py

@@ -44,7 +44,7 @@ fuzz_rule = ''  # fuzz域名的正则 示例：[a-z][0-9] 第一位是字母 第
 ips_appear_maximum = 10  # 同一IP集合出现次数超过10认为是泛解析
 
 # 代理设置
-enable_proxy = False  # 是否使用代理(全局开关)
+enable_proxy = True  # 是否使用代理(全局开关)
 proxy_all_module = False  # 代理所有模块
 proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
                         'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
@@ -59,7 +59,7 @@ proxy_pool = [{'http': 'http://127.0.0.1:1080',
 # 网络请求设置
 enable_fake_header = True  # 启用伪造请求头
 request_delay = 1  # 请求时延
-request_timeout = 30  # 请求超时
+request_timeout = 60  # 请求超时
 request_verify = True  # 请求SSL验证
 
 # 搜索模块设置
@@ -78,6 +78,7 @@ resolver_nameservers = [
 ]  # 指定查询的DNS域名服务器
 resolver_timeout = 5.0  # 解析超时时间
 resolver_lifetime = 30.0  # 解析存活时间
+limit_resolve_conn = 50  # 限制同一时间解析的数量(默认50)
 
 # http探测设置
 small_ports = {80, 443}
@@ -97,7 +98,7 @@ ports = {'small': small_ports, 'medium': medium_ports,
 verify_ssl = False
 # aiohttp 支持 HTTP/HTTPS形式的代理
 get_proxy = None  # proxy="http://user:pass@some.proxy.com"
-get_timeout = 10  # http请求探测总超时时间 None或者0则表示不检测超时
+get_timeout = 120  # http请求探测总超时时间 None或者0则表示不检测超时
 get_redirects = True  # 允许请求跳转
 fake_header = True  # 使用伪造请求头
 limit_open_conn = 100  # 限制同一时间打开的连接数(默认100)，0表示不限制

oneforall/modules/check/sitemap.py

@@ -43,7 +43,7 @@ class CheckRobots(Module):
         for url in urls:
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
-            self.timeout = 5
+            self.timeout = 10
             response = self.get(url, allow_redirects=False)
             if response:
                 break

oneforall/modules/datasets/netcraft.py

@@ -25,7 +25,7 @@ class NetCraft(Query):
         resp = self.get(self.init)
         if not resp:
             return None
-        self.cookie = self.get(self.init).cookies
+        self.cookie = resp.cookies
         cookie_value = self.cookie['netcraft_js_verification_challenge']
         cookie_encode = parse.unquote(cookie_value).encode('utf-8')
         verify_taken = hashlib.sha1(cookie_encode).hexdigest()

oneforall/oneforall.py

@@ -9,6 +9,8 @@ OneForAll是一款功能强大的子域收集工具
 """
 
 import asyncio
+import sys
+
 import fire
 import config
 import dbexport