增加HEAD请求功能

oneforall/common/request.py

@@ -40,7 +40,6 @@ def get_ports(port):
 def gen_new_datas(datas, ports):
     logger.log('INFOR', f'正在生成请求地址')
     new_datas = []
-    protocols = ['http://', 'https://']
     for data in datas:
         valid = data.get('valid')
         if valid is None:  # 子域有效性未知的才进行http请求探测
@@ -68,14 +67,23 @@ async def fetch(session, url):
     请求
 
     :param session: session对象
-    :param url: url地址
+    :param str url: url地址
     :return: 响应对象和响应文本
     """
+    method = config.request_method.upper()
     timeout = aiohttp.ClientTimeout(total=None,
                                     connect=None,
                                     sock_read=config.sockread_timeout,
                                     sock_connect=config.sockconn_timeout)
     try:
+        if method == 'HEAD':
+            async with session.head(url,
+                                    ssl=config.verify_ssl,
+                                    allow_redirects=config.allow_redirects,
+                                    timeout=timeout,
+                                    proxy=config.aiohttp_proxy) as resp:
+                text = await resp.text()
+        else:
             async with session.get(url,
                                    ssl=config.verify_ssl,
                                    allow_redirects=config.allow_redirects,
@@ -152,9 +160,10 @@ def request_callback(future, index, datas):
                           'Via': headers.get('Via'),
                           'X-Powered-By': headers.get('X-Powered-By')})
             datas[index]['banner'] = banner[1:-1]
+            datas[index]['header'] = str(dict(headers))[1:-1]
+            if isinstance(text, str):
                 title = get_title(text).strip()
                 datas[index]['title'] = utils.remove_string(title)
-            datas[index]['header'] = str(dict(headers))[1:-1]
                 datas[index]['response'] = utils.remove_string(text)
 
 
@@ -173,10 +182,12 @@ def get_header():
     return header
 
 
-async def bulk_get_request(datas, port):
+async def bulk_request(datas, port):
     ports = get_ports(port)
     new_datas = gen_new_datas(datas, ports)
-    logger.log('INFOR', f'正在异步进行子域的GET请求')
+    method = config.request_method
+    logger.log('INFOR', f'使用{method}请求方法')
+    logger.log('INFOR', f'正在进行异步子域请求')
     connector = get_connector()
     header = get_header()
     async with ClientSession(connector=connector, headers=header) as session:
@@ -203,5 +214,5 @@ async def bulk_get_request(datas, port):
 
 
 def run_bulk_query(datas, port):
-    new_datas = asyncio.run(bulk_get_request(datas, port))
+    new_datas = asyncio.run(bulk_request(datas, port))
     return new_datas

oneforall/config.py

@@ -117,6 +117,8 @@ aiohttp_proxy = None  # proxy="http://user:pass@some.proxy.com"
 allow_redirects = True  # 允许请求跳转
 fake_header = True  # 使用伪造请求头
 # 为了保证请求质量 请谨慎更改以下设置
+# request_method只能是HEAD或GET,HEAD请求方法更快，但是不能获取响应体并提取从中提取
+request_method = 'GET'  # 使用请求方法，默认GET
 sockread_timeout = 5  # 每个请求socket读取超时时间，默认5秒
 sockconn_timeout = 5  # 每个请求socket连接超时时间，默认5秒
 # 限制同一时间打开的连接总数

oneforall/oneforall.py

@@ -161,7 +161,7 @@ class OneForAll(object):
             return
 
         # 请求子域
-        task = request.bulk_get_request(self.data, self.port)
+        task = request.bulk_request(self.data, self.port)
         self.data = loop.run_until_complete(task)
         self.datas.extend(self.data)
         # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间