实现从跳转历史URL收集子域功能

common/database.py

@@ -71,6 +71,7 @@ class Database(object):
                    f'title text,'
                    f'banner text,'
                    f'header text,'
+                   f'history text,'
                    f'response text,'
                    f'times text,'
                    f'ttl text,'
@@ -99,15 +100,16 @@ class Database(object):
         if results:
             try:
                 self.conn.bulk_query(
-                    f'insert into "{table_name}" (id, alive, resolve, request, new,'
-                    f'url, subdomain, port, level, cname, content, public, cdn, status,'
-                    f'reason, title, banner, header, response, times, ttl, cidr, asn, org,'
-                    f' ip2region, ip2location, resolver, module, source, elapse, find) '
+                    f'insert into "{table_name}" '
+                    f'(id, alive, resolve, request, new, url, subdomain, port, level,'
+                    f'cname, content, public, cdn, status, reason, title, banner, header,'
+                    f'history, response, times, ttl, cidr, asn, org, ip2region,'
+                    f'ip2location, resolver, module, source, elapse, find) '
                     f'values (:id, :alive, :resolve, :request, :new, :url, '
-                    f':subdomain, :port, :level, :cname, :content, :public, :cdn, :status,'
-                    f':reason, :title, :banner, :header, :response, :times, :ttl, :cidr,'
-                    f':asn, :org, :ip2region, :ip2location, :resolver, :module, :source,'
-                    f':elapse, :find)', results)
+                    f':subdomain, :port, :level, :cname, :content, :public, :cdn,'
+                    f':status, :reason, :title, :banner, :header, :history, :response,'
+                    f':times, :ttl, :cidr, :asn, :org, :ip2region, :ip2location,'
+                    f':resolver, :module, :source, :elapse, :find)', results)
             except Exception as e:
                 logger.log('ERROR', e)
 

common/module.py

@@ -272,6 +272,7 @@ class Module(object):
                       'title': None,
                       'banner': None,
                       'header': None,
+                      'history': None,
                       'response': None,
                       'times': None,
                       'ttl': None,
@@ -322,6 +323,7 @@ class Module(object):
                           'title': None,
                           'banner': None,
                           'header': None,
+                          'history': None,
                           'response': None,
                           'times': times,
                           'ttl': ttl,

common/request.py

@@ -81,7 +81,7 @@ async def fetch(session, method, url):
     :return: 响应对象和响应文本
     """
     timeout = aiohttp.ClientTimeout(total=None,
-                                    connect=5.0,
+                                    connect=None,
                                     sock_read=settings.sockread_timeout,
                                     sock_connect=settings.sockconn_timeout)
     try:
@@ -175,6 +175,8 @@ def request_callback(future, index, datas):
         if settings.enable_banner_identify:
             datas[index]['banner'] = utils.get_sample_banner(headers)
         datas[index]['header'] = json.dumps(dict(headers))
+        history = resp.history
+        datas[index]['history'] = str(history)
         if isinstance(text, str):
             title = get_title(text).strip()
             datas[index]['title'] = utils.remove_invalid_string(title)
@@ -229,7 +231,7 @@ async def bulk_request(data, port):
                                                  datas=to_req_data))
         tasks.append(task)
     if tasks:
-        futures = asyncio.as_completed(tasks, timeout=1*60)
+        futures = asyncio.as_completed(tasks)
         for future in tqdm.tqdm(futures,
                                 total=len(tasks),
                                 desc='Request Progress',