新增new,header,response字段

oneforall/common/database.py

@@ -53,8 +53,11 @@ class Database(object):
                             f'status int,'
                             f'reason text,'
                             f'valid int,'
+                            f'new int,'
                             f'title text,'
                             f'banner text,'
+                            f'header text,'
+                            f'response text,'
                             f'module text,'
                             f'source text,'
                             f'elapsed float,'
@@ -78,10 +81,11 @@ class Database(object):
                 self.conn.bulk_query(
                     f'insert into "{table_name}" ('
                     f'id, url, subdomain, port, ips, status, reason, valid,'
-                    f'title, banner, module, source, elapsed, count)'
+                    f'new, title, banner, header, response, module, source, '
+                    f'elapsed, count)'
                     f'values (:id, :url, :subdomain, :port, :ips, :status,'
-                    f':reason, :valid, :title, :banner, :module, :source,'
-                    f':elapsed, :count)',
+                    f':reason, :valid, :new, :title, :banner, :header,'
+                    f':response, :module, :source,:elapsed, :count)',
                     results)
             except Exception as e:
                 logger.log('ERROR', e)
@@ -190,18 +194,22 @@ class Database(object):
         else:
             return rows
 
-    def get_subdomain(self, table_name, valid):
+    def export_data(self, table_name, valid):
         """
-        获取表中的子域数据
+        获取表中的部分数据
 
         :param str table_name: 表名
-        :param int valid: 是否有效
+        :param any valid: 有效性
         """
         table_name = table_name.replace('.', '_')
+        query = f'select id, url, subdomain, port, ips, status, reason,' \
+                f'valid, new, title, banner from "{table_name}"'
+        if valid == 0 or valid == 1:
+            where = f' where valid = {valid}'
+            query += where
         logger.log('DEBUG', f'获取{table_name}表中的所有数据')
         try:
-            rows = self.conn.query(
-                f'select * from "{table_name}" where valid = {valid}')
+            rows = self.conn.query(query)
         except Exception as e:
             logger.log('ERROR', e)
         else:

oneforall/common/module.py

@@ -220,8 +220,11 @@ class Module(object):
                       'status': None,
                       'reason': None,
                       'valid': None,
+                      'new': None,
                       'title': None,
                       'banner': None,
+                      'header': None,
+                      'response': None,
                       'module': self.module,
                       'source': self.source,
                       'elapsed': self.elapsed,
@@ -240,9 +243,12 @@ class Module(object):
                           'status': None,
                           'reason': None,
                           'valid': None,
+                          'new': None,
                           'title': None,
                           'banner': None,
                           'module': self.module,
+                          'header': None,
+                          'response': None,
                           'source': self.source,
                           'elapsed': self.elapsed,
                           'count': len(self.subdomains)}

oneforall/common/request.py

@@ -166,6 +166,8 @@ def request_callback(future, index, datas):
                               'X-Powered-By': headers.get('X-Powered-By')})
                 datas[index]['banner'] = banner
                 datas[index]['title'] = get_title(text)
+                datas[index]['header'] = str(dict(headers))
+                datas[index]['response'] = text
         else:
             datas[index]['reason'] = 'Something error'
             datas[index]['valid'] = 0

oneforall/dbexport.py

@@ -30,19 +30,14 @@ def export(table, db=None, valid=None, dpath=None, format='csv', show=False):
     :param str table:   要导出的表
     :param str db:      要导出的数据库路径(默认为results/result.sqlite3)
     :param int valid:   导出子域的有效性(默认None)
-    :param str format:  导出格式(默认xls)
+    :param str format:  导出格式(默认csv)
     :param str dpath:    导出目录(默认None)
     :param bool show:   终端显示导出数据(默认False)
     """
     format = utils.check_format(format)
     dpath = utils.check_dpath(dpath)
     database = Database(db)
-    if valid is None:
-        rows = database.get_data(table)
-    elif isinstance(valid, int):
-        rows = database.get_subdomain(table, valid)
-    else:
-        rows = database.get_data(table)  # 意外情况导出全部子域
+    rows = database.export_data(table, valid)  # 意外情况导出全部子域
     if show:
         print(rows.dataset)
     if format == 'txt':
@@ -50,7 +45,7 @@ def export(table, db=None, valid=None, dpath=None, format='csv', show=False):
     else:
         data = rows.export(format)
     database.close()
-    fpath = dpath.joinpath(f'{table}.{format}')
+    fpath = dpath.joinpath(f'{table}_subdomain.{format}')
     utils.save_data(fpath, data)
 
 

oneforall/takeover.py

@@ -83,7 +83,8 @@ class Takeover(Module):
             data = str(self.results)
         else:
             data = self.results.export(self.format)
-        fpath = self.dpath.joinpath(f'takeover.{self.format}')
+        ts = int(time.time())
+        fpath = self.dpath.joinpath(f'takeover_{ts}.{self.format}')
         utils.save_data(fpath, data)
 
     def compare(self, subdomain, cname, responses):