去掉type字段

brute.py

@@ -292,13 +292,13 @@ def check_dict():
         exit(0)
 
 
-def gen_records(items, records, subdomains, ip_times, wc_ips, wc_ttl):
+def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
     qname = items.get('name')[:-1]  # 去除最右边的`.`点号
     reason = items.get('status')
     resolver = items.get('resolver')
     data = items.get('data')
     answers = data.get('answers')
-    record = dict()
+    info = dict()
     cname = list()
     ips = list()
     public = list()
@@ -327,17 +327,17 @@ def gen_records(items, records, subdomains, ip_times, wc_ips, wc_ttl):
         logger.log('TRACE', f'All query result of {qname} no A record{answers}')
     # 为了优化内存 只添加有A记录且通过判断的子域到记录中
     if have_a_record and all(is_valid_flags):
-        record['resolve'] = 1
+        info['resolve'] = 1
-        record['reason'] = reason
+        info['reason'] = reason
-        record['ttl'] = ttls
+        info['ttl'] = ttls
-        record['cname'] = cname
+        info['cname'] = cname
-        record['content'] = ips
+        info['content'] = ips
-        record['public'] = public
+        info['public'] = public
-        record['times'] = times
+        info['times'] = times
-        record['resolver'] = resolver
+        info['resolver'] = resolver
-        records[qname] = record
+        infos[qname] = info
         subdomains.append(qname)
-    return records, subdomains
+    return infos, subdomains
 
 
 def stat_ip_times(result_paths):
@@ -373,7 +373,7 @@ def stat_ip_times(result_paths):
 
 def deal_output(output_paths, ip_times, wildcard_ips, wildcard_ttl):
     logger.log('INFOR', f'Processing result')
-    records = dict()  # 用来记录所有域名解析数据
+    infos = dict()  # 用来记录所有域名有关信息
     subdomains = list()  # 用来保存所有通过有效性检查的子域
     for output_path in output_paths:
         logger.log('DEBUG', f'Processing {output_path}')
@@ -396,10 +396,10 @@ def deal_output(output_paths, ip_times, wildcard_ips, wildcard_ttl):
                 if 'answers' not in data:
                     logger.log('TRACE', f'Processing {line}, {qname} no response')
                     continue
-                records, subdomains = gen_records(items, records, subdomains,
+                infos, subdomains = gen_result_infos(items, infos, subdomains,
-                                                  ip_times, wildcard_ips,
+                                                     ip_times, wildcard_ips,
-                                                  wildcard_ttl)
+                                                     wildcard_ttl)
-    return records, subdomains
+    return infos, subdomains
 
 
 def check_by_compare(ip, ttl, wc_ips, wc_ttl):
@@ -633,8 +633,8 @@ class Brute(Module):
                 output_path = temp_dir.joinpath(output_name)
                 output_paths.append(output_path)
         ip_times = stat_ip_times(output_paths)
-        self.records, self.subdomains = deal_output(output_paths, ip_times,
+        self.infos, self.subdomains = deal_output(output_paths, ip_times,
-                                                    wildcard_ips, wildcard_ttl)
+                                                  wildcard_ips, wildcard_ttl)
         delete_file(dict_path, output_paths)
         end = time.time()
         self.elapse = round(end - start, 1)

common/database.py

@@ -54,7 +54,6 @@ class Database(object):
         logger.log('TRACE', f'Creating {table_name} table')
         self.query(f'create table "{table_name}" ('
                    f'id integer primary key,'
-                   f'type text,'
                    f'alive int,'
                    f'request int,'
                    f'resolve int,'
@@ -102,12 +101,12 @@ class Database(object):
         if results:
             try:
                 self.conn.bulk_query(
-                    f'insert into "{table_name}" (id, type, alive, resolve, request, new,'
+                    f'insert into "{table_name}" (id, alive, resolve, request, new,'
                     f'url, subdomain, port, level, cname, content, public, cdn, status,'
                     f'reason, title, banner, header, response, times, ttl, cidr, asn, org,'
                     f' ip2region, ip2location, resolver, module, source, elapse, find,'
                     f'brute, valid) '
-                    f'values (:id, :type, :alive, :resolve, :request, :new, :url, '
+                    f'values (:id, :alive, :resolve, :request, :new, :url, '
                     f':subdomain, :port, :level, :cname, :content, :public, :cdn, :status,'
                     f':reason, :title, :banner, :header, :response, :times, :ttl, :cidr,'
                     f':asn, :org, :ip2region, :ip2location, :resolver, :module, :source,'
@@ -231,7 +230,7 @@ class Database(object):
         :param str limit: limit value
         """
         table_name = table_name.replace('.', '_')
-        query = f'select id, type, new, alive, request, resolve, url, subdomain, level,' \
+        query = f'select id, new, alive, request, resolve, url, subdomain, level,' \
                 f'cname, content, public, cdn, port, status, reason, title, banner,' \
                 f'times, ttl, cidr, asn, org, ip2region, ip2location, resolver, module,' \
                 f'source, elapse, find, brute, valid from "{table_name}"'

common/lookup.py

@@ -1,5 +1,6 @@
 from common.module import Module
 from common import utils
+from config.log import logger
 
 
 class Lookup(Module):
@@ -9,18 +10,19 @@ class Lookup(Module):
 
     def __init__(self):
         Module.__init__(self)
+        self.qtype = ''
 
     def query(self):
         """
         Query the TXT record of domain
         :return: query result
         """
-        answer = utils.dns_query(self.domain, self.type)
+        answer = utils.dns_query(self.domain, self.qtype)
         if answer is None:
             return None
         for item in answer:
             record = item.to_text()
             subdomains = self.match_subdomains(record)
             self.subdomains = self.subdomains.union(subdomains)
-            self.gen_record(subdomains, record)
+            logger.log('DEBUG', record)
         return self.subdomains

common/module.py

@@ -26,9 +26,8 @@ class Module(object):
         self.timeout = settings.request_timeout  # 请求超时时间
         self.verify = settings.request_verify  # 请求SSL验证
         self.domain = str()  # 当前进行子域名收集的主域
-        self.type = 'A'  # 对主域进行子域收集时利用的DNS记录查询类型(默认利用A记录)
         self.subdomains = set()  # 存放发现的子域
-        self.records = dict()  # 存放子域解析记录
+        self.infos = dict()  # 存放子域有关信息
         self.results = list()  # 存放模块结果
         self.start = time.time()  # 模块开始执行时间
         self.end = None  # 模块结束执行时间
@@ -244,19 +243,10 @@ class Module(object):
                       'elapse': self.elapse,
                       'find': len(self.subdomains),
                       'subdomains': list(self.subdomains),
-                      'records': self.records}
+                      'infos': self.infos}
             json.dump(result, file, ensure_ascii=False, indent=4)
         return True
 
-    def gen_record(self, subdomains, record):
-        """
-        Generate record dictionary
-        """
-        item = dict()
-        item['content'] = record
-        for subdomain in subdomains:
-            self.records[subdomain] = item
-
     def gen_result(self, find=0, brute=None, valid=0):
         """
         Generate results
@@ -265,7 +255,6 @@ class Module(object):
         if not len(self.subdomains):  # 该模块一个子域都没有发现的情况
             logger.log('DEBUG', f'{self.source} module result is empty')
             result = {'id': None,
-                      'type': self.type,
                       'alive': None,
                       'request': None,
                       'resolve': None,
@@ -303,29 +292,14 @@ class Module(object):
             for subdomain in self.subdomains:
                 url = 'http://' + subdomain
                 level = subdomain.count('.') - self.domain.count('.')
-                record = self.records.get(subdomain)
+                info = self.infos.get(subdomain)
-                if record is None:
+                if info is None:
-                    record = dict()
+                    info = dict()
-                resolve = record.get('resolve')
+                cname = info.get('cname')
-                request = record.get('request')
+                content = info.get('content')
-                alive = record.get('alive')
+                times = info.get('times')
-                if self.type != 'A':  # 不是利用的DNS记录的A记录查询子域默认都有效
+                ttl = info.get('ttl')
-                    resolve = 1
+                public = info.get('public')
-                    request = 1
-                    alive = 1
-                reason = record.get('reason')
-                resolver = record.get('resolver')
-                cname = record.get('cname')
-                content = record.get('content')
-                times = record.get('times')
-                ttl = record.get('ttl')
-                public = record.get('public')
-                cdn = record.get('cdn')
-                cidr = record.get('cidr')
-                asn = record.get('asn')
-                org = record.get('org')
-                ip2region = record.get('ip2region')
-                ip2location = record.get('ip2location')
                 if isinstance(cname, list):
                     cname = ','.join(cname)
                     content = ','.join(content)
@@ -333,10 +307,9 @@ class Module(object):
                     ttl = ','.join([str(num) for num in ttl])
                     public = ','.join([str(num) for num in public])
                 result = {'id': None,
-                          'type': self.type,
+                          'alive': info.get('alive'),
-                          'alive': alive,
+                          'request': info.get('request'),
-                          'request': request,
+                          'resolve': info.get('resolve'),
-                          'resolve': resolve,
                           'new': None,
                           'url': url,
                           'subdomain': subdomain,
@@ -345,21 +318,21 @@ class Module(object):
                           'cname': cname,
                           'content': content,
                           'public': public,
-                          'cdn': cdn,
+                          'cdn': info.get('cdn'),
                           'status': None,
-                          'reason': reason,
+                          'reason': info.get('reason'),
                           'title': None,
                           'banner': None,
                           'header': None,
                           'response': None,
                           'times': times,
                           'ttl': ttl,
-                          'cidr': cidr,
+                          'cidr': info.get('cidr'),
-                          'asn': asn,
+                          'asn': info.get('asn'),
-                          'org': org,
+                          'org': info.get('org'),
-                          'ip2region': ip2region,
+                          'ip2region': info.get('ip2region'),
-                          'ip2location': ip2location,
+                          'ip2location': info.get('ip2location'),
-                          'resolver': resolver,
+                          'resolver': info.get('resolver'),
                           'module': self.module,
                           'source': self.source,
                           'elapse': self.elapse,

docs/field.md

@@ -2,10 +2,6 @@
 
 标识作用无意义
 
-### type
-
-DNS记录类型
-
 ### new
 
 标记是否是新发现的子域名

modules/dnsquery/ns.py

@@ -7,7 +7,7 @@ class QueryNS(Lookup):
         self.domain = domain
         self.module = 'dnsquery'
         self.source = "QueryNS"
-        self.type = 'NS'  # 利用的DNS记录的NS记录收集子域
+        self.qtype = 'NS'  # 利用的DNS记录的NS记录收集子域
 
     def run(self):
         """

modules/dnsquery/soa.py

@@ -7,7 +7,7 @@ class QuerySOA(Lookup):
         self.domain = domain
         self.module = 'dnsquery'
         self.source = "QuerySOA"
-        self.type = 'SOA'  # 利用的DNS记录的SOA记录收集子域
+        self.qtype = 'SOA'  # 利用的DNS记录的SOA记录收集子域
 
     def run(self):
         """

modules/dnsquery/spf.py

@@ -7,7 +7,7 @@ class QuerySPF(Lookup):
         self.domain = domain
         self.module = 'dnsquery'
         self.source = "QuerySPF"
-        self.type = 'SPF'  # 利用的DNS记录的SPF记录收集子域
+        self.qtype = 'SPF'  # 利用的DNS记录的SPF记录收集子域
 
     def run(self):
         """

modules/dnsquery/srv.py

@@ -17,7 +17,7 @@ class BruteSRV(Module):
         self.domain = domain
         self.module = 'dnsquery'
         self.source = "BruteSRV"
-        self.type = 'SRV'  # 利用的DNS记录的SRV记录查询子域
+        self.qtype = 'SRV'  # 利用的DNS记录的SRV记录查询子域
         self.thread_num = 10
         self.names_que = queue.Queue()
         self.answers_que = queue.Queue()

modules/dnsquery/txt.py

@@ -7,7 +7,7 @@ class QueryTXT(Lookup):
         self.domain = domain
         self.module = 'dnsquery'
         self.source = "QueryTXT"
-        self.type = 'TXT'  # 利用的DNS记录的TXT记录收集子域
+        self.qtype = 'TXT'  # 利用的DNS记录的TXT记录收集子域
 
     def run(self):
         """