diff --git a/brute.py b/brute.py index dd41833..fc9783c 100644 --- a/brute.py +++ b/brute.py @@ -292,13 +292,13 @@ def check_dict(): exit(0) -def gen_records(items, records, subdomains, ip_times, wc_ips, wc_ttl): +def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl): qname = items.get('name')[:-1] # 去除最右边的`.`点号 reason = items.get('status') resolver = items.get('resolver') data = items.get('data') answers = data.get('answers') - record = dict() + info = dict() cname = list() ips = list() public = list() @@ -327,17 +327,17 @@ def gen_records(items, records, subdomains, ip_times, wc_ips, wc_ttl): logger.log('TRACE', f'All query result of {qname} no A record{answers}') # 为了优化内存 只添加有A记录且通过判断的子域到记录中 if have_a_record and all(is_valid_flags): - record['resolve'] = 1 - record['reason'] = reason - record['ttl'] = ttls - record['cname'] = cname - record['content'] = ips - record['public'] = public - record['times'] = times - record['resolver'] = resolver - records[qname] = record + info['resolve'] = 1 + info['reason'] = reason + info['ttl'] = ttls + info['cname'] = cname + info['content'] = ips + info['public'] = public + info['times'] = times + info['resolver'] = resolver + infos[qname] = info subdomains.append(qname) - return records, subdomains + return infos, subdomains def stat_ip_times(result_paths): @@ -373,7 +373,7 @@ def stat_ip_times(result_paths): def deal_output(output_paths, ip_times, wildcard_ips, wildcard_ttl): logger.log('INFOR', f'Processing result') - records = dict() # 用来记录所有域名解析数据 + infos = dict() # 用来记录所有域名有关信息 subdomains = list() # 用来保存所有通过有效性检查的子域 for output_path in output_paths: logger.log('DEBUG', f'Processing {output_path}') @@ -396,10 +396,10 @@ def deal_output(output_paths, ip_times, wildcard_ips, wildcard_ttl): if 'answers' not in data: logger.log('TRACE', f'Processing {line}, {qname} no response') continue - records, subdomains = gen_records(items, records, subdomains, - ip_times, wildcard_ips, - wildcard_ttl) - return records, subdomains + infos, subdomains = gen_result_infos(items, infos, subdomains, + ip_times, wildcard_ips, + wildcard_ttl) + return infos, subdomains def check_by_compare(ip, ttl, wc_ips, wc_ttl): @@ -633,8 +633,8 @@ class Brute(Module): output_path = temp_dir.joinpath(output_name) output_paths.append(output_path) ip_times = stat_ip_times(output_paths) - self.records, self.subdomains = deal_output(output_paths, ip_times, - wildcard_ips, wildcard_ttl) + self.infos, self.subdomains = deal_output(output_paths, ip_times, + wildcard_ips, wildcard_ttl) delete_file(dict_path, output_paths) end = time.time() self.elapse = round(end - start, 1) diff --git a/common/database.py b/common/database.py index 40b3ae2..7b20858 100644 --- a/common/database.py +++ b/common/database.py @@ -54,7 +54,6 @@ class Database(object): logger.log('TRACE', f'Creating {table_name} table') self.query(f'create table "{table_name}" (' f'id integer primary key,' - f'type text,' f'alive int,' f'request int,' f'resolve int,' @@ -102,12 +101,12 @@ class Database(object): if results: try: self.conn.bulk_query( - f'insert into "{table_name}" (id, type, alive, resolve, request, new,' + f'insert into "{table_name}" (id, alive, resolve, request, new,' f'url, subdomain, port, level, cname, content, public, cdn, status,' f'reason, title, banner, header, response, times, ttl, cidr, asn, org,' f' ip2region, ip2location, resolver, module, source, elapse, find,' f'brute, valid) ' - f'values (:id, :type, :alive, :resolve, :request, :new, :url, ' + f'values (:id, :alive, :resolve, :request, :new, :url, ' f':subdomain, :port, :level, :cname, :content, :public, :cdn, :status,' f':reason, :title, :banner, :header, :response, :times, :ttl, :cidr,' f':asn, :org, :ip2region, :ip2location, :resolver, :module, :source,' @@ -231,7 +230,7 @@ class Database(object): :param str limit: limit value """ table_name = table_name.replace('.', '_') - query = f'select id, type, new, alive, request, resolve, url, subdomain, level,' \ + query = f'select id, new, alive, request, resolve, url, subdomain, level,' \ f'cname, content, public, cdn, port, status, reason, title, banner,' \ f'times, ttl, cidr, asn, org, ip2region, ip2location, resolver, module,' \ f'source, elapse, find, brute, valid from "{table_name}"' diff --git a/common/lookup.py b/common/lookup.py index 29d53d4..66945b0 100644 --- a/common/lookup.py +++ b/common/lookup.py @@ -1,5 +1,6 @@ from common.module import Module from common import utils +from config.log import logger class Lookup(Module): @@ -9,18 +10,19 @@ class Lookup(Module): def __init__(self): Module.__init__(self) + self.qtype = '' def query(self): """ Query the TXT record of domain :return: query result """ - answer = utils.dns_query(self.domain, self.type) + answer = utils.dns_query(self.domain, self.qtype) if answer is None: return None for item in answer: record = item.to_text() subdomains = self.match_subdomains(record) self.subdomains = self.subdomains.union(subdomains) - self.gen_record(subdomains, record) + logger.log('DEBUG', record) return self.subdomains diff --git a/common/module.py b/common/module.py index 9521231..47aa322 100644 --- a/common/module.py +++ b/common/module.py @@ -26,9 +26,8 @@ class Module(object): self.timeout = settings.request_timeout # 请求超时时间 self.verify = settings.request_verify # 请求SSL验证 self.domain = str() # 当前进行子域名收集的主域 - self.type = 'A' # 对主域进行子域收集时利用的DNS记录查询类型(默认利用A记录) self.subdomains = set() # 存放发现的子域 - self.records = dict() # 存放子域解析记录 + self.infos = dict() # 存放子域有关信息 self.results = list() # 存放模块结果 self.start = time.time() # 模块开始执行时间 self.end = None # 模块结束执行时间 @@ -244,19 +243,10 @@ class Module(object): 'elapse': self.elapse, 'find': len(self.subdomains), 'subdomains': list(self.subdomains), - 'records': self.records} + 'infos': self.infos} json.dump(result, file, ensure_ascii=False, indent=4) return True - def gen_record(self, subdomains, record): - """ - Generate record dictionary - """ - item = dict() - item['content'] = record - for subdomain in subdomains: - self.records[subdomain] = item - def gen_result(self, find=0, brute=None, valid=0): """ Generate results @@ -265,7 +255,6 @@ class Module(object): if not len(self.subdomains): # 该模块一个子域都没有发现的情况 logger.log('DEBUG', f'{self.source} module result is empty') result = {'id': None, - 'type': self.type, 'alive': None, 'request': None, 'resolve': None, @@ -303,29 +292,14 @@ class Module(object): for subdomain in self.subdomains: url = 'http://' + subdomain level = subdomain.count('.') - self.domain.count('.') - record = self.records.get(subdomain) - if record is None: - record = dict() - resolve = record.get('resolve') - request = record.get('request') - alive = record.get('alive') - if self.type != 'A': # 不是利用的DNS记录的A记录查询子域默认都有效 - resolve = 1 - request = 1 - alive = 1 - reason = record.get('reason') - resolver = record.get('resolver') - cname = record.get('cname') - content = record.get('content') - times = record.get('times') - ttl = record.get('ttl') - public = record.get('public') - cdn = record.get('cdn') - cidr = record.get('cidr') - asn = record.get('asn') - org = record.get('org') - ip2region = record.get('ip2region') - ip2location = record.get('ip2location') + info = self.infos.get(subdomain) + if info is None: + info = dict() + cname = info.get('cname') + content = info.get('content') + times = info.get('times') + ttl = info.get('ttl') + public = info.get('public') if isinstance(cname, list): cname = ','.join(cname) content = ','.join(content) @@ -333,10 +307,9 @@ class Module(object): ttl = ','.join([str(num) for num in ttl]) public = ','.join([str(num) for num in public]) result = {'id': None, - 'type': self.type, - 'alive': alive, - 'request': request, - 'resolve': resolve, + 'alive': info.get('alive'), + 'request': info.get('request'), + 'resolve': info.get('resolve'), 'new': None, 'url': url, 'subdomain': subdomain, @@ -345,21 +318,21 @@ class Module(object): 'cname': cname, 'content': content, 'public': public, - 'cdn': cdn, + 'cdn': info.get('cdn'), 'status': None, - 'reason': reason, + 'reason': info.get('reason'), 'title': None, 'banner': None, 'header': None, 'response': None, 'times': times, 'ttl': ttl, - 'cidr': cidr, - 'asn': asn, - 'org': org, - 'ip2region': ip2region, - 'ip2location': ip2location, - 'resolver': resolver, + 'cidr': info.get('cidr'), + 'asn': info.get('asn'), + 'org': info.get('org'), + 'ip2region': info.get('ip2region'), + 'ip2location': info.get('ip2location'), + 'resolver': info.get('resolver'), 'module': self.module, 'source': self.source, 'elapse': self.elapse, diff --git a/docs/field.md b/docs/field.md index f32e8ed..f22f081 100644 --- a/docs/field.md +++ b/docs/field.md @@ -2,10 +2,6 @@ 标识作用无意义 -### type - -DNS记录类型 - ### new 标记是否是新发现的子域名 diff --git a/modules/dnsquery/ns.py b/modules/dnsquery/ns.py index be7d267..baa91f8 100644 --- a/modules/dnsquery/ns.py +++ b/modules/dnsquery/ns.py @@ -7,7 +7,7 @@ class QueryNS(Lookup): self.domain = domain self.module = 'dnsquery' self.source = "QueryNS" - self.type = 'NS' # 利用的DNS记录的NS记录收集子域 + self.qtype = 'NS' # 利用的DNS记录的NS记录收集子域 def run(self): """ diff --git a/modules/dnsquery/soa.py b/modules/dnsquery/soa.py index 1917c6b..a02d0cb 100644 --- a/modules/dnsquery/soa.py +++ b/modules/dnsquery/soa.py @@ -7,7 +7,7 @@ class QuerySOA(Lookup): self.domain = domain self.module = 'dnsquery' self.source = "QuerySOA" - self.type = 'SOA' # 利用的DNS记录的SOA记录收集子域 + self.qtype = 'SOA' # 利用的DNS记录的SOA记录收集子域 def run(self): """ diff --git a/modules/dnsquery/spf.py b/modules/dnsquery/spf.py index 3b1ff17..7de44ab 100644 --- a/modules/dnsquery/spf.py +++ b/modules/dnsquery/spf.py @@ -7,7 +7,7 @@ class QuerySPF(Lookup): self.domain = domain self.module = 'dnsquery' self.source = "QuerySPF" - self.type = 'SPF' # 利用的DNS记录的SPF记录收集子域 + self.qtype = 'SPF' # 利用的DNS记录的SPF记录收集子域 def run(self): """ diff --git a/modules/dnsquery/srv.py b/modules/dnsquery/srv.py index 38e474e..88ba9dc 100644 --- a/modules/dnsquery/srv.py +++ b/modules/dnsquery/srv.py @@ -17,7 +17,7 @@ class BruteSRV(Module): self.domain = domain self.module = 'dnsquery' self.source = "BruteSRV" - self.type = 'SRV' # 利用的DNS记录的SRV记录查询子域 + self.qtype = 'SRV' # 利用的DNS记录的SRV记录查询子域 self.thread_num = 10 self.names_que = queue.Queue() self.answers_que = queue.Queue() diff --git a/modules/dnsquery/txt.py b/modules/dnsquery/txt.py index b015193..8d1721c 100644 --- a/modules/dnsquery/txt.py +++ b/modules/dnsquery/txt.py @@ -7,7 +7,7 @@ class QueryTXT(Lookup): self.domain = domain self.module = 'dnsquery' self.source = "QueryTXT" - self.type = 'TXT' # 利用的DNS记录的TXT记录收集子域 + self.qtype = 'TXT' # 利用的DNS记录的TXT记录收集子域 def run(self): """