diff --git a/brute.py b/brute.py index ca2cb2e..db2e768 100644 --- a/brute.py +++ b/brute.py @@ -218,7 +218,8 @@ def collect_wildcard_record(domain, authoritative_ns): ip, ttl = get_wildcard_record(random_subdomain, resolver) except Exception as e: logger.log('DEBUG', e.args) - logger.log('ALERT', f'Multiple query errors, try to query a new random subdomain') + logger.log('ALERT', f'Multiple query errors,' + f'try to query a new random subdomain') continue if ip is None: continue @@ -255,7 +256,8 @@ def check_dict(): if not setting.enable_check_dict: return sec = setting.check_time - logger.log('ALERT', f'You have {sec} seconds to check whether the configuration is correct or not') + logger.log('ALERT', f'You have {sec} seconds to check ' + f'whether the configuration is correct or not') logger.log('ALERT', f'If you want to exit, please use `Ctrl + C`') try: time.sleep(sec) @@ -324,7 +326,8 @@ def stat_ip_times(result_paths): items = json.loads(line) except Exception as e: logger.log('ERROR', e.args) - logger.log('ERROR', f'Error parsing {result_path} line {line} Skip this line') + logger.log('ERROR', f'Error parsing {result_path} ' + f'line {line} Skip this line') continue status = items.get('status') if status != 'NOERROR': @@ -360,7 +363,8 @@ def deal_output(output_paths, ip_times, wildcard_ips, wildcard_ttl): qname = items.get('name')[:-1] # 去除最右边的`.`点号 status = items.get('status') if status != 'NOERROR': - logger.log('TRACE', f'Found {qname}\'s result {status} while processing {line}') + logger.log('TRACE', f'Found {qname}\'s result {status} ' + f'while processing {line}') continue data = items.get('data') if 'answers' not in data: @@ -504,7 +508,7 @@ class Brute(Module): if self.place is None: self.place = '*.' + domain wordlist = self.wordlist - main_domain = self.register(domain) + main_domain = self.get_maindomain(domain) if domain != main_domain: wordlist = self.recursive_nextlist if self.word: @@ -519,7 +523,8 @@ class Brute(Module): count = len(dict_set) logger.log('INFOR', f'Dictionary size: {count}') if count > 10000000: - logger.log('ALERT', f'The dictionary generated is too large:{count} > 10000000') + logger.log('ALERT', f'The generated dictionary is ' + f'too large {count} > 10000000') return dict_set def check_brute_params(self): diff --git a/common/database.py b/common/database.py index f333c1c..b4e7de9 100644 --- a/common/database.py +++ b/common/database.py @@ -97,22 +97,21 @@ class Database(object): :param list results: results list :param str module_name: mo """ - logger.log('TRACE', - f'Saving the subdomain results of {table_name} found by module {module_name} into database') + logger.log('TRACE', f'Saving the subdomain results of {table_name} ' + f'found by module {module_name} into database') table_name = table_name.replace('.', '_') if results: try: self.conn.bulk_query( - f'insert into "{table_name}" (' - f'id, type, alive, resolve, request, new, url, subdomain,' - f'port, level, cname, content, public, status, reason,' - f'title, banner, header, response, times, ttl, cidr, asn,' - f'ip2region, ip2location, resolver, module, source, elapse, find, brute, valid) ' - f'values (:id, :type, :alive, :resolve, :request, :new,' - f':url, :subdomain, :port, :level, :cname, :content,' - f':public, :status, :reason, :title, :banner, :header,' - f':response, :times, :ttl, :cidr, :asn, :ip2region, :ip2location, :resolver,' - f':module, :source, :elapse, :find, :brute, :valid)', results) + f'insert into "{table_name}" (id, type, alive, resolve, request, new,' + f'url, subdomain, port, level, cname, content, public, status, reason,' + f'title, banner, header, response, times, ttl, cidr, asn, ip2region,' + f'ip2location, resolver, module, source, elapse, find, brute, valid) ' + f'values (:id, :type, :alive, :resolve, :request, :new, :url, ' + f':subdomain, :port, :level, :cname, :content, :public, :status,' + f':reason, :title, :banner, :header, :response, :times, :ttl, :cidr,' + f':asn, :ip2region, :ip2location, :resolver, :module, :source,' + f':elapse, :find, :brute, :valid)', results) except Exception as e: logger.log('ERROR', e) diff --git a/common/lookup.py b/common/lookup.py index 3f8b20a..29d53d4 100644 --- a/common/lookup.py +++ b/common/lookup.py @@ -20,7 +20,7 @@ class Lookup(Module): return None for item in answer: record = item.to_text() - subdomains = self.match_subdomains(self.domain, record) + subdomains = self.match_subdomains(record) self.subdomains = self.subdomains.union(subdomains) self.gen_record(subdomains, record) return self.subdomains diff --git a/common/module.py b/common/module.py index 8ed7cb2..4bd7060 100644 --- a/common/module.py +++ b/common/module.py @@ -53,7 +53,8 @@ class Module(object): """ begin log """ - logger.log('DEBUG', f'Start {self.source} module to collect subdomains of {self.domain}') + logger.log('DEBUG', f'Start {self.source} module to ' + f'collect subdomains of {self.domain}') def finish(self): """ @@ -61,7 +62,8 @@ class Module(object): """ self.end = time.time() self.elapse = round(self.end - self.start, 1) - logger.log('DEBUG', f'Finished {self.source} module to collect {self.domain}\'s subdomains') + logger.log('DEBUG', f'Finished {self.source} module to ' + f'collect {self.domain}\'s subdomains') logger.log('INFOR', f'The {self.source} module took {self.elapse} seconds ' f'found {len(self.subdomains)} subdomains') logger.log('DEBUG', f'{self.source} module found subdomains of {self.domain}\n' @@ -209,35 +211,48 @@ class Module(object): logger.log('TRACE', f'{module} module does not use proxy') return self.proxy - @staticmethod - def match_subdomains(domain, text, distinct=True): + def match_subdomains(self, html, distinct=True, fuzzy=True): """ Use regexp to match subdomains - :param str domain: domain - :param str text: text + :param str html: response html text :param bool distinct: deduplicate results or not (default True) + :param bool fuzzy: fuzzy match subdomain or not (default True) :return set/list: result set or list """ logger.log('TRACE', f'Use regexp to match subdomains in the response body') - regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ - + domain.replace('.', r'\.') - result = re.findall(regexp, text, re.I) + if fuzzy: + regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + + self.domain.replace('.', r'\.') + result = re.findall(regexp, html, re.I) + if not result: + return set() + deal = map(lambda s: s.lower(), result) + if distinct: + return set(deal) + else: + return list(deal) + else: + regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \ + r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + + self.domain.replace('.', r'\.') + result = re.findall(regexp, html, re.I) if not result: return set() - deal = map(lambda s: s.lower(), result) + regexp = r'(?:http://|https://)' + deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result) if distinct: return set(deal) else: return list(deal) @staticmethod - def register(domain): + def get_maindomain(domain): """ - Get registered domain + Get main domain :param str domain: domain - :return: registered domain + :return: main domain """ return Domain(domain).registered() @@ -249,7 +264,8 @@ class Module(object): """ if not setting.save_module_result: return False - logger.log('TRACE', f'Save the subdomain results found by {self.source} module as a json file') + logger.log('TRACE', f'Save the subdomain results found by ' + f'{self.source} module as a json file') path = setting.result_save_dir.joinpath(self.domain, self.module) path.mkdir(parents=True, exist_ok=True) name = self.source + '.json' diff --git a/common/request.py b/common/request.py index f97b5de..1849ca5 100644 --- a/common/request.py +++ b/common/request.py @@ -281,7 +281,7 @@ def urls_request(urls): return data -def save_data(name, data): +def save_db(name, data): """ Save request results to database diff --git a/common/resolve.py b/common/resolve.py index f4f986b..73e68a6 100644 --- a/common/resolve.py +++ b/common/resolve.py @@ -48,7 +48,7 @@ def update_data(data, records): return data -def save_data(name, data): +def save_db(name, data): """ 保存解析结果到数据库 diff --git a/common/search.py b/common/search.py index 932bf52..48e3b09 100644 --- a/common/search.py +++ b/common/search.py @@ -36,13 +36,12 @@ class Search(Module): subdomains_temp[i:i + 2])))) return statements_list - def match_location(self, domain, url): + def match_location(self, url): """ 匹配跳转之后的url 针对部分搜索引擎(如百度搜索)搜索展示url时有显示不全的情况 此函数会向每条结果的链接发送head请求获取响应头的location值并做子域匹配 - :param str domain: 域名 :param str url: 展示结果的url链接 :return: 匹配的子域 :rtype set @@ -53,28 +52,4 @@ class Search(Module): location = resp.headers.get('location') if not location: return set() - return set(self.match_subdomains(domain, location)) - - @staticmethod - def match_subdomains(domain, html, distinct=True): - """ - Use regexp to match subdomains - - :param str domain: domain - :param str html: response html text - :param bool distinct: deduplicate results or not (default True) - :return set/list: result set or list - """ - logger.log('TRACE', f'Use regexp to match subdomains in the response body') - regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \ - r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ - + domain.replace('.', r'\.') - result = re.findall(regexp, html, re.I) - if not result: - return set() - regexp = r'(?:http://|https://)' - deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result) - if distinct: - return set(deal) - else: - return list(deal) + return set(self.match_subdomains(location)) diff --git a/common/utils.py b/common/utils.py index e2052a9..23fe160 100644 --- a/common/utils.py +++ b/common/utils.py @@ -237,6 +237,21 @@ def save_data(path, data): return False +def remove_data(path): + """ + 删除保存数据的文件 + + :param path: 路径 + :return: 删除成功与否 + """ + try: + path.unlink() + except Exception as e: + logger.log('ERROR', e.args) + return False + return True + + def check_response(method, resp): """ 检查响应 输出非正常响应返回json的信息 diff --git a/config/setting.py b/config/setting.py index 73f3ba9..ac01035 100644 --- a/config/setting.py +++ b/config/setting.py @@ -12,6 +12,7 @@ module_dir = relative_directory.joinpath('modules') # OneForAll模块目录 third_party_dir = relative_directory.joinpath('thirdparty') # 三方工具目录 data_storage_dir = relative_directory.joinpath('data') # 数据存放目录 result_save_dir = relative_directory.joinpath('results') # 结果保存目录 +temp_save_dir = result_save_dir.joinpath('temp') # OneForAll入口参数设置 enable_check_version = True # 开启最新版本检查 diff --git a/modules/certificates/censys_api.py b/modules/certificates/censys_api.py index 263cebc..04f8306 100644 --- a/modules/certificates/censys_api.py +++ b/modules/certificates/censys_api.py @@ -6,7 +6,7 @@ from config.log import logger class CensysAPI(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Certificate' self.source = "CensysAPIQuery" self.addr = 'https://www.censys.io/api/v1/search/certificates' @@ -33,7 +33,7 @@ class CensysAPI(Query): if status != 'ok': logger.log('ALERT', f'{self.source} module {status}') return - subdomains = self.match_subdomains(self.domain, str(json)) + subdomains = self.match_subdomains(resp.text) self.subdomains = self.subdomains.union(subdomains) pages = json.get('metadata').get('pages') for page in range(2, pages + 1): @@ -41,7 +41,7 @@ class CensysAPI(Query): resp = self.post(self.addr, json=data, auth=(self.id, self.secret)) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/certificates/certspotter.py b/modules/certificates/certspotter.py index 76906c6..61435a4 100644 --- a/modules/certificates/certspotter.py +++ b/modules/certificates/certspotter.py @@ -22,7 +22,7 @@ class CertSpotter(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/certificates/crtsh.py b/modules/certificates/crtsh.py index ba555ac..4dd9593 100644 --- a/modules/certificates/crtsh.py +++ b/modules/certificates/crtsh.py @@ -5,7 +5,7 @@ from common.query import Query class Crtsh(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Certificate' self.source = 'CrtshQuery' self.addr = 'https://crt.sh/' @@ -21,7 +21,7 @@ class Crtsh(Query): if not resp: return text = resp.text.replace(r'\n', ' ') - subdomains = self.match_subdomains(self.domain, text) + subdomains = self.match_subdomains(text) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/certificates/google.py b/modules/certificates/google.py index a7ad98f..bb83a6b 100644 --- a/modules/certificates/google.py +++ b/modules/certificates/google.py @@ -5,7 +5,7 @@ from common.query import Query class Google(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Certificate' self.source = 'GoogleQuery' self.addr = 'https://transparencyreport.google.com/' \ @@ -23,7 +23,7 @@ class Google(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/check/axfr.py b/modules/check/axfr.py index adea42b..5babfcb 100644 --- a/modules/check/axfr.py +++ b/modules/check/axfr.py @@ -22,7 +22,7 @@ class CheckAXFR(Module): def __init__(self, domain: str): Module.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Check' self.source = 'AXFRCheck' self.results = [] @@ -45,7 +45,7 @@ class CheckAXFR(Module): names = zone.nodes.keys() for name in names: full_domain = str(name) + '.' + self.domain - subdomain = self.match_subdomains(self.domain, full_domain) + subdomain = self.match_subdomains(full_domain) self.subdomains = self.subdomains.union(subdomain) record = zone[name].to_text(name) self.results.append(record) diff --git a/modules/check/cdx.py b/modules/check/cdx.py index 80bd9a0..2da2a5f 100644 --- a/modules/check/cdx.py +++ b/modules/check/cdx.py @@ -12,7 +12,7 @@ class CheckCDX(Module): """ def __init__(self, domain: str): Module.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Check' self.source = "CrossDomainXml" @@ -31,7 +31,7 @@ class CheckCDX(Module): if not resp: return if resp and len(resp.content): - self.subdomains = self.match_subdomains(self.domain, resp.text) + self.subdomains = self.match_subdomains(resp.text) def run(self): """ diff --git a/modules/check/cert.py b/modules/check/cert.py index 659fe43..05a81ea 100644 --- a/modules/check/cert.py +++ b/modules/check/cert.py @@ -14,7 +14,7 @@ from config.log import logger class CheckCert(Module): def __init__(self, domain): Module.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.port = 443 # ssl port self.module = 'Check' self.source = 'CertInfo' @@ -32,7 +32,7 @@ class CheckCert(Module): except Exception as e: logger.log('DEBUG', e.args) return - subdomains = self.match_subdomains(self.domain, str(cert_dict)) + subdomains = self.match_subdomains(str(cert_dict)) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/check/csp.py b/modules/check/csp.py index bc5e773..77721a2 100644 --- a/modules/check/csp.py +++ b/modules/check/csp.py @@ -14,7 +14,7 @@ class CheckCSP(Module): """ def __init__(self, domain, header): Module.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Check' self.source = 'ContentSecurityPolicy' self.csp_header = header @@ -52,7 +52,7 @@ class CheckCSP(Module): if not csp: logger.log('DEBUG', f'There is no Content-Security-Policy in the header of {self.domain}') return - self.subdomains = self.match_subdomains(self.domain, csp) + self.subdomains = self.match_subdomains(csp) def run(self): """ diff --git a/modules/check/nsec.py b/modules/check/nsec.py index f2912ed..51c33fd 100644 --- a/modules/check/nsec.py +++ b/modules/check/nsec.py @@ -8,7 +8,7 @@ from common import utils class CheckNSEC(Module): def __init__(self, domain): Module.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'check' self.source = "CheckNSEC" @@ -21,7 +21,7 @@ class CheckNSEC(Module): subdomain = str() for item in answer: record = item.to_text() - subdomains = self.match_subdomains(self.domain, record) + subdomains = self.match_subdomains(record) subdomain = ''.join(subdomains) # 其实这里的subdomains的长度为1 也就是说只会有一个子域 self.subdomains = self.subdomains.union(subdomains) self.gen_record(subdomains, record) diff --git a/modules/check/robots.py b/modules/check/robots.py index a67eb5d..416b101 100644 --- a/modules/check/robots.py +++ b/modules/check/robots.py @@ -11,7 +11,7 @@ class CheckRobots(Module): """ def __init__(self, domain): Module.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Check' self.source = 'Robots' @@ -30,7 +30,7 @@ class CheckRobots(Module): if not resp: return if resp and len(resp.content): - self.subdomains = self.match_subdomains(self.domain, resp.text) + self.subdomains = self.match_subdomains(resp.text) def run(self): """ diff --git a/modules/check/sitemap.py b/modules/check/sitemap.py index 066de7d..64f71f4 100644 --- a/modules/check/sitemap.py +++ b/modules/check/sitemap.py @@ -11,7 +11,7 @@ class CheckRobots(Module): """ def __init__(self, domain): Module.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Check' self.source = 'Sitemap' @@ -35,7 +35,7 @@ class CheckRobots(Module): if not resp: return if resp and len(resp.content): - self.subdomains = self.match_subdomains(self.domain, resp.text) + self.subdomains = self.match_subdomains(resp.text) def run(self): """ diff --git a/modules/crawl/archivecrawl.py b/modules/crawl/archivecrawl.py index 70ac9a1..8eec194 100644 --- a/modules/crawl/archivecrawl.py +++ b/modules/crawl/archivecrawl.py @@ -26,7 +26,7 @@ class ArchiveCrawl(Crawl): for resp in cdx.iter(url, limit=limit): if resp.data.get('status') not in ['301', '302']: url = resp.data.get('url') - subdomains = self.match_subdomains(self.register(domain), + subdomains = self.match_subdomains(self.get_maindomain(domain), url + resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/crawl/commoncrawl.py b/modules/crawl/commoncrawl.py index 6504910..07a74d6 100644 --- a/modules/crawl/commoncrawl.py +++ b/modules/crawl/commoncrawl.py @@ -26,7 +26,7 @@ class CommonCrawl(Crawl): for resp in tqdm(cdx.iter(url, limit=limit), total=limit): if resp.data.get('status') not in ['301', '302']: - subdomains = self.match_subdomains(self.register(domain), resp.text) + subdomains = self.match_subdomains(self.get_maindomain(domain), resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/binaryedge_api.py b/modules/datasets/binaryedge_api.py index 59ffdc2..51b5d28 100644 --- a/modules/datasets/binaryedge_api.py +++ b/modules/datasets/binaryedge_api.py @@ -5,7 +5,7 @@ from common.query import Query class BinaryEdgeAPI(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'BinaryEdgeAPIQuery' self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/' @@ -22,7 +22,7 @@ class BinaryEdgeAPI(Query): resp = self.get(url) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/datasets/bufferover.py b/modules/datasets/bufferover.py index e9c9401..c58d94f 100644 --- a/modules/datasets/bufferover.py +++ b/modules/datasets/bufferover.py @@ -6,7 +6,7 @@ from config.log import logger class BufferOver(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'BufferOverQuery' self.addr = 'https://dns.bufferover.run/dns?q=' @@ -26,7 +26,7 @@ class BufferOver(Query): return if resp.status_code != 200: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/cebaidu.py b/modules/datasets/cebaidu.py index f8363dd..c529871 100644 --- a/modules/datasets/cebaidu.py +++ b/modules/datasets/cebaidu.py @@ -4,7 +4,7 @@ from common.query import Query class CeBaidu(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'CeBaiduQuery' self.addr = 'https://ce.baidu.com/index/getRelatedSites' @@ -19,7 +19,7 @@ class CeBaidu(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/chinaz.py b/modules/datasets/chinaz.py index 82a0128..6b0b655 100644 --- a/modules/datasets/chinaz.py +++ b/modules/datasets/chinaz.py @@ -4,7 +4,7 @@ from common.query import Query class Chinaz(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'ChinazQuery' self.addr = 'https://alexa.chinaz.com/' @@ -19,7 +19,7 @@ class Chinaz(Query): resp = self.get(self.addr) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/chinaz_api.py b/modules/datasets/chinaz_api.py index f878702..038d791 100644 --- a/modules/datasets/chinaz_api.py +++ b/modules/datasets/chinaz_api.py @@ -5,7 +5,7 @@ from common.query import Query class ChinazAPI(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'ChinazAPIQuery' self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa' @@ -21,7 +21,7 @@ class ChinazAPI(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/circl_api.py b/modules/datasets/circl_api.py index e4d6a36..5b9f3d2 100644 --- a/modules/datasets/circl_api.py +++ b/modules/datasets/circl_api.py @@ -5,7 +5,7 @@ from common.query import Query class CirclAPI(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'CirclAPIQuery' self.addr = 'https://www.circl.lu/pdns/query/' @@ -21,7 +21,7 @@ class CirclAPI(Query): resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd)) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/cloudflare_api.py b/modules/datasets/cloudflare_api.py index 7bb870a..35d82c3 100644 --- a/modules/datasets/cloudflare_api.py +++ b/modules/datasets/cloudflare_api.py @@ -79,7 +79,7 @@ class CloudFlareAPI(Query): logger.log('DEBUG', f'{list_dns_resp.status_code} {list_dns_resp.text}') return - subdomains = self.match_subdomains(self.domain, list_dns_resp.text) + subdomains = self.match_subdomains(list_dns_resp.text) self.subdomains = self.subdomains.union(subdomains) if not self.subdomains: # waiting for cloudflare enumerate subdomains @@ -94,7 +94,7 @@ class CloudFlareAPI(Query): f'{list_dns_resp.status_code} {list_dns_resp.text}') return total_pages = list_dns_resp.json()['result_info']['total_pages'] - subdomains = (self.match_subdomains(self.domain, list_dns_resp.text)) + subdomains = (self.match_subdomains(list_dns_resp.text)) self.subdomains = self.subdomains.union(subdomains) page += 1 if page > total_pages: diff --git a/modules/datasets/dnsdb_api.py b/modules/datasets/dnsdb_api.py index 6d4f839..798ae4e 100644 --- a/modules/datasets/dnsdb_api.py +++ b/modules/datasets/dnsdb_api.py @@ -6,7 +6,7 @@ from common.query import Query class DNSdbAPI(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'DNSdbAPIQuery' self.addr = 'https://api.dnsdb.info/lookup/rrset/name/' @@ -23,7 +23,7 @@ class DNSdbAPI(Query): resp = self.get(url) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/dnsdumpster.py b/modules/datasets/dnsdumpster.py index 1b0e415..33828b5 100644 --- a/modules/datasets/dnsdumpster.py +++ b/modules/datasets/dnsdumpster.py @@ -5,7 +5,7 @@ from common.query import Query class DNSdumpster(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = "DNSdumpsterQuery" self.addr = 'https://dnsdumpster.com/' @@ -26,7 +26,7 @@ class DNSdumpster(Query): resp = self.post(self.addr, data) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) if subdomains: # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/hackertarget.py b/modules/datasets/hackertarget.py index 967aad5..14013c1 100644 --- a/modules/datasets/hackertarget.py +++ b/modules/datasets/hackertarget.py @@ -5,7 +5,7 @@ from common.query import Query class HackerTarget(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = "HackerTargetQuery" self.addr = 'https://api.hackertarget.com/hostsearch/' @@ -21,7 +21,7 @@ class HackerTarget(Query): if not resp: return if resp.status_code == 200: - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) if subdomains: # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/ip138.py b/modules/datasets/ip138.py index 54072fc..da1cc79 100644 --- a/modules/datasets/ip138.py +++ b/modules/datasets/ip138.py @@ -4,7 +4,7 @@ from common.query import Query class IP138(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'IP138Query' self.addr = 'https://site.ip138.com/{domain}/domain.htm' @@ -19,7 +19,7 @@ class IP138(Query): resp = self.get(self.addr) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/ipv4info_api.py b/modules/datasets/ipv4info_api.py index 681a50e..ad605b8 100644 --- a/modules/datasets/ipv4info_api.py +++ b/modules/datasets/ipv4info_api.py @@ -6,7 +6,7 @@ from config.log import logger class IPv4InfoAPI(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'IPv4InfoAPIQuery' self.addr = ' http://ipv4info.com/api_v1/' @@ -32,7 +32,7 @@ class IPv4InfoAPI(Query): except Exception as e: logger.log('DEBUG', e.args) break - subdomains = self.match_subdomains(self.domain, str(json)) + subdomains = self.match_subdomains(str(json)) if not subdomains: break # 合并搜索子域名搜索结果 diff --git a/modules/datasets/netcraft.py b/modules/datasets/netcraft.py index 60999e9..de81156 100644 --- a/modules/datasets/netcraft.py +++ b/modules/datasets/netcraft.py @@ -9,7 +9,7 @@ from common.query import Query class NetCraft(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'NetCraftQuery' self.init = 'https://searchdns.netcraft.com/' @@ -49,7 +49,7 @@ class NetCraft(Query): resp = self.get(self.addr + last, params) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break # 合并搜索子域名搜索结果 diff --git a/modules/datasets/passivedns_api.py b/modules/datasets/passivedns_api.py index ff5805b..987daba 100644 --- a/modules/datasets/passivedns_api.py +++ b/modules/datasets/passivedns_api.py @@ -5,7 +5,7 @@ from common.query import Query class PassiveDnsAPI(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'PassiveDnsQuery' self.addr = api.passivedns_api_addr or 'http://api.passivedns.cn' @@ -22,7 +22,7 @@ class PassiveDnsAPI(Query): resp = self.get(url) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/phonebook.py b/modules/datasets/phonebook.py index 9a63640..065dd83 100644 --- a/modules/datasets/phonebook.py +++ b/modules/datasets/phonebook.py @@ -5,7 +5,7 @@ from common.query import Query class PhoneBook(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'PhoneBookQuery' @@ -35,7 +35,7 @@ class PhoneBook(Query): resp = self.get(url) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/datasets/qianxun.py b/modules/datasets/qianxun.py index f261689..7f47b20 100644 --- a/modules/datasets/qianxun.py +++ b/modules/datasets/qianxun.py @@ -27,7 +27,7 @@ class QianXun(Query): resp = self.post(url, data) if not resp: break - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) self.subdomains = self.subdomains.union(subdomains) if '
' not in resp.text: break diff --git a/modules/datasets/rapiddns.py b/modules/datasets/rapiddns.py index 058599f..755d2cc 100644 --- a/modules/datasets/rapiddns.py +++ b/modules/datasets/rapiddns.py @@ -5,7 +5,7 @@ from common.query import Query class RapidDNS(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'RapidDNSQuery' @@ -20,7 +20,7 @@ class RapidDNS(Query): resp = self.get(url, params) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/riddler.py b/modules/datasets/riddler.py index 109ebb4..8d21540 100644 --- a/modules/datasets/riddler.py +++ b/modules/datasets/riddler.py @@ -4,7 +4,7 @@ from common.query import Query class Riddler(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'RiddlerQuery' self.addr = 'https://riddler.io/search' @@ -19,7 +19,7 @@ class Riddler(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/robtex.py b/modules/datasets/robtex.py index 3a9f22e..4358b02 100644 --- a/modules/datasets/robtex.py +++ b/modules/datasets/robtex.py @@ -7,7 +7,7 @@ from common.query import Query class Robtex(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = "RobtexQuery" self.addr = 'https://freeapi.robtex.com/pdns/' @@ -32,7 +32,7 @@ class Robtex(Query): resp = self.get(url) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) if subdomains: # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/securitytrails_api.py b/modules/datasets/securitytrails_api.py index 18d9f96..8b52bfb 100644 --- a/modules/datasets/securitytrails_api.py +++ b/modules/datasets/securitytrails_api.py @@ -5,7 +5,7 @@ from common.query import Query class SecurityTrailsAPI(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'SecurityTrailsAPIQuery' self.addr = 'https://api.securitytrails.com/v1/domain/' diff --git a/modules/datasets/sitedossier.py b/modules/datasets/sitedossier.py index 81cb9e9..923160d 100644 --- a/modules/datasets/sitedossier.py +++ b/modules/datasets/sitedossier.py @@ -5,7 +5,7 @@ from common.query import Query class SiteDossier(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'SiteDossierQuery' self.addr = 'http://www.sitedossier.com/parentdomain/' @@ -24,7 +24,7 @@ class SiteDossier(Query): resp = self.get(url) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break # 合并搜索子域名搜索结果 diff --git a/modules/datasets/spyse_api.py b/modules/datasets/spyse_api.py index ba99178..64a4ef9 100644 --- a/modules/datasets/spyse_api.py +++ b/modules/datasets/spyse_api.py @@ -27,7 +27,7 @@ class SpyseAPI(Query): if not resp: return json = resp.json() - subdomains = self.match_subdomains(self.domain, str(json)) + subdomains = self.match_subdomains(str(json)) if not subdomains: # 搜索没有发现子域名则停止搜索 break # 合并搜索子域名搜索结果 diff --git a/modules/datasets/sublist3r.py b/modules/datasets/sublist3r.py index 235d8eb..a96eb9b 100644 --- a/modules/datasets/sublist3r.py +++ b/modules/datasets/sublist3r.py @@ -4,7 +4,7 @@ from common.query import Query class Sublist3r(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'Sublist3rQuery' @@ -19,7 +19,7 @@ class Sublist3r(Query): resp = self.get(addr, param) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/datasets/threatcrowd.py b/modules/datasets/threatcrowd.py index 0971a43..523de7d 100644 --- a/modules/datasets/threatcrowd.py +++ b/modules/datasets/threatcrowd.py @@ -7,7 +7,7 @@ from config.log import logger class ThreatCrowd(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'ThreatCrowdQuery' self.addr = 'https://www.threatcrowd.org/searchApi' \ @@ -25,7 +25,7 @@ class ThreatCrowd(Query): return if resp.status_code != 200: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/wzpc.py b/modules/datasets/wzpc.py index ec60237..5be8255 100644 --- a/modules/datasets/wzpc.py +++ b/modules/datasets/wzpc.py @@ -6,7 +6,7 @@ from common.query import Query class WZPCQuery(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'WZPCQuery' @@ -31,7 +31,7 @@ class WZPCQuery(Query): break if not resp: break - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) self.subdomains = self.subdomains.union(subdomains) if not subdomains: break diff --git a/modules/datasets/ximcx.py b/modules/datasets/ximcx.py index 57a1a09..8e104f4 100644 --- a/modules/datasets/ximcx.py +++ b/modules/datasets/ximcx.py @@ -4,7 +4,7 @@ from common.query import Query class Ximcx(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Dataset' self.source = 'XimcxQuery' self.addr = 'http://sbd.ximcx.cn/DomainServlet' @@ -20,7 +20,7 @@ class Ximcx(Query): if not resp: return json = resp.json() - subdomains = self.match_subdomains(self.domain, str(json)) + subdomains = self.match_subdomains(str(json)) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/dnsquery/mx.py b/modules/dnsquery/mx.py index f4a9819..b2a5514 100644 --- a/modules/dnsquery/mx.py +++ b/modules/dnsquery/mx.py @@ -4,7 +4,7 @@ from common.lookup import Lookup class QueryMX(Lookup): def __init__(self, domain): Lookup.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'dnsquery' self.source = "QueryMX" self.type = 'MX' # 利用的DNS记录的MX记录收集子域 diff --git a/modules/dnsquery/ns.py b/modules/dnsquery/ns.py index 55d3a43..b79e232 100644 --- a/modules/dnsquery/ns.py +++ b/modules/dnsquery/ns.py @@ -4,7 +4,7 @@ from common.lookup import Lookup class QueryNS(Lookup): def __init__(self, domain): Lookup.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'dnsquery' self.source = "QueryNS" self.type = 'NS' # 利用的DNS记录的NS记录收集子域 diff --git a/modules/dnsquery/soa.py b/modules/dnsquery/soa.py index 51696de..b688cfa 100644 --- a/modules/dnsquery/soa.py +++ b/modules/dnsquery/soa.py @@ -4,7 +4,7 @@ from common.lookup import Lookup class QuerySOA(Lookup): def __init__(self, domain): Lookup.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'dnsquery' self.source = "QuerySOA" self.type = 'SOA' # 利用的DNS记录的SOA记录收集子域 diff --git a/modules/dnsquery/spf.py b/modules/dnsquery/spf.py index 86a0a2c..8bc89d1 100644 --- a/modules/dnsquery/spf.py +++ b/modules/dnsquery/spf.py @@ -4,7 +4,7 @@ from common.lookup import Lookup class QuerySPF(Lookup): def __init__(self, domain): Lookup.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'dnsquery' self.source = "QuerySPF" self.type = 'SPF' # 利用的DNS记录的SPF记录收集子域 diff --git a/modules/dnsquery/srv.py b/modules/dnsquery/srv.py index 1f00ec7..fa29780 100644 --- a/modules/dnsquery/srv.py +++ b/modules/dnsquery/srv.py @@ -14,7 +14,7 @@ from config.setting import data_storage_dir class BruteSRV(Module): def __init__(self, domain): Module.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'dnsquery' self.source = "BruteSRV" self.type = 'SRV' # 利用的DNS记录的SRV记录查询子域 @@ -50,7 +50,7 @@ class BruteSRV(Module): continue for item in answer: record = str(item) - subdomains = self.match_subdomains(self.domain, record) + subdomains = self.match_subdomains(record) self.subdomains = self.subdomains.union(subdomains) self.gen_record(subdomains, record) diff --git a/modules/dnsquery/txt.py b/modules/dnsquery/txt.py index bd30ad8..c7ce963 100644 --- a/modules/dnsquery/txt.py +++ b/modules/dnsquery/txt.py @@ -4,7 +4,7 @@ from common.lookup import Lookup class QueryTXT(Lookup): def __init__(self, domain): Lookup.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'dnsquery' self.source = "QueryTXT" self.type = 'TXT' # 利用的DNS记录的TXT记录收集子域 diff --git a/modules/intelligence/alienvault.py b/modules/intelligence/alienvault.py index 925beec..ee2d8c8 100644 --- a/modules/intelligence/alienvault.py +++ b/modules/intelligence/alienvault.py @@ -4,7 +4,7 @@ from common.query import Query class AlienVault(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Intelligence' self.source = 'AlienVaultQuery' @@ -21,7 +21,7 @@ class AlienVault(Query): if not resp: return json = resp.json() - subdomains = self.match_subdomains(self.domain, str(json)) + subdomains = self.match_subdomains(str(json)) self.subdomains = self.subdomains.union(subdomains) url = f'{base}/{self.domain}/url_list' @@ -29,7 +29,7 @@ class AlienVault(Query): if not resp: return json = resp.json() - subdomains = self.match_subdomains(self.domain, str(json)) + subdomains = self.match_subdomains(str(json)) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/intelligence/riskiq_api.py b/modules/intelligence/riskiq_api.py index 0f7adcd..6ec320c 100644 --- a/modules/intelligence/riskiq_api.py +++ b/modules/intelligence/riskiq_api.py @@ -5,7 +5,7 @@ from common.query import Query class RiskIQ(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Intelligence' self.source = 'RiskIQAPIQuery' self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains' diff --git a/modules/intelligence/threatbook_api.py b/modules/intelligence/threatbook_api.py index 409c399..87f1161 100644 --- a/modules/intelligence/threatbook_api.py +++ b/modules/intelligence/threatbook_api.py @@ -5,7 +5,7 @@ from common.query import Query class ThreatBookAPI(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Intelligence' self.source = 'ThreatBookAPIQuery' self.addr = 'https://api.threatbook.cn/v3/domain/sub_domains' @@ -22,7 +22,7 @@ class ThreatBookAPI(Query): resp = self.post(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/intelligence/threatminer.py b/modules/intelligence/threatminer.py index 8832d78..98a2294 100644 --- a/modules/intelligence/threatminer.py +++ b/modules/intelligence/threatminer.py @@ -4,7 +4,7 @@ from common.query import Query class ThreatMiner(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Intelligence' self.source = 'ThreatMinerQuery' self.addr = 'https://www.threatminer.org/getData.php' @@ -20,7 +20,7 @@ class ThreatMiner(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/intelligence/virustotal.py b/modules/intelligence/virustotal.py index a9f3a76..2307d9e 100644 --- a/modules/intelligence/virustotal.py +++ b/modules/intelligence/virustotal.py @@ -11,7 +11,7 @@ class VirusTotal(Query): self.source = 'VirusTotalQuery' self.module = 'Intelligence' self.addr = 'https://www.virustotal.com/ui/domains/{}/subdomains' - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) def query(self): """ diff --git a/modules/intelligence/virustotal_api.py b/modules/intelligence/virustotal_api.py index ad5288f..93649b4 100644 --- a/modules/intelligence/virustotal_api.py +++ b/modules/intelligence/virustotal_api.py @@ -5,7 +5,7 @@ from common.query import Query class VirusTotalAPI(Query): def __init__(self, domain): Query.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Intelligence' self.source = 'VirusTotalAPIQuery' self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report' diff --git a/modules/search/ask.py b/modules/search/ask.py index a8fd235..cd1b115 100644 --- a/modules/search/ask.py +++ b/modules/search/ask.py @@ -30,7 +30,7 @@ class Ask(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(domain, resp.text) + subdomains = self.match_subdomains(resp.text, fuzzy=False) if not subdomains: break if not full_search: diff --git a/modules/search/baidu.py b/modules/search/baidu.py index a58a31a..a6d2464 100644 --- a/modules/search/baidu.py +++ b/modules/search/baidu.py @@ -54,7 +54,7 @@ class Baidu(Search): # 获取百度跳转URL响应头的Location字段获取直链 subdomains = self.redirect_match(domain, resp.text) else: - subdomains = self.match_subdomains(domain, resp.text) + subdomains = self.match_subdomains(resp.text, fuzzy=False) if not subdomains: # 搜索没有发现子域名则停止搜索 break if not full_search: diff --git a/modules/search/bing.py b/modules/search/bing.py index 461c8b8..324faf6 100644 --- a/modules/search/bing.py +++ b/modules/search/bing.py @@ -36,7 +36,7 @@ class Bing(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(domain, resp.text) + subdomains = self.match_subdomains(resp.text, fuzzy=False) if not subdomains: # 搜索没有发现子域名则停止搜索 break if not full_search: diff --git a/modules/search/bing_api.py b/modules/search/bing_api.py index 8e46ee3..5a683dc 100644 --- a/modules/search/bing_api.py +++ b/modules/search/bing_api.py @@ -37,7 +37,7 @@ class BingAPI(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(domain, resp.text) + subdomains = self.match_subdomains(resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break if not full_search: diff --git a/modules/search/fofa_api.py b/modules/search/fofa_api.py index 9e65e5f..370520c 100644 --- a/modules/search/fofa_api.py +++ b/modules/search/fofa_api.py @@ -37,7 +37,7 @@ class FoFa(Search): if not resp: return resp_json = resp.json() - subdomains = self.match_subdomains(self.domain, str(resp_json)) + subdomains = self.match_subdomains(resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/search/gitee.py b/modules/search/gitee.py index f24a054..9eef654 100644 --- a/modules/search/gitee.py +++ b/modules/search/gitee.py @@ -10,7 +10,7 @@ class Gitee(Search): self.source = 'GiteeSearch' self.module = 'Search' self.addr = 'https://search.gitee.com/' - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.header = self.get_header() def search(self, full_search=False): @@ -34,7 +34,7 @@ class Gitee(Search): if 'class="empty-box"' in resp.text: break soup = BeautifulSoup(resp.text, 'html.parser') - subdomains = self.match_subdomains(self.domain, soup.text) + subdomains = self.match_subdomains(soup.text, fuzzy=False) if not subdomains: break if not full_search: diff --git a/modules/search/github_api.py b/modules/search/github_api.py index a17b403..3559d8b 100644 --- a/modules/search/github_api.py +++ b/modules/search/github_api.py @@ -10,7 +10,7 @@ class GithubAPI(Search): self.source = 'GithubAPISearch' self.module = 'Search' self.addr = 'https://api.github.com/search/code' - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.session = requests.Session() self.auth_url = 'https://api.github.com' self.token = api.github_api_token @@ -60,7 +60,7 @@ class GithubAPI(Search): if resp.status_code != 200: logger.log('ERROR', f'{self.source} module query failed') break - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) if not subdomains: break self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/search/google.py b/modules/search/google.py index 457cb34..b405d87 100644 --- a/modules/search/google.py +++ b/modules/search/google.py @@ -40,7 +40,7 @@ class Google(Search): resp = self.get(url=self.addr, params=payload) if not resp: return - subdomains = self.match_subdomains(domain, resp.text) + subdomains = self.match_subdomains(resp.text, fuzzy=False) if not subdomains: break if not full_search: diff --git a/modules/search/google_api.py b/modules/search/google_api.py index 86dd50e..1a0baef 100644 --- a/modules/search/google_api.py +++ b/modules/search/google_api.py @@ -35,7 +35,7 @@ class GoogleAPI(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(domain, resp.text) + subdomains = self.match_subdomains(resp.text) if not subdomains: break if not full_search: diff --git a/modules/search/shodan_api.py b/modules/search/shodan_api.py index 36628cf..909462b 100644 --- a/modules/search/shodan_api.py +++ b/modules/search/shodan_api.py @@ -5,7 +5,7 @@ from common.search import Search class ShodanAPI(Search): def __init__(self, domain): Search.__init__(self) - self.domain = self.register(domain) + self.domain = self.get_maindomain(domain) self.module = 'Search' self.source = 'ShodanAPISearch' self.addr = 'https://api.shodan.io/shodan/host/search' @@ -25,7 +25,7 @@ class ShodanAPI(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break if subdomains: diff --git a/modules/search/so.py b/modules/search/so.py index b2d88ab..9d68d94 100644 --- a/modules/search/so.py +++ b/modules/search/so.py @@ -31,7 +31,7 @@ class So(Search): resp = self.get(url=self.addr, params=payload) if not resp: return - subdomains = self.match_subdomains(domain, resp.text) + subdomains = self.match_subdomains(resp.text, fuzzy=False) if not subdomains: break if not full_search: diff --git a/modules/search/sogou.py b/modules/search/sogou.py index 481498b..1a91f98 100644 --- a/modules/search/sogou.py +++ b/modules/search/sogou.py @@ -28,7 +28,7 @@ class Sogou(Search): resp = self.get(self.addr, payload) if not resp: return - subdomains = self.match_subdomains(domain, resp.text) + subdomains = self.match_subdomains(resp.text, fuzzy=False) if not subdomains: break if not full_search: diff --git a/modules/search/yahoo.py b/modules/search/yahoo.py index cb790fd..18e7664 100644 --- a/modules/search/yahoo.py +++ b/modules/search/yahoo.py @@ -36,7 +36,7 @@ class Yahoo(Search): if not resp: return text = resp.text.replace('', '').replace('', '') - subdomains = self.match_subdomains(domain, text) + subdomains = self.match_subdomains(text, fuzzy=False) if not subdomains: # 搜索没有发现子域名则停止搜索 break if not full_search: diff --git a/modules/search/yandex.py b/modules/search/yandex.py index b7af641..4d63225 100644 --- a/modules/search/yandex.py +++ b/modules/search/yandex.py @@ -36,7 +36,7 @@ class Yandex(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(domain, resp.text) + subdomains = self.match_subdomains(resp.text, fuzzy=False) if not subdomains: # 搜索没有发现子域名则停止搜索 break if not full_search: diff --git a/modules/search/zoomeye_api.py b/modules/search/zoomeye_api.py index 9578f49..3f99969 100644 --- a/modules/search/zoomeye_api.py +++ b/modules/search/zoomeye_api.py @@ -48,7 +48,7 @@ class ZoomEyeAPI(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match_subdomains(self.domain, resp.text) + subdomains = self.match_subdomains(resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break self.subdomains = self.subdomains.union(subdomains) diff --git a/oneforall.py b/oneforall.py index 048e4c4..65285cf 100644 --- a/oneforall.py +++ b/oneforall.py @@ -198,7 +198,7 @@ class OneForAll(object): # Resolve subdomains self.data = resolve.run_resolve(self.domain, self.data) # Save resolve results - resolve.save_data(self.resolve_table, self.data) + resolve.save_db(self.resolve_table, self.data) # Export results without HTTP request if not self.req: @@ -207,7 +207,7 @@ class OneForAll(object): # HTTP request self.data = request.run_request(self.domain, self.data, self.port) # Save HTTP request result - request.save_data(self.domain, self.data) + request.save_db(self.domain, self.data) # Add the final result list to the total data list self.datas.extend(self.data)