优化

brute.py

@@ -218,7 +218,8 @@ def collect_wildcard_record(domain, authoritative_ns):
             ip, ttl = get_wildcard_record(random_subdomain, resolver)
         except Exception as e:
             logger.log('DEBUG', e.args)
-            logger.log('ALERT', f'Multiple query errors, try to query a new random subdomain')
+            logger.log('ALERT', f'Multiple query errors,'
+                                f'try to query a new random subdomain')
             continue
         if ip is None:
             continue
@@ -255,7 +256,8 @@ def check_dict():
     if not setting.enable_check_dict:
         return
     sec = setting.check_time
-    logger.log('ALERT', f'You have {sec} seconds to check whether the configuration is correct or not')
+    logger.log('ALERT', f'You have {sec} seconds to check '
+                        f'whether the configuration is correct or not')
     logger.log('ALERT', f'If you want to exit, please use `Ctrl + C`')
     try:
         time.sleep(sec)
@@ -324,7 +326,8 @@ def stat_ip_times(result_paths):
                     items = json.loads(line)
                 except Exception as e:
                     logger.log('ERROR', e.args)
-                    logger.log('ERROR', f'Error parsing {result_path} line {line} Skip this line')
+                    logger.log('ERROR', f'Error parsing {result_path} '
+                                        f'line {line} Skip this line')
                     continue
                 status = items.get('status')
                 if status != 'NOERROR':
@@ -360,7 +363,8 @@ def deal_output(output_paths, ip_times, wildcard_ips, wildcard_ttl):
                 qname = items.get('name')[:-1]  # 去除最右边的`.`点号
                 status = items.get('status')
                 if status != 'NOERROR':
-                    logger.log('TRACE', f'Found {qname}\'s result {status} while processing {line}')
+                    logger.log('TRACE', f'Found {qname}\'s result {status} '
+                                        f'while processing {line}')
                     continue
                 data = items.get('data')
                 if 'answers' not in data:
@@ -504,7 +508,7 @@ class Brute(Module):
         if self.place is None:
             self.place = '*.' + domain
         wordlist = self.wordlist
-        main_domain = self.register(domain)
+        main_domain = self.get_maindomain(domain)
         if domain != main_domain:
             wordlist = self.recursive_nextlist
         if self.word:
@@ -519,7 +523,8 @@ class Brute(Module):
         count = len(dict_set)
         logger.log('INFOR', f'Dictionary size: {count}')
         if count > 10000000:
-            logger.log('ALERT', f'The dictionary generated is too large：{count} > 10000000')
+            logger.log('ALERT', f'The generated dictionary is '
+                                f'too large {count} > 10000000')
         return dict_set
 
     def check_brute_params(self):

common/database.py

@@ -97,22 +97,21 @@ class Database(object):
         :param list results: results list
         :param str module_name: mo
         """
-        logger.log('TRACE',
+        logger.log('TRACE', f'Saving the subdomain results of {table_name} '
-                   f'Saving the subdomain results of {table_name} found by module {module_name} into database')
+                            f'found by module {module_name} into database')
         table_name = table_name.replace('.', '_')
         if results:
             try:
                 self.conn.bulk_query(
-                    f'insert into "{table_name}" ('
+                    f'insert into "{table_name}" (id, type, alive, resolve, request, new,'
-                    f'id, type, alive, resolve, request, new, url, subdomain,'
+                    f'url, subdomain, port, level, cname, content, public, status, reason,'
-                    f'port, level, cname, content, public, status, reason,'
+                    f'title, banner, header, response, times, ttl, cidr, asn, ip2region,'
-                    f'title, banner, header, response, times, ttl, cidr, asn,'
+                    f'ip2location, resolver, module, source, elapse, find, brute, valid) '
-                    f'ip2region, ip2location, resolver, module, source, elapse, find, brute, valid) '
+                    f'values (:id, :type, :alive, :resolve, :request, :new, :url, '
-                    f'values (:id, :type, :alive, :resolve, :request, :new,'
+                    f':subdomain, :port, :level, :cname, :content, :public, :status,'
-                    f':url, :subdomain, :port, :level, :cname, :content,'
+                    f':reason, :title, :banner, :header, :response, :times, :ttl, :cidr,'
-                    f':public, :status, :reason, :title, :banner, :header,'
+                    f':asn, :ip2region, :ip2location, :resolver, :module, :source,'
-                    f':response, :times, :ttl, :cidr, :asn, :ip2region, :ip2location, :resolver,'
+                    f':elapse, :find, :brute, :valid)', results)
-                    f':module, :source, :elapse, :find, :brute, :valid)', results)
             except Exception as e:
                 logger.log('ERROR', e)
 

common/lookup.py

@@ -20,7 +20,7 @@ class Lookup(Module):
             return None
         for item in answer:
             record = item.to_text()
-            subdomains = self.match_subdomains(self.domain, record)
+            subdomains = self.match_subdomains(record)
             self.subdomains = self.subdomains.union(subdomains)
             self.gen_record(subdomains, record)
         return self.subdomains

common/module.py

@@ -53,7 +53,8 @@ class Module(object):
         """
         begin log
         """
-        logger.log('DEBUG', f'Start {self.source} module to collect subdomains of {self.domain}')
+        logger.log('DEBUG', f'Start {self.source} module to '
+                            f'collect subdomains of {self.domain}')
 
     def finish(self):
         """
@@ -61,7 +62,8 @@ class Module(object):
         """
         self.end = time.time()
         self.elapse = round(self.end - self.start, 1)
-        logger.log('DEBUG', f'Finished {self.source} module to collect {self.domain}\'s subdomains')
+        logger.log('DEBUG', f'Finished {self.source} module to '
+                            f'collect {self.domain}\'s subdomains')
         logger.log('INFOR', f'The {self.source} module took {self.elapse} seconds '
                             f'found {len(self.subdomains)} subdomains')
         logger.log('DEBUG', f'{self.source} module found subdomains of {self.domain}\n'
@@ -209,35 +211,48 @@ class Module(object):
             logger.log('TRACE', f'{module} module does not use proxy')
             return self.proxy
 
-    @staticmethod
+    def match_subdomains(self, html, distinct=True, fuzzy=True):
-    def match_subdomains(domain, text, distinct=True):
         """
         Use regexp to match subdomains
 
-        :param  str domain: domain
+        :param  str html: response html text
-        :param  str text: text
         :param  bool distinct: deduplicate results or not (default True)
+        :param  bool fuzzy: fuzzy match subdomain or not (default True)
         :return set/list: result set or list
         """
         logger.log('TRACE', f'Use regexp to match subdomains in the response body')
-        regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+        if fuzzy:
-                 + domain.replace('.', r'\.')
+            regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
-        result = re.findall(regexp, text, re.I)
+                     + self.domain.replace('.', r'\.')
+            result = re.findall(regexp, html, re.I)
+            if not result:
+                return set()
+            deal = map(lambda s: s.lower(), result)
+            if distinct:
+                return set(deal)
+            else:
+                return list(deal)
+        else:
+            regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
+                     r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+                     + self.domain.replace('.', r'\.')
+            result = re.findall(regexp, html, re.I)
         if not result:
             return set()
-        deal = map(lambda s: s.lower(), result)
+        regexp = r'(?:http://|https://)'
+        deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
         if distinct:
             return set(deal)
         else:
             return list(deal)
 
     @staticmethod
-    def register(domain):
+    def get_maindomain(domain):
         """
-        Get registered domain
+        Get main domain
 
         :param str domain: domain
-        :return: registered domain
+        :return: main domain
         """
         return Domain(domain).registered()
 
@@ -249,7 +264,8 @@ class Module(object):
         """
         if not setting.save_module_result:
             return False
-        logger.log('TRACE', f'Save the subdomain results found by {self.source} module as a json file')
+        logger.log('TRACE', f'Save the subdomain results found by '
+                            f'{self.source} module as a json file')
         path = setting.result_save_dir.joinpath(self.domain, self.module)
         path.mkdir(parents=True, exist_ok=True)
         name = self.source + '.json'

common/request.py

@@ -281,7 +281,7 @@ def urls_request(urls):
     return data
 
 
-def save_data(name, data):
+def save_db(name, data):
     """
     Save request results to database
 

common/resolve.py

@@ -48,7 +48,7 @@ def update_data(data, records):
     return data
 
 
-def save_data(name, data):
+def save_db(name, data):
     """
     保存解析结果到数据库
 

common/search.py

@@ -36,13 +36,12 @@ class Search(Module):
                                                    subdomains_temp[i:i + 2]))))
         return statements_list
 
-    def match_location(self, domain, url):
+    def match_location(self, url):
         """
         匹配跳转之后的url
         针对部分搜索引擎(如百度搜索)搜索展示url时有显示不全的情况
         此函数会向每条结果的链接发送head请求获取响应头的location值并做子域匹配
 
-        :param str domain: 域名
         :param str url: 展示结果的url链接
         :return: 匹配的子域
         :rtype set
@@ -53,28 +52,4 @@ class Search(Module):
         location = resp.headers.get('location')
         if not location:
             return set()
-        return set(self.match_subdomains(domain, location))
+        return set(self.match_subdomains(location))
-
-    @staticmethod
-    def match_subdomains(domain, html, distinct=True):
-        """
-        Use regexp to match subdomains
-
-        :param  str domain: domain
-        :param  str html: response html text
-        :param  bool distinct: deduplicate results or not (default True)
-        :return set/list: result set or list
-        """
-        logger.log('TRACE', f'Use regexp to match subdomains in the response body')
-        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
-                 r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
-                 + domain.replace('.', r'\.')
-        result = re.findall(regexp, html, re.I)
-        if not result:
-            return set()
-        regexp = r'(?:http://|https://)'
-        deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
-        if distinct:
-            return set(deal)
-        else:
-            return list(deal)

common/utils.py

@@ -237,6 +237,21 @@ def save_data(path, data):
         return False
 
 
+def remove_data(path):
+    """
+    删除保存数据的文件
+
+    :param path: 路径
+    :return: 删除成功与否
+    """
+    try:
+        path.unlink()
+    except Exception as e:
+        logger.log('ERROR', e.args)
+        return False
+    return True
+
+
 def check_response(method, resp):
     """
     检查响应 输出非正常响应返回json的信息

config/setting.py

@@ -12,6 +12,7 @@ module_dir = relative_directory.joinpath('modules')  # OneForAll模块目录
 third_party_dir = relative_directory.joinpath('thirdparty')  # 三方工具目录
 data_storage_dir = relative_directory.joinpath('data')  # 数据存放目录
 result_save_dir = relative_directory.joinpath('results')  # 结果保存目录
+temp_save_dir = result_save_dir.joinpath('temp')
 
 # OneForAll入口参数设置
 enable_check_version = True  # 开启最新版本检查

modules/certificates/censys_api.py

@@ -6,7 +6,7 @@ from config.log import logger
 class CensysAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Certificate'
         self.source = "CensysAPIQuery"
         self.addr = 'https://www.censys.io/api/v1/search/certificates'
@@ -33,7 +33,7 @@ class CensysAPI(Query):
         if status != 'ok':
             logger.log('ALERT', f'{self.source} module {status}')
             return
-        subdomains = self.match_subdomains(self.domain, str(json))
+        subdomains = self.match_subdomains(resp.text)
         self.subdomains = self.subdomains.union(subdomains)
         pages = json.get('metadata').get('pages')
         for page in range(2, pages + 1):
@@ -41,7 +41,7 @@ class CensysAPI(Query):
             resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
             if not resp:
                 return
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/certificates/certspotter.py

@@ -22,7 +22,7 @@ class CertSpotter(Query):
         resp = self.get(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/certificates/crtsh.py

@@ -5,7 +5,7 @@ from common.query import Query
 class Crtsh(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Certificate'
         self.source = 'CrtshQuery'
         self.addr = 'https://crt.sh/'
@@ -21,7 +21,7 @@ class Crtsh(Query):
         if not resp:
             return
         text = resp.text.replace(r'\n', ' ')
-        subdomains = self.match_subdomains(self.domain, text)
+        subdomains = self.match_subdomains(text)
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/certificates/google.py

@@ -5,7 +5,7 @@ from common.query import Query
 class Google(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Certificate'
         self.source = 'GoogleQuery'
         self.addr = 'https://transparencyreport.google.com/' \
@@ -23,7 +23,7 @@ class Google(Query):
         resp = self.get(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/check/axfr.py

@@ -22,7 +22,7 @@ class CheckAXFR(Module):
 
     def __init__(self, domain: str):
         Module.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Check'
         self.source = 'AXFRCheck'
         self.results = []
@@ -45,7 +45,7 @@ class CheckAXFR(Module):
         names = zone.nodes.keys()
         for name in names:
             full_domain = str(name) + '.' + self.domain
-            subdomain = self.match_subdomains(self.domain, full_domain)
+            subdomain = self.match_subdomains(full_domain)
             self.subdomains = self.subdomains.union(subdomain)
             record = zone[name].to_text(name)
             self.results.append(record)

modules/check/cdx.py

@@ -12,7 +12,7 @@ class CheckCDX(Module):
     """
     def __init__(self, domain: str):
         Module.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Check'
         self.source = "CrossDomainXml"
 
@@ -31,7 +31,7 @@ class CheckCDX(Module):
             if not resp:
                 return
             if resp and len(resp.content):
-                self.subdomains = self.match_subdomains(self.domain, resp.text)
+                self.subdomains = self.match_subdomains(resp.text)
 
     def run(self):
         """

modules/check/cert.py

@@ -14,7 +14,7 @@ from config.log import logger
 class CheckCert(Module):
     def __init__(self, domain):
         Module.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.port = 443  # ssl port
         self.module = 'Check'
         self.source = 'CertInfo'
@@ -32,7 +32,7 @@ class CheckCert(Module):
         except Exception as e:
             logger.log('DEBUG', e.args)
             return
-        subdomains = self.match_subdomains(self.domain, str(cert_dict))
+        subdomains = self.match_subdomains(str(cert_dict))
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/check/csp.py

@@ -14,7 +14,7 @@ class CheckCSP(Module):
     """
     def __init__(self, domain, header):
         Module.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Check'
         self.source = 'ContentSecurityPolicy'
         self.csp_header = header
@@ -52,7 +52,7 @@ class CheckCSP(Module):
         if not csp:
             logger.log('DEBUG', f'There is no Content-Security-Policy in the header of {self.domain}')
             return
-        self.subdomains = self.match_subdomains(self.domain, csp)
+        self.subdomains = self.match_subdomains(csp)
 
     def run(self):
         """

modules/check/nsec.py

@@ -8,7 +8,7 @@ from common import utils
 class CheckNSEC(Module):
     def __init__(self, domain):
         Module.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'check'
         self.source = "CheckNSEC"
 
@@ -21,7 +21,7 @@ class CheckNSEC(Module):
             subdomain = str()
             for item in answer:
                 record = item.to_text()
-                subdomains = self.match_subdomains(self.domain, record)
+                subdomains = self.match_subdomains(record)
                 subdomain = ''.join(subdomains)  # 其实这里的subdomains的长度为1 也就是说只会有一个子域
                 self.subdomains = self.subdomains.union(subdomains)
                 self.gen_record(subdomains, record)

modules/check/robots.py

@@ -11,7 +11,7 @@ class CheckRobots(Module):
     """
     def __init__(self, domain):
         Module.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Check'
         self.source = 'Robots'
 
@@ -30,7 +30,7 @@ class CheckRobots(Module):
             if not resp:
                 return
             if resp and len(resp.content):
-                self.subdomains = self.match_subdomains(self.domain, resp.text)
+                self.subdomains = self.match_subdomains(resp.text)
 
     def run(self):
         """

modules/check/sitemap.py

@@ -11,7 +11,7 @@ class CheckRobots(Module):
     """
     def __init__(self, domain):
         Module.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Check'
         self.source = 'Sitemap'
 
@@ -35,7 +35,7 @@ class CheckRobots(Module):
             if not resp:
                 return
             if resp and len(resp.content):
-                self.subdomains = self.match_subdomains(self.domain, resp.text)
+                self.subdomains = self.match_subdomains(resp.text)
 
     def run(self):
         """

modules/crawl/archivecrawl.py

@@ -26,7 +26,7 @@ class ArchiveCrawl(Crawl):
         for resp in cdx.iter(url, limit=limit):
             if resp.data.get('status') not in ['301', '302']:
                 url = resp.data.get('url')
-                subdomains = self.match_subdomains(self.register(domain),
+                subdomains = self.match_subdomains(self.get_maindomain(domain),
                                                    url + resp.text)
                 # 合并搜索子域名搜索结果
                 self.subdomains = self.subdomains.union(subdomains)

modules/crawl/commoncrawl.py

@@ -26,7 +26,7 @@ class CommonCrawl(Crawl):
 
         for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
             if resp.data.get('status') not in ['301', '302']:
-                subdomains = self.match_subdomains(self.register(domain), resp.text)
+                subdomains = self.match_subdomains(self.get_maindomain(domain), resp.text)
                 # 合并搜索子域名搜索结果
                 self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/binaryedge_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class BinaryEdgeAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'BinaryEdgeAPIQuery'
         self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/'
@@ -22,7 +22,7 @@ class BinaryEdgeAPI(Query):
         resp = self.get(url)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/datasets/bufferover.py

@@ -6,7 +6,7 @@ from config.log import logger
 class BufferOver(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'BufferOverQuery'
         self.addr = 'https://dns.bufferover.run/dns?q='
@@ -26,7 +26,7 @@ class BufferOver(Query):
             return
         if resp.status_code != 200:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/cebaidu.py

@@ -4,7 +4,7 @@ from common.query import Query
 class CeBaidu(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'CeBaiduQuery'
         self.addr = 'https://ce.baidu.com/index/getRelatedSites'
@@ -19,7 +19,7 @@ class CeBaidu(Query):
         resp = self.get(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/chinaz.py

@@ -4,7 +4,7 @@ from common.query import Query
 class Chinaz(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'ChinazQuery'
         self.addr = 'https://alexa.chinaz.com/'
@@ -19,7 +19,7 @@ class Chinaz(Query):
         resp = self.get(self.addr)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/chinaz_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class ChinazAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'ChinazAPIQuery'
         self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa'
@@ -21,7 +21,7 @@ class ChinazAPI(Query):
         resp = self.get(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/circl_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class CirclAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'CirclAPIQuery'
         self.addr = 'https://www.circl.lu/pdns/query/'
@@ -21,7 +21,7 @@ class CirclAPI(Query):
         resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/cloudflare_api.py

@@ -79,7 +79,7 @@ class CloudFlareAPI(Query):
             logger.log('DEBUG',
                        f'{list_dns_resp.status_code} {list_dns_resp.text}')
             return
-        subdomains = self.match_subdomains(self.domain, list_dns_resp.text)
+        subdomains = self.match_subdomains(list_dns_resp.text)
         self.subdomains = self.subdomains.union(subdomains)
         if not self.subdomains:
             # waiting for cloudflare enumerate subdomains
@@ -94,7 +94,7 @@ class CloudFlareAPI(Query):
                                f'{list_dns_resp.status_code} {list_dns_resp.text}')
                     return
                 total_pages = list_dns_resp.json()['result_info']['total_pages']
-                subdomains = (self.match_subdomains(self.domain, list_dns_resp.text))
+                subdomains = (self.match_subdomains(list_dns_resp.text))
                 self.subdomains = self.subdomains.union(subdomains)
                 page += 1
                 if page > total_pages:

modules/datasets/dnsdb_api.py

@@ -6,7 +6,7 @@ from common.query import Query
 class DNSdbAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'DNSdbAPIQuery'
         self.addr = 'https://api.dnsdb.info/lookup/rrset/name/'
@@ -23,7 +23,7 @@ class DNSdbAPI(Query):
         resp = self.get(url)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/dnsdumpster.py

@@ -5,7 +5,7 @@ from common.query import Query
 class DNSdumpster(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = "DNSdumpsterQuery"
         self.addr = 'https://dnsdumpster.com/'
@@ -26,7 +26,7 @@ class DNSdumpster(Query):
         resp = self.post(self.addr, data)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         if subdomains:
             # 合并搜索子域名搜索结果
             self.subdomains = self.subdomains.union(subdomains)

modules/datasets/hackertarget.py

@@ -5,7 +5,7 @@ from common.query import Query
 class HackerTarget(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = "HackerTargetQuery"
         self.addr = 'https://api.hackertarget.com/hostsearch/'
@@ -21,7 +21,7 @@ class HackerTarget(Query):
         if not resp:
             return
         if resp.status_code == 200:
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             if subdomains:
                 # 合并搜索子域名搜索结果
                 self.subdomains = self.subdomains.union(subdomains)

modules/datasets/ip138.py

@@ -4,7 +4,7 @@ from common.query import Query
 class IP138(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'IP138Query'
         self.addr = 'https://site.ip138.com/{domain}/domain.htm'
@@ -19,7 +19,7 @@ class IP138(Query):
         resp = self.get(self.addr)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/ipv4info_api.py

@@ -6,7 +6,7 @@ from config.log import logger
 class IPv4InfoAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'IPv4InfoAPIQuery'
         self.addr = ' http://ipv4info.com/api_v1/'
@@ -32,7 +32,7 @@ class IPv4InfoAPI(Query):
             except Exception as e:
                 logger.log('DEBUG', e.args)
                 break
-            subdomains = self.match_subdomains(self.domain, str(json))
+            subdomains = self.match_subdomains(str(json))
             if not subdomains:
                 break
             # 合并搜索子域名搜索结果

modules/datasets/netcraft.py

@@ -9,7 +9,7 @@ from common.query import Query
 class NetCraft(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'NetCraftQuery'
         self.init = 'https://searchdns.netcraft.com/'
@@ -49,7 +49,7 @@ class NetCraft(Query):
             resp = self.get(self.addr + last, params)
             if not resp:
                 return
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             # 合并搜索子域名搜索结果

modules/datasets/passivedns_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class PassiveDnsAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'PassiveDnsQuery'
         self.addr = api.passivedns_api_addr or 'http://api.passivedns.cn'
@@ -22,7 +22,7 @@ class PassiveDnsAPI(Query):
         resp = self.get(url)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/phonebook.py

@@ -5,7 +5,7 @@ from common.query import Query
 class PhoneBook(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'PhoneBookQuery'
 
@@ -35,7 +35,7 @@ class PhoneBook(Query):
         resp = self.get(url)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/datasets/qianxun.py

@@ -27,7 +27,7 @@ class QianXun(Query):
             resp = self.post(url, data)
             if not resp:
                 break
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             self.subdomains = self.subdomains.union(subdomains)
             if '<div id="page" class="pagelist">' not in resp.text:
                 break

modules/datasets/rapiddns.py

@@ -5,7 +5,7 @@ from common.query import Query
 class RapidDNS(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'RapidDNSQuery'
 
@@ -20,7 +20,7 @@ class RapidDNS(Query):
         resp = self.get(url, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/riddler.py

@@ -4,7 +4,7 @@ from common.query import Query
 class Riddler(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'RiddlerQuery'
         self.addr = 'https://riddler.io/search'
@@ -19,7 +19,7 @@ class Riddler(Query):
         resp = self.get(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/robtex.py

@@ -7,7 +7,7 @@ from common.query import Query
 class Robtex(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = "RobtexQuery"
         self.addr = 'https://freeapi.robtex.com/pdns/'
@@ -32,7 +32,7 @@ class Robtex(Query):
                 resp = self.get(url)
                 if not resp:
                     return
-                subdomains = self.match_subdomains(self.domain, resp.text)
+                subdomains = self.match_subdomains(resp.text)
                 if subdomains:
                     # 合并搜索子域名搜索结果
                     self.subdomains = self.subdomains.union(subdomains)

modules/datasets/securitytrails_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class SecurityTrailsAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'SecurityTrailsAPIQuery'
         self.addr = 'https://api.securitytrails.com/v1/domain/'

modules/datasets/sitedossier.py

@@ -5,7 +5,7 @@ from common.query import Query
 class SiteDossier(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'SiteDossierQuery'
         self.addr = 'http://www.sitedossier.com/parentdomain/'
@@ -24,7 +24,7 @@ class SiteDossier(Query):
             resp = self.get(url)
             if not resp:
                 return
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             # 合并搜索子域名搜索结果

modules/datasets/spyse_api.py

@@ -27,7 +27,7 @@ class SpyseAPI(Query):
             if not resp:
                 return
             json = resp.json()
-            subdomains = self.match_subdomains(self.domain, str(json))
+            subdomains = self.match_subdomains(str(json))
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             # 合并搜索子域名搜索结果

modules/datasets/sublist3r.py

@@ -4,7 +4,7 @@ from common.query import Query
 class Sublist3r(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'Sublist3rQuery'
 
@@ -19,7 +19,7 @@ class Sublist3r(Query):
         resp = self.get(addr, param)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/datasets/threatcrowd.py

@@ -7,7 +7,7 @@ from config.log import logger
 class ThreatCrowd(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'ThreatCrowdQuery'
         self.addr = 'https://www.threatcrowd.org/searchApi' \
@@ -25,7 +25,7 @@ class ThreatCrowd(Query):
             return
         if resp.status_code != 200:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/wzpc.py

@@ -6,7 +6,7 @@ from common.query import Query
 class WZPCQuery(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'WZPCQuery'
 
@@ -31,7 +31,7 @@ class WZPCQuery(Query):
                 break
             if not resp:
                 break
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             self.subdomains = self.subdomains.union(subdomains)
             if not subdomains:
                 break

modules/datasets/ximcx.py

@@ -4,7 +4,7 @@ from common.query import Query
 class Ximcx(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'XimcxQuery'
         self.addr = 'http://sbd.ximcx.cn/DomainServlet'
@@ -20,7 +20,7 @@ class Ximcx(Query):
         if not resp:
             return
         json = resp.json()
-        subdomains = self.match_subdomains(self.domain, str(json))
+        subdomains = self.match_subdomains(str(json))
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/dnsquery/mx.py

@@ -4,7 +4,7 @@ from common.lookup import Lookup
 class QueryMX(Lookup):
     def __init__(self, domain):
         Lookup.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'dnsquery'
         self.source = "QueryMX"
         self.type = 'MX'  # 利用的DNS记录的MX记录收集子域

modules/dnsquery/ns.py

@@ -4,7 +4,7 @@ from common.lookup import Lookup
 class QueryNS(Lookup):
     def __init__(self, domain):
         Lookup.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'dnsquery'
         self.source = "QueryNS"
         self.type = 'NS'  # 利用的DNS记录的NS记录收集子域

modules/dnsquery/soa.py

@@ -4,7 +4,7 @@ from common.lookup import Lookup
 class QuerySOA(Lookup):
     def __init__(self, domain):
         Lookup.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'dnsquery'
         self.source = "QuerySOA"
         self.type = 'SOA'  # 利用的DNS记录的SOA记录收集子域

modules/dnsquery/spf.py

@@ -4,7 +4,7 @@ from common.lookup import Lookup
 class QuerySPF(Lookup):
     def __init__(self, domain):
         Lookup.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'dnsquery'
         self.source = "QuerySPF"
         self.type = 'SPF'  # 利用的DNS记录的SPF记录收集子域

modules/dnsquery/srv.py

@@ -14,7 +14,7 @@ from config.setting import data_storage_dir
 class BruteSRV(Module):
     def __init__(self, domain):
         Module.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'dnsquery'
         self.source = "BruteSRV"
         self.type = 'SRV'  # 利用的DNS记录的SRV记录查询子域
@@ -50,7 +50,7 @@ class BruteSRV(Module):
                 continue
             for item in answer:
                 record = str(item)
-                subdomains = self.match_subdomains(self.domain, record)
+                subdomains = self.match_subdomains(record)
                 self.subdomains = self.subdomains.union(subdomains)
                 self.gen_record(subdomains, record)
 

modules/dnsquery/txt.py

@@ -4,7 +4,7 @@ from common.lookup import Lookup
 class QueryTXT(Lookup):
     def __init__(self, domain):
         Lookup.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'dnsquery'
         self.source = "QueryTXT"
         self.type = 'TXT'  # 利用的DNS记录的TXT记录收集子域

modules/intelligence/alienvault.py

@@ -4,7 +4,7 @@ from common.query import Query
 class AlienVault(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Intelligence'
         self.source = 'AlienVaultQuery'
 
@@ -21,7 +21,7 @@ class AlienVault(Query):
         if not resp:
             return
         json = resp.json()
-        subdomains = self.match_subdomains(self.domain, str(json))
+        subdomains = self.match_subdomains(str(json))
         self.subdomains = self.subdomains.union(subdomains)
 
         url = f'{base}/{self.domain}/url_list'
@@ -29,7 +29,7 @@ class AlienVault(Query):
         if not resp:
             return
         json = resp.json()
-        subdomains = self.match_subdomains(self.domain, str(json))
+        subdomains = self.match_subdomains(str(json))
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/intelligence/riskiq_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class RiskIQ(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Intelligence'
         self.source = 'RiskIQAPIQuery'
         self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'

modules/intelligence/threatbook_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class ThreatBookAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Intelligence'
         self.source = 'ThreatBookAPIQuery'
         self.addr = 'https://api.threatbook.cn/v3/domain/sub_domains'
@@ -22,7 +22,7 @@ class ThreatBookAPI(Query):
         resp = self.post(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/intelligence/threatminer.py

@@ -4,7 +4,7 @@ from common.query import Query
 class ThreatMiner(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Intelligence'
         self.source = 'ThreatMinerQuery'
         self.addr = 'https://www.threatminer.org/getData.php'
@@ -20,7 +20,7 @@ class ThreatMiner(Query):
         resp = self.get(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/intelligence/virustotal.py

@@ -11,7 +11,7 @@ class VirusTotal(Query):
         self.source = 'VirusTotalQuery'
         self.module = 'Intelligence'
         self.addr = 'https://www.virustotal.com/ui/domains/{}/subdomains'
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
 
     def query(self):
         """

modules/intelligence/virustotal_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class VirusTotalAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Intelligence'
         self.source = 'VirusTotalAPIQuery'
         self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'

modules/search/ask.py

@@ -30,7 +30,7 @@ class Ask(Search):
             resp = self.get(self.addr, params)
             if not resp:
                 return
-            subdomains = self.match_subdomains(domain, resp.text)
+            subdomains = self.match_subdomains(resp.text, fuzzy=False)
             if not subdomains:
                 break
             if not full_search:

modules/search/baidu.py

@@ -54,7 +54,7 @@ class Baidu(Search):
                 # 获取百度跳转URL响应头的Location字段获取直链
                 subdomains = self.redirect_match(domain, resp.text)
             else:
-                subdomains = self.match_subdomains(domain, resp.text)
+                subdomains = self.match_subdomains(resp.text, fuzzy=False)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             if not full_search:

modules/search/bing.py

@@ -36,7 +36,7 @@ class Bing(Search):
             resp = self.get(self.addr, params)
             if not resp:
                 return
-            subdomains = self.match_subdomains(domain, resp.text)
+            subdomains = self.match_subdomains(resp.text, fuzzy=False)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             if not full_search:

modules/search/bing_api.py

@@ -37,7 +37,7 @@ class BingAPI(Search):
             resp = self.get(self.addr, params)
             if not resp:
                 return
-            subdomains = self.match_subdomains(domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             if not full_search:

modules/search/fofa_api.py

@@ -37,7 +37,7 @@ class FoFa(Search):
             if not resp:
                 return
             resp_json = resp.json()
-            subdomains = self.match_subdomains(self.domain, str(resp_json))
+            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             self.subdomains = self.subdomains.union(subdomains)

modules/search/gitee.py

@@ -10,7 +10,7 @@ class Gitee(Search):
         self.source = 'GiteeSearch'
         self.module = 'Search'
         self.addr = 'https://search.gitee.com/'
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.header = self.get_header()
 
     def search(self, full_search=False):
@@ -34,7 +34,7 @@ class Gitee(Search):
             if 'class="empty-box"' in resp.text:
                 break
             soup = BeautifulSoup(resp.text, 'html.parser')
-            subdomains = self.match_subdomains(self.domain, soup.text)
+            subdomains = self.match_subdomains(soup.text, fuzzy=False)
             if not subdomains:
                 break
             if not full_search:

modules/search/github_api.py

@@ -10,7 +10,7 @@ class GithubAPI(Search):
         self.source = 'GithubAPISearch'
         self.module = 'Search'
         self.addr = 'https://api.github.com/search/code'
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.session = requests.Session()
         self.auth_url = 'https://api.github.com'
         self.token = api.github_api_token
@@ -60,7 +60,7 @@ class GithubAPI(Search):
             if resp.status_code != 200:
                 logger.log('ERROR', f'{self.source} module query failed')
                 break
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             if not subdomains:
                 break
             self.subdomains = self.subdomains.union(subdomains)

modules/search/google.py

@@ -40,7 +40,7 @@ class Google(Search):
             resp = self.get(url=self.addr, params=payload)
             if not resp:
                 return
-            subdomains = self.match_subdomains(domain, resp.text)
+            subdomains = self.match_subdomains(resp.text, fuzzy=False)
             if not subdomains:
                 break
             if not full_search:

modules/search/google_api.py

@@ -35,7 +35,7 @@ class GoogleAPI(Search):
             resp = self.get(self.addr, params)
             if not resp:
                 return
-            subdomains = self.match_subdomains(domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             if not subdomains:
                 break
             if not full_search:

modules/search/shodan_api.py

@@ -5,7 +5,7 @@ from common.search import Search
 class ShodanAPI(Search):
     def __init__(self, domain):
         Search.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Search'
         self.source = 'ShodanAPISearch'
         self.addr = 'https://api.shodan.io/shodan/host/search'
@@ -25,7 +25,7 @@ class ShodanAPI(Search):
             resp = self.get(self.addr, params)
             if not resp:
                 return
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             if subdomains:

modules/search/so.py

@@ -31,7 +31,7 @@ class So(Search):
             resp = self.get(url=self.addr, params=payload)
             if not resp:
                 return
-            subdomains = self.match_subdomains(domain, resp.text)
+            subdomains = self.match_subdomains(resp.text, fuzzy=False)
             if not subdomains:
                 break
             if not full_search:

modules/search/sogou.py

@@ -28,7 +28,7 @@ class Sogou(Search):
             resp = self.get(self.addr, payload)
             if not resp:
                 return
-            subdomains = self.match_subdomains(domain, resp.text)
+            subdomains = self.match_subdomains(resp.text, fuzzy=False)
             if not subdomains:
                 break
             if not full_search:

modules/search/yahoo.py

@@ -36,7 +36,7 @@ class Yahoo(Search):
             if not resp:
                 return
             text = resp.text.replace('<b>', '').replace('</b>', '')
-            subdomains = self.match_subdomains(domain, text)
+            subdomains = self.match_subdomains(text, fuzzy=False)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             if not full_search:

modules/search/yandex.py

@@ -36,7 +36,7 @@ class Yandex(Search):
             resp = self.get(self.addr, params)
             if not resp:
                 return
-            subdomains = self.match_subdomains(domain, resp.text)
+            subdomains = self.match_subdomains(resp.text, fuzzy=False)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             if not full_search:

modules/search/zoomeye_api.py

@@ -48,7 +48,7 @@ class ZoomEyeAPI(Search):
             resp = self.get(self.addr, params)
             if not resp:
                 return
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             self.subdomains = self.subdomains.union(subdomains)

oneforall.py

@@ -198,7 +198,7 @@ class OneForAll(object):
         # Resolve subdomains
         self.data = resolve.run_resolve(self.domain, self.data)
         # Save resolve results
-        resolve.save_data(self.resolve_table, self.data)
+        resolve.save_db(self.resolve_table, self.data)
 
         # Export results without HTTP request
         if not self.req:
@@ -207,7 +207,7 @@ class OneForAll(object):
         # HTTP request
         self.data = request.run_request(self.domain, self.data, self.port)
         # Save HTTP request result
-        request.save_data(self.domain, self.data)
+        request.save_db(self.domain, self.data)
 
         # Add the final result list to the total data list
         self.datas.extend(self.data)