优化

modules/datasets/binaryedge_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class BinaryEdgeAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'BinaryEdgeAPIQuery'
         self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/'
@@ -22,7 +22,7 @@ class BinaryEdgeAPI(Query):
         resp = self.get(url)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/datasets/bufferover.py

@@ -6,7 +6,7 @@ from config.log import logger
 class BufferOver(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'BufferOverQuery'
         self.addr = 'https://dns.bufferover.run/dns?q='
@@ -26,7 +26,7 @@ class BufferOver(Query):
             return
         if resp.status_code != 200:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/cebaidu.py

@@ -4,7 +4,7 @@ from common.query import Query
 class CeBaidu(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'CeBaiduQuery'
         self.addr = 'https://ce.baidu.com/index/getRelatedSites'
@@ -19,7 +19,7 @@ class CeBaidu(Query):
         resp = self.get(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/chinaz.py

@@ -4,7 +4,7 @@ from common.query import Query
 class Chinaz(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'ChinazQuery'
         self.addr = 'https://alexa.chinaz.com/'
@@ -19,7 +19,7 @@ class Chinaz(Query):
         resp = self.get(self.addr)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/chinaz_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class ChinazAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'ChinazAPIQuery'
         self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa'
@@ -21,7 +21,7 @@ class ChinazAPI(Query):
         resp = self.get(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/circl_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class CirclAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'CirclAPIQuery'
         self.addr = 'https://www.circl.lu/pdns/query/'
@@ -21,7 +21,7 @@ class CirclAPI(Query):
         resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/cloudflare_api.py

@@ -79,7 +79,7 @@ class CloudFlareAPI(Query):
             logger.log('DEBUG',
                        f'{list_dns_resp.status_code} {list_dns_resp.text}')
             return
-        subdomains = self.match_subdomains(self.domain, list_dns_resp.text)
+        subdomains = self.match_subdomains(list_dns_resp.text)
         self.subdomains = self.subdomains.union(subdomains)
         if not self.subdomains:
             # waiting for cloudflare enumerate subdomains
@@ -94,7 +94,7 @@ class CloudFlareAPI(Query):
                                f'{list_dns_resp.status_code} {list_dns_resp.text}')
                     return
                 total_pages = list_dns_resp.json()['result_info']['total_pages']
-                subdomains = (self.match_subdomains(self.domain, list_dns_resp.text))
+                subdomains = (self.match_subdomains(list_dns_resp.text))
                 self.subdomains = self.subdomains.union(subdomains)
                 page += 1
                 if page > total_pages:

modules/datasets/dnsdb_api.py

@@ -6,7 +6,7 @@ from common.query import Query
 class DNSdbAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'DNSdbAPIQuery'
         self.addr = 'https://api.dnsdb.info/lookup/rrset/name/'
@@ -23,7 +23,7 @@ class DNSdbAPI(Query):
         resp = self.get(url)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/dnsdumpster.py

@@ -5,7 +5,7 @@ from common.query import Query
 class DNSdumpster(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = "DNSdumpsterQuery"
         self.addr = 'https://dnsdumpster.com/'
@@ -26,7 +26,7 @@ class DNSdumpster(Query):
         resp = self.post(self.addr, data)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         if subdomains:
             # 合并搜索子域名搜索结果
             self.subdomains = self.subdomains.union(subdomains)

modules/datasets/hackertarget.py

@@ -5,7 +5,7 @@ from common.query import Query
 class HackerTarget(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = "HackerTargetQuery"
         self.addr = 'https://api.hackertarget.com/hostsearch/'
@@ -21,7 +21,7 @@ class HackerTarget(Query):
         if not resp:
             return
         if resp.status_code == 200:
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             if subdomains:
                 # 合并搜索子域名搜索结果
                 self.subdomains = self.subdomains.union(subdomains)

modules/datasets/ip138.py

@@ -4,7 +4,7 @@ from common.query import Query
 class IP138(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'IP138Query'
         self.addr = 'https://site.ip138.com/{domain}/domain.htm'
@@ -19,7 +19,7 @@ class IP138(Query):
         resp = self.get(self.addr)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/ipv4info_api.py

@@ -6,7 +6,7 @@ from config.log import logger
 class IPv4InfoAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'IPv4InfoAPIQuery'
         self.addr = ' http://ipv4info.com/api_v1/'
@@ -32,7 +32,7 @@ class IPv4InfoAPI(Query):
             except Exception as e:
                 logger.log('DEBUG', e.args)
                 break
-            subdomains = self.match_subdomains(self.domain, str(json))
+            subdomains = self.match_subdomains(str(json))
             if not subdomains:
                 break
             # 合并搜索子域名搜索结果

modules/datasets/netcraft.py

@@ -9,7 +9,7 @@ from common.query import Query
 class NetCraft(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'NetCraftQuery'
         self.init = 'https://searchdns.netcraft.com/'
@@ -49,7 +49,7 @@ class NetCraft(Query):
             resp = self.get(self.addr + last, params)
             if not resp:
                 return
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             # 合并搜索子域名搜索结果

modules/datasets/passivedns_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class PassiveDnsAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'PassiveDnsQuery'
         self.addr = api.passivedns_api_addr or 'http://api.passivedns.cn'
@@ -22,7 +22,7 @@ class PassiveDnsAPI(Query):
         resp = self.get(url)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/phonebook.py

@@ -5,7 +5,7 @@ from common.query import Query
 class PhoneBook(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'PhoneBookQuery'
 
@@ -35,7 +35,7 @@ class PhoneBook(Query):
         resp = self.get(url)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/datasets/qianxun.py

@@ -27,7 +27,7 @@ class QianXun(Query):
             resp = self.post(url, data)
             if not resp:
                 break
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             self.subdomains = self.subdomains.union(subdomains)
             if '<div id="page" class="pagelist">' not in resp.text:
                 break

modules/datasets/rapiddns.py

@@ -5,7 +5,7 @@ from common.query import Query
 class RapidDNS(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'RapidDNSQuery'
 
@@ -20,7 +20,7 @@ class RapidDNS(Query):
         resp = self.get(url, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/riddler.py

@@ -4,7 +4,7 @@ from common.query import Query
 class Riddler(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'RiddlerQuery'
         self.addr = 'https://riddler.io/search'
@@ -19,7 +19,7 @@ class Riddler(Query):
         resp = self.get(self.addr, params)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/robtex.py

@@ -7,7 +7,7 @@ from common.query import Query
 class Robtex(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = "RobtexQuery"
         self.addr = 'https://freeapi.robtex.com/pdns/'
@@ -32,7 +32,7 @@ class Robtex(Query):
                 resp = self.get(url)
                 if not resp:
                     return
-                subdomains = self.match_subdomains(self.domain, resp.text)
+                subdomains = self.match_subdomains(resp.text)
                 if subdomains:
                     # 合并搜索子域名搜索结果
                     self.subdomains = self.subdomains.union(subdomains)

modules/datasets/securitytrails_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class SecurityTrailsAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'SecurityTrailsAPIQuery'
         self.addr = 'https://api.securitytrails.com/v1/domain/'

modules/datasets/sitedossier.py

@@ -5,7 +5,7 @@ from common.query import Query
 class SiteDossier(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'SiteDossierQuery'
         self.addr = 'http://www.sitedossier.com/parentdomain/'
@@ -24,7 +24,7 @@ class SiteDossier(Query):
             resp = self.get(url)
             if not resp:
                 return
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             # 合并搜索子域名搜索结果

modules/datasets/spyse_api.py

@@ -27,7 +27,7 @@ class SpyseAPI(Query):
             if not resp:
                 return
             json = resp.json()
-            subdomains = self.match_subdomains(self.domain, str(json))
+            subdomains = self.match_subdomains(str(json))
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             # 合并搜索子域名搜索结果

modules/datasets/sublist3r.py

@@ -4,7 +4,7 @@ from common.query import Query
 class Sublist3r(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'Sublist3rQuery'
 
@@ -19,7 +19,7 @@ class Sublist3r(Query):
         resp = self.get(addr, param)
         if not resp:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):

modules/datasets/threatcrowd.py

@@ -7,7 +7,7 @@ from config.log import logger
 class ThreatCrowd(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'ThreatCrowdQuery'
         self.addr = 'https://www.threatcrowd.org/searchApi' \
@@ -25,7 +25,7 @@ class ThreatCrowd(Query):
             return
         if resp.status_code != 200:
             return
-        subdomains = self.match_subdomains(self.domain, resp.text)
+        subdomains = self.match_subdomains(resp.text)
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)
 

modules/datasets/wzpc.py

@@ -6,7 +6,7 @@ from common.query import Query
 class WZPCQuery(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'WZPCQuery'
 
@@ -31,7 +31,7 @@ class WZPCQuery(Query):
                 break
             if not resp:
                 break
-            subdomains = self.match_subdomains(self.domain, resp.text)
+            subdomains = self.match_subdomains(resp.text)
             self.subdomains = self.subdomains.union(subdomains)
             if not subdomains:
                 break

modules/datasets/ximcx.py

@@ -4,7 +4,7 @@ from common.query import Query
 class Ximcx(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.register(domain)
+        self.domain = self.get_maindomain(domain)
         self.module = 'Dataset'
         self.source = 'XimcxQuery'
         self.addr = 'http://sbd.ximcx.cn/DomainServlet'
@@ -20,7 +20,7 @@ class Ximcx(Query):
         if not resp:
             return
         json = resp.json()
-        subdomains = self.match_subdomains(self.domain, str(json))
+        subdomains = self.match_subdomains(str(json))
         # 合并搜索子域名搜索结果
         self.subdomains = self.subdomains.union(subdomains)