Merge remote-tracking branch 'origin/master'

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,38 +1,42 @@
 ---
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
+name: Please use this English template to submit Bug
+about: "Be sure to submit the Bug according to the template\U0001F64F"
+title: Please fill in the BUG title
+labels: bug
+assignees: shmilylty
 
 ---
 
-**Describe the bug**
-A clear and concise description of what the bug is.
+**Whether the latest code is used**
+Yes or no (if not, try to clone the latest code and run again!)
 
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
+**Bug description**
+Clear and concise Bug description(required)
 
-**Expected behavior**
-A clear and concise description of what you expected to happen.
+**Operation environment**
+ - System information: [e.g. Windows 10 x64] (required)
+ - Python version: [e.g. 3.7.1] (required)
+ - OneForAll version: [e.g. 0.3.0] (required)
  
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
+**How to reproduce**
+1. Step (optional)
 
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
+2. Command (required)
 
-**Smartphone (please complete the following information):**
- - Device: [e.g. iPhone6]
- - OS: [e.g. iOS8.1]
- - Browser [e.g. stock browser, safari]
- - Version [e.g. 22]
+**Error text**
+Copy the complete error text (required)
 
-**Additional context**
-Add any other context about the problem here.
+**Expected results**
+A clear and concise description of the expected results (optional, such as what a normal situation should look like)
+
+**Actual results**
+A clear and concise description of the actual results (optional, such as any errors)
+
+**Screenshot**
+Screenshot of complete OneForAll execution process (recommended upload)
+
+**Log upload**
+Upload oneforall.log log files (it is recommended to upload logs in case of complex problems)
+
+**Supplementary information**
+Some other supplementary notes about bug

.github/ISSUE_TEMPLATE/bug_report_zh.md

@@ -16,7 +16,7 @@ assignees: shmilylty
 **运行环境**
  - 系统：[例如Windows 10 x64]（必写）
  - Python版本：[例如3.7.1]（必写）
- - OneForAll版本：[例如0.0.6]（必写）
+ - OneForAll版本：[例如0.3.0]（必写）
  
 **如何复现**
 复现步骤（选写）
@@ -37,3 +37,6 @@ assignees: shmilylty
 
 **日志上传**
 上传oneforall.log日志文件（复杂问题建议上传）
+
+**其他补充**
+关于bug的其他一些补充说明

.github/ISSUE_TEMPLATE/custom.md

@@ -1,10 +0,0 @@
----
-name: Custom issue template
-about: Describe this issue template's purpose here.
-title: ''
-labels: ''
-assignees: ''
-
----
-
-

README.md

@@ -212,7 +212,7 @@ FLAGS
 
 为了解决以上痛点，此项目应用而生，正如其名，我希望OneForAll是一款集百家之长，功能强大的全面快速子域收集终极神器🔨。
 
-目前OneForAll还在开发中，肯定有不少问题和需要改进的地方，欢迎大佬们提交[Issues](https://github.com/shmilylty/OneForAll/issues)和[PR](https://github.com/shmilylty/OneForAll/pulls)，用着还行给个小星星✨吧，目前有一个专门用于OneForAll交流和反馈QQ群👨‍👨‍👦‍👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)（加群验证：我的英雄学院）。
+目前OneForAll还在开发中，肯定有不少问题和需要改进的地方，欢迎大佬们提交[Issues](https://github.com/shmilylty/OneForAll/issues)和[PR](https://github.com/shmilylty/OneForAll/pulls)，用着还行给个小星星✨吧，目前有一个专门用于OneForAll交流和反馈QQ群👨‍👨‍👦‍👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)（加群验证：信息收集）。
 
 ## 👍功能特性
 

brute.py

@@ -157,7 +157,7 @@ def query_domain_ns_a(ns_list):
 
 def query_domain_ns(domain):
     logger.log('INFOR', f'Querying NS records of {domain}')
-    domain = utils.get_maindomain(domain)
+    domain = utils.get_main_domain(domain)
     resolver = utils.dns_resolver()
     try:
         answer = resolver.query(domain, 'NS')
@@ -508,7 +508,7 @@ class Brute(Module):
         if self.place is None:
             self.place = '*.' + domain
         wordlist = self.wordlist
-        main_domain = self.get_maindomain(domain)
+        main_domain = utils.get_main_domain(domain)
         if domain != main_domain:
             wordlist = self.recursive_nextlist
         if self.word:

common/module.py

@@ -213,16 +213,6 @@ class Module(object):
     def match_subdomains(self, html, distinct=True, fuzzy=True):
         return utils.match_subdomains(self.domain, html, distinct, fuzzy)
 
-    @staticmethod
-    def get_maindomain(domain):
-        """
-        Get main domain
-
-        :param str domain: domain
-        :return: main domain
-        """
-        return Domain(domain).registered()
-
     def save_json(self):
         """
         Save the results of each module as a json file

common/utils.py

@@ -622,7 +622,7 @@ def check_version(local):
         logger.log('INFOR', f'The current version {local} is already the latest version')
 
 
-def get_maindomain(domain):
+def get_main_domain(domain):
     return Domain(domain).registered()
 
 

modules/autotake/github.py

@@ -23,12 +23,12 @@ def github_takeover(url):
     print('[*]正在读取配置文件')
     user = api.github_api_user
     token = api.github_api_token
-    CHECK_HEADERS = {
+    headers = {
         "Authorization": 'token ' + token,
         "Accept": "application/vnd.github.switcheroo-preview+json"
     }
     repos_url = 'https://api.github.com/repos/' + user + '/' + repo_name
-    repos_r = requests.get(url=repos_url, headers=CHECK_HEADERS)
+    repos_r = requests.get(url=repos_url, headers=headers)
     # 验证token是否正确
     if 'message' in repos_r.json():
         if repos_r.json()['message'] == 'Bad credentials':
@@ -41,7 +41,7 @@ def github_takeover(url):
             }
             creat_repo_url = 'https://api.github.com/user/repos'
             creat_repo_r = requests.post(url=creat_repo_url,
-                                         headers=CHECK_HEADERS,
+                                         headers=headers,
                                          data=json.dumps(creat_repo_dict))
             creat_repo_status = creat_repo_r.status_code
             if creat_repo_status == 201:
@@ -76,9 +76,9 @@ def github_takeover(url):
                 html_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/index.html'
                 url_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/CNAME'
                 html_r = requests.put(url=html_url, data=json.dumps(html_dict),
-                                      headers=CHECK_HEADERS)  # 上传index.html
+                                      headers=headers)  # 上传index.html
                 cname_r = requests.put(url=url_url, data=json.dumps(url_dict),
-                                       headers=CHECK_HEADERS)  # 上传CNAME
+                                       headers=headers)  # 上传CNAME
                 rs = cname_r.status_code
                 if rs == 201:
                     print('[*]生成接管库成功，正在开启Github pages')
@@ -90,7 +90,7 @@ def github_takeover(url):
                     }
                     page_r = requests.post(url=page_url,
                                            data=json.dumps(page_dict),
-                                           headers=CHECK_HEADERS)  # 开启page
+                                           headers=headers)  # 开启page
                     if page_r.status_code == 201:
                         print('[+]自动接管成功，请稍后访问http://' + str(url) + '查看结果')
                     else:

modules/certificates/censys_api.py

@@ -6,7 +6,7 @@ from config.log import logger
 class CensysAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Certificate'
         self.source = "CensysAPIQuery"
         self.addr = 'https://www.censys.io/api/v1/search/certificates'

modules/certificates/certspotter.py

@@ -1,4 +1,3 @@
-from common import utils
 from common.query import Query
 
 

modules/certificates/crtsh.py

@@ -1,11 +1,10 @@
-from common import utils
 from common.query import Query
 
 
 class Crtsh(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Certificate'
         self.source = 'CrtshQuery'
         self.addr = 'https://crt.sh/'

modules/certificates/google.py

@@ -1,11 +1,10 @@
-from common import utils
 from common.query import Query
 
 
 class Google(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Certificate'
         self.source = 'GoogleQuery'
         self.addr = 'https://transparencyreport.google.com/' \

modules/check/axfr.py

@@ -22,7 +22,7 @@ class CheckAXFR(Module):
 
     def __init__(self, domain: str):
         Module.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Check'
         self.source = 'AXFRCheck'
         self.results = []

modules/check/cdx.py

@@ -3,7 +3,6 @@
 """
 
 from common.module import Module
-from common import utils
 
 
 class CheckCDX(Module):
@@ -12,7 +11,7 @@ class CheckCDX(Module):
     """
     def __init__(self, domain: str):
         Module.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Check'
         self.source = "CrossDomainXml"
 

modules/check/cert.py

@@ -6,7 +6,6 @@
 import socket
 import ssl
 
-from common import utils
 from common.module import Module
 from config.log import logger
 
@@ -14,7 +13,7 @@ from config.log import logger
 class CheckCert(Module):
     def __init__(self, domain):
         Module.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.port = 443  # ssl port
         self.module = 'Check'
         self.source = 'CertInfo'

modules/check/csp.py

@@ -3,7 +3,6 @@ Collect subdomains from ContentSecurityPolicy
 """
 import requests
 
-from common import utils
 from common.module import Module
 from config.log import logger
 
@@ -14,7 +13,7 @@ class CheckCSP(Module):
     """
     def __init__(self, domain, header):
         Module.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Check'
         self.source = 'ContentSecurityPolicy'
         self.csp_header = header
@@ -45,7 +44,7 @@ class CheckCSP(Module):
         """
         if not self.csp_header:
             self.csp_header = self.grab_header()
-        csp = self.header.get('Content-Security-Policy')
+        csp = self.csp_header.get('Content-Security-Policy')
         if not self.csp_header:
             logger.log('DEBUG', f'Failed to get header of {self.domain} domain')
             return
@@ -79,4 +78,4 @@ def do(domain, header=None):  # 统一入口名字 方便多线程调用
 
 if __name__ == '__main__':
     resp = requests.get('https://content-security-policy.com/')
-    do('google-analytics.com', resp.headers)
+    do('google-analytics.com', dict(resp.headers))

modules/check/nsec.py

@@ -8,7 +8,7 @@ from common import utils
 class CheckNSEC(Module):
     def __init__(self, domain):
         Module.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'check'
         self.source = "CheckNSEC"
 

modules/check/robots.py

@@ -2,7 +2,6 @@
 检查内容安全策略收集子域名收集子域名
 """
 from common.module import Module
-from common import utils
 
 
 class CheckRobots(Module):
@@ -11,7 +10,7 @@ class CheckRobots(Module):
     """
     def __init__(self, domain):
         Module.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Check'
         self.source = 'Robots'
 

modules/check/sitemap.py

@@ -2,7 +2,6 @@
 检查内容安全策略收集子域名收集子域名
 """
 from common.module import Module
-from common import utils
 
 
 class CheckRobots(Module):
@@ -11,7 +10,7 @@ class CheckRobots(Module):
     """
     def __init__(self, domain):
         Module.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Check'
         self.source = 'Sitemap'
 

modules/collect.py

@@ -5,6 +5,7 @@ import importlib
 import dbexport
 from config.log import logger
 from config import setting
+from common import utils
 
 
 class Collect(object):
@@ -13,7 +14,7 @@ class Collect(object):
     """
 
     def __init__(self, domain, export=True):
-        self.domain = domain
+        self.domain = utils.get_main_domain(domain)
         self.elapse = 0.0
         self.modules = []
         self.collect_funcs = []

modules/datasets/binaryedge_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class BinaryEdgeAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'BinaryEdgeAPIQuery'
         self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/'

modules/datasets/bufferover.py

@@ -6,7 +6,7 @@ from config.log import logger
 class BufferOver(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'BufferOverQuery'
         self.addr = 'https://dns.bufferover.run/dns?q='

modules/datasets/cebaidu.py

@@ -4,7 +4,7 @@ from common.query import Query
 class CeBaidu(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'CeBaiduQuery'
         self.addr = 'https://ce.baidu.com/index/getRelatedSites'

modules/datasets/chinaz.py

@@ -4,7 +4,7 @@ from common.query import Query
 class Chinaz(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'ChinazQuery'
         self.addr = 'https://alexa.chinaz.com/'

modules/datasets/chinaz_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class ChinazAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'ChinazAPIQuery'
         self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa'

modules/datasets/circl_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class CirclAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'CirclAPIQuery'
         self.addr = 'https://www.circl.lu/pdns/query/'

modules/datasets/cloudflare_api.py

@@ -42,8 +42,6 @@ class CloudFlareAPI(Query):
                     else:
                         return
                 elif zones_resp.json()['success']:
-                    zone_id = zones_resp.json()['result'][0]['id']
-                    delete_zone_resp = self.delete(self.addr + f'zones/{zone_id}', check=False)
                     zone_id = self.create_zone(account_id)
                     if zone_id:
                         self.list_dns(zone_id)

modules/datasets/dnsdb_api.py

@@ -1,12 +1,11 @@
 from config import api
-from common import utils
 from common.query import Query
 
 
 class DNSdbAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'DNSdbAPIQuery'
         self.addr = 'https://api.dnsdb.info/lookup/rrset/name/'

modules/datasets/dnsdumpster.py

@@ -1,11 +1,10 @@
-from common import utils
 from common.query import Query
 
 
 class DNSdumpster(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = "DNSdumpsterQuery"
         self.addr = 'https://dnsdumpster.com/'

modules/datasets/hackertarget.py

@@ -1,11 +1,10 @@
-from common import utils
 from common.query import Query
 
 
 class HackerTarget(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = "HackerTargetQuery"
         self.addr = 'https://api.hackertarget.com/hostsearch/'

modules/datasets/ip138.py

@@ -4,7 +4,7 @@ from common.query import Query
 class IP138(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'IP138Query'
         self.addr = 'https://site.ip138.com/{domain}/domain.htm'

modules/datasets/ipv4info_api.py

@@ -6,7 +6,7 @@ from config.log import logger
 class IPv4InfoAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'IPv4InfoAPIQuery'
         self.addr = ' http://ipv4info.com/api_v1/'

modules/datasets/netcraft.py

@@ -9,7 +9,7 @@ from common.query import Query
 class NetCraft(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'NetCraftQuery'
         self.init = 'https://searchdns.netcraft.com/'

modules/datasets/passivedns_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class PassiveDnsAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'PassiveDnsQuery'
         self.addr = api.passivedns_api_addr or 'http://api.passivedns.cn'

modules/datasets/phonebook.py

@@ -5,7 +5,7 @@ from common.query import Query
 class PhoneBook(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'PhoneBookQuery'
 

modules/datasets/rapiddns.py

@@ -1,11 +1,10 @@
-from common import utils
 from common.query import Query
 
 
 class RapidDNS(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'RapidDNSQuery'
 

modules/datasets/riddler.py

@@ -4,7 +4,7 @@ from common.query import Query
 class Riddler(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'RiddlerQuery'
         self.addr = 'https://riddler.io/search'

modules/datasets/robtex.py

@@ -7,7 +7,7 @@ from common.query import Query
 class Robtex(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = "RobtexQuery"
         self.addr = 'https://freeapi.robtex.com/pdns/'

modules/datasets/securitytrails_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class SecurityTrailsAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'SecurityTrailsAPIQuery'
         self.addr = 'https://api.securitytrails.com/v1/domain/'

modules/datasets/sitedossier.py

@@ -5,7 +5,7 @@ from common.query import Query
 class SiteDossier(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'SiteDossierQuery'
         self.addr = 'http://www.sitedossier.com/parentdomain/'

modules/datasets/spyse_api.py

@@ -1,5 +1,4 @@
 from config import api
-from common import utils
 from common.query import Query
 
 
@@ -21,7 +20,7 @@ class SpyseAPI(Query):
             self.header = self.get_header()
             self.header.update({'Authorization': 'Bearer ' + self.token})
             self.proxy = self.get_proxy(self.source)
-            addr = 'https://api.spyse.com/v2/data/domain/subdomain'
+            addr = 'https://api.spyse.com/v3/data/domain/subdomain'
             params = {'domain': self.domain, 'offset': offset, 'limit': limit}
             resp = self.get(addr, params)
             if not resp:

modules/datasets/sublist3r.py

@@ -4,7 +4,7 @@ from common.query import Query
 class Sublist3r(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'Sublist3rQuery'
 

modules/datasets/threatcrowd.py

@@ -7,7 +7,7 @@ from config.log import logger
 class ThreatCrowd(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'ThreatCrowdQuery'
         self.addr = 'https://www.threatcrowd.org/searchApi' \

modules/datasets/wzpc.py

@@ -6,7 +6,7 @@ from common.query import Query
 class WZPCQuery(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'WZPCQuery'
 

modules/datasets/ximcx.py

@@ -4,7 +4,7 @@ from common.query import Query
 class Ximcx(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Dataset'
         self.source = 'XimcxQuery'
         self.addr = 'http://sbd.ximcx.cn/DomainServlet'

modules/dnsquery/mx.py

@@ -4,7 +4,7 @@ from common.lookup import Lookup
 class QueryMX(Lookup):
     def __init__(self, domain):
         Lookup.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'dnsquery'
         self.source = "QueryMX"
         self.type = 'MX'  # 利用的DNS记录的MX记录收集子域

modules/dnsquery/ns.py

@@ -4,7 +4,7 @@ from common.lookup import Lookup
 class QueryNS(Lookup):
     def __init__(self, domain):
         Lookup.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'dnsquery'
         self.source = "QueryNS"
         self.type = 'NS'  # 利用的DNS记录的NS记录收集子域

modules/dnsquery/soa.py

@@ -4,7 +4,7 @@ from common.lookup import Lookup
 class QuerySOA(Lookup):
     def __init__(self, domain):
         Lookup.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'dnsquery'
         self.source = "QuerySOA"
         self.type = 'SOA'  # 利用的DNS记录的SOA记录收集子域

modules/dnsquery/spf.py

@@ -4,7 +4,7 @@ from common.lookup import Lookup
 class QuerySPF(Lookup):
     def __init__(self, domain):
         Lookup.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'dnsquery'
         self.source = "QuerySPF"
         self.type = 'SPF'  # 利用的DNS记录的SPF记录收集子域

modules/dnsquery/srv.py

@@ -14,7 +14,7 @@ from config.setting import data_storage_dir
 class BruteSRV(Module):
     def __init__(self, domain):
         Module.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'dnsquery'
         self.source = "BruteSRV"
         self.type = 'SRV'  # 利用的DNS记录的SRV记录查询子域

modules/dnsquery/txt.py

@@ -4,7 +4,7 @@ from common.lookup import Lookup
 class QueryTXT(Lookup):
     def __init__(self, domain):
         Lookup.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'dnsquery'
         self.source = "QueryTXT"
         self.type = 'TXT'  # 利用的DNS记录的TXT记录收集子域

modules/intelligence/alienvault.py

@@ -4,7 +4,7 @@ from common.query import Query
 class AlienVault(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Intelligence'
         self.source = 'AlienVaultQuery'
 

modules/intelligence/riskiq_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class RiskIQ(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Intelligence'
         self.source = 'RiskIQAPIQuery'
         self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'

modules/intelligence/threatbook_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class ThreatBookAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Intelligence'
         self.source = 'ThreatBookAPIQuery'
         self.addr = 'https://api.threatbook.cn/v3/domain/sub_domains'

modules/intelligence/threatminer.py

@@ -4,7 +4,7 @@ from common.query import Query
 class ThreatMiner(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Intelligence'
         self.source = 'ThreatMinerQuery'
         self.addr = 'https://www.threatminer.org/getData.php'

modules/intelligence/virustotal.py

@@ -11,7 +11,7 @@ class VirusTotal(Query):
         self.source = 'VirusTotalQuery'
         self.module = 'Intelligence'
         self.addr = 'https://www.virustotal.com/ui/domains/{}/subdomains'
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
 
     def query(self):
         """

modules/intelligence/virustotal_api.py

@@ -5,7 +5,7 @@ from common.query import Query
 class VirusTotalAPI(Query):
     def __init__(self, domain):
         Query.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Intelligence'
         self.source = 'VirusTotalAPIQuery'
         self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'

modules/search/baidu.py

@@ -13,11 +13,10 @@ class Baidu(Search):
         self.domain = domain
         self.limit_num = 750  # 限制搜索条数
 
-    def redirect_match(self, domain, html):
+    def redirect_match(self, html):
         """
         获取跳转地址并传递地址进行跳转head请求
 
-        :param domain: 域名
         :param html: 响应体
         :return: 子域
         """
@@ -26,7 +25,7 @@ class Baidu(Search):
         # 获取搜索结果中所有的跳转URL地址
         for find_res in bs.find_all('a', {'class': 'c-showurl'}):
             url = find_res.get('href')
-            subdomains = self.match_location(domain, url)
+            subdomains = self.match_location(url)
             subdomains_all = subdomains_all.union(subdomains)
         return subdomains_all
 
@@ -52,7 +51,7 @@ class Baidu(Search):
                 return
             if len(domain) > 12:  # 解决百度搜索结果中域名过长会显示不全的问题
                 # 获取百度跳转URL响应头的Location字段获取直链
-                subdomains = self.redirect_match(domain, resp.text)
+                subdomains = self.redirect_match(resp.text)
             else:
                 subdomains = self.match_subdomains(resp.text, fuzzy=False)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
@@ -106,4 +105,4 @@ def do(domain):  # 统一入口名字 方便多线程调用
 
 
 if __name__ == '__main__':
-    do('huayunshuzi.com')
+    do('mi.com')

modules/search/gitee.py

@@ -10,7 +10,7 @@ class Gitee(Search):
         self.source = 'GiteeSearch'
         self.module = 'Search'
         self.addr = 'https://search.gitee.com/'
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.header = self.get_header()
 
     def search(self, full_search=False):

modules/search/github_api.py

@@ -10,7 +10,7 @@ class GithubAPI(Search):
         self.source = 'GithubAPISearch'
         self.module = 'Search'
         self.addr = 'https://api.github.com/search/code'
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.session = requests.Session()
         self.auth_url = 'https://api.github.com'
         self.token = api.github_api_token

modules/search/shodan_api.py

@@ -5,7 +5,7 @@ from common.search import Search
 class ShodanAPI(Search):
     def __init__(self, domain):
         Search.__init__(self)
-        self.domain = self.get_maindomain(domain)
+        self.domain = domain
         self.module = 'Search'
         self.source = 'ShodanAPISearch'
         self.addr = 'https://api.shodan.io/shodan/host/search'