重构项目目录结构

modules/datasets/binaryedge_api.py

@@ -0,0 +1,53 @@
+from config import api
+from common.query import Query
+
+
+class BinaryEdgeAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'BinaryEdgeAPIQuery'
+        self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/'
+        self.api = api.binaryedge_api
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({'X-Key': self.api})
+        self.proxy = self.get_proxy(self.source)
+        url = self.addr + self.domain
+        resp = self.get(url)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.api):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = BinaryEdgeAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/bufferover.py

@@ -0,0 +1,57 @@
+import cloudscraper
+from common.query import Query
+from config.log import logger
+
+
+class BufferOver(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'BufferOverQuery'
+        self.addr = 'https://dns.bufferover.run/dns?q='
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        # 绕过cloudFlare验证
+        scraper = cloudscraper.create_scraper()
+        scraper.interpreter = 'js2py'
+        scraper.proxies = self.get_proxy(self.source)
+        url = self.addr + self.domain
+        try:
+            resp = scraper.get(url, timeout=self.timeout)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return
+        if resp.status_code != 200:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = BufferOver(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/cebaidu.py

@@ -0,0 +1,49 @@
+from common.query import Query
+
+
+class CeBaidu(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'CeBaiduQuery'
+        self.addr = 'https://ce.baidu.com/index/getRelatedSites'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'site_address': self.domain}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = CeBaidu(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/chinaz.py

@@ -0,0 +1,49 @@
+from common.query import Query
+
+
+class Chinaz(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'ChinazQuery'
+        self.addr = 'https://alexa.chinaz.com/'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        self.addr = self.addr + self.domain
+        resp = self.get(self.addr)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Chinaz(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/chinaz_api.py

@@ -0,0 +1,53 @@
+from config import api
+from common.query import Query
+
+
+class ChinazAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'ChinazAPIQuery'
+        self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa'
+        self.api = api.chinaz_api
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'key': self.api, 'domainName': self.domain}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.api):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = ChinazAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/circl_api.py

@@ -0,0 +1,53 @@
+from config import api
+from common.query import Query
+
+
+class CirclAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'CirclAPIQuery'
+        self.addr = 'https://www.circl.lu/pdns/query/'
+        self.user = api.circl_api_username
+        self.pwd = api.circl_api_password
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.user, self.pwd):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = CirclAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/dnsdb_api.py

@@ -0,0 +1,56 @@
+from config import api
+from common import utils
+from common.query import Query
+
+
+class DNSdbAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'DNSdbAPIQuery'
+        self.addr = 'https://api.dnsdb.info/lookup/rrset/name/'
+        self.api = api.dnsdb_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({'X-API-Key': self.api})
+        self.proxy = self.get_proxy(self.source)
+        url = f'{self.addr}*.{self.domain}'
+        resp = self.get(url)
+        if not resp:
+            return
+        subdomains = utils.match_subdomain(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.api):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+
+    """
+    query = DNSdbAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/dnsdumpster.py

@@ -0,0 +1,58 @@
+from common import utils
+from common.query import Query
+
+
+class DNSdumpster(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = "DNSdumpsterQuery"
+        self.addr = 'https://dnsdumpster.com/'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({'Referer': 'https://dnsdumpster.com'})
+        self.proxy = self.get_proxy(self.source)
+        resp = self.get(self.addr)
+        if not resp:
+            return
+        self.cookie = resp.cookies
+        data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'),
+                'targetip': self.domain}
+        resp = self.post(self.addr, data)
+        if not resp:
+            return
+        subdomains = utils.match_subdomain(self.domain, resp.text)
+        if subdomains:
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = DNSdumpster(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+
+    do('example.com')

modules/datasets/hackertarget.py

@@ -0,0 +1,52 @@
+from common import utils
+from common.query import Query
+
+
+class HackerTarget(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = "HackerTargetQuery"
+        self.addr = 'https://api.hackertarget.com/hostsearch/'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'q': self.domain}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        if resp.status_code == 200:
+            subdomains = utils.match_subdomain(self.domain, resp.text)
+            if subdomains:
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = HackerTarget(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/ip138.py

@@ -0,0 +1,49 @@
+from common.query import Query
+
+
+class IP138(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'IP138Query'
+        self.addr = 'https://site.ip138.com/{domain}/domain.htm'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        self.addr = self.addr.format(domain=self.domain)
+        resp = self.get(self.addr)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = IP138(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/ipv4info_api.py

@@ -0,0 +1,75 @@
+from config import api
+from common.query import Query
+from config.log import logger
+
+
+class IPv4InfoAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'IPv4InfoAPIQuery'
+        self.addr = ' http://ipv4info.com/api_v1/'
+        self.api = api.ipv4info_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        page = 0
+        while True:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            params = {'type': 'SUBDOMAINS', 'key': self.api,
+                      'value': self.domain, 'page': page}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            if resp.status_code != 200:
+                break  # 请求不正常通常网络是有问题，不再继续请求下去
+            try:
+                json = resp.json()
+            except Exception as e:
+                logger.log('DEBUG', e.args)
+                break
+            subdomains = self.match(self.domain, str(json))
+            if not subdomains:
+                break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
+            subdomains = json.get('Subdomains')
+            if subdomains:
+                # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
+                if len(subdomains) < 300:
+                    break
+            page += 1
+            if page >= 50:  # ipv4info子域查询接口最多允许查询50页
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.api):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = IPv4InfoAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/netcraft.py

@@ -0,0 +1,83 @@
+import hashlib
+import re
+import time
+from urllib import parse
+
+from common.query import Query
+
+
+class NetCraft(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'NetCraftQuery'
+        self.init = 'https://searchdns.netcraft.com/'
+        self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains'
+        self.page_num = 1
+        self.per_page_num = 20
+
+    def bypass_verification(self):
+        """
+        绕过NetCraft的JS验证
+        """
+        self.header = self.get_header()  # Netcraft会检查User-Agent
+        resp = self.get(self.init)
+        if not resp:
+            return False
+        self.cookie = resp.cookies
+        cookie_value = self.cookie['netcraft_js_verification_challenge']
+        cookie_encode = parse.unquote(cookie_value).encode('utf-8')
+        verify_taken = hashlib.sha1(cookie_encode).hexdigest()
+        self.cookie['netcraft_js_verification_response'] = verify_taken
+        return True
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        if not self.bypass_verification():
+            return
+        last = ''
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            params = {'host': '*.' + self.domain, 'from': self.page_num}
+            resp = self.get(self.addr + last, params)
+            if not resp:
+                return
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            if 'Next page' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+                break
+            last = re.search(r'&last=.*' + self.domain, resp.text).group(0)
+            self.page_num += self.per_page_num
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = NetCraft(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/passivedns_api.py

@@ -0,0 +1,54 @@
+from config import api
+from common.query import Query
+
+
+class PassiveDnsAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'PassiveDnsQuery'
+        self.addr = api.passivedns_api_addr or 'http://api.passivedns.cn'
+        self.token = api.passivedns_api_token
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({'X-AuthToken': self.token})
+        self.proxy = self.get_proxy(self.source)
+        url = self.addr + '/flint/rrset/*.' + self.domain
+        resp = self.get(url)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.addr):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = PassiveDnsAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/ptrarchive.py

@@ -0,0 +1,56 @@
+import random
+
+from common import utils
+from common.query import Query
+
+
+class PTRArchive(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = "PTRArchiveQuery"
+        self.addr = 'http://ptrarchive.com/tools/search4.htm'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        # 绕过主页前端JS验证
+        self.cookie = {'pa_id': str(random.randint(0, 1000000000))}
+        params = {'label': self.domain, 'date': 'ALL'}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        if resp.status_code == 200:
+            subdomains = utils.match_subdomain(self.domain, resp.text)
+            if subdomains:
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = PTRArchive(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/qianxun.py

@@ -0,0 +1,61 @@
+from common.query import Query
+
+
+class QianXun(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Query'
+        self.source = 'QianXunQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+
+        num = 1
+        while True:
+            data = {'ecmsfrom': '',
+                    'show': '',
+                    'num': '',
+                    'classid': '0',
+                    'keywords': self.domain}
+            url = f'https://www.dnsscan.cn/dns.html?' \
+                  f'keywords={self.domain}&page={num}'
+            resp = self.post(url, data)
+            if not resp:
+                break
+            subdomains = self.match(self.domain, resp.text)
+            self.subdomains = self.subdomains.union(subdomains)
+            if '<div id="page" class="pagelist">' not in resp.text:
+                break
+            if '<li class="disabled"><span>&raquo;</span></li>' in resp.text:
+                break
+            num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = QianXun(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/rapiddns.py

@@ -0,0 +1,50 @@
+from common import utils
+from common.query import Query
+
+
+class RapidDNS(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'RapidDNSQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        url = f'http://rapiddns.io/subdomain/{self.domain}'
+        params = {'full': '1'}
+        resp = self.get(url, params)
+        if not resp:
+            return
+        subdomains = utils.match_subdomain(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = RapidDNS(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/riddler.py

@@ -0,0 +1,49 @@
+from common.query import Query
+
+
+class Riddler(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'RiddlerQuery'
+        self.addr = 'https://riddler.io/search'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'q': 'pld:' + self.domain}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Riddler(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/robtex.py

@@ -0,0 +1,63 @@
+import json
+import time
+
+from common.query import Query
+
+
+class Robtex(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = "RobtexQuery"
+        self.addr = 'https://freeapi.robtex.com/pdns/'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        url = self.addr + 'forward/' + self.domain
+        resp = self.get(url)
+        if not resp:
+            return
+        text_list = resp.text.splitlines()
+        text_json = list(map(lambda x: json.loads(x), text_list))
+        for record in text_json:
+            if record.get('rrtype') in ['A', 'AAAA']:
+                time.sleep(self.delay)  # Robtex有查询频率限制
+                ip = record.get('rrdata')
+                url = self.addr + 'reverse/' + ip
+                resp = self.get(url)
+                if not resp:
+                    return
+                subdomains = self.match(self.domain, resp.text)
+                if subdomains:
+                    # 合并搜索子域名搜索结果
+                    self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Robtex(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/securitytrails_api.py

@@ -0,0 +1,57 @@
+from config import api
+from common.query import Query
+
+
+class SecurityTrailsAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'SecurityTrailsAPIQuery'
+        self.addr = 'https://api.securitytrails.com/v1/domain/'
+        self.api = api.securitytrails_api
+        self.delay = 2  # SecurityTrails查询时延至少2秒
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'apikey': self.api}
+        url = f'{self.addr}{self.domain}/subdomains'
+        resp = self.get(url, params)
+        if not resp:
+            return
+        prefixs = resp.json()['subdomains']
+        subdomains = [f'{prefix}.{self.domain}' for prefix in prefixs]
+        if subdomains:
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.api):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = SecurityTrailsAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/sitedossier.py

@@ -0,0 +1,60 @@
+
+from common.query import Query
+
+
+class SiteDossier(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'SiteDossierQuery'
+        self.addr = 'http://www.sitedossier.com/parentdomain/'
+        self.delay = 2
+        self.page_num = 1
+        self.per_page_num = 100
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        while True:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            url = f'{self.addr}{self.domain}/{self.page_num}'
+            resp = self.get(url)
+            if not resp:
+                return
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            # 搜索页面没有出现下一页时停止搜索
+            if 'Show next 100 items' not in resp.text:
+                break
+            self.page_num += self.per_page_num
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = SiteDossier(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/datasets/threatcrowd.py

@@ -0,0 +1,56 @@
+import cloudscraper
+
+from common.query import Query
+from config.log import logger
+
+
+class ThreatCrowd(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'ThreatCrowdQuery'
+        self.addr = 'https://www.threatcrowd.org/searchApi' \
+                    '/v2/domain/report?domain='
+
+    def query(self):
+        # 绕过cloudFlare验证
+        scraper = cloudscraper.create_scraper()
+        scraper.interpreter = 'js2py'
+        scraper.proxies = self.get_proxy(self.source)
+        url = self.addr + self.domain
+        try:
+            resp = scraper.get(url, timeout=self.timeout)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return
+        if resp.status_code != 200:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = ThreatCrowd(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('mi.com')

modules/datasets/wzpc.py

@@ -0,0 +1,66 @@
+import time
+from config.log import logger
+from common.query import Query
+
+
+class WZPCQuery(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'WZPCQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+
+        base_addr = 'http://114.55.181.28/check_web/' \
+                    'databaseInfo_mainSearch.action'
+        page_num = 1
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            params = {'isSearch': 'true', 'searchType': 'url',
+                      'term': self.domain, 'pageNo': page_num}
+            try:
+                resp = self.get(base_addr, params)
+            except Exception as e:
+                logger.log('ERROR', e.args)
+                break
+            if not resp:
+                break
+            subdomains = self.match(self.domain, resp.text)
+            self.subdomains = self.subdomains.union(subdomains)
+            if not subdomains:
+                break
+            if page_num > 10:
+                break
+            page_num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = WZPCQuery(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('sc.gov.cn')
+    do('bkzy.org')

modules/datasets/ximcx.py

@@ -0,0 +1,50 @@
+from common.query import Query
+
+
+class Ximcx(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'XimcxQuery'
+        self.addr = 'http://sbd.ximcx.cn/DomainServlet'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        data = {'domain': self.domain}
+        resp = self.post(self.addr, data=data)
+        if not resp:
+            return
+        json = resp.json()
+        subdomains = self.match(self.domain, str(json))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Ximcx(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')