重构项目目录结构

modules/search/ask.py

@@ -0,0 +1,81 @@
+import time
+from common.search import Search
+
+
+class Ask(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'AskSearch'
+        self.addr = 'https://www.search.ask.com/web'
+        self.limit_num = 200  # 限制搜索条数
+        self.per_page_num = 10  # 默认每页显示10页
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 1
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'q': query, 'page': self.page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
+                break
+            if not full_search:
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
+                    break
+            self.subdomains = self.subdomains.union(subdomains)
+            self.page_num += 1
+            if '>Next<' not in resp.text:
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.search(self.domain, full_search=True)
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
+                for subdomain in self.subdomains:
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
+                        self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Ask(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/baidu.py

@@ -0,0 +1,109 @@
+import time
+from bs4 import BeautifulSoup
+from common.search import Search
+
+
+class Baidu(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.module = 'Search'
+        self.source = 'BaiduSearch'
+        self.init = 'https://www.baidu.com/'
+        self.addr = 'https://www.baidu.com/s'
+        self.domain = domain
+        self.limit_num = 750  # 限制搜索条数
+
+    def redirect_match(self, domain, html):
+        """
+        获取跳转地址并传递地址进行跳转head请求
+
+        :param domain: 域名
+        :param html: 响应体
+        :return: 子域
+        """
+        bs = BeautifulSoup(html, 'html.parser')
+        subdomains_all = set()
+        # 获取搜索结果中所有的跳转URL地址
+        for find_res in bs.find_all('a', {'class': 'c-showurl'}):
+            url = find_res.get('href')
+            subdomains = self.match_location(domain, url)
+            subdomains_all = subdomains_all.union(subdomains)
+        return subdomains_all
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 0  # 二次搜索重新置0
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'wd': query,
+                      'pn': self.page_num,
+                      'rn': self.per_page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            if len(domain) > 12:  # 解决百度搜索结果中域名过长会显示不全的问题
+                # 获取百度跳转URL响应头的Location字段获取直链
+                subdomains = self.redirect_match(domain, resp.text)
+            else:
+                subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            if not full_search:
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
+                    break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            self.page_num += self.per_page_num
+            # 搜索页面没有出现下一页时停止搜索
+            if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:
+                break
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.search(self.domain, full_search=True)
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
+                for subdomain in self.subdomains:
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
+                        self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Baidu(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('huayunshuzi.com')

modules/search/bing.py

@@ -0,0 +1,92 @@
+import time
+from common.search import Search
+
+
+class Bing(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'BingSearch'
+        self.init = 'https://www.bing.com/'
+        self.addr = 'https://www.bing.com/search'
+        self.limit_num = 1000  # 限制搜索条数
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 0  # 二次搜索重新置0
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        resp = self.get(self.init)
+        if not resp:
+            return
+        self.cookie = resp.cookies  # 获取cookie bing在搜索时需要带上cookie
+        while True:
+            time.sleep(self.delay)
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'q': query, 'first': self.page_num,
+                      'count': self.per_page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            if not full_search:
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
+                    break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            # 搜索页面没有出现下一页时停止搜索
+            if '<div class="sw_next">' not in resp.text:
+                break
+            self.page_num += self.per_page_num
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
+                for subdomain in self.subdomains:
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
+                        self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Bing(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/bing_api.py

@@ -0,0 +1,92 @@
+import time
+from config import api
+from common.search import Search
+
+
+class BingAPI(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'BingAPISearch'
+        self.addr = 'https://api.cognitive.microsoft.com/' \
+                    'bing/v7.0/search'
+        self.id = api.bing_api_id
+        self.key = api.bing_api_key
+        self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数
+        self.delay = 1  # 必应自定义搜索限制时延1秒
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 0  # 二次搜索重新置0
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.header = {'Ocp-Apim-Subscription-Key': self.key}
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'q': query, 'safesearch': 'Off',
+                      'count': self.per_page_num,
+                      'offset': self.page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomains = self.match(domain, str(resp.json()))
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            if not full_search:
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
+                    break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            self.page_num += self.per_page_num
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.id, self.key):
+            return
+        self.begin()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
+                for subdomain in self.subdomains:
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
+                        self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = BingAPI(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/exalead.py

@@ -0,0 +1,87 @@
+import random
+import time
+from common.search import Search
+
+
+class Exalead(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = "ExaleadSearch"
+        self.addr = "http://www.exalead.com/search/web/results/"
+        self.per_page_num = 30
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 0
+        while True:
+            self.delay = random.randint(1, 5)
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'q': query, 'elements_per_page': '30',
+                      "start_index": self.page_num}
+            resp = self.get(url=self.addr, params=params)
+            if not resp:
+                return
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
+                break
+            if not full_search:
+                if subdomains.issubset(self.subdomains):
+                    break
+            self.subdomains = self.subdomains.union(subdomains)
+            self.page_num += self.per_page_num
+            if self.page_num > 1999:
+                break
+            if 'title="Go to the next page"' not in resp.text:
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            statement = statement.replace('-site', 'and -site')
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
+                for subdomain in self.subdomains:
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
+                        self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Exalead(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/fofa_api.py

@@ -0,0 +1,73 @@
+import base64
+import time
+
+from config import api
+from common.search import Search
+
+
+class FoFa(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'FoFaAPISearch'
+        self.addr = 'https://fofa.so/api/v1/search/all'
+        self.delay = 1
+        self.email = api.fofa_api_email
+        self.key = api.fofa_api_key
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        self.page_num = 1
+        subdomain_encode = f'domain={self.domain}'.encode('utf-8')
+        query_data = base64.b64encode(subdomain_encode)
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = {'email': self.email,
+                     'key': self.key,
+                     'qbase64': query_data,
+                     'page': self.page_num,
+                     'size': 10000}
+            resp = self.get(self.addr, query)
+            if not resp:
+                return
+            resp_json = resp.json()
+            subdomains = self.match(self.domain, str(resp_json))
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains = self.subdomains.union(subdomains)
+            size = resp_json.get('size')
+            if size < 10000:
+                break
+            self.page_num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.email, self.key):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = FoFa(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/gitee.py

@@ -0,0 +1,74 @@
+import time
+from bs4 import BeautifulSoup
+from common.search import Search
+from config.log import logger
+
+
+class Gitee(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.source = 'GiteeSearch'
+        self.module = 'Search'
+        self.addr = 'https://search.gitee.com/'
+        self.domain = self.register(domain)
+        self.header = self.get_header()
+
+    def search(self, full_search=False):
+        """
+        向接口查询子域并做子域匹配
+        """
+        page_num = 1
+        while True:
+            time.sleep(self.delay)
+            params = {'pageno': page_num, 'q': self.domain, 'type': 'code'}
+            try:
+                resp = self.get(self.addr, params=params)
+            except Exception as e:
+                logger.log('ERROR', e.args)
+                break
+            if not resp:
+                break
+            if resp.status_code != 200:
+                logger.log('ERROR', f'{self.source}模块搜索出错')
+                break
+            if 'class="empty-box"' in resp.text:
+                break
+            soup = BeautifulSoup(resp.text, 'html.parser')
+            subdomains = self.match(self.domain, soup.text)
+            self.subdomains = self.subdomains.union(subdomains)
+            if not subdomains:
+                break
+            if not full_search:
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
+                    break
+            if '<li class="disabled"><a href="###">' in resp.text:
+                break
+            if page_num > 100:
+                break
+            page_num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Gitee(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/github_api.py

@@ -0,0 +1,107 @@
+import requests
+from config import api
+from common.utils import match_subdomain
+from common.search import Search
+from config.log import logger
+
+
+class GithubAPI(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.source = 'GithubAPISearch'
+        self.module = 'Search'
+        self.addr = 'https://api.github.com/search/code'
+        self.domain = self.register(domain)
+        self.session = requests.Session()
+        self.auth_url = 'https://api.github.com'
+        self.token = api.github_api_token
+
+    def auth_github(self):
+        """
+        github api 认证
+
+        :return: 认证失败返回False 成功返回True
+        """
+        self.session.headers.update({'Authorization': 'token ' + self.token})
+        try:
+            resp = self.session.get(self.auth_url)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return False
+        if resp.status_code != 200:
+            resp_json = resp.json()
+            msg = resp_json.get('message')
+            logger.log('ERROR', msg)
+            return False
+        else:
+            return True
+
+    def search(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.session.headers = self.get_header()
+        self.session.proxies = self.get_proxy(self.source)
+        self.session.verify = self.verify
+        self.session.headers.update(
+            {'Accept': 'application/vnd.github.v3.text-match+json'})
+
+        if not self.auth_github():
+            logger.log('ERROR', f'{self.source}模块登录失败')
+            return
+        page = 1
+        while True:
+            params = {'q': self.domain, 'per_page': 100,
+                      'page': page, 'sort': 'indexed'}
+            try:
+                resp = self.session.get(self.addr, params=params)
+            except Exception as e:
+                logger.log('ERROR', e.args)
+                break
+            if resp.status_code != 200:
+                logger.log('ERROR', f'{self.source}模块搜索出错')
+                break
+            subdomains = match_subdomain(self.domain, resp.text)
+            if not subdomains:
+                break
+            self.subdomains = self.subdomains.union(subdomains)
+            page += 1
+            try:
+                resp_json = resp.json()
+            except Exception as e:
+                logger.log('ERROR', e.args)
+                break
+            total_count = resp_json.get('total_count')
+            if not isinstance(total_count, int):
+                break
+            if page * 100 > total_count:
+                break
+            if page * 100 > 1000:
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.token):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = GithubAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('exmaple.com')

modules/search/google.py

@@ -0,0 +1,94 @@
+import random
+import time
+from common.search import Search
+
+
+class Google(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'GoogleSearch'
+        self.init = 'https://www.google.com/'
+        self.addr = 'https://www.google.com/search'
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        page_num = 1
+        per_page_num = 50
+        self.header = self.get_header()
+        self.header.update({'User-Agent': 'Googlebot',
+                            'Referer': 'https://www.google.com'})
+        self.proxy = self.get_proxy(self.source)
+        resp = self.get(self.init)
+        if not resp:
+            return
+        self.cookie = resp.cookies
+        while True:
+            self.delay = random.randint(1, 5)
+            time.sleep(self.delay)
+            self.proxy = self.get_proxy(self.source)
+            word = 'site:.' + domain + filtered_subdomain
+            payload = {'q': word, 'start': page_num, 'num': per_page_num,
+                       'filter': '0', 'btnG': 'Search', 'gbv': '1', 'hl': 'en'}
+            resp = self.get(url=self.addr, params=payload)
+            if not resp:
+                return
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
+                break
+            if not full_search:
+                if subdomains.issubset(self.subdomains):
+                    break
+            self.subdomains = self.subdomains.union(subdomains)
+            page_num += per_page_num
+            if 'start=' + str(page_num) not in resp.text:
+                break
+            if '302 Moved' in resp.text:
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
+                for subdomain in self.subdomains:
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
+                        self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Google(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/google_api.py

@@ -0,0 +1,88 @@
+import time
+from config import api
+from common.search import Search
+
+
+class GoogleAPI(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'GoogleAPISearch'
+        self.addr = 'https://www.googleapis.com/customsearch/v1'
+        self.delay = 1
+        self.key = api.google_api_key
+        self.cx = api.google_api_cx
+        self.per_page_num = 10  # 每次只能请求10个结果
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 1
+        while True:
+            word = 'site:.' + domain + filtered_subdomain
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            params = {'key': self.key, 'cx': self.cx,
+                      'q': word, 'fields': 'items/link',
+                      'start': self.page_num, 'num': self.per_page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomains = self.match(domain, str(resp.json()))
+            if not subdomains:
+                break
+            if not full_search:
+                if subdomains.issubset(self.subdomains):
+                    break
+            self.subdomains = self.subdomains.union(subdomains)
+            self.page_num += self.per_page_num
+            if self.page_num > 100:  # 免费的API只能查询前100条结果
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.cx, self.key):
+            return
+        self.begin()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
+                for subdomain in self.subdomains:
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
+                        self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = GoogleAPI(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/shodan_api.py

@@ -0,0 +1,60 @@
+from config import api
+from common.search import Search
+
+
+class ShodanAPI(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Search'
+        self.source = 'ShodanAPISearch'
+        self.addr = 'https://api.shodan.io/shodan/host/search'
+        self.key = api.shodan_api_key
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        query = 'hostname:.' + self.domain
+        page = 1
+        while True:
+            params = {'key': self.key, 'page': page, 'query': query,
+                      'minify': True, 'facets': {'hostnames'}}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            if subdomains:
+                self.subdomains = self.subdomains.union(subdomains)
+            page += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.key):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = ShodanAPI(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/so.py

@@ -0,0 +1,87 @@
+import time
+
+from common.search import Search
+
+
+class So(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'SoSearch'
+        self.addr = 'https://www.so.com/s'
+        self.limit_num = 640  # 限制搜索条数
+        self.per_page_num = 10  # 默认每页显示10页
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        page_num = 1
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            word = 'site:.' + domain + filtered_subdomain
+            payload = {'q': word, 'pn': page_num}
+            resp = self.get(url=self.addr, params=payload)
+            if not resp:
+                return
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
+                break
+            if not full_search:
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
+                    break
+            self.subdomains = self.subdomains.union(subdomains)
+            page_num += 1
+            # 搜索页面没有出现下一页时停止搜索
+            if '<a id="snext"' not in resp.text:
+                break
+            # 搜索条数限制
+            if self.page_num * self.per_page_num >= self.limit_num:
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
+                for subdomain in self.subdomains:
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
+                        self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = So(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/sogou.py

@@ -0,0 +1,85 @@
+from common.search import Search
+
+
+class Sogou(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'SogouSearch'
+        self.addr = 'https://www.sogou.com/web'
+        self.limit_num = 1000  # 限制搜索条数
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 1
+        while True:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            word = 'site:.' + domain + filtered_subdomain
+            payload = {'query': word, 'page': self.page_num,
+                       "num": self.per_page_num}
+            resp = self.get(self.addr, payload)
+            if not resp:
+                return
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
+                break
+            if not full_search:
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
+                    break
+            self.subdomains = self.subdomains.union(subdomains)
+            self.page_num += 1
+            # 搜索页面没有出现下一页时停止搜索
+            if '<a id="sogou_next"' not in resp.text:
+                break
+            # 搜索条数限制
+            if self.page_num * self.per_page_num >= self.limit_num:
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
+                for subdomain in self.subdomains:
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
+                        self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Sogou(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/yahoo.py

@@ -0,0 +1,92 @@
+import time
+from common.search import Search
+
+
+class Yahoo(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'YahooSearch'
+        self.init = 'https://search.yahoo.com/'
+        self.addr = 'https://search.yahoo.com/search'
+        self.limit_num = 1000  # Yahoo限制搜索条数
+        self.delay = 2
+        self.per_page_num = 30  # Yahoo每次搜索最大条数
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        resp = self.get(self.init)
+        if not resp:
+            return
+        self.cookie = resp.cookies  # 获取cookie Yahoo在搜索时需要带上cookie
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'p': query, 'b': self.page_num, 'pz': self.per_page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            text = resp.text.replace('<b>', '').replace('</b>', '')
+            subdomains = self.match(domain, text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            if not full_search:
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
+                    break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            if '>Next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+                break
+            self.page_num += self.per_page_num
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
+                for subdomain in self.subdomains:
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
+                        self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Yahoo(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/yandex.py

@@ -0,0 +1,92 @@
+import time
+from common.search import Search
+
+
+class Yandex(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'YandexSearch'
+        self.init = 'https://yandex.com/'
+        self.addr = 'https://yandex.com/search'
+        self.limit_num = 1000  # 限制搜索条数
+        self.delay = 5
+
+    def search(self, domain, filtered_subdomain='', full_search=False):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        :param bool full_search: 全量搜索
+        """
+        self.page_num = 0  # 二次搜索重新置0
+        resp = self.get(self.init)
+        if not resp:
+            return
+        self.cookie = resp.cookies  # 获取cookie
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'text': query, 'p': self.page_num,
+                      'numdoc': self.per_page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            if not full_search:
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
+                    break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            if '>next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+                break
+            self.page_num += 1
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+
+        self.search(self.domain, full_search=True)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
+                for subdomain in self.subdomains:
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
+                        self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Yandex(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')

modules/search/zoomeye_api.py

@@ -0,0 +1,86 @@
+import time
+from config import api
+from common.search import Search
+from config.log import logger
+
+
+class ZoomEyeAPI(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'ZoomEyeAPISearch'
+        self.addr = 'https://api.zoomeye.org/web/search'
+        self.delay = 2
+        self.user = api.zoomeye_api_usermail
+        self.pwd = api.zoomeye_api_password
+
+    def login(self):
+        """
+        登陆获取查询taken
+        """
+        url = 'https://api.zoomeye.org/user/login'
+        data = {'username': self.user, 'password': self.pwd}
+        resp = self.post(url=url, json=data)
+        if not resp:
+            logger.log('FATAL', f'登录失败无法获取{self.source}的访问token')
+            exit(1)
+        data = resp.json()
+        if resp.status_code == 200:
+            logger.log('DEBUG', f'{self.source}模块登录成功')
+            return data.get('access_token')
+        else:
+            logger.log('ALERT', data.get('message'))
+            exit(1)
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        page_num = 1
+        access_token = self.login()
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            self.header.update({'Authorization': 'JWT ' + access_token})
+            params = {'query': 'hostname:' + self.domain, 'page': page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains = self.subdomains.union(subdomains)
+            page_num += 1
+            if page_num > 500:
+                break
+            if resp.status_code == 403:
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.user, self.pwd):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = ZoomEyeAPI(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('mi.com')