模块优化

modules/search/ask.py

@@ -12,13 +12,12 @@ class Ask(Search):
         self.limit_num = 200  # 限制搜索条数
         self.per_page_num = 10  # 默认每页显示10页
 
-    def search(self, domain, filtered_subdomain='', full_search=False):
+    def search(self, domain, filtered_subdomain=''):
         """
         发送搜索请求并做子域匹配
 
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
-        :param bool full_search: 全量搜索
         """
         self.page_num = 1
         while True:
@@ -28,13 +27,8 @@ class Ask(Search):
             query = 'site:.' + domain + filtered_subdomain
             params = {'q': query, 'page': self.page_num}
             resp = self.get(self.addr, params)
-            if not resp:
-                return
-            subdomains = self.match_subdomains(resp.text, fuzzy=False)
-            if not subdomains:
-                break
-            if not full_search and subdomains.issubset(self.subdomains):
-                # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
+            subdomains = self.match_subdomains(resp, fuzzy=False)
+            if not self.check_subdomains(subdomains):
                 break
             self.subdomains = self.subdomains.union(subdomains)
             self.page_num += 1
@@ -46,20 +40,15 @@ class Ask(Search):
         类执行入口
         """
         self.begin()
-        self.search(self.domain, full_search=True)
+        self.search(self.domain)
         # 排除同一子域搜索结果过多的子域以发现新的子域
         for statement in self.filter(self.domain, self.subdomains):
             self.search(self.domain, filtered_subdomain=statement)
 
         # 递归搜索下一层的子域
         if self.recursive_search:
-            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
-            for layer_num in range(1, self.recursive_times):
-                for subdomain in self.subdomains:
-                    # 进行下一层子域搜索的限制条件
-                    count = subdomain.count('.') - self.domain.count('.')
-                    if count == layer_num:
-                        self.search(subdomain)
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
         self.finish()
         self.save_json()
         self.gen_result()

modules/search/baidu.py

@@ -28,13 +28,12 @@ class Baidu(Search):
             subdomains_all = subdomains_all.union(subdomains)
         return subdomains_all
 
-    def search(self, domain, filtered_subdomain='', full_search=False):
+    def search(self, domain, filtered_subdomain=''):
         """
         发送搜索请求并做子域匹配
 
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
-        :param bool full_search: 全量搜索
         """
         self.page_num = 0  # 二次搜索重新置0
         while True:
@@ -52,11 +51,8 @@ class Baidu(Search):
                 # 获取百度跳转URL响应头的Location字段获取直链
                 subdomains = self.redirect_match(resp.text)
             else:
-                subdomains = self.match_subdomains(resp.text, fuzzy=False)
-            if not subdomains:  # 搜索没有发现子域名则停止搜索
-                break
-            if not full_search and subdomains.issubset(self.subdomains):
-                # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
+                subdomains = self.match_subdomains(resp, fuzzy=False)
+            if not self.check_subdomains(subdomains):
                 break
             self.subdomains = self.subdomains.union(subdomains)
             self.page_num += self.per_page_num
@@ -71,20 +67,15 @@ class Baidu(Search):
         类执行入口
         """
         self.begin()
-        self.search(self.domain, full_search=True)
+        self.search(self.domain)
         # 排除同一子域搜索结果过多的子域以发现新的子域
         for statement in self.filter(self.domain, self.subdomains):
             self.search(self.domain, filtered_subdomain=statement)
 
         # 递归搜索下一层的子域
         if self.recursive_search:
-            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
-            for layer_num in range(1, self.recursive_times):
-                for subdomain in self.subdomains:
-                    # 进行下一层子域搜索的限制条件
-                    count = subdomain.count('.') - self.domain.count('.')
-                    if count == layer_num:
-                        self.search(subdomain)
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
         self.finish()
         self.save_json()
         self.gen_result()

modules/search/bing.py

@@ -12,13 +12,12 @@ class Bing(Search):
         self.addr = 'https://www.bing.com/search'
         self.limit_num = 1000  # 限制搜索条数
 
-    def search(self, domain, filtered_subdomain='', full_search=False):
+    def search(self, domain, filtered_subdomain=''):
         """
         发送搜索请求并做子域匹配
 
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
-        :param bool full_search: 全量搜索
         """
         self.page_num = 0  # 二次搜索重新置0
         self.header = self.get_header()
@@ -34,13 +33,8 @@ class Bing(Search):
             params = {'q': query, 'first': self.page_num,
                       'count': self.per_page_num}
             resp = self.get(self.addr, params)
-            if not resp:
-                return
-            subdomains = self.match_subdomains(resp.text, fuzzy=False)
-            if not subdomains:  # 搜索没有发现子域名则停止搜索
-                break
-            if not full_search and subdomains.issubset(self.subdomains):
-                # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
+            subdomains = self.match_subdomains(resp, fuzzy=False)
+            if not self.check_subdomains(subdomains):
                 break
             self.subdomains = self.subdomains.union(subdomains)
             # 搜索页面没有出现下一页时停止搜索
@@ -55,7 +49,7 @@ class Bing(Search):
         类执行入口
         """
         self.begin()
-        self.search(self.domain, full_search=True)
+        self.search(self.domain)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
         for statement in self.filter(self.domain, self.subdomains):
@@ -63,13 +57,8 @@ class Bing(Search):
 
         # 递归搜索下一层的子域
         if self.recursive_search:
-            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
-            for layer_num in range(1, self.recursive_times):
-                for subdomain in self.subdomains:
-                    # 进行下一层子域搜索的限制条件
-                    count = subdomain.count('.') - self.domain.count('.')
-                    if count == layer_num:
-                        self.search(subdomain)
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
         self.finish()
         self.save_json()
         self.gen_result()

modules/search/bing_api.py

@@ -15,13 +15,12 @@ class BingAPI(Search):
         self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数
         self.delay = 1  # 必应自定义搜索限制时延1秒
 
-    def search(self, domain, filtered_subdomain='', full_search=False):
+    def search(self, domain, filtered_subdomain=''):
         """
         发送搜索请求并做子域匹配
 
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
-        :param bool full_search: 全量搜索
         """
         self.page_num = 0  # 二次搜索重新置0
         while True:
@@ -35,10 +34,7 @@ class BingAPI(Search):
                       'offset': self.page_num}
             resp = self.get(self.addr, params)
             subdomains = self.match_subdomains(resp)
-            if not subdomains:  # 搜索没有发现子域名则停止搜索
-                break
-            if not full_search and subdomains.issubset(self.subdomains):
-                # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
+            if not self.check_subdomains(subdomains):
                 break
             self.subdomains = self.subdomains.union(subdomains)
             self.page_num += self.per_page_num
@@ -52,7 +48,7 @@ class BingAPI(Search):
         if not self.check(self.id, self.key):
             return
         self.begin()
-        self.search(self.domain, full_search=True)
+        self.search(self.domain)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
         for statement in self.filter(self.domain, self.subdomains):
@@ -60,13 +56,8 @@ class BingAPI(Search):
 
         # 递归搜索下一层的子域
         if self.recursive_search:
-            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
-            for layer_num in range(1, self.recursive_times):
-                for subdomain in self.subdomains:
-                    # 进行下一层子域搜索的限制条件
-                    count = subdomain.count('.') - self.domain.count('.')
-                    if count == layer_num:
-                        self.search(subdomain)
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
         self.finish()
         self.save_json()
         self.gen_result()

modules/search/fofa_api.py

@@ -37,7 +37,7 @@ class FoFa(Search):
             if not resp:
                 return
             resp_json = resp.json()
-            subdomains = self.match_subdomains(resp.text)
+            subdomains = self.match_subdomains(resp)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             self.subdomains = self.subdomains.union(subdomains)

modules/search/gitee.py

@@ -34,11 +34,8 @@ class Gitee(Search):
             if 'class="empty-box"' in resp.text:
                 break
             soup = BeautifulSoup(resp.text, 'html.parser')
-            subdomains = self.match_subdomains(soup.text, fuzzy=False)
-            if not subdomains:
-                break
-            if not full_search and subdomains.issubset(self.subdomains):
-                # 在全搜索中发现搜索出的结果有完全重复的结果就停止搜索
+            subdomains = self.match_subdomains(soup, fuzzy=False)
+            if not self.check_subdomains(subdomains):
                 break
             self.subdomains = self.subdomains.union(subdomains)
             if '<li class="disabled"><a href="###">' in resp.text:

modules/search/github_api.py

@@ -60,7 +60,7 @@ class GithubAPI(Search):
             if resp.status_code != 200:
                 logger.log('ERROR', f'{self.source} module query failed')
                 break
-            subdomains = self.match_subdomains(resp.text)
+            subdomains = self.match_subdomains(resp)
             if not subdomains:
                 break
             self.subdomains = self.subdomains.union(subdomains)

modules/search/google.py

@@ -12,13 +12,12 @@ class Google(Search):
         self.init = 'https://www.google.com/'
         self.addr = 'https://www.google.com/search'
 
-    def search(self, domain, filtered_subdomain='', full_search=False):
+    def search(self, domain, filtered_subdomain=''):
         """
         发送搜索请求并做子域匹配
 
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
-        :param bool full_search: 全量搜索
         """
         page_num = 1
         per_page_num = 50
@@ -38,13 +37,8 @@ class Google(Search):
             payload = {'q': word, 'start': page_num, 'num': per_page_num,
                        'filter': '0', 'btnG': 'Search', 'gbv': '1', 'hl': 'en'}
             resp = self.get(url=self.addr, params=payload)
-            if not resp:
-                return
-            subdomains = self.match_subdomains(resp.text, fuzzy=False)
-            if not subdomains:
-                break
-            if not full_search and subdomains.issubset(self.subdomains):
-                # 在全搜索中发现搜索出的结果有完全重复的结果就停止搜索
+            subdomains = self.match_subdomains(resp, fuzzy=False)
+            if not self.check_subdomains(subdomains):
                 break
             self.subdomains = self.subdomains.union(subdomains)
             page_num += per_page_num
@@ -59,7 +53,7 @@ class Google(Search):
         """
         self.begin()
 
-        self.search(self.domain, full_search=True)
+        self.search(self.domain)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
         for statement in self.filter(self.domain, self.subdomains):
@@ -67,13 +61,8 @@ class Google(Search):
 
         # 递归搜索下一层的子域
         if self.recursive_search:
-            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
-            for layer_num in range(1, self.recursive_times):
-                for subdomain in self.subdomains:
-                    # 进行下一层子域搜索的限制条件
-                    count = subdomain.count('.') - self.domain.count('.')
-                    if count == layer_num:
-                        self.search(subdomain)
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
         self.finish()
         self.save_json()
         self.gen_result()

modules/search/google_api.py

@@ -15,13 +15,12 @@ class GoogleAPI(Search):
         self.id = api.google_api_id
         self.per_page_num = 10  # 每次只能请求10个结果
 
-    def search(self, domain, filtered_subdomain='', full_search=False):
+    def search(self, domain, filtered_subdomain=''):
         """
         发送搜索请求并做子域匹配
 
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
-        :param bool full_search: 全量搜索
         """
         self.page_num = 1
         while True:
@@ -34,10 +33,7 @@ class GoogleAPI(Search):
                       'start': self.page_num, 'num': self.per_page_num}
             resp = self.get(self.addr, params)
             subdomains = self.match_subdomains(resp)
-            if not subdomains:
-                break
-            if not full_search and subdomains.issubset(self.subdomains):
-                # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
+            if not self.check_subdomains(subdomains):
                 break
             self.subdomains = self.subdomains.union(subdomains)
             self.page_num += self.per_page_num
@@ -51,7 +47,7 @@ class GoogleAPI(Search):
         if not self.check(self.id, self.key):
             return
         self.begin()
-        self.search(self.domain, full_search=True)
+        self.search(self.domain)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
         for statement in self.filter(self.domain, self.subdomains):
@@ -59,13 +55,8 @@ class GoogleAPI(Search):
 
         # 递归搜索下一层的子域
         if self.recursive_search:
-            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
-            for layer_num in range(1, self.recursive_times):
-                for subdomain in self.subdomains:
-                    # 进行下一层子域搜索的限制条件
-                    count = subdomain.count('.') - self.domain.count('.')
-                    if count == layer_num:
-                        self.search(subdomain)
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
         self.finish()
         self.save_json()
         self.gen_result()

modules/search/shodan_api.py

@@ -26,8 +26,7 @@ class ShodanAPI(Search):
             subdomains = self.match_subdomains(resp)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
-            if subdomains:
-                self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains = self.subdomains.union(subdomains)
             page += 1
 
     def run(self):

modules/search/so.py

@@ -13,13 +13,12 @@ class So(Search):
         self.limit_num = 640  # 限制搜索条数
         self.per_page_num = 10  # 默认每页显示10页
 
-    def search(self, domain, filtered_subdomain='', full_search=False):
+    def search(self, domain, filtered_subdomain=''):
         """
         发送搜索请求并做子域匹配
 
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
-        :param bool full_search: 全量搜索
         """
         page_num = 1
         while True:
@@ -29,13 +28,8 @@ class So(Search):
             word = 'site:.' + domain + filtered_subdomain
             payload = {'q': word, 'pn': page_num}
             resp = self.get(url=self.addr, params=payload)
-            if not resp:
-                return
-            subdomains = self.match_subdomains(resp.text, fuzzy=False)
-            if not subdomains:
-                break
-            if not full_search and subdomains.issubset(self.subdomains):
-                # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
+            subdomains = self.match_subdomains(resp, fuzzy=False)
+            if not self.check_subdomains(subdomains):
                 break
             self.subdomains = self.subdomains.union(subdomains)
             page_num += 1
@@ -51,7 +45,7 @@ class So(Search):
         类执行入口
         """
         self.begin()
-        self.search(self.domain, full_search=True)
+        self.search(self.domain)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
         for statement in self.filter(self.domain, self.subdomains):
@@ -59,13 +53,8 @@ class So(Search):
 
         # 递归搜索下一层的子域
         if self.recursive_search:
-            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
-            for layer_num in range(1, self.recursive_times):
-                for subdomain in self.subdomains:
-                    # 进行下一层子域搜索的限制条件
-                    count = subdomain.count('.') - self.domain.count('.')
-                    if count == layer_num:
-                        self.search(subdomain)
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
         self.finish()
         self.save_json()
         self.gen_result()

modules/search/sogou.py

@@ -10,13 +10,12 @@ class Sogou(Search):
         self.addr = 'https://www.sogou.com/web'
         self.limit_num = 1000  # 限制搜索条数
 
-    def search(self, domain, filtered_subdomain='', full_search=False):
+    def search(self, domain, filtered_subdomain=''):
         """
         发送搜索请求并做子域匹配
 
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
-        :param bool full_search: 全量搜索
         """
         self.page_num = 1
         while True:
@@ -26,13 +25,8 @@ class Sogou(Search):
             payload = {'query': word, 'page': self.page_num,
                        "num": self.per_page_num}
             resp = self.get(self.addr, payload)
-            if not resp:
-                return
-            subdomains = self.match_subdomains(resp.text, fuzzy=False)
-            if not subdomains:
-                break
-            if not full_search and subdomains.issubset(self.subdomains):
-                # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
+            subdomains = self.match_subdomains(resp, fuzzy=False)
+            if not self.check_subdomains(subdomains):
                 break
             self.subdomains = self.subdomains.union(subdomains)
             self.page_num += 1
@@ -48,8 +42,7 @@ class Sogou(Search):
         类执行入口
         """
         self.begin()
-
-        self.search(self.domain, full_search=True)
+        self.search(self.domain)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
         for statement in self.filter(self.domain, self.subdomains):
@@ -57,13 +50,8 @@ class Sogou(Search):
 
         # 递归搜索下一层的子域
         if self.recursive_search:
-            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
-            for layer_num in range(1, self.recursive_times):
-                for subdomain in self.subdomains:
-                    # 进行下一层子域搜索的限制条件
-                    count = subdomain.count('.') - self.domain.count('.')
-                    if count == layer_num:
-                        self.search(subdomain)
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
         self.finish()
         self.save_json()
         self.gen_result()

modules/search/yahoo.py

@@ -14,13 +14,12 @@ class Yahoo(Search):
         self.delay = 2
         self.per_page_num = 30  # Yahoo每次搜索最大条数
 
-    def search(self, domain, filtered_subdomain='', full_search=False):
+    def search(self, domain, filtered_subdomain=''):
         """
         发送搜索请求并做子域匹配
 
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
-        :param bool full_search: 全量搜索
         """
         resp = self.get(self.init)
         if not resp:
@@ -37,10 +36,7 @@ class Yahoo(Search):
                 return
             text = resp.text.replace('<b>', '').replace('</b>', '')
             subdomains = self.match_subdomains(text, fuzzy=False)
-            if not subdomains:  # 搜索没有发现子域名则停止搜索
-                break
-            if not full_search and subdomains.issubset(self.subdomains):
-                # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
+            if not self.check_subdomains(subdomains):
                 break
             self.subdomains = self.subdomains.union(subdomains)
             if '>Next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
@@ -54,8 +50,7 @@ class Yahoo(Search):
         类执行入口
         """
         self.begin()
-
-        self.search(self.domain, full_search=True)
+        self.search(self.domain)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
         for statement in self.filter(self.domain, self.subdomains):
@@ -63,13 +58,8 @@ class Yahoo(Search):
 
         # 递归搜索下一层的子域
         if self.recursive_search:
-            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
-            for layer_num in range(1, self.recursive_times):
-                for subdomain in self.subdomains:
-                    # 进行下一层子域搜索的限制条件
-                    count = subdomain.count('.') - self.domain.count('.')
-                    if count == layer_num:
-                        self.search(subdomain)
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
         self.finish()
         self.save_json()
         self.gen_result()

modules/search/yandex.py

@@ -13,13 +13,12 @@ class Yandex(Search):
         self.limit_num = 1000  # 限制搜索条数
         self.delay = 5
 
-    def search(self, domain, filtered_subdomain='', full_search=False):
+    def search(self, domain, filtered_subdomain=''):
         """
         发送搜索请求并做子域匹配
 
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
-        :param bool full_search: 全量搜索
         """
         self.page_num = 0  # 二次搜索重新置0
         resp = self.get(self.init)
@@ -34,13 +33,8 @@ class Yandex(Search):
             params = {'text': query, 'p': self.page_num,
                       'numdoc': self.per_page_num}
             resp = self.get(self.addr, params)
-            if not resp:
-                return
-            subdomains = self.match_subdomains(resp.text, fuzzy=False)
-            if not subdomains:  # 搜索没有发现子域名则停止搜索
-                break
-            if not full_search and subdomains.issubset(self.subdomains):
-                # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
+            subdomains = self.match_subdomains(resp, fuzzy=False)
+            if not self.check_subdomains(subdomains):
                 break
             self.subdomains = self.subdomains.union(subdomains)
             if '>next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
@@ -55,7 +49,7 @@ class Yandex(Search):
         """
         self.begin()
 
-        self.search(self.domain, full_search=True)
+        self.search(self.domain)
 
         # 排除同一子域搜索结果过多的子域以发现新的子域
         for statement in self.filter(self.domain, self.subdomains):
@@ -63,13 +57,8 @@ class Yandex(Search):
 
         # 递归搜索下一层的子域
         if self.recursive_search:
-            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
-            for layer_num in range(1, self.recursive_times):
-                for subdomain in self.subdomains:
-                    # 进行下一层子域搜索的限制条件
-                    count = subdomain.count('.') - self.domain.count('.')
-                    if count == layer_num:
-                        self.search(subdomain)
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
         self.finish()
         self.save_json()
         self.gen_result()

modules/search/zoomeye_api.py

@@ -23,7 +23,8 @@ class ZoomEyeAPI(Search):
         data = {'username': self.user, 'password': self.pwd}
         resp = self.post(url=url, json=data)
         if not resp:
-            logger.log('FATAL', f'{self.source} module login failed, can not get access token')
+            logger.log('FATAL', f'{self.source} module login failed,'
+                                f' can not get access token')
             exit(1)
         data = resp.json()
         if resp.status_code == 200: