Merge remote-tracking branch 'origin/master'

modules/search/baidu.py

@@ -57,7 +57,7 @@ class Baidu(Search):
             self.subdomains.update(subdomains)
             self.page_num += self.per_page_num
             # 搜索页面没有出现下一页时停止搜索
-            if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:
+            if f'&pn={self.page_num}&' not in resp.text:
                 break
             if self.page_num >= self.limit_num:  # 搜索条数限制
                 break

modules/search/bing.py

@@ -19,9 +19,9 @@ class Bing(Search):
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
         """
-        self.page_num = 0  # 二次搜索重新置0
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
+        self.page_num = 0  # 二次搜索重新置0
         resp = self.get(self.init)
         if not resp:
             return

modules/search/fofa_api.py

@@ -21,7 +21,7 @@ class FoFa(Search):
         发送搜索请求并做子域匹配
         """
         self.page_num = 1
-        subdomain_encode = f'domain={self.domain} || cert={self.domain}'.encode('utf-8')
+        subdomain_encode = f'domain="{self.domain}"'.encode('utf-8')
         query_data = base64.b64encode(subdomain_encode)
         while True:
             time.sleep(self.delay)
@@ -32,7 +32,7 @@ class FoFa(Search):
                      'qbase64': query_data,
                      'page': self.page_num,
                      'full': 'true',
-                     'size': 5000}
+                     'size': 1000}
             resp = self.get(self.addr, query)
             if not resp:
                 return
@@ -42,7 +42,7 @@ class FoFa(Search):
                 break
             self.subdomains.update(subdomains)
             size = resp_json.get('size')
-            if size < 5000:
+            if size < 1000:
                 break
             self.page_num += 1
 

modules/search/gitee.py

@@ -11,7 +11,6 @@ class Gitee(Search):
         self.module = 'Search'
         self.addr = 'https://search.gitee.com/'
         self.domain = domain
-        self.header = self.get_header()
 
     def search(self):
         """
@@ -20,6 +19,8 @@ class Gitee(Search):
         page_num = 1
         while True:
             time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
             params = {'pageno': page_num, 'q': self.domain, 'type': 'code'}
             try:
                 resp = self.get(self.addr, params=params)

modules/search/github_api.py

@@ -1,4 +1,5 @@
-import requests
+import time
+
 from config import settings
 from common.search import Search
 from config.log import logger
@@ -11,53 +12,30 @@ class GithubAPI(Search):
         self.module = 'Search'
         self.addr = 'https://api.github.com/search/code'
         self.domain = domain
-        self.session = requests.Session()
-        self.session.trust_env = False
-        self.auth_url = 'https://api.github.com'
+        self.delay = 5
         self.token = settings.github_api_token
 
-    def auth_github(self):
-        """
-        github api 认证
-
-        :return: 认证失败返回False 成功返回True
-        """
-        self.session.headers.update({'Authorization': 'token ' + self.token})
-        try:
-            resp = self.session.get(self.auth_url)
-        except Exception as e:
-            logger.log('ERROR', e.args)
-            return False
-        if resp.status_code != 200:
-            resp_json = resp.json()
-            msg = resp_json.get('message')
-            logger.log('ERROR', msg)
-            return False
-        return True
-
     def search(self):
         """
         向接口查询子域并做子域匹配
         """
-        self.session.headers = self.get_header()
-        self.session.proxies = self.get_proxy(self.source)
-        self.session.verify = self.verify
-        self.session.headers.update(
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        self.header.update(
             {'Accept': 'application/vnd.github.v3.text-match+json'})
 
-        if not self.auth_github():
-            logger.log('ERROR', f'{self.source} module login failed')
-            return
         page = 1
         while True:
+            time.sleep(self.delay)
             params = {'q': self.domain, 'per_page': 100,
-                      'page': page, 'sort': 'indexed'}
+                      'page': page, 'sort': 'indexed',
+                      'access_token': self.token}
             try:
-                resp = self.session.get(self.addr, params=params)
+                resp = self.get(self.addr, params=params)
             except Exception as e:
                 logger.log('ERROR', e.args)
                 break
-            if resp.status_code != 200:
+            if not resp or resp.status_code != 200:
                 logger.log('ERROR', f'{self.source} module query failed')
                 break
             subdomains = self.match_subdomains(resp)
@@ -103,4 +81,4 @@ def run(domain):
 
 
 if __name__ == '__main__':
-    run('example.com')
+    run('freebuf.com')

modules/search/yahoo.py

@@ -21,13 +21,14 @@ class Yahoo(Search):
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
         """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
         resp = self.get(self.init)
         if not resp:
             return
         self.cookie = resp.cookies  # 获取cookie Yahoo在搜索时需要带上cookie
         while True:
             time.sleep(self.delay)
-            self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             query = 'site:.' + domain + filtered_subdomain
             params = {'p': query, 'b': self.page_num, 'pz': self.per_page_num}

modules/search/yandex.py

@@ -20,6 +20,8 @@ class Yandex(Search):
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
         """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
         self.page_num = 0  # 二次搜索重新置0
         resp = self.get(self.init)
         if not resp:
@@ -27,7 +29,6 @@ class Yandex(Search):
         self.cookie = resp.cookies  # 获取cookie
         while True:
             time.sleep(self.delay)
-            self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             query = 'site:.' + domain + filtered_subdomain
             params = {'text': query, 'p': self.page_num,