Merge pull request #411 from Sci-Twi/patch-1

exporting utf-8 csv files

.dockerignore

@@ -0,0 +1 @@
+.git/

common/domain.py

@@ -55,6 +55,8 @@ class Domain(object):
 
         :return: registered domain result
         """
+        if not settings.use_tld_extract:
+            return self.string
         result = self.extract()
         if result:
             return result.registered_domain

common/utils.py

@@ -248,11 +248,11 @@ def save_to_file(path, data):
     :return: 保存成功与否
     """
     try:
-        with open(path, 'w', errors='ignore', newline='') as file:
+        with open(path, 'w', errors='ignore', newline='', encoding='utf-8') as file:
             file.write(data)
             return True
     except TypeError:
-        with open(path, 'wb') as file:
+        with open(path, 'wb', encoding='utf-8') as file:
             file.write(data)
             return True
     except Exception as e:

config/api.py

@@ -89,3 +89,6 @@ quake_api_key = ''
 
 #https://www.racent.com/ctlog F2>Network抓包获取Token
 racent_api_token = ''
+
+# https://windvane.lichoin.com/
+windvane_api_token = ''

config/default.py

@@ -250,3 +250,6 @@ hunter_api_key = ''
 
 # https://api-docs.fullhunt.io/
 fullhunt_api_key = ''
+
+# https://windvane.lichoin.com/
+windvane_api_token = ''

config/setting.py

@@ -100,3 +100,9 @@ enable_random_ua = True  # 使用随机UA(默认True，开启可以覆盖request
 # 搜索模块设置
 enable_recursive_search = False  # 递归搜索子域
 search_recursive_times = 2  # 递归搜索层数
+
+# 网络空间测绘引擎设置
+cam_records_maximum_per_domain = 1000   # 对于单个主域名，在测绘引擎中的最多查询多少条记录，防止泛解析和CDN浪费积分，对 fofa, hunter, quake, zoomeye 生效，最低为100
+
+# 是否从输入的数据中使用tldextract提取主域名。若设为 False，OneForAll会直接将输入的域名作为主域名，比如北京分公司的域名 beijing.10086.com 就不会被解析成母公司的域名 10086.com
+use_tld_extract = True

modules/certificates/censys_api.py

@@ -9,7 +9,7 @@ class CensysAPI(Query):
         self.domain = domain
         self.module = 'Certificate'
         self.source = "CensysAPIQuery"
-        self.addr = 'https://search.censys.io/api/v1/search/certificates'
+        self.addr = 'https://search.censys.io/api/v2/certificates/search'
         self.id = settings.censys_api_id
         self.secret = settings.censys_api_secret
         self.delay = 3.0  # Censys 接口查询速率限制 最快2.5秒查1次
@@ -20,26 +20,30 @@ class CensysAPI(Query):
         """
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
-        data = {
-            'query': f'parsed.names: {self.domain}',
-            'page': 1,
-            'fields': ['parsed.subject_dn', 'parsed.names'],
-            'flatten': True}
-        resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
+        params = {
+            'q': f'names: {self.domain}',
+            'per_page': 100,
+        }
+        resp = self.get(self.addr, params=params, auth=(self.id, self.secret))
         if not resp:
             return
         json = resp.json()
         status = json.get('status')
-        if status != 'ok':
+        if status != 'OK':
             logger.log('ALERT', f'{self.source} module {status}')
             return
         subdomains = self.match_subdomains(resp.text)
         self.subdomains.update(subdomains)
-        pages = json.get('metadata').get('pages')
-        for page in range(2, pages + 1):
-            data['page'] = page
-            resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
-            self.subdomains = self.collect_subdomains(resp)
+        next_cursor = json.get("result").get("links").get("next")
+        while next_cursor:
+            tmp_params = {
+                'q': f'names: {self.domain}',
+                'per_page': 100,
+                "cursor": next_cursor
+            }
+            tmp_resp = self.get(self.addr, params=tmp_params, auth=(self.id, self.secret))
+            self.subdomains = self.collect_subdomains(tmp_resp)
+            next_cursor = tmp_resp.json().get("result").get("links").get("next")
 
     def run(self):
         """

modules/datasets/windvane.py

@@ -0,0 +1,92 @@
+from config import settings
+from common.query import Query
+
+
+class Windvane(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = "WindvaneQuery"
+        self.addr = 'https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService/ListSubDomain'
+        self.api_key = settings.windvane_api_token
+        self.page_size = 1000
+        
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({
+            'Content-Type': 'application/json',
+            'Referer': 'https://windvane.lichoin.com'
+        })
+        
+        if self.api_key:
+            self.header.update({'X-Api-Key': self.api_key})
+            
+        self.proxy = self.get_proxy(self.source)
+        
+        page = 1
+        total_pages = 1
+        all_subdomains = []
+        
+        while page <= total_pages:
+            data = {
+                "domain": self.domain,
+                "page_request": {
+                    "page": page,
+                    "count": self.page_size
+                }
+            }
+            
+            resp = self.post(self.addr, json=data)
+            if not resp:
+                break
+                
+            try:
+                result = resp.json()
+                
+                if result.get('code') != 0:
+                    break
+                
+                data_section = result.get('data', {})
+
+                subdomains = self.match_subdomains(resp)
+                if not subdomains:
+                    break
+                self.subdomains.update(subdomains)
+                
+                page_info = data_section.get('page_response', {})
+                total_pages = int(page_info.get('total_page', 1))
+                
+                page += 1
+                
+            except:
+                break
+        
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Windvane(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('baidu.com')

modules/dnsquery/mx.py

@@ -7,7 +7,7 @@ class QueryMX(Lookup):
         self.domain = domain
         self.module = 'dnsquery'
         self.source = "QueryMX"
-        self.type = 'MX'  # 利用的DNS记录的MX记录收集子域
+        self.qtype = 'MX'  # 利用的DNS记录的MX记录收集子域
 
     def run(self):
         """

modules/search/fofa_api.py

@@ -23,7 +23,7 @@ class FoFa(Search):
         self.page_num = 1
         subdomain_encode = f'domain="{self.domain}"'.encode('utf-8')
         query_data = base64.b64encode(subdomain_encode)
-        while True:
+        while 100 * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
@@ -32,7 +32,7 @@ class FoFa(Search):
                      'qbase64': query_data,
                      'page': self.page_num,
                      'full': 'true',
-                     'size': 1000}
+                     'size': min(1000, settings.cam_records_maximum_per_domain)}
             resp = self.get(self.addr, query)
             if not resp:
                 return
@@ -42,7 +42,7 @@ class FoFa(Search):
                 break
             self.subdomains.update(subdomains)
             size = resp_json.get('size')
-            if size < 1000:
+            if size < min(1000, settings.cam_records_maximum_per_domain):
                 break
             self.page_num += 1
 

modules/search/hunter_api.py

@@ -22,7 +22,7 @@ class Hunter(Search):
         self.page_num = 1
         subdomain_encode = f'domain_suffix="{self.domain}"'.encode('utf-8')
         query_data = base64.b64encode(subdomain_encode)
-        while True:
+        while 100 * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)

modules/search/quake_api.py

@@ -20,7 +20,7 @@ class Quake(Search):
         """
         self.per_page_num = 100
         self.page_num = 0
-        while True:
+        while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.header.update({'Content-Type': 'application/json'})

modules/search/zoomeye_api.py

@@ -19,7 +19,7 @@ class ZoomEyeAPI(Search):
         """
         self.per_page_num = 30
         self.page_num = 1
-        while True:
+        while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.header.update({'API-KEY': self.key})

requirements.txt

@@ -4,7 +4,7 @@ certifi==2022.06.15
 chardet==5.0.0
 colorama==0.4.4
 dnspython==2.2.1
-exrex==0.10.5
+exrex==0.11.0
 fire==0.4.0
 future==0.18.2
 idna==3.3
@@ -20,3 +20,4 @@ tqdm==4.64.0
 treelib==1.6.1
 urllib3==1.26.9
 win32-setctime==1.1.0
+setuptools