Merge pull request #396 from lovelyjuice/master

增加空间测绘引擎最大查询数量限制，允许用户不使用 tldextract 提取主域名，升级exrex版本，修复`No module named 'distutils.util'`的bug

common/domain.py

@@ -55,6 +55,8 @@ class Domain(object):
 
         :return: registered domain result
         """
+        if not settings.use_tld_extract:
+            return self.string
         result = self.extract()
         if result:
             return result.registered_domain

config/setting.py

@@ -100,3 +100,9 @@ enable_random_ua = True  # 使用随机UA(默认True，开启可以覆盖request
 # 搜索模块设置
 enable_recursive_search = False  # 递归搜索子域
 search_recursive_times = 2  # 递归搜索层数
+
+# 网络空间测绘引擎设置
+cam_records_maximum_per_domain = 1000   # 对于单个主域名，在测绘引擎中的最多查询多少条记录，防止泛解析和CDN浪费积分，对 fofa, hunter, quake, zoomeye 生效，最低为100
+
+# 是否从输入的数据中使用tldextract提取主域名。若设为 False，OneForAll会直接将输入的域名作为主域名，比如北京分公司的域名 beijing.10086.com 就不会被解析成母公司的域名 10086.com
+use_tld_extract = True

modules/search/fofa_api.py

@@ -23,7 +23,7 @@ class FoFa(Search):
         self.page_num = 1
         subdomain_encode = f'domain="{self.domain}"'.encode('utf-8')
         query_data = base64.b64encode(subdomain_encode)
-        while True:
+        while 100 * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
@@ -32,7 +32,7 @@ class FoFa(Search):
                      'qbase64': query_data,
                      'page': self.page_num,
                      'full': 'true',
-                     'size': 1000}
+                     'size': min(1000, settings.cam_records_maximum_per_domain)}
             resp = self.get(self.addr, query)
             if not resp:
                 return
@@ -42,7 +42,7 @@ class FoFa(Search):
                 break
             self.subdomains.update(subdomains)
             size = resp_json.get('size')
-            if size < 1000:
+            if size < min(1000, settings.cam_records_maximum_per_domain):
                 break
             self.page_num += 1
 

modules/search/hunter_api.py

@@ -22,7 +22,7 @@ class Hunter(Search):
         self.page_num = 1
         subdomain_encode = f'domain_suffix="{self.domain}"'.encode('utf-8')
         query_data = base64.b64encode(subdomain_encode)
-        while True:
+        while 100 * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)

modules/search/quake_api.py

@@ -20,7 +20,7 @@ class Quake(Search):
         """
         self.per_page_num = 100
         self.page_num = 0
-        while True:
+        while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.header.update({'Content-Type': 'application/json'})

modules/search/zoomeye_api.py

@@ -19,7 +19,7 @@ class ZoomEyeAPI(Search):
         """
         self.per_page_num = 30
         self.page_num = 1
-        while True:
+        while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.header.update({'API-KEY': self.key})

requirements.txt

@@ -4,7 +4,7 @@ certifi==2022.06.15
 chardet==5.0.0
 colorama==0.4.4
 dnspython==2.2.1
-exrex==0.10.5
+exrex==0.11.0
 fire==0.4.0
 future==0.18.2
 idna==3.3
@@ -20,3 +20,4 @@ tqdm==4.64.0
 treelib==1.6.1
 urllib3==1.26.9
 win32-setctime==1.1.0
+setuptools