1. 增加空间测绘引擎最大查询数量限制，防止泛解析和CDN浪费积分。对 fofa, hunter, quake, zoomeye 生效

2. 允许用户不使用 tldextract 提取主域名，防止收集分公司域名时错误收集到主公司的域名
3. 升级exrex版本，兼容python3.12
4. 修复`No module named 'distutils.util'`的bug，关联issue: https://github.com/shmilylty/OneForAll/issues/394

modules/search/fofa_api.py

@@ -23,7 +23,7 @@ class FoFa(Search):
         self.page_num = 1
         subdomain_encode = f'domain="{self.domain}"'.encode('utf-8')
         query_data = base64.b64encode(subdomain_encode)
-        while True:
+        while 100 * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
@@ -32,7 +32,7 @@ class FoFa(Search):
                      'qbase64': query_data,
                      'page': self.page_num,
                      'full': 'true',
-                     'size': 1000}
+                     'size': min(1000, settings.cam_records_maximum_per_domain)}
             resp = self.get(self.addr, query)
             if not resp:
                 return
@@ -42,7 +42,7 @@ class FoFa(Search):
                 break
             self.subdomains.update(subdomains)
             size = resp_json.get('size')
-            if size < 1000:
+            if size < min(1000, settings.cam_records_maximum_per_domain):
                 break
             self.page_num += 1
 

modules/search/hunter_api.py

@@ -22,7 +22,7 @@ class Hunter(Search):
         self.page_num = 1
         subdomain_encode = f'domain_suffix="{self.domain}"'.encode('utf-8')
         query_data = base64.b64encode(subdomain_encode)
-        while True:
+        while 100 * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)

modules/search/quake_api.py

@@ -20,7 +20,7 @@ class Quake(Search):
         """
         self.per_page_num = 100
         self.page_num = 0
-        while True:
+        while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.header.update({'Content-Type': 'application/json'})

modules/search/zoomeye_api.py

@@ -19,7 +19,7 @@ class ZoomEyeAPI(Search):
         """
         self.per_page_num = 30
         self.page_num = 1
-        while True:
+        while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.header.update({'API-KEY': self.key})