为搜索模块线程设置超时

oneforall/collect.py

@@ -14,7 +14,7 @@ class Collect(object):
         self.domain = domain
         self.elapsed = 0.0
         self.modules = []
-        self.collect_func = []
+        self.collect_funcs = []
         self.path = None
         self.export = export
         self.format = 'csv'
@@ -44,8 +44,9 @@ class Collect(object):
         导入脚本的do函数
         """
         for package, name in self.modules:
-            import_object = importlib.import_module('.'+name, package)
+            import_object = importlib.import_module('.' + name, package)
-            self.collect_func.append(getattr(import_object, 'do'))
+            func = getattr(import_object, 'do')
+            self.collect_funcs.append([func, name])
 
     def run(self):
         """
@@ -58,8 +59,10 @@ class Collect(object):
 
         threads = []
         # 创建多个子域收集线程
-        for collect_func in self.collect_func:
+        for collect in self.collect_funcs:
-            thread = threading.Thread(target=collect_func,
+            func_obj, func_name = collect
+            thread = threading.Thread(target=func_obj,
+                                      name=func_name,
                                       args=(self.domain,),
                                       daemon=True)
             threads.append(thread)
@@ -68,7 +71,13 @@ class Collect(object):
             thread.start()
         # 等待所有线程完成
         for thread in threads:
-            thread.join()
+            # 挨个线程判断超时 最坏情况主线程阻塞时间=线程数*module_thread_timeout
+            # 超时线程将脱离主线程 由于创建线程时已添加守护属于 所有超时线程会随着主线程结束
+            thread.join(config.module_thread_timeout)
+
+        for thread in threads:
+            if thread.is_alive():
+                logger.log('ALERT', f'{thread.name}模块线程发生超时')
 
         # 数据库导出
         if self.export:

oneforall/common/utils.py

@@ -166,7 +166,7 @@ def check_path(path, name, format):
         path = default_path
     else:
         if path.exists():
-            logger.log('ALERT', f'存在{path}路径将会覆盖')
+            logger.log('ALERT', f'存在{path}文件将会覆盖')
         parent_path = path.parent
         if not parent_path.exists():
             logger.log('ALERT', f'不存在{parent_path}目录将会新建')

oneforall/config.py

@@ -27,7 +27,7 @@ enable_partial_module = []  # 启用部分模块 必须禁用enable_all_module
 # 只使用ask和baidu搜索引擎收集子域
 # enable_partial_module = [('modules.search', 'ask')
 #                          ('modules.search', 'baidu')]
-
+module_thread_timeout = 360.0  # 每个收集模块线程超时时间(默认6分钟)
 
 # 爆破模块设置
 enable_brute_module = False  # 使用爆破模块(默认禁用)
@@ -141,7 +141,7 @@ stdout_fmt = '<cyan>{time:HH:mm:ss,SSS}</cyan> ' \
 logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
               '[<level>{level: <5}</level>] ' \
               '<cyan>{process.name}({process.id})</cyan>:' \
-              '<cyan>{thread.name: <10}({thread.id: <5})</cyan> | ' \
+              '<cyan>{thread.name: <18}({thread.id: <5})</cyan> | ' \
               '<blue>{module}</blue>.<blue>{function}</blue>:' \
               '<blue>{line}</blue> - <level>{message}</level>'