diff --git a/oneforall/collect.py b/oneforall/collect.py index 70e5279..4f75bf9 100644 --- a/oneforall/collect.py +++ b/oneforall/collect.py @@ -14,7 +14,7 @@ class Collect(object): self.domain = domain self.elapsed = 0.0 self.modules = [] - self.collect_func = [] + self.collect_funcs = [] self.path = None self.export = export self.format = 'csv' @@ -44,8 +44,9 @@ class Collect(object): 导入脚本的do函数 """ for package, name in self.modules: - import_object = importlib.import_module('.'+name, package) - self.collect_func.append(getattr(import_object, 'do')) + import_object = importlib.import_module('.' + name, package) + func = getattr(import_object, 'do') + self.collect_funcs.append([func, name]) def run(self): """ @@ -58,8 +59,10 @@ class Collect(object): threads = [] # 创建多个子域收集线程 - for collect_func in self.collect_func: - thread = threading.Thread(target=collect_func, + for collect in self.collect_funcs: + func_obj, func_name = collect + thread = threading.Thread(target=func_obj, + name=func_name, args=(self.domain,), daemon=True) threads.append(thread) @@ -68,7 +71,13 @@ class Collect(object): thread.start() # 等待所有线程完成 for thread in threads: - thread.join() + # 挨个线程判断超时 最坏情况主线程阻塞时间=线程数*module_thread_timeout + # 超时线程将脱离主线程 由于创建线程时已添加守护属于 所有超时线程会随着主线程结束 + thread.join(config.module_thread_timeout) + + for thread in threads: + if thread.is_alive(): + logger.log('ALERT', f'{thread.name}模块线程发生超时') # 数据库导出 if self.export: diff --git a/oneforall/common/utils.py b/oneforall/common/utils.py index 6503b0b..bebae00 100644 --- a/oneforall/common/utils.py +++ b/oneforall/common/utils.py @@ -166,7 +166,7 @@ def check_path(path, name, format): path = default_path else: if path.exists(): - logger.log('ALERT', f'存在{path}路径将会覆盖') + logger.log('ALERT', f'存在{path}文件将会覆盖') parent_path = path.parent if not parent_path.exists(): logger.log('ALERT', f'不存在{parent_path}目录将会新建') diff --git a/oneforall/config.py b/oneforall/config.py index 43138f8..286d96f 100644 --- a/oneforall/config.py +++ b/oneforall/config.py @@ -27,7 +27,7 @@ enable_partial_module = [] # 启用部分模块 必须禁用enable_all_module # 只使用ask和baidu搜索引擎收集子域 # enable_partial_module = [('modules.search', 'ask') # ('modules.search', 'baidu')] - +module_thread_timeout = 360.0 # 每个收集模块线程超时时间(默认6分钟) # 爆破模块设置 enable_brute_module = False # 使用爆破模块(默认禁用) @@ -141,7 +141,7 @@ stdout_fmt = '{time:HH:mm:ss,SSS} ' \ logfile_fmt = '{time:YYYY-MM-DD HH:mm:ss,SSS} ' \ '[{level: <5}] ' \ '{process.name}({process.id}):' \ - '{thread.name: <10}({thread.id: <5}) | ' \ + '{thread.name: <18}({thread.id: <5}) | ' \ '{module}.{function}:' \ '{line} - {message}'