From faf56a7ff84478d3bc47a9612c5cb93ed55ada8e Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Thu, 12 Nov 2020 20:47:37 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BA=BF=E7=A8=8B=E6=95=B0?= =?UTF-8?q?=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/request.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/common/request.py b/common/request.py index 3923cf9..8f04101 100644 --- a/common/request.py +++ b/common/request.py @@ -217,16 +217,19 @@ def bulk_request(domain, req_data, ret=False): logger.log('INFOR', 'Requesting urls in bulk') resp_queue = Queue() urls_queue = Queue() - total = len(req_data) + task_count = len(req_data) for index, info in enumerate(req_data): url = info.get('url') urls_queue.put((index, url)) session = get_session() thread_count = req_thread_count() - bar = get_progress_bar(total) + if task_count <= req_thread_count(): + # 如果请求任务数很小不用创建很多线程了 + thread_count = task_count + bar = get_progress_bar(task_count) progress_thread = Thread(target=progress, name='ProgressThread', - args=(bar, total, urls_queue), daemon=True) + args=(bar, task_count, urls_queue), daemon=True) progress_thread.start() for i in range(thread_count): @@ -237,7 +240,7 @@ def bulk_request(domain, req_data, ret=False): urls_queue.join() return resp_queue save_thread = Thread(target=save, name=f'SaveThread', - args=(domain, total, req_data, resp_queue), daemon=True) + args=(domain, task_count, req_data, resp_queue), daemon=True) save_thread.start() urls_queue.join() save_thread.join()