river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-10-25 00:56:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

优化线程数量

Browse Source
This commit is contained in:
Jing Ling
2020-11-12 20:47:37 +08:00
parent 6e9c5b4f74
commit faf56a7ff8
1 changed files with 7 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/request.py
+7 -4
View File
@@ -217,16 +217,19 @@ def bulk_request(domain, req_data, ret=False):
logger.log('INFOR', 'Requesting urls in bulk') logger.log('INFOR', 'Requesting urls in bulk')
resp_queue = Queue() resp_queue = Queue()
urls_queue = Queue() urls_queue = Queue()
total = len(req_data) task_count = len(req_data)
for index, info in enumerate(req_data): for index, info in enumerate(req_data):
url = info.get('url') url = info.get('url')
urls_queue.put((index, url)) urls_queue.put((index, url))
session = get_session() session = get_session()
thread_count = req_thread_count() thread_count = req_thread_count()
bar = get_progress_bar(total) if task_count <= req_thread_count():
# 如果请求任务数很小不用创建很多线程了
thread_count = task_count
bar = get_progress_bar(task_count)
progress_thread = Thread(target=progress, name='ProgressThread', progress_thread = Thread(target=progress, name='ProgressThread',
args=(bar, total, urls_queue), daemon=True) args=(bar, task_count, urls_queue), daemon=True)
progress_thread.start() progress_thread.start()
for i in range(thread_count): for i in range(thread_count):
@@ -237,7 +240,7 @@ def bulk_request(domain, req_data, ret=False):
urls_queue.join() urls_queue.join()
return resp_queue return resp_queue
save_thread = Thread(target=save, name=f'SaveThread', save_thread = Thread(target=save, name=f'SaveThread',
args=(domain, total, req_data, resp_queue), daemon=True) args=(domain, task_count, req_data, resp_queue), daemon=True)
save_thread.start() save_thread.start()
urls_queue.join() urls_queue.join()
save_thread.join() save_thread.join()