river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-10-24 16:46:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

优化线程数量

Browse Source
This commit is contained in:
Jing Ling
2020-11-12 20:47:37 +08:00
parent 6e9c5b4f74
commit faf56a7ff8
1 changed files with 7 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/request.py
+7 -4
View File
@@ -217,16 +217,19 @@ def bulk_request(domain, req_data, ret=False):
logger.log('INFOR', 'Requesting urls in bulk')
resp_queue = Queue()
urls_queue = Queue()
total = len(req_data)
task_count = len(req_data)
for index, info in enumerate(req_data):
url = info.get('url')
urls_queue.put((index, url))
session = get_session()
thread_count = req_thread_count()
bar = get_progress_bar(total)
if task_count <= req_thread_count():
# 如果请求任务数很小不用创建很多线程了
thread_count = task_count
bar = get_progress_bar(task_count)
progress_thread = Thread(target=progress, name='ProgressThread',
args=(bar, total, urls_queue), daemon=True)
args=(bar, task_count, urls_queue), daemon=True)
progress_thread.start()
for i in range(thread_count):
@@ -237,7 +240,7 @@ def bulk_request(domain, req_data, ret=False):
urls_queue.join()
return resp_queue
save_thread = Thread(target=save, name=f'SaveThread',
args=(domain, total, req_data, resp_queue), daemon=True)
args=(domain, task_count, req_data, resp_queue), daemon=True)
save_thread.start()
urls_queue.join()
save_thread.join()