尝试解决macOS队列大小限制问题

This commit is contained in:
Jing Ling
2020-03-07 19:11:00 +08:00
parent 57dd7d8c1e
commit 598f91c6f7
+13
View File
@@ -1,6 +1,7 @@
import socket import socket
import asyncio import asyncio
import functools import functools
import sys
from multiprocessing import Manager from multiprocessing import Manager
import tqdm import tqdm
@@ -169,6 +170,18 @@ async def aio_resolve(subdomain_list, process_num, coroutine_num):
loop.run_in_executor(None, resolve_progress_func, loop.run_in_executor(None, resolve_progress_func,
done_obj, len(subdomain_list)) done_obj, len(subdomain_list))
wrapped_resolve_func = functools.partial(do_resolve, done_obj) wrapped_resolve_func = functools.partial(do_resolve, done_obj)
result_list = list()
# macOS上队列大小不能超过2**15 - 1 = 32767
# https://stackoverflow.com/questions/5900985/multiprocessing-queue-maxsize-limit-is-32767
if sys.platform == 'darwin':
split_subdomain_list = utils.split_list(subdomain_list, 32767)
for current_subdomain_list in split_subdomain_list:
async with aiomp.Pool(processes=process_num,
childconcurrency=coroutine_num) as pool:
result = await pool.map(wrapped_resolve_func,
current_subdomain_list)
result_list.extend(result)
return result_list
async with aiomp.Pool(processes=process_num, async with aiomp.Pool(processes=process_num,
childconcurrency=coroutine_num) as pool: childconcurrency=coroutine_num) as pool:
result_list = await pool.map(wrapped_resolve_func, subdomain_list) result_list = await pool.map(wrapped_resolve_func, subdomain_list)