From 598f91c6f72c6635fa72183f88c45487eea8d031 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Sat, 7 Mar 2020 19:11:00 +0800 Subject: [PATCH] =?UTF-8?q?=E5=B0=9D=E8=AF=95=E8=A7=A3=E5=86=B3macOS?= =?UTF-8?q?=E9=98=9F=E5=88=97=E5=A4=A7=E5=B0=8F=E9=99=90=E5=88=B6=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- oneforall/common/resolve.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/oneforall/common/resolve.py b/oneforall/common/resolve.py index 2ac06e4..7c12861 100644 --- a/oneforall/common/resolve.py +++ b/oneforall/common/resolve.py @@ -1,6 +1,7 @@ import socket import asyncio import functools +import sys from multiprocessing import Manager import tqdm @@ -169,6 +170,18 @@ async def aio_resolve(subdomain_list, process_num, coroutine_num): loop.run_in_executor(None, resolve_progress_func, done_obj, len(subdomain_list)) wrapped_resolve_func = functools.partial(do_resolve, done_obj) + result_list = list() + # macOS上队列大小不能超过2**15 - 1 = 32767 + # https://stackoverflow.com/questions/5900985/multiprocessing-queue-maxsize-limit-is-32767 + if sys.platform == 'darwin': + split_subdomain_list = utils.split_list(subdomain_list, 32767) + for current_subdomain_list in split_subdomain_list: + async with aiomp.Pool(processes=process_num, + childconcurrency=coroutine_num) as pool: + result = await pool.map(wrapped_resolve_func, + current_subdomain_list) + result_list.extend(result) + return result_list async with aiomp.Pool(processes=process_num, childconcurrency=coroutine_num) as pool: result_list = await pool.map(wrapped_resolve_func, subdomain_list)