mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-26 12:57:50 +08:00
尝试解决macOS队列大小限制问题
This commit is contained in:
@@ -1,6 +1,7 @@
|
|||||||
import socket
|
import socket
|
||||||
import asyncio
|
import asyncio
|
||||||
import functools
|
import functools
|
||||||
|
import sys
|
||||||
from multiprocessing import Manager
|
from multiprocessing import Manager
|
||||||
|
|
||||||
import tqdm
|
import tqdm
|
||||||
@@ -169,6 +170,18 @@ async def aio_resolve(subdomain_list, process_num, coroutine_num):
|
|||||||
loop.run_in_executor(None, resolve_progress_func,
|
loop.run_in_executor(None, resolve_progress_func,
|
||||||
done_obj, len(subdomain_list))
|
done_obj, len(subdomain_list))
|
||||||
wrapped_resolve_func = functools.partial(do_resolve, done_obj)
|
wrapped_resolve_func = functools.partial(do_resolve, done_obj)
|
||||||
|
result_list = list()
|
||||||
|
# macOS上队列大小不能超过2**15 - 1 = 32767
|
||||||
|
# https://stackoverflow.com/questions/5900985/multiprocessing-queue-maxsize-limit-is-32767
|
||||||
|
if sys.platform == 'darwin':
|
||||||
|
split_subdomain_list = utils.split_list(subdomain_list, 32767)
|
||||||
|
for current_subdomain_list in split_subdomain_list:
|
||||||
|
async with aiomp.Pool(processes=process_num,
|
||||||
|
childconcurrency=coroutine_num) as pool:
|
||||||
|
result = await pool.map(wrapped_resolve_func,
|
||||||
|
current_subdomain_list)
|
||||||
|
result_list.extend(result)
|
||||||
|
return result_list
|
||||||
async with aiomp.Pool(processes=process_num,
|
async with aiomp.Pool(processes=process_num,
|
||||||
childconcurrency=coroutine_num) as pool:
|
childconcurrency=coroutine_num) as pool:
|
||||||
result_list = await pool.map(wrapped_resolve_func, subdomain_list)
|
result_list = await pool.map(wrapped_resolve_func, subdomain_list)
|
||||||
|
|||||||
Reference in New Issue
Block a user