diff --git a/oneforall/common/request.py b/oneforall/common/request.py
index e5a3d5b..40c3baf 100644
--- a/oneforall/common/request.py
+++ b/oneforall/common/request.py
@@ -46,26 +46,24 @@ def gen_new_datas(datas, ports):
return new_datas
-async def fetch(session, url):
+async def fetch(session, url, semaphore):
"""
请求
:param session: session对象
:param url: url地址
+ :param semaphore: 并发信号量
:return: 响应对象和响应文本
"""
- header = None
- if config.fake_header:
- header = utils.gen_fake_header()
timeout = aiohttp.ClientTimeout(total=config.get_timeout)
- async with session.get(url,
- headers=header,
- ssl=config.verify_ssl,
- allow_redirects=config.get_redirects,
- timeout=timeout,
- proxy=config.get_proxy) as resp:
- text = await resp.text()
- return resp, text
+ async with semaphore:
+ async with session.get(url,
+ ssl=config.verify_ssl,
+ allow_redirects=config.get_redirects,
+ timeout=timeout,
+ proxy=config.get_proxy) as resp:
+ text = await resp.text()
+ return resp, text
def deal_results(datas, results):
@@ -79,7 +77,7 @@ def deal_results(datas, results):
resp, text = result
datas[index]['reason'] = resp.reason
datas[index]['status'] = resp.status
- if resp.status == 400 or resp.status >= 500:
+ if resp.status >= 500:
datas[index]['valid'] = 0
else:
datas[index]['valid'] = 1
@@ -95,7 +93,7 @@ def deal_results(datas, results):
datas[index]['title'] = title.text
elif head:
datas[index]['title'] = head.text
- else:
+ elif len(text) <= 200:
datas[index]['title'] = text
return datas
@@ -105,19 +103,25 @@ async def bulk_get_request(datas, port):
new_datas = gen_new_datas(datas, ports)
logger.log('INFOR', f'正在异步进行子域的GET请求')
+ limit_open_conn = config.limit_open_conn
+ if not limit_open_conn:
+ limit_open_conn = utils.get_semaphore()
# 使用异步域名解析器 自定义域名服务器
resolver = AsyncResolver(nameservers=config.resolver_nameservers)
conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
- limit=config.limit_open_conn,
+ limit=limit_open_conn,
limit_per_host=config.limit_per_host,
resolver=resolver)
- # semaphore = asyncio.Semaphore(utils.get_semaphore())
- async with ClientSession(connector=conn) as session:
+
+ semaphore = asyncio.Semaphore(limit_open_conn)
+ header = None
+ if config.fake_header:
+ header = utils.gen_fake_header()
+ async with ClientSession(connector=conn, headers=header) as session:
tasks = []
for i, data in enumerate(new_datas):
url = data.get('url')
- # task = asyncio.ensure_future(fetch(session, url, semaphore))
- task = asyncio.ensure_future(fetch(session, url))
+ task = asyncio.ensure_future(fetch(session, url, semaphore))
tasks.append(task)
if tasks: # 任务列表里有任务不空时才进行解析
# 等待所有task完成 错误聚合到结果列表里
diff --git a/oneforall/config.py b/oneforall/config.py
index a4437e9..bf3a92e 100644
--- a/oneforall/config.py
+++ b/oneforall/config.py
@@ -44,7 +44,7 @@ fuzz_rule = '' # fuzz域名的正则 示例:[a-z][0-9] 第一位是字母 第
ips_appear_maximum = 10 # 同一IP集合出现次数超过10认为是泛解析
# 代理设置
-enable_proxy = True # 是否使用代理(全局开关)
+enable_proxy = False # 是否使用代理(全局开关)
proxy_all_module = False # 代理所有模块
proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
@@ -101,9 +101,10 @@ get_proxy = None # proxy="http://user:pass@some.proxy.com"
get_timeout = 120 # http请求探测总超时时间 None或者0则表示不检测超时
get_redirects = True # 允许请求跳转
fake_header = True # 使用伪造请求头
-limit_open_conn = 100 # 限制同一时间打开的连接数(默认100),0表示不限制
+# 限制同一时间打开的连接数(默认None,根据系统不同设置,Windows系统500 其他系统1000)
+limit_open_conn = None
# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
-limit_per_host = 0 # 默认0表示不限制
+limit_per_host = 10 # 默认0表示不限制
# 模块API配置
@@ -180,8 +181,8 @@ stdout_fmt = '{time:HH:mm:ss,SSS} ' \
logfile_fmt = '{time:YYYY-MM-DD HH:mm:ss,SSS} ' \
'[{level: <5}] ' \
'{process.name}:{thread.name: <10} | ' \
- '{module}.{function}:{line} - ' \
- '{message}'
+ '{module}.{function}:' \
+ '{line} - {message}'
log_path = result_save_path.joinpath('oneforall.log')