river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-26 12:57:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

pip8格式化 使用ssl=False

Browse Source
This commit is contained in:
shmilylty
2019-08-07 15:17:40 +08:00
parent f74c45a67d
commit 18851408c2
1 changed files with 13 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
oneforall/common/request.py
+13 -6
View File
@@ -58,7 +58,8 @@ async def fetch(session, url, semaphore):
timeout = aiohttp.ClientTimeout(total=config.get_timeout) timeout = aiohttp.ClientTimeout(total=config.get_timeout)
async with semaphore: async with semaphore:
async with session.get(url, allow_redirects=config.get_redirects, async with session.get(url, allow_redirects=config.get_redirects,
timeout=timeout, proxy=config.get_proxy) as resp: timeout=timeout,
proxy=config.get_proxy) as resp:
text = await resp.text() text = await resp.text()
return resp, text return resp, text
@@ -77,7 +78,8 @@ def request_callback(future, index, datas):
datas[index]['valid'] = 0 datas[index]['valid'] = 0
else: else:
headers = resp.headers headers = resp.headers
banner = str({'Server': headers.get('Server'), 'Via': headers.get('Via'), banner = str({'Server': headers.get('Server'),
'Via': headers.get('Via'),
'X-Powered-By': headers.get('X-Powered-By')}) 'X-Powered-By': headers.get('X-Powered-By')})
datas[index]['banner'] = banner datas[index]['banner'] = banner
soup = BeautifulSoup(text, 'lxml') soup = BeautifulSoup(text, 'lxml')
@@ -98,16 +100,21 @@ async def bulk_get_request(datas, port):
header = None header = None
if config.fake_header: if config.fake_header:
header = utils.gen_fake_header() header = utils.gen_fake_header()
resolver = AsyncResolver(nameservers=config.resolver_nameservers) # 使用异步域名解析器 自定义域名服务器 # 使用异步域名解析器 自定义域名服务器
conn = aiohttp.TCPConnector(verify_ssl=config.verify_ssl, limit=config.limit_open_conn, resolver = AsyncResolver(nameservers=config.resolver_nameservers)
limit_per_host=config.limit_per_host, resolver=resolver) conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
limit=config.limit_open_conn,
limit_per_host=config.limit_per_host,
resolver=resolver)
semaphore = asyncio.Semaphore(utils.get_semaphore()) semaphore = asyncio.Semaphore(utils.get_semaphore())
async with ClientSession(connector=conn, headers=header) as session: async with ClientSession(connector=conn, headers=header) as session:
tasks = [] tasks = []
for i, data in enumerate(new_datas): for i, data in enumerate(new_datas):
url = data.get('url') url = data.get('url')
task = asyncio.ensure_future(fetch(session, url, semaphore)) task = asyncio.ensure_future(fetch(session, url, semaphore))
task.add_done_callback(functools.partial(request_callback, index=i, datas=new_datas)) task.add_done_callback(functools.partial(request_callback,
index=i,
datas=new_datas))
tasks.append(task) tasks.append(task)
if tasks: # 任务列表里有任务不空时才进行解析 if tasks: # 任务列表里有任务不空时才进行解析
await asyncio.wait(tasks) # 等待所有task完成 await asyncio.wait(tasks) # 等待所有task完成