mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-25 20:37:48 +08:00
Jing Ling
274 lines
8.7 KiB
Python
274 lines
8.7 KiB
Python
import asyncio
|
|
import functools
|
|
|
|
import aiohttp
|
|
import tqdm
|
|
from aiohttp import ClientSession
|
|
from bs4 import BeautifulSoup
|
|
|
|
from common import utils
|
|
from config.log import logger
|
|
from config import setting
|
|
from common.database import Database
|
|
|
|
|
|
def get_limit_conn():
|
|
limit_open_conn = setting.limit_open_conn
|
|
if limit_open_conn is None: # 默认情况
|
|
limit_open_conn = utils.get_semaphore()
|
|
elif not isinstance(limit_open_conn, int): # 如果传入不是数字的情况
|
|
limit_open_conn = utils.get_semaphore()
|
|
return limit_open_conn
|
|
|
|
|
|
def get_ports(port):
|
|
logger.log('DEBUG', f'Getting port range')
|
|
ports = set()
|
|
if isinstance(port, (set, list, tuple)):
|
|
ports = port
|
|
elif isinstance(port, int):
|
|
if 0 <= port <= 65535:
|
|
ports = {port}
|
|
elif port in {'default', 'small', 'large'}:
|
|
logger.log('DEBUG', f'{port} port range')
|
|
ports = setting.ports.get(port)
|
|
if not ports: # 意外情况
|
|
logger.log('ERROR', f'The specified request port range is incorrect')
|
|
ports = {80}
|
|
logger.log('INFOR', f'Port range:{ports}')
|
|
return set(ports)
|
|
|
|
|
|
def gen_req_data(data, ports):
|
|
logger.log('INFOR', f'Generating request urls')
|
|
new_data = []
|
|
for data in data:
|
|
resolve = data.get('resolve')
|
|
# 解析失败(0)的子域不进行http请求探测
|
|
if resolve == 0:
|
|
continue
|
|
subdomain = data.get('subdomain')
|
|
for port in ports:
|
|
if str(port).endswith('443'):
|
|
url = f'https://{subdomain}:{port}'
|
|
if port == 443:
|
|
url = f'https://{subdomain}'
|
|
data['id'] = None
|
|
data['url'] = url
|
|
data['port'] = port
|
|
new_data.append(data)
|
|
data = dict(data) # 需要生成一个新的字典对象
|
|
else:
|
|
url = f'http://{subdomain}:{port}'
|
|
if port == 80:
|
|
url = f'http://{subdomain}'
|
|
data['id'] = None
|
|
data['url'] = url
|
|
data['port'] = port
|
|
new_data.append(data)
|
|
data = dict(data) # 需要生成一个新的字典对象
|
|
return new_data
|
|
|
|
|
|
async def fetch(session, url):
|
|
"""
|
|
请求
|
|
|
|
:param session: session对象
|
|
:param str url: url地址
|
|
:return: 响应对象和响应文本
|
|
"""
|
|
method = setting.request_method.upper()
|
|
timeout = aiohttp.ClientTimeout(total=None,
|
|
connect=None,
|
|
sock_read=setting.sockread_timeout,
|
|
sock_connect=setting.sockconn_timeout)
|
|
try:
|
|
if method == 'HEAD':
|
|
async with session.head(url,
|
|
ssl=setting.verify_ssl,
|
|
allow_redirects=setting.allow_redirects,
|
|
timeout=timeout,
|
|
proxy=setting.aiohttp_proxy) as resp:
|
|
text = await resp.text()
|
|
else:
|
|
async with session.get(url,
|
|
ssl=setting.verify_ssl,
|
|
allow_redirects=setting.allow_redirects,
|
|
timeout=timeout,
|
|
proxy=setting.aiohttp_proxy) as resp:
|
|
|
|
try:
|
|
# 先尝试用utf-8解码
|
|
text = await resp.text(encoding='utf-8', errors='strict')
|
|
except UnicodeError:
|
|
try:
|
|
# 再尝试用gb18030解码
|
|
text = await resp.text(encoding='gb18030',
|
|
errors='strict')
|
|
except UnicodeError:
|
|
# 最后尝试自动解码
|
|
text = await resp.text(encoding=None,
|
|
errors='ignore')
|
|
return resp, text
|
|
except Exception as e:
|
|
return e
|
|
|
|
|
|
def get_title(markup):
|
|
"""
|
|
获取标题
|
|
|
|
:param markup: html标签
|
|
:return: 标题
|
|
"""
|
|
soup = BeautifulSoup(markup, 'html.parser')
|
|
|
|
title = soup.title
|
|
if title:
|
|
return title.text
|
|
|
|
h1 = soup.h1
|
|
if h1:
|
|
return h1.text
|
|
|
|
h2 = soup.h2
|
|
if h2:
|
|
return h2.text
|
|
|
|
h3 = soup.h3
|
|
if h2:
|
|
return h3.text
|
|
|
|
desc = soup.find('meta', attrs={'name': 'description'})
|
|
if desc:
|
|
return desc['content']
|
|
|
|
word = soup.find('meta', attrs={'name': 'keywords'})
|
|
if word:
|
|
return word['content']
|
|
|
|
text = soup.text
|
|
if len(text) <= 200:
|
|
return text
|
|
|
|
return 'None'
|
|
|
|
|
|
def request_callback(future, index, datas):
|
|
result = future.result()
|
|
if isinstance(result, BaseException):
|
|
logger.log('TRACE', result.args)
|
|
name = utils.get_classname(result)
|
|
datas[index]['reason'] = name + ' ' + str(result)
|
|
datas[index]['request'] = 0
|
|
datas[index]['alive'] = 0
|
|
elif isinstance(result, tuple):
|
|
resp, text = result
|
|
datas[index]['reason'] = resp.reason
|
|
datas[index]['status'] = resp.status
|
|
if resp.status == 400 or resp.status >= 500:
|
|
datas[index]['request'] = 0
|
|
datas[index]['alive'] = 0
|
|
else:
|
|
datas[index]['request'] = 1
|
|
datas[index]['alive'] = 1
|
|
headers = resp.headers
|
|
datas[index]['banner'] = utils.get_sample_banner(headers)
|
|
datas[index]['header'] = str(dict(headers))[1:-1]
|
|
if isinstance(text, str):
|
|
title = get_title(text).strip()
|
|
datas[index]['title'] = utils.remove_invalid_string(title)
|
|
datas[index]['response'] = utils.remove_invalid_string(text)
|
|
|
|
|
|
def get_connector():
|
|
limit_open_conn = get_limit_conn()
|
|
return aiohttp.TCPConnector(ttl_dns_cache=300,
|
|
ssl=setting.verify_ssl,
|
|
limit=limit_open_conn,
|
|
limit_per_host=setting.limit_per_host)
|
|
|
|
|
|
def get_header():
|
|
header = None
|
|
if setting.fake_header:
|
|
header = utils.gen_fake_header()
|
|
return header
|
|
|
|
|
|
async def bulk_request(data, port):
|
|
ports = get_ports(port)
|
|
no_req_data = utils.get_filtered_data(data)
|
|
to_req_data = gen_req_data(data, ports)
|
|
method = setting.request_method
|
|
logger.log('INFOR', f'Use {method} method to request')
|
|
logger.log('INFOR', f'Async subdomains request in progress')
|
|
connector = get_connector()
|
|
header = get_header()
|
|
async with ClientSession(connector=connector, headers=header) as session:
|
|
tasks = []
|
|
for i, data in enumerate(to_req_data):
|
|
url = data.get('url')
|
|
task = asyncio.ensure_future(fetch(session, url))
|
|
task.add_done_callback(functools.partial(request_callback,
|
|
index=i,
|
|
datas=to_req_data))
|
|
tasks.append(task)
|
|
# 任务列表里有任务不空时才进行解析
|
|
if tasks:
|
|
# 等待所有task完成 错误聚合到结果列表里
|
|
futures = asyncio.as_completed(tasks)
|
|
for future in tqdm.tqdm(futures,
|
|
total=len(tasks),
|
|
desc='Request Progress',
|
|
ncols=80):
|
|
await future
|
|
return to_req_data + no_req_data
|
|
|
|
|
|
def set_loop_policy():
|
|
try:
|
|
import uvloop
|
|
except ImportError:
|
|
pass
|
|
else:
|
|
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
|
|
|
|
|
|
def run_request(domain, data, port):
|
|
"""
|
|
HTTP request entrance
|
|
|
|
:param str domain: domain to be requested
|
|
:param list data: subdomains data to be requested
|
|
:param str port: range of ports to be requested
|
|
:return list: result
|
|
"""
|
|
logger.log('INFOR', f'Start subdomain request module')
|
|
set_loop_policy()
|
|
loop = asyncio.get_event_loop()
|
|
asyncio.set_event_loop(loop)
|
|
data = utils.set_id_none(data)
|
|
request_coroutine = bulk_request(data, port)
|
|
data = loop.run_until_complete(request_coroutine)
|
|
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
|
|
loop.run_until_complete(asyncio.sleep(0.25))
|
|
count = utils.count_alive(data)
|
|
logger.log('INFOR', f'Request module found {domain} have {count} alive subdomains')
|
|
return data
|
|
|
|
|
|
def save_data(name, data):
|
|
"""
|
|
Save request results to database
|
|
|
|
:param str name: table name
|
|
:param list data: data to be saved
|
|
"""
|
|
db = Database()
|
|
db.drop_table(name)
|
|
db.create_table(name)
|
|
db.save_db(name, data, 'request')
|
|
db.close()
|