mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-26 04:47:48 +08:00
v0.0.1
This commit is contained in:
Jing Ling
@@ -0,0 +1 @@
|
||||
# coding=utf-8
|
||||
@@ -0,0 +1,10 @@
|
||||
# coding=utf-8
|
||||
from .module import Module
|
||||
|
||||
|
||||
class Crawl(Module):
|
||||
"""
|
||||
|
||||
"""
|
||||
def __init__(self):
|
||||
Module.__init__(self)
|
||||
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
# coding=utf-8
|
||||
|
||||
"""
|
||||
SQLite数据库初始化和操作
|
||||
"""
|
||||
|
||||
import records
|
||||
import config
|
||||
from records import Connection
|
||||
from config import logger
|
||||
|
||||
|
||||
def connect_db(db_path=None):
|
||||
"""
|
||||
获取数据库对象
|
||||
|
||||
:param db_path: 数据库连接或路径
|
||||
:return: SQLite数据库
|
||||
"""
|
||||
logger.log('DEBUG', f'正在获取数据库连接')
|
||||
if isinstance(db_path, Connection):
|
||||
return db_path
|
||||
protocol = 'sqlite:///'
|
||||
if not db_path: # 数据库路径为空连接默认数据库
|
||||
db_path = f'{protocol}{config.result_save_path}/result.sqlite3'
|
||||
else:
|
||||
db_path = protocol + db_path
|
||||
db = records.Database(db_path) # 不存在数据库时会新建一个数据库
|
||||
logger.log('DEBUG', f'使用数据库: {db_path}')
|
||||
return db.get_connection()
|
||||
|
||||
|
||||
def create_table(db_conn, table_name):
|
||||
"""
|
||||
初始化数据库
|
||||
|
||||
:param db_conn: 数据库连接
|
||||
:param str table_name: 要创建的表名
|
||||
"""
|
||||
logger.log('DEBUG', f'正在创建{table_name}表')
|
||||
try:
|
||||
db_conn.query(f'create table if not exists {table_name} ('
|
||||
f'id integer primary key,'
|
||||
f'url text,'
|
||||
f'subdomain text,'
|
||||
f'port int,'
|
||||
f'ips text,'
|
||||
f'status int,'
|
||||
f'reason text,'
|
||||
f'valid int,'
|
||||
f'title text,'
|
||||
f'banner text,'
|
||||
f'module text,'
|
||||
f'source text,'
|
||||
f'elapsed float,'
|
||||
f'count int)')
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
|
||||
|
||||
def save_db(db_conn, table_name, results, module_name=None):
|
||||
"""
|
||||
将各模块结果存入数据库
|
||||
|
||||
:param db_conn: 数据库连接
|
||||
:param str table_name: 表名
|
||||
:param list results: 结果列表
|
||||
:param str module_name: 模块名
|
||||
"""
|
||||
logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域结果存入数据库')
|
||||
if results:
|
||||
try:
|
||||
db_conn.bulk_query(f'insert into {table_name} (id, url, subdomain, port, ips, status,'
|
||||
f'reason, valid, title, banner, module, source, elapsed, count)'
|
||||
f'values (:id, :url, :subdomain, :port, :ips, :status, :reason, :valid,'
|
||||
f':title, :banner, :module, :source, :elapsed, :count)', results)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
|
||||
|
||||
def copy_table(db_conn, table_name):
|
||||
"""
|
||||
复制表创建备份
|
||||
|
||||
:param db_conn: 数据库连接
|
||||
:param str table_name: 表名
|
||||
"""
|
||||
new_table_name = table_name + '_bak'
|
||||
logger.log('DEBUG', f'正在将{table_name}表复制到{new_table_name}新表')
|
||||
try:
|
||||
db_conn.query(f'drop table if exists {new_table_name}')
|
||||
db_conn.query(f'create table {new_table_name} as select * from {table_name}')
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
|
||||
|
||||
def clear_table(db_conn, table_name):
|
||||
"""
|
||||
清空表中数据
|
||||
|
||||
:param db_conn: 数据库连接
|
||||
:param str table_name: 表名
|
||||
"""
|
||||
logger.log('DEBUG', f'正在清空{table_name}表中的数据')
|
||||
try:
|
||||
db_conn.query(f'delete from {table_name}')
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
|
||||
|
||||
def deduplicate_subdomain(db_conn, table_name):
|
||||
"""
|
||||
去重表中的子域并删除空值和无效值
|
||||
|
||||
:param db_conn: 数据库连接
|
||||
:param str table_name: 表名
|
||||
"""
|
||||
logger.log('DEBUG', f'正在去重{table_name}表中的子域')
|
||||
try:
|
||||
db_conn.query(f'delete from {table_name} where id not in (select min(id) from {table_name} group by subdomain)')
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
|
||||
|
||||
def remove_invalid(db_conn, table_name):
|
||||
"""
|
||||
去除表中的空值或无效子域
|
||||
|
||||
:param db_conn: 数据库连接
|
||||
:param str table_name: 表名
|
||||
"""
|
||||
logger.log('DEBUG', f'正在去除{table_name}表中的无效子域')
|
||||
try:
|
||||
db_conn.query(f'delete from {table_name} where subdomain is null or valid == 0')
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
|
||||
|
||||
def get_data(db_conn, table_name):
|
||||
"""
|
||||
获取表中的所有数据
|
||||
|
||||
:param db_conn: 数据库连接
|
||||
:param str table_name: 表名
|
||||
"""
|
||||
logger.log('DEBUG', f'获取{table_name}表中的所有数据')
|
||||
try:
|
||||
rows = db_conn.query(f'select * from {table_name}')
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
else:
|
||||
return rows
|
||||
|
||||
|
||||
def get_subdomain(db_conn, table_name, valid):
|
||||
"""
|
||||
获取表中的子域数据
|
||||
|
||||
:param db_conn: 数据库连接
|
||||
:param str table_name: 表名
|
||||
:param int valid: 是否有效
|
||||
"""
|
||||
logger.log('DEBUG', f'获取{table_name}表中的所有数据')
|
||||
try:
|
||||
rows = db_conn.query(f'select * from {table_name} where valid = {valid}')
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
else:
|
||||
return rows
|
||||
@@ -0,0 +1,64 @@
|
||||
# coding=utf-8
|
||||
import re
|
||||
import tldextract
|
||||
import config
|
||||
|
||||
|
||||
class Domain(object):
|
||||
"""
|
||||
域名处理类
|
||||
|
||||
:param str string: 传入的字符串
|
||||
"""
|
||||
def __init__(self, string):
|
||||
self.string = str(string)
|
||||
self.regexp = r'\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b'
|
||||
self.domain = None
|
||||
|
||||
def match(self):
|
||||
"""
|
||||
域名匹配
|
||||
|
||||
:return: 匹配结果
|
||||
"""
|
||||
result = re.search(self.regexp, self.string, re.I)
|
||||
if result:
|
||||
return result.group()
|
||||
else:
|
||||
return None
|
||||
|
||||
def extract(self):
|
||||
"""
|
||||
域名导出
|
||||
|
||||
>>>d = Domain('www.example.com')
|
||||
<domain.Domain object>
|
||||
>>>d.extract()
|
||||
ExtractResult(subdomain='www', domain='example', suffix='com')
|
||||
|
||||
:return: 导出结果
|
||||
"""
|
||||
extract_cache_file = config.data_storage_path.joinpath('public_suffix_list.dat')
|
||||
tldext = tldextract.TLDExtract(extract_cache_file)
|
||||
result = self.match()
|
||||
if result:
|
||||
return tldext(result)
|
||||
else:
|
||||
return None
|
||||
|
||||
def registered(self):
|
||||
"""
|
||||
获取注册域名
|
||||
|
||||
>>>d = Domain('www.example.com')
|
||||
<domain.Domain object>
|
||||
>>>d.registered()
|
||||
example.com
|
||||
|
||||
:return: 注册域名
|
||||
"""
|
||||
result = self.extract()
|
||||
if result:
|
||||
return result.registered_domain
|
||||
else:
|
||||
return None
|
||||
@@ -0,0 +1,191 @@
|
||||
# coding=utf-8
|
||||
"""
|
||||
模块基类
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
|
||||
import requests
|
||||
import config
|
||||
from config import logger
|
||||
from . import utils
|
||||
from .domain import Domain
|
||||
from common import database
|
||||
|
||||
|
||||
lock = threading.Lock()
|
||||
|
||||
|
||||
class Module(object):
|
||||
def __init__(self):
|
||||
self.module = 'Module'
|
||||
self.source = 'BaseModule'
|
||||
self.cookie = None
|
||||
self.header = dict()
|
||||
self.proxy = None
|
||||
self.delay = config.request_delay # 请求睡眠时延
|
||||
self.timeout = config.request_timeout # 请求超时时间
|
||||
self.verify = config.request_verify # 请求SSL验证
|
||||
self.domain = '' # 要进行子域名收集的域名
|
||||
self.subdomains = set() # 存放发现的子域
|
||||
self.records = dict() # 存放子域解析记录
|
||||
self.results = list() # 存放模块结果
|
||||
self.elapsed = 0.0 # 模块执行耗时
|
||||
|
||||
def get(self, url, params=None, **kwargs):
|
||||
"""
|
||||
自定义get请求
|
||||
|
||||
:param str url: 请求地址
|
||||
:param dict params: 请求参数
|
||||
:param kwargs: 其他参数
|
||||
:return: requests响应对象
|
||||
"""
|
||||
try:
|
||||
resp = requests.get(url, params=params, cookies=self.cookie, headers=self.header,
|
||||
proxies=self.proxy, timeout=self.timeout, verify=self.verify, **kwargs)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
return None
|
||||
if resp.status_code != 200:
|
||||
logger.log('ALERT', f'GET {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
|
||||
content_type = resp.headers.get('Content-Type')
|
||||
if content_type:
|
||||
if 'json' in content_type:
|
||||
logger.log('ALERT', resp.json())
|
||||
return None
|
||||
logger.log('DEBUG', f'GET {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
|
||||
return resp
|
||||
|
||||
def post(self, url, data=None, **kwargs):
|
||||
"""
|
||||
自定义post请求
|
||||
|
||||
:param str url: 请求地址
|
||||
:param dict data: 请求数据
|
||||
:param kwargs: 其他参数
|
||||
:return: requests响应对象
|
||||
"""
|
||||
try:
|
||||
resp = requests.post(url, data=data, cookies=self.cookie, headers=self.header,
|
||||
proxies=self.proxy, timeout=self.timeout, verify=self.verify, **kwargs)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
return None
|
||||
if resp.status_code != 200:
|
||||
content_type = resp.headers.get('Content-Type')
|
||||
if content_type:
|
||||
if 'json' in content_type:
|
||||
logger.log('ALERT', resp.json())
|
||||
return None
|
||||
logger.log('DEBUG', f'POST {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
|
||||
return resp
|
||||
|
||||
def get_header(self):
|
||||
"""
|
||||
获取请求头
|
||||
|
||||
:return: 请求头
|
||||
"""
|
||||
# logger.log('DEBUG', f'获取请求头')
|
||||
if config.enable_fake_header:
|
||||
return utils.gen_fake_header()
|
||||
else:
|
||||
return self.header
|
||||
|
||||
def get_proxy(self, module):
|
||||
"""
|
||||
获取代理
|
||||
|
||||
:param str module: 模块名
|
||||
:return: 代理字典
|
||||
"""
|
||||
if not config.enable_proxy:
|
||||
logger.log('DEBUG', f'所有模块不使用代理')
|
||||
return self.proxy
|
||||
if config.proxy_all_module:
|
||||
logger.log('DEBUG', f'{module}模块使用代理')
|
||||
return utils.get_random_proxy()
|
||||
if module in config.proxy_partial_module:
|
||||
logger.log('DEBUG', f'{module}模块使用代理')
|
||||
return utils.get_random_proxy()
|
||||
else:
|
||||
logger.log('DEBUG', f'{module}模块不使用代理')
|
||||
return self.proxy
|
||||
|
||||
@staticmethod
|
||||
def match(domain, html, distinct=True):
|
||||
"""
|
||||
正则匹配出子域
|
||||
|
||||
:param str domain: 域名
|
||||
:param str html: 要匹配的html响应体
|
||||
:param bool distinct: 匹配结果去除
|
||||
:return: 匹配出的子域集合或列表
|
||||
:rtype: set or list
|
||||
"""
|
||||
logger.log('DEBUG', f'正则匹配响应体中的子域')
|
||||
regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
|
||||
+ domain.replace('.', r'\.')
|
||||
result = re.findall(regexp, html, re.I)
|
||||
if not result:
|
||||
return set()
|
||||
deal = map(lambda s: re.sub(r'(?:http://|https://)', '', s[1:].lower(), flags=re.I), result)
|
||||
if distinct:
|
||||
return set(deal)
|
||||
else:
|
||||
return list(deal)
|
||||
|
||||
@staticmethod
|
||||
def register(domain):
|
||||
"""
|
||||
获取注册域名
|
||||
|
||||
:param str domain: 域名
|
||||
:return: 注册域名
|
||||
"""
|
||||
return Domain(domain).registered()
|
||||
|
||||
def save_json(self):
|
||||
"""
|
||||
将各模块结果保存为json文件
|
||||
"""
|
||||
logger.log('DEBUG', f'将{self.source}模块发现的子域结果保存为json文件')
|
||||
if config.save_module_result:
|
||||
dirpath = config.result_save_path.joinpath(self.domain, self.module)
|
||||
dirpath.mkdir(parents=True, exist_ok=True)
|
||||
name = self.source + '.json'
|
||||
path = dirpath.joinpath(name)
|
||||
with open(path, mode='w', encoding='utf-8') as file:
|
||||
result = {'domain': self.domain, 'name': self.module, 'source': self.source, 'elapsed': self.elapsed,
|
||||
'count': len(self.subdomains), 'subdomains': list(self.subdomains), 'records': self.records}
|
||||
json.dump(result, file, ensure_ascii=False, indent=4)
|
||||
|
||||
def gen_result(self):
|
||||
results = list()
|
||||
if not len(self.subdomains): # 一个子域都没有发现的情况
|
||||
result = {'id': None, 'url': None, 'subdomain': None, 'port': None, 'ips': None, 'status': None,
|
||||
'reason': None, 'valid': 1, 'title': None, 'banner': None, 'module': self.module,
|
||||
'source': self.source, 'elapsed': self.elapsed, 'count': 0}
|
||||
results.append(result)
|
||||
self.results = (self.source, results)
|
||||
else:
|
||||
for subdomain in self.subdomains:
|
||||
url = 'http://' + subdomain
|
||||
ips = self.records.get(subdomain)
|
||||
result = {'id': None, 'url': url, 'subdomain': subdomain, 'port': None, 'ips': ips, 'status': None,
|
||||
'reason': None, 'valid': 1, 'title': None, 'banner': None, 'module': self.module,
|
||||
'source': self.source, 'elapsed': self.elapsed, 'count': len(self.subdomains)}
|
||||
results.append(result)
|
||||
self.results = (self.source, results)
|
||||
|
||||
def save_db(self):
|
||||
lock.acquire()
|
||||
db_conn = database.connect_db()
|
||||
table_name = self.domain.replace('.', '_')
|
||||
database.create_table(db_conn, table_name)
|
||||
source, results = self.results
|
||||
database.save_db(db_conn, table_name, results, source) # 将结果存入数据库中
|
||||
lock.release()
|
||||
@@ -0,0 +1,10 @@
|
||||
# coding=utf-8
|
||||
from .module import Module
|
||||
|
||||
|
||||
class Query(Module):
|
||||
"""
|
||||
查询基类
|
||||
"""
|
||||
def __init__(self):
|
||||
Module.__init__(self)
|
||||
@@ -0,0 +1,115 @@
|
||||
# coding=utf-8
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import aiohttp
|
||||
from aiohttp import ClientSession
|
||||
from aiohttp.resolver import AsyncResolver
|
||||
from bs4 import BeautifulSoup
|
||||
import config
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
def get_ports(port):
|
||||
logger.log('INFOR', f'正在获取请求端口范围')
|
||||
ports = set()
|
||||
if isinstance(port, set):
|
||||
ports = port
|
||||
elif isinstance(port, str):
|
||||
if port not in {'small', 'medium', 'large', 'xlarge'}:
|
||||
logger.log('ERROR', f'不存在{port}等端口范围')
|
||||
port = 'medium'
|
||||
ports = config.ports.get(port)
|
||||
logger.log('INFOR', f'使用{port}等端口范围')
|
||||
if not ports: # 意外情况 ports_range为空使用使用中等端口范围
|
||||
logger.log('ALERT', f'使用medium等端口范围')
|
||||
ports = config.ports.get('medium')
|
||||
return ports
|
||||
|
||||
|
||||
def gen_new_datas(datas, ports):
|
||||
logger.log('INFOR', f'正在生成请求地址')
|
||||
new_datas = []
|
||||
protocols = ['http://', 'https://']
|
||||
for data in datas:
|
||||
if data.get('valid'): # 有效的子域才进行http请求探测
|
||||
subdomain = data.get('subdomain')
|
||||
for port in ports:
|
||||
for protocol in protocols:
|
||||
url = f'{protocol}{subdomain}:{port}'
|
||||
data['id'] = None
|
||||
data['url'] = url
|
||||
data['port'] = port
|
||||
new_datas.append(data)
|
||||
data = dict(data) # 需要生成一个新的字典对象
|
||||
return new_datas
|
||||
|
||||
|
||||
async def fetch(session, url, semaphore):
|
||||
"""
|
||||
请求
|
||||
|
||||
:param session: session对象
|
||||
:param url: url地址
|
||||
:param semaphore: 同步对象（控制并发量）
|
||||
:return: 响应对象和响应文本
|
||||
"""
|
||||
timeout = aiohttp.ClientTimeout(total=config.get_timeout)
|
||||
async with semaphore:
|
||||
async with session.get(url, allow_redirects=config.get_redirects,
|
||||
timeout=timeout, proxy=config.get_proxy) as resp:
|
||||
text = await resp.text()
|
||||
return resp, text
|
||||
|
||||
|
||||
def request_callback(future, index, datas):
|
||||
try:
|
||||
resp, text = future.result()
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
datas[index]['reason'] = str(e.args)
|
||||
datas[index]['valid'] = 0
|
||||
else:
|
||||
datas[index]['reason'] = resp.reason
|
||||
datas[index]['status'] = resp.status
|
||||
if resp.status == 400 or resp.status >= 500:
|
||||
datas[index]['valid'] = 0
|
||||
else:
|
||||
headers = resp.headers
|
||||
banner = str({'Server': headers.get('Server'), 'Via': headers.get('Via'),
|
||||
'X-Powered-By': headers.get('X-Powered-By')})
|
||||
datas[index]['banner'] = banner
|
||||
soup = BeautifulSoup(text, 'lxml')
|
||||
title = soup.title
|
||||
head = soup.head
|
||||
if title:
|
||||
datas[index]['title'] = title.text
|
||||
elif head:
|
||||
datas[index]['title'] = head.text
|
||||
else:
|
||||
datas[index]['title'] = text
|
||||
|
||||
|
||||
async def bulk_get_request(datas, port):
|
||||
logger.log('INFOR', f'正在异步进行子域的GET请求')
|
||||
ports = get_ports(port)
|
||||
new_datas = gen_new_datas(datas, ports)
|
||||
header = None
|
||||
if config.fake_header:
|
||||
header = utils.gen_fake_header()
|
||||
resolver = AsyncResolver(nameservers=config.resolver_nameservers) # 使用异步域名解析器 自定义域名服务器
|
||||
conn = aiohttp.TCPConnector(verify_ssl=config.verify_ssl, limit=config.limit_open_conn,
|
||||
limit_per_host=config.limit_per_host, resolver=resolver)
|
||||
semaphore = asyncio.Semaphore(utils.get_semaphore())
|
||||
async with ClientSession(connector=conn, headers=header) as session:
|
||||
tasks = []
|
||||
for i, data in enumerate(new_datas):
|
||||
url = data.get('url')
|
||||
task = asyncio.ensure_future(fetch(session, url, semaphore))
|
||||
task.add_done_callback(functools.partial(request_callback, index=i, datas=new_datas))
|
||||
tasks.append(task)
|
||||
if tasks: # 任务列表里有任务不空时才进行解析
|
||||
await asyncio.wait(tasks) # 等待所有task完成
|
||||
logger.log('INFOR', f'完成异步进行子域的GET请求')
|
||||
return new_datas
|
||||
@@ -0,0 +1,88 @@
|
||||
# coding=utf-8
|
||||
import asyncio
|
||||
import functools
|
||||
|
||||
import dns.resolver
|
||||
import aiodns
|
||||
import config
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
def dns_resolver():
|
||||
"""
|
||||
dns解析器
|
||||
"""
|
||||
resolver = dns.resolver.Resolver()
|
||||
resolver.nameservers = config.resolver_nameservers
|
||||
resolver.timeout = config.resolver_timeout
|
||||
resolver.lifetime = config.resolver_lifetime
|
||||
return resolver
|
||||
|
||||
|
||||
def dns_query_a(hostname):
|
||||
"""
|
||||
查询A记录
|
||||
|
||||
:param str hostname: 主机名
|
||||
:return: 查询结果
|
||||
"""
|
||||
resolver = dns_resolver()
|
||||
return resolver.query(hostname, 'A')
|
||||
|
||||
|
||||
def aiodns_resolver():
|
||||
"""
|
||||
异步dns解析器
|
||||
"""
|
||||
return aiodns.DNSResolver(nameservers=config.resolver_nameservers,
|
||||
timeout=config.resolver_timeout)
|
||||
|
||||
|
||||
async def aiodns_query_a(hostname, semaphore):
|
||||
"""
|
||||
异步查询A记录
|
||||
|
||||
:param str hostname: 主机名
|
||||
:param semaphore: 并发查询数量
|
||||
:return: 主机名或查询结果或查询异常
|
||||
"""
|
||||
async with semaphore:
|
||||
resolver = aiodns_resolver()
|
||||
answers = await resolver.query(hostname, 'A')
|
||||
return hostname, answers
|
||||
|
||||
|
||||
def resolve_callback(future, index, datas):
|
||||
try:
|
||||
result = future.result()
|
||||
except aiodns.error.DNSError as e:
|
||||
datas[index]['ips'] = str(e.args)
|
||||
datas[index]['valid'] = 0
|
||||
else:
|
||||
if isinstance(result, tuple):
|
||||
_, answers = result
|
||||
ips = {record.host for record in answers}
|
||||
datas[index]['ips'] = str(ips)
|
||||
|
||||
|
||||
async def bulk_query_a(datas):
|
||||
"""
|
||||
批量查询A记录
|
||||
|
||||
:param datas: 待查的数据集
|
||||
:return: 查询过得到的结果集
|
||||
"""
|
||||
logger.log('INFOR', '正在异步查询子域的A记录')
|
||||
tasks = []
|
||||
semaphore = asyncio.Semaphore(utils.get_semaphore())
|
||||
for i, data in enumerate(datas):
|
||||
if not data.get('ips'):
|
||||
subdomain = data.get('subdomain')
|
||||
task = asyncio.ensure_future(aiodns_query_a(subdomain, semaphore))
|
||||
task.add_done_callback(functools.partial(resolve_callback, index=i, datas=datas)) # 回调
|
||||
tasks.append(task)
|
||||
if tasks: # 任务列表里有任务不空时才进行解析
|
||||
await asyncio.wait(tasks) # 等待所有task完成
|
||||
logger.log('INFOR', '完成异步查询子域的A记录')
|
||||
return datas
|
||||
@@ -0,0 +1,50 @@
|
||||
# coding=utf-8
|
||||
import requests
|
||||
import config
|
||||
from .module import Module
|
||||
from . import utils
|
||||
|
||||
|
||||
class Search(Module):
|
||||
"""
|
||||
搜索基类
|
||||
"""
|
||||
def __init__(self):
|
||||
Module.__init__(self)
|
||||
self.page_num = 0 # 要显示搜索起始条数
|
||||
self.per_page_num = 50 # 每页显示搜索条数
|
||||
self.recursive_search = config.enable_recursive_search
|
||||
self.recursive_times = config.search_recursive_times
|
||||
|
||||
@staticmethod
|
||||
def filter(domain, subdomain):
|
||||
"""
|
||||
生成搜索过滤语句
|
||||
使用搜索引擎支持的-site:语法过滤掉搜索页面较多的子域以发现新域
|
||||
|
||||
:param str domain: 域名
|
||||
:param set subdomain: 子域名集合
|
||||
:return: 过滤语句
|
||||
:rtype: str
|
||||
"""
|
||||
statements_list = []
|
||||
subdomains_temp = set(map(lambda x: x + '.' + domain, config.subdomains_common))
|
||||
subdomains_temp = list(subdomain.intersection(subdomains_temp))
|
||||
for i in range(0, len(subdomains_temp), 2): # 同时排除2个子域
|
||||
statements_list.append(''.join(set(map(lambda s: ' -site:' + s, subdomains_temp[i:i + 2]))))
|
||||
return statements_list
|
||||
|
||||
def match_location(self, domain, url):
|
||||
"""
|
||||
匹配跳转之后的url
|
||||
针对部分搜索引擎（如百度搜索）搜索展示url时有显示不全的情况
|
||||
此函数会向每条结果的链接发送head请求获取响应头的location值并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str url: 展示结果的url链接
|
||||
:return: 匹配的子域
|
||||
:rtype set
|
||||
"""
|
||||
resp = requests.head(url, headers=self.header, proxies=self.proxy, timeout=self.timeout, allow_redirects=False)
|
||||
location = resp.headers.get('location')
|
||||
return set(utils.match_subdomain(domain, location))
|
||||
@@ -0,0 +1,130 @@
|
||||
# coding=utf-8
|
||||
import re
|
||||
import pathlib
|
||||
import random
|
||||
import ipaddress
|
||||
import platform
|
||||
import config
|
||||
from fake_useragent import UserAgent
|
||||
from common.domain import Domain
|
||||
from config import logger
|
||||
|
||||
|
||||
def match_subdomain(domain, text, distinct=True):
|
||||
"""
|
||||
匹配text中domain的子域名
|
||||
|
||||
:param str domain: 域名
|
||||
:param str text: 响应文本
|
||||
:param bool distinct: 结果去重
|
||||
:return: 匹配结果
|
||||
:rtype: set or list
|
||||
"""
|
||||
regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' + domain.replace('.', r'\.')
|
||||
result = re.findall(regexp, text, re.I)
|
||||
if not result:
|
||||
return set()
|
||||
deal = map(lambda s: s.lower(), result)
|
||||
if distinct:
|
||||
return set(deal)
|
||||
else:
|
||||
return list(deal)
|
||||
|
||||
|
||||
def gen_random_ip():
|
||||
"""
|
||||
生成随机的点分十进制的IP字符串
|
||||
"""
|
||||
while True:
|
||||
ip = ipaddress.IPv4Address(random.randint(0, 2 ** 32 - 1))
|
||||
if ip.is_global:
|
||||
return ip.exploded
|
||||
|
||||
|
||||
def gen_fake_header():
|
||||
"""
|
||||
生成伪造请求头
|
||||
"""
|
||||
ua = UserAgent()
|
||||
ip = gen_random_ip()
|
||||
headers = {
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
|
||||
'Cache-Control': 'max-age=0',
|
||||
'Connection': 'keep-alive',
|
||||
'DNT': '1',
|
||||
'Referer': 'https://www.google.com/',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'User-Agent': ua.random,
|
||||
'X-Forwarded-For': ip,
|
||||
'X-Real-IP': ip
|
||||
}
|
||||
return headers
|
||||
|
||||
|
||||
def get_random_proxy():
|
||||
"""
|
||||
获取随机代理
|
||||
"""
|
||||
try:
|
||||
return random.choice(config.proxy_pool)
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
|
||||
def split_list(ls, size):
|
||||
"""
|
||||
将ls列表按size大小划分并返回新的划分结果列表
|
||||
|
||||
:param list ls: 要划分的列表
|
||||
:param int size: 划分大小
|
||||
:return 划分结果
|
||||
|
||||
>>> split_list([1, 2, 3, 4], 3)
|
||||
[[1, 2, 3], [4]]
|
||||
"""
|
||||
if size == 0:
|
||||
return ls
|
||||
return [ls[i:i+size] for i in range(0, len(ls), size)]
|
||||
|
||||
|
||||
def get_domains(target):
|
||||
"""
|
||||
获取域名
|
||||
|
||||
:param set or str target:
|
||||
:return: 域名集合
|
||||
"""
|
||||
domains = set()
|
||||
logger.log('INFOR', f'正在获取域名')
|
||||
if isinstance(target, set):
|
||||
domains = target
|
||||
elif isinstance(target, str):
|
||||
path = pathlib.Path(target)
|
||||
if path.is_file():
|
||||
with open(target) as file:
|
||||
for line in file:
|
||||
domain = Domain(line.strip()).match()
|
||||
if domain:
|
||||
domains.add(domain)
|
||||
if Domain(target).match():
|
||||
domains = {target}
|
||||
logger.log('INFOR', f'获取到{len(domains)}个域名')
|
||||
return domains
|
||||
|
||||
|
||||
def get_semaphore():
|
||||
"""
|
||||
获取查询并发值
|
||||
|
||||
:return: 并发整型值
|
||||
"""
|
||||
system = platform.system()
|
||||
if system == 'Windows':
|
||||
return 500
|
||||
elif system == 'Linux':
|
||||
return 1000
|
||||
elif system == 'Darwin':
|
||||
return 1000
|
||||
|
||||
Reference in New Issue
Block a user