river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-26 04:47:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

重构:OneForAll入口逻辑更加清晰

Browse Source
This commit is contained in:
Jing Ling
2020-02-23 04:23:25 +08:00
parent 36bfa0b7a0
commit 094fedc22a
13 changed files with 287 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
oneforall/aiobrute.py
+22 -23
View File
@@ -10,15 +10,11 @@ OneForAll多进程多协程异步子域爆破模块
import time
import queue
import signal
import asyncio
import secrets
import functools
from multiprocessing import Manager
import aiomultiprocess as aiomp
import exrex
import fire
import tqdm
import config
import dbexport
@@ -89,6 +85,13 @@ def wildcard_by_times(ips, ips_times):
def gen_fuzz_domains(domain, rule):
"""
生成fuzz模式下即将用于爆破的子域集合
:param str domain: 待爆破的主域
:param str rule: 用于爆破的正则规则
:return: 用于爆破的子域集合
"""
domains = set()
if '{fuzz}' not in domain:
logger.log('FATAL', f'没有指定fuzz位置')
@@ -118,6 +121,13 @@ def gen_fuzz_domains(domain, rule):
def gen_brute_domains(domain, path):
"""
生成基于字典爆破的子域数据
:param str domain: 待爆破的主域
:param str path: 字典路径
:return: 用于爆破的子域集合
"""
domains = set()
with open(path, encoding='utf-8', errors='ignore') as file:
for line in file:
@@ -146,7 +156,7 @@ class AIOBrute(Module):
参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
参数path默认None使用OneForAll结果目录生成路径
参数path默认None使用OneForAll结果目录自动生成路径
:param str target: 单个域名或者每行一个域名的文件路径
:param int process: 爆破的进程数(默认CPU核心数)
@@ -158,7 +168,7 @@ class AIOBrute(Module):
:param bool fuzz: 是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
:param str rule: fuzz模式使用的正则规则(默认使用config.py配置)
:param bool export: 是否导出爆破结果(默认True)
:param int valid: 导出子域的有效性(默认None)
:param bool valid: 只导出有效的子域结果(默认False)
:param str format: 导出格式(默认csv)
:param str path: 导出路径(默认None)
:param bool show: 终端显示导出数据(默认False)
@@ -248,8 +258,6 @@ class AIOBrute(Module):
= detect_wildcard(domain)
tasks = self.gen_tasks(domain)
logger.log('INFOR', f'正在爆破{domain}的域名')
# for task in tqdm.tqdm(tasks, total=len(tasks),
# desc='Progress'):
results = await resolve.aio_resolve(tasks, self.process, self.coroutine)
self.deal_results(results)
self.save_json()
@@ -303,7 +311,7 @@ class AIOBrute(Module):
f'{self.subdomains}')
if not self.path:
name = f'{self.domain}_brute_result.{self.format}'
self.path = config.result_save_path.joinpath(name)
self.path = config.result_save_dir.joinpath(name)
# 数据库导出
if self.export:
dbexport.export(self.domain,
@@ -313,18 +321,9 @@ class AIOBrute(Module):
show=self.show)
def do(domain, result): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
:param result: 结果集队列
"""
brute = AIOBrute(domain)
brute.run(result)
if __name__ == '__main__':
fire.Fire(AIOBrute)
# result_queue = queue.Queue()
# do('example.com', result_queue)
# domain = 'example.com'
# result = queue.Queue()
# brute = AIOBrute(domain)
# brute.run(result)
oneforall/collect.py
+7 -6
View File
@@ -1,6 +1,7 @@
import time
import threading
import importlib
import config
import dbexport
from config import logger
@@ -31,7 +32,7 @@ class Collect(object):
'dnsquery', 'intelligence', 'search']
# modules = ['intelligence'] # crawl模块还有点问题
for module in modules:
module_path = config.oneforall_module_path.joinpath(module)
module_path = config.module_dir.joinpath(module)
for path in module_path.rglob('*.py'):
# 需要导入的类
import_module = ('modules.' + module, path.stem)
@@ -59,8 +60,8 @@ class Collect(object):
threads = []
# 创建多个子域收集线程
for collect in self.collect_funcs:
func_obj, func_name = collect
for collect_func in self.collect_funcs:
func_obj, func_name = collect_func
thread = threading.Thread(target=func_obj,
name=func_name,
args=(self.domain,),
@@ -83,12 +84,12 @@ class Collect(object):
if self.export:
if not self.path:
name = f'{self.domain}.{self.format}'
self.path = config.result_save_path.joinpath(name)
self.path = config.result_save_dir.joinpath(name)
dbexport.export(self.domain, path=self.path, format=self.format)
end = time.time()
self.elapsed = round(end - start, 1)
if __name__ == '__main__':
a = Collect('example.com')
a.run()
collect = Collect('example.com')
collect.run()
oneforall/common/database.py
+18 -3
View File
@@ -6,6 +6,7 @@ SQLite数据库初始化和操作
"""
import records
import config
from records import Connection
from config import logger
@@ -28,7 +29,7 @@ class Database(object):
return db_path
protocol = 'sqlite:///'
if not db_path: # 数据库路径为空连接默认数据库
db_path = f'{protocol}{config.result_save_path}/result.sqlite3'
db_path = f'{protocol}{config.result_save_dir}/result.sqlite3'
else:
db_path = protocol + db_path
db = records.Database(db_path) # 不存在数据库时会新建一个数据库
@@ -185,6 +186,17 @@ class Database(object):
self.query(f'delete from "{table_name}" where '
f'subdomain is null or valid == 0')
def deal_table(self, deal_table_name, backup_table_name):
"""
收集任务完成时对表进行处理
:param str deal_table_name: 待处理的表名
:param str backup_table_name: 备份的表名
"""
self.copy_table(deal_table_name, backup_table_name)
self.remove_invalid(deal_table_name)
self.deduplicate_subdomain(deal_table_name)
def get_data(self, table_name):
"""
获取表中的所有数据
@@ -205,11 +217,14 @@ class Database(object):
table_name = table_name.replace('.', '_')
query = f'select id, url, subdomain, port, ips, status, reason,' \
f'valid, new, title, banner from "{table_name}"'
if valid == 0 or valid == 1:
where = f' where valid = {valid}'
if valid:
where = f' where valid = 1'
query += where
logger.log('TRACE', f'获取{table_name}表中的所有数据')
return self.query(query)
def close(self):
"""
关闭数据库连接
"""
self.conn.close()
oneforall/common/domain.py
+1 -1
View File
@@ -38,7 +38,7 @@ class Domain(object):
:return: 导出结果
"""
extract_cache_file = config.data_storage_path.joinpath('public_suffix_list.dat')
extract_cache_file = config.data_storage_dir.joinpath('public_suffix_list.dat')
tldext = tldextract.TLDExtract(extract_cache_file)
result = self.match()
if result:
oneforall/common/module.py
+1 -1
View File
@@ -226,7 +226,7 @@ class Module(object):
if not config.save_module_result:
return False
logger.log('TRACE', f'将{self.source}模块发现的子域结果保存为json文件')
path = config.result_save_path.joinpath(self.domain, self.module)
path = config.result_save_dir.joinpath(self.domain, self.module)
path.mkdir(parents=True, exist_ok=True)
name = self.source + '.json'
path = path.joinpath(name)
oneforall/common/request.py
+35 -3
View File
@@ -5,9 +5,11 @@ import aiohttp
import tqdm
from aiohttp import ClientSession
from bs4 import BeautifulSoup
import config
from common import utils
from config import logger
from common.database import Database
def get_limit_conn():
@@ -220,6 +222,36 @@ async def bulk_request(datas, port):
return new_datas
def run_bulk_query(datas, port):
new_datas = asyncio.run(bulk_request(datas, port))
return new_datas
def run_request(domain, data, port):
"""
调用子域请求入口函数
:param str domain: 待请求的主域
:param list data: 待请求的子域数据
:param str port: 待请求的端口范围
:return: 请求后得到的结果列表
:rtype: list
"""
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
request_coroutine = bulk_request(data, port)
data = loop.run_until_complete(request_coroutine)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
count = utils.count_valid(data)
logger.log('INFOR', f'经验证{domain}有效子域{count}个')
return data
def save_data(name, data):
"""
保存请求结果到数据库
:param str name: 保存表名
:param list data: 待保存的数据
"""
db = Database()
db.drop_table(name)
db.create_table(name)
db.save_db(name, data, 'request')
db.close()
oneforall/common/resolve.py
+39 -7
View File
@@ -10,6 +10,7 @@ from dns.resolver import Resolver
import config
from config import logger
from common.database import Database
def dns_resolver():
@@ -99,11 +100,27 @@ def update_data(data_list, results_dict):
return data_list
def init_worker():
signal.signal(signal.SIGINT, signal.SIG_IGN)
def save_data(name, data):
"""
保存解析结果到数据库
:param str name: 保存表名
:param list data: 待保存的数据
"""
db = Database()
db.drop_table(name)
db.create_table(name)
db.save_db(name, data, 'resolve')
db.close()
def query_progress(pr_queue, total):
def resolve_progress(pr_queue, total):
"""
解析进度
:param pr_queue: 进度队列
:param int total: 待解析的子域个数
"""
bar = tqdm.tqdm()
bar.total = total
bar.desc = 'Resolve Progress'
@@ -143,10 +160,9 @@ async def aio_resolve(subdomain_list, process_num, coroutine_num):
m = Manager()
pr_queue = m.Queue()
loop = asyncio.get_event_loop()
loop.run_in_executor(None, query_progress, pr_queue, len(subdomain_list))
loop.run_in_executor(None, resolve_progress, pr_queue, len(subdomain_list))
wrapped_query = functools.partial(aio_query, pr_queue)
async with aiomp.Pool(processes=process_num,
initializer=init_worker,
childconcurrency=coroutine_num) as pool:
results = await pool.map(wrapped_query, subdomain_list)
return results
@@ -156,8 +172,8 @@ async def bulk_resolve(data_list):
"""
批量解析A记录并返回解析结果
:param list data_list: 待查的数据列表
:return: 查询过得到的结果列表
:param list data_list: 待解析的数据列表
:return: 解析得到的结果列表
"""
logger.log('INFOR', '正在异步查询子域的A记录')
# semaphore = asyncio.Semaphore(config.limit_resolve_conn)
@@ -169,3 +185,19 @@ async def bulk_resolve(data_list):
data_list = update_data(data_list, results_dict)
logger.log('INFOR', '完成异步查询子域的A记录')
return data_list
def run_resolve(data):
"""
调用子域解析入口函数
:param list data: 待解析的子域数据列表
:return: 解析得到的结果列表
:rtype: list
"""
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
resolve_coroutine = bulk_resolve(data)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
return loop.run_until_complete(resolve_coroutine)
oneforall/common/utils.py
+18 -11
View File
@@ -5,6 +5,7 @@ import time
import random
import ipaddress
import platform
import config
from pathlib import Path
from records import Record, RecordCollection
@@ -157,7 +158,7 @@ def check_path(path, name, format):
:return: 保存路径
"""
filename = f'{name}.{format}'
default_path = config.result_save_path.joinpath(filename)
default_path = config.result_save_dir.joinpath(filename)
if path is None:
path = default_path
try:
@@ -250,30 +251,32 @@ def check_response(method, resp):
return False
def mark_subdomain(old_data, new_data):
def mark_subdomain(old_data, now_data):
"""
标记新增子域并返回新的数据集
:param old_data: 之前数据集
:param new_data: 现在数据集
:return: 已标记的新的数据集
:param list old_data: 之前子域数据
:param list now_data: 现在子域数据
:return: 标记后的的子域数据
:rtype: list
"""
# 第一次收集子域的情况
mark_data = now_data.copy()
if not old_data:
for index, item in enumerate(new_data):
for index, item in enumerate(mark_data):
item['new'] = 1
new_data[index] = item
return new_data
mark_data[index] = item
return mark_data
# 非第一次收集子域的情况
old_subdomains = {item.get('subdomain') for item in old_data}
for index, item in enumerate(new_data):
for index, item in enumerate(mark_data):
subdomain = item.get('subdomain')
if subdomain in old_subdomains:
item['new'] = 0
else:
item['new'] = 1
new_data[index] = item
return new_data
mark_data[index] = item
return mark_data
def remove_string(string):
@@ -335,3 +338,7 @@ def python_version():
def count_valid(data):
return len(list(filter(lambda item: item.get('valid') == 1, data)))
def get_subdomains(data):
return set(map(lambda item: item.get('subdomain'), data))
oneforall/config.py
+25 -12
View File
@@ -10,36 +10,49 @@ import urllib3
from loguru import logger
# 路径设置
oneforall_relpath = pathlib.Path(__file__).parent # oneforall代码相对路径
oneforall_abspath = oneforall_relpath.resolve() # oneforall代码绝对路径
oneforall_module_path = oneforall_relpath.joinpath('modules') # oneforall模块目录
data_storage_path = oneforall_relpath.joinpath('data') # 数据存放目录
result_save_path = oneforall_relpath.joinpath('results') # 结果保存目录
relative_directory = pathlib.Path(__file__).parent # OneForAll代码相对路径
module_dir = relative_directory.joinpath('modules') # OneForAll模块目录
data_storage_dir = relative_directory.joinpath('data') # 数据存放目录
result_save_dir = relative_directory.joinpath('results') # 结果保存目录
# 模块设置
# OneForAll入口参数设置
enable_dns_resolve = True # 使用DNS解析子域(默认True)
enable_http_request = True # 使用HTTP请求子域(默认True)
enable_takeover_check = False # 开启子域接管风险检查(默认False)
# 参数port可选值有'default', 'small', 'large'
http_request_port = 'default' # HTTP请求子域(默认'default'，探测80端口)
# 参数valid可选值True，False分别表示导出有效，全部子域结果
result_export_valid = False # 只导出有效的子域结果(默认False)
# 参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
# 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
result_save_format = 'csv' # 子域结果保存文件格式(默认csv)
# 参数path默认None使用OneForAll结果目录自动生成路径
result_save_path = None # 子域结果保存文件路径(默认None)
# 收集模块设置
save_module_result = False # 保存各模块发现结果为json文件(默认False)
enable_all_module = True # 启用所有模块(默认True)
enable_partial_module = [] # 启用部分模块 必须禁用enable_all_module才能生效
# 只使用ask和baidu搜索引擎收集子域
# 只使用ask和baidu搜索引擎收集子域的示例
# enable_partial_module = [('modules.search', 'ask')
# ('modules.search', 'baidu')]
module_thread_timeout = 360.0 # 每个收集模块线程超时时间(默认6分钟)
# 爆破模块设置
enable_brute_module = False # 使用爆破模块(默认False)
enable_dns_resolve = True # DNS解析子域(默认True)
enable_http_request = True # HTTP请求子域(默认True)
enable_wildcard_check = True # 开启泛解析检测(默认True)
enable_wildcard_deal = True # 开启泛解析处理(默认True)
# 爆破时使用的进程数(根据系统中CPU数量情况设置 不宜大于CPU数量 默认为系统中的CPU数量)
brute_process_num = os.cpu_count()
brute_coroutine_num = 1024 # 爆破时每个进程下的协程数
# 爆破所使用的字典路径 默认data/subdomains.txt
brute_wordlist_path = data_storage_path.joinpath('subnames.txt')
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
enable_recursive_brute = False # 是否使用递归爆破(默认禁用)
brute_recursive_depth = 2 # 递归爆破深度(默认2层)
# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt
recursive_namelist_path = data_storage_path.joinpath('next_subnames.txt')
recursive_namelist_path = data_storage_dir.joinpath('next_subnames.txt')
enable_fuzz = False # 是否使用fuzz模式枚举域名
fuzz_rule = '' # fuzz域名的正则 示例：[a-z][0-9] 第一位是字母 第二位是数字
ips_appear_maximum = 10 # 同一IP集合出现次数超过10认为是泛解析
@@ -142,7 +155,7 @@ logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
'<blue>{module}</blue>.<blue>{function}</blue>:' \
'<blue>{line}</blue> - <level>{message}</level>'
log_path = result_save_path.joinpath('oneforall.log')
log_path = result_save_dir.joinpath('oneforall.log')
logger.remove()
logger.level(name='TRACE', no=5, color='<cyan><bold>', icon='✏️')
oneforall/dbexport.py
+6 -4
View File
@@ -9,11 +9,12 @@ OneForAll数据库导出模块
"""
import fire
from common import utils
from common.database import Database
def export(table, db=None, valid=None, path=None, format='csv', show=False):
def export(table, db=None, valid=False, path=None, format='csv', show=False):
"""
OneForAll数据库导出模块
@@ -22,14 +23,13 @@ def export(table, db=None, valid=None, path=None, format='csv', show=False):
python3 dbexport.py --db result.db --table name --show False
Note:
参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
参数path默认None使用OneForAll结果目录生成路径
参数path默认None使用OneForAll结果目录自动生成路径
:param str table: 要导出的表
:param str db: 要导出的数据库路径(默认为results/result.sqlite3)
:param int valid: 导出子域的有效性(默认None)
:param bool valid: 只导出有效的子域结果(默认False)
:param str format: 导出文件格式(默认csv)
:param str path: 导出文件路径(默认None)
:param bool show: 终端显示导出数据(默认False)
@@ -47,6 +47,8 @@ def export(table, db=None, valid=None, path=None, format='csv', show=False):
data = rows.export(format)
database.close()
utils.save_data(path, data)
data_dict = rows.as_dict()
return data_dict
if __name__ == '__main__':
oneforall/modules/dnsquery/srv.py
+2 -2
View File
@@ -11,7 +11,7 @@ import threading
from common import utils
from common import resolve
from common.module import Module
from config import data_storage_path, logger
from config import data_storage_dir, logger
class BruteSRV(Module):
@@ -25,7 +25,7 @@ class BruteSRV(Module):
self.answers_que = queue.Queue()
def gen_names(self):
path = data_storage_path.joinpath('srv_prefixes.json')
path = data_storage_dir.joinpath('srv_prefixes.json')
with open(path, encoding='utf-8', errors='ignore') as file:
prefixes = json.load(file)
names = map(lambda prefix: prefix + self.domain, prefixes)
oneforall/oneforall.py
+112 -85
View File
@@ -8,9 +8,8 @@ OneForAll是一款功能强大的子域收集工具
:license: GNU General Public License v3.0, see LICENSE for more details.
"""
import asyncio
import fire
import config
import dbexport
from datetime import datetime
@@ -62,50 +61,112 @@ class OneForAll(object):
python3 oneforall.py --target example.com --show True run
Note:
参数valid可选值1，0，None分别表示导出有效，无效，全部子域
# 参数valid可选值True，False分别表示导出有效，全部子域结果
参数port可选值有'default', 'small', 'large', 详见config.py配置
参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
参数path默认None使用OneForAll结果目录生成路径
参数path默认None使用OneForAll结果目录自动生成路径
:param str target: 单个域名或者每行一个域名的文件路径(必需参数)
:param bool brute: 使用爆破模块(默认False)
:param bool dns: DNS解析子域(默认True)
:param bool req: HTTP请求子域(默认True)
:param str port: 请求验证子域的端口范围(默认只探测80端口)
:param int valid: 导出子域的有效性(默认None)
:param str format: 导出文件格式(默认csv)
:param str path: 导出文件路径(默认None)
:param str port: 请求验证子域的端口范围(默认探测80端口)
:param bool valid: 只导出有效的子域结果(默认False)
:param str format: 结果保存格式(默认csv)
:param str path: 结果保存路径(默认None)
:param bool takeover: 检查子域接管(默认False)
:param bool show: 终端显示导出数据(默认False)
"""
def __init__(self, target, brute=None, dns=None, req=None,
port='default', valid=None, format='csv', path=None,
takeover=False, show=False):
def __init__(self, target, brute=None, dns=None, req=None, port=None,
valid=None, format=None, path=None, takeover=None):
self.target = target
self.port = port
self.domains = set()
self.domain = str()
self.data = list()
self.datas = list()
self.brute = brute
self.dns = dns
self.req = req
self.takeover = takeover
self.port = port
self.valid = valid
self.format = format
self.path = path
self.show = show
self.takeover = takeover
self.domain = str()
self.domains = set()
self.data = list()
self.datas = list()
self.old_table = self.domain + '_last_result'
self.new_table = self.domain + '_now_result'
self.origin_table = self.domain + '_origin_result'
self.resolve_table = self.domain + '_resolve_result'
def config(self):
"""
配置参数
"""
if self.brute is None:
self.brute = bool(config.enable_brute_module)
if self.dns is None:
self.dns = bool(config.enable_dns_resolve)
if self.req is None:
self.req = bool(config.enable_http_request)
if self.takeover is None:
self.takeover = bool(config.enable_takeover_check)
if self.port is None:
self.port = config.http_request_port
if self.valid is None:
self.valid = bool(config.result_export_valid)
if self.format is None:
self.format = config.result_save_format
if self.path is None:
self.path = config.result_save_path
def export(self, table):
"""
从数据库中导出数据并做一些后续数据库善后处理
:param table: 要导出的表名
:return: 导出的数据
:rtype: list
"""
db = Database()
data = dbexport.export(table, valid=self.valid, format=self.format)
db.drop_table(self.new_table)
db.rename_table(self.domain, self.new_table)
db.close()
return data
def deal_db(self):
"""
收集任务完成时对数据库进行处理
"""
db = Database()
db.deal_table(self.domain, self.origin_table)
db.close()
def mark(self):
"""
标记新发现子域
:return: 标记后的的子域数据
:rtype: list
"""
db = Database()
old_data = list()
now_data = db.get_data(self.domain).as_dict()
# 非第一次收集子域的情况时数据库预处理
if db.exist_table(self.new_table):
db.drop_table(self.old_table) # 如果存在上次收集结果表就先删除
db.rename_table(self.new_table, self.old_table) # 新表重命名为旧表
old_data = db.get_data(self.old_table).as_dict()
db.close()
marked_data = utils.mark_subdomain(old_data, now_data)
return marked_data
def main(self):
if self.brute is None:
self.brute = config.enable_brute_module
if self.dns is None:
self.dns = config.enable_dns_resolve
if self.req is None:
self.req = config.enable_http_request
old_table = self.domain + '_last_result'
new_table = self.domain + '_now_result'
"""
OneForAll实际运行主流程
:return: 子域结果
:rtype: list
"""
collect = Collect(self.domain, export=False)
collect.run()
if self.brute:
@@ -113,91 +174,56 @@ class OneForAll(object):
brute = AIOBrute(self.domain, export=False)
brute.run()
db = Database()
original_table = self.domain + '_original_result'
db.copy_table(self.domain, original_table)
db.remove_invalid(self.domain)
db.deduplicate_subdomain(self.domain)
old_data = []
# 非第一次收集子域的情况时数据库预处理
if db.exist_table(new_table):
db.drop_table(old_table) # 如果存在上次收集结果表就先删除
db.rename_table(new_table, old_table) # 新表重命名为旧表
old_data = db.get_data(old_table).as_dict()
# 有关数据库处理
self.deal_db()
# 标记新发现子域
self.data = self.mark()
# 不解析子域直接导出结果
if not self.dns:
# 数据库导出
dbexport.export(self.domain, valid=self.valid,
format=self.format, show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
return
self.data = db.get_data(self.domain).as_dict()
# 标记新发现子域
self.data = utils.mark_subdomain(old_data, self.data)
# 获取事件循环
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
return self.export(self.domain)
# 解析子域
task = resolve.bulk_resolve(self.data)
self.data = loop.run_until_complete(task)
self.data = resolve.run_resolve(self.data)
# 保存解析结果
resolve_table = self.domain + '_resolve_result'
db.drop_table(resolve_table)
db.create_table(resolve_table)
db.save_db(resolve_table, self.data, 'resolve')
resolve.save_data(self.resolve_table, self.data)
# 不请求子域直接导出结果
if not self.req:
# 数据库导出
dbexport.export(resolve_table, valid=self.valid,
format=self.format, show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
return
return self.export(self.resolve_table)
# 请求子域
task = request.bulk_request(self.data, self.port)
self.data = loop.run_until_complete(task)
self.datas.extend(self.data)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
count = utils.count_valid(self.data)
logger.log('INFOR', f'经验证{self.domain}有效子域{count}个')
self.data = request.run_request(self.domain, self.data, self.port)
# 保存请求结果
db.clear_table(self.domain)
db.save_db(self.domain, self.data, 'request')
request.save_data(self.domain, self.data)
# 将最终结果列表添加到总的数据列表中
self.datas.extend(self.data)
# 数据库导出
dbexport.export(self.domain, valid=self.valid,
format=self.format, show=self.show)
db.drop_table(new_table)
db.rename_table(self.domain, new_table)
db.close()
self.export(self.domain)
# 子域接管检查
if self.takeover:
subdomains = set(map(lambda x: x.get('subdomain'), self.data))
subdomains = utils.get_subdomains(self.data)
takeover = Takeover(subdomains)
takeover.run()
return self.data
def run(self):
"""
OneForAll运行入口
:return: 总的子域结果
:rtype: list
"""
print(banner)
dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f'[*] Starting OneForAll @ {dt}\n')
logger.log('DEBUG', 'Python ' + utils.python_version())
logger.log('DEBUG', 'OneForAll ' + version)
logger.log('INFOR', f'开始运行OneForAll')
self.config()
self.domains = utils.get_domains(self.target)
if self.domains:
for self.domain in self.domains:
@@ -206,6 +232,7 @@ class OneForAll(object):
else:
logger.log('FATAL', f'获取域名失败')
logger.log('INFOR', f'结束运行OneForAll')
return self.datas
@staticmethod
def version():
oneforall/takeover.py
+1 -1
View File
@@ -24,7 +24,7 @@ from common.domain import Domain
def get_fingerprint():
path = config.data_storage_path.joinpath('fingerprints.json')
path = config.data_storage_dir.joinpath('fingerprints.json')
with open(path, encoding='utf-8', errors='ignore') as file:
fingerprints = json.load(file)
return fingerprints