river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-26 04:47:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

translate

Browse Source
This commit is contained in:
JrD
2020-05-11 18:36:50 +08:00
parent 8e0daa48a1
commit 038b91b52f
14 changed files with 230 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
brute.py
+8 -8
View File
@@ -132,7 +132,7 @@ def gen_word_subdomains(expression, path):
def query_domain_ns_a(ns_list):
logger.log('INFOR', f'Querying authoritative name server {ns_list} A record')
logger.log('INFOR', f'Querying A record from authoritative name server: {ns_list} ')
if not isinstance(ns_list, list):
return list()
ns_ip_list = []
@@ -147,7 +147,7 @@ def query_domain_ns_a(ns_list):
if answer:
for item in answer:
ns_ip_list.append(item.address)
logger.log('INFOR', f'Authoritative name server A record: {ns_ip_list}')
logger.log('INFOR', f'Authoritative name server A record result: {ns_ip_list}')
return ns_ip_list
@@ -162,7 +162,7 @@ def query_domain_ns(domain):
logger.log('ERROR', f'Querying NS records of {domain} error')
return list()
ns = [item.to_text() for item in answer]
logger.log('INFOR', f'{domain}\'s authoritative name server: {ns}')
logger.log('INFOR', f'{domain}\'s authoritative name server is {ns}')
return ns
@@ -186,7 +186,7 @@ def get_wildcard_record(domain, resolver):
exit(1)
else:
if answer.rrset is None:
logger.log('DEBUG', f'No record of query results')
logger.log('DEBUG', f'No record of query result')
return None, None
name = answer.name
ip = {item.address for item in answer}
@@ -282,7 +282,7 @@ def gen_records(items, records, subdomains, ip_times, wc_ips, wc_ttl):
have_a_record = True
ttl = answer.get('ttl')
ttls.append(ttl)
cname.append(answer.get('name')[:-1]) # 去出最右边的`.`点号
cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号
ip = answer.get('data')
ips.append(ip)
public.append(utils.ip_is_public(ip))
@@ -339,7 +339,7 @@ def stat_ip_times(result_paths):
def deal_output(output_paths, ip_times, wildcard_ips, wildcard_ttl):
logger.log('INFOR', f'Processing results...')
logger.log('INFOR', f'Processing result...')
records = dict() # 用来记录所有域名解析数据
subdomains = list() # 用来保存所有通过有效性检查的子域
for output_path in output_paths:
@@ -598,8 +598,8 @@ class Brute(Module):
delete_file(dict_path, output_paths)
end = time.time()
self.elapse = round(end - start, 1)
logger.log('INFOR', f'{self.source} module spends {self.elapse} seconds'
f'Found {len(self.subdomains)} subdomains of {domain}')
logger.log('INFOR', f'{self.source} module spends {self.elapse} seconds, '
f'found {len(self.subdomains)} subdomains of {domain}')
logger.log('DEBUG', f'{self.source} module found subdomains of {domain}:\n'
f'{self.subdomains}')
self.gen_result(brute=dict_len, valid=len(self.subdomains))
common/crawl.py
+2 -1
View File
@@ -3,7 +3,8 @@ from .module import Module
class Crawl(Module):
"""
爬虫基类
Crawl base class
"""
def __init__(self):
Module.__init__(self)
common/database.py
+51 -51
View File
@@ -2,7 +2,7 @@
# coding=utf-8
"""
SQLite数据库初始化和操作
SQLite database initialization and operation
"""
import records
@@ -19,12 +19,12 @@ class Database(object):
@staticmethod
def get_conn(db_path):
"""
获取数据库对象
Get database connection
:param db_path: 数据库连接或路径
:return: SQLite数据库
:param db_path: Database path
:return: db_conn: SQLite database connection
"""
logger.log('TRACE', f'正在获取数据库连接')
logger.log('TRACE', f'Establishing database connection...')
if isinstance(db_path, Connection):
return db_path
protocol = 'sqlite:///'
@@ -33,7 +33,7 @@ class Database(object):
else:
db_path = protocol + db_path
db = records.Database(db_path) # 不存在数据库时会新建一个数据库
logger.log('TRACE', f'使用数据库: {db_path}')
logger.log('TRACE', f'Use the database: {db_path}')
return db.get_connection()
def query(self, sql):
@@ -46,15 +46,15 @@ class Database(object):
def create_table(self, table_name):
"""
创建表结构
Create table
:param str table_name: 要创建的表名
:param str table_name: table name
"""
table_name = table_name.replace('.', '_')
if self.exist_table(table_name):
logger.log('TRACE', f'已经存在{table_name}表')
logger.log('TRACE', f'{table_name} table already exists')
return
logger.log('TRACE', f'正在创建{table_name}表')
logger.log('TRACE', f'Creating {table_name} table')
self.query(f'create table "{table_name}" ('
f'id integer primary key,'
f'type text,'
@@ -87,14 +87,14 @@ class Database(object):
def save_db(self, table_name, results, module_name=None):
"""
将各模块结果存入数据库
Save the results of each module in the database
:param str table_name: 表名
:param list results: 结果列表
:param str module_name: 模块名
:param str table_name: table name
:param list results: results list
:param str module_name: mo
"""
logger.log('TRACE', f'正在将{module_name}模块发现{table_name}的子域'
'结果存入数据库')
logger.log('TRACE',
f'Saving the subdomain results of {table_name} found by module {module_name} into database...')
table_name = table_name.replace('.', '_')
if results:
try:
@@ -114,13 +114,13 @@ class Database(object):
def exist_table(self, table_name):
"""
判断是否存在某表
Determine table exists
:param str table_name: 表名
:return: 是否存在某表
:param str table_name: table name
:return bool: Whether table exists
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在查询是否存在{table_name}表')
logger.log('TRACE', f'Determining whether the {table_name} table exists')
results = self.query(f'select count() from sqlite_master '
f'where type = "table" and '
f'name = "{table_name}"')
@@ -131,80 +131,80 @@ class Database(object):
def copy_table(self, table_name, bak_table_name):
"""
复制表创建备份
Copy table to create backup
:param str table_name: 表名
:param str bak_table_name: 新表名
:param str table_name: table name
:param str bak_table_name: new table name
"""
table_name = table_name.replace('.', '_')
bak_table_name = bak_table_name.replace('.', '_')
logger.log('TRACE', f'正在将{table_name}表复制到{bak_table_name}新表')
logger.log('TRACE', f'Copying {table_name} table to {bak_table_name} new table')
self.query(f'drop table if exists "{bak_table_name}"')
self.query(f'create table "{bak_table_name}" '
f'as select * from "{table_name}"')
def clear_table(self, table_name):
"""
清空表中数据
Clear the table
:param str table_name: 表名
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在清空{table_name}表中的数据')
logger.log('TRACE', f'Clearing data in table {table_name}')
self.query(f'delete from "{table_name}"')
def drop_table(self, table_name):
"""
删除表
Delete table
:param str table_name: 表名
:param str table_name: table name
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在删除{table_name}表')
logger.log('TRACE', f'Deleting {table_name} table')
self.query(f'drop table if exists "{table_name}"')
def rename_table(self, table_name, new_table_name):
"""
重命名表名
Rename table name
:param str table_name: 表名
:param str new_table_name: 新表名
:param str table_name: old table name
:param str new_table_name: new table name
"""
table_name = table_name.replace('.', '_')
new_table_name = new_table_name.replace('.', '_')
logger.log('TRACE', f'正在将{table_name}表重命名为{table_name}表')
logger.log('TRACE', f'Renaming {table_name} table to {new_table_name} table')
self.query(f'alter table "{table_name}" '
f'rename to "{new_table_name}"')
def deduplicate_subdomain(self, table_name):
"""
去重表中的子域
Deduplicates of subdomains in the table
:param str table_name: 表名
:param str table_name: table name
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在去重{table_name}表中的子域')
logger.log('TRACE', f'Deduplicating subdomains in {table_name} table')
self.query(f'delete from "{table_name}" where '
f'id not in (select min(id) '
f'from "{table_name}" group by subdomain)')
def remove_invalid(self, table_name):
"""
去除表中的空值或无效子域
Remove nulls or invalid subdomains in the table
:param str table_name: 表名
:param str table_name: table name
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在去除{table_name}表中的无效子域')
logger.log('TRACE', f'Removing invalid subdomains in {table_name} table')
self.query(f'delete from "{table_name}" where '
f'subdomain is null or resolve == 0')
def deal_table(self, deal_table_name, backup_table_name):
"""
收集任务完成时对表进行处理
Process the table when the collection task is complete
:param str deal_table_name: 待处理的表名
:param str backup_table_name: 备份的表名
:param str deal_table_name: Pending table name
:param str backup_table_name: Table name for backup
"""
self.copy_table(deal_table_name, backup_table_name)
self.remove_invalid(deal_table_name)
@@ -212,21 +212,21 @@ class Database(object):
def get_data(self, table_name):
"""
获取表中的所有数据
Get all the data in the table
:param str table_name: 表名
:param str table_name: table name
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'获取{table_name}表中的所有数据')
logger.log('TRACE', f'Get all the data from {table_name} table')
return self.query(f'select * from "{table_name}"')
def export_data(self, table_name, alive, limit):
"""
获取表中的部分数据
Get part of the data in the table
:param str table_name: 表名
:param any alive: 存活
:param str limit: 限制字段
:param str table_name: table name
:param any alive: alive flag
:param str limit: limit value
"""
table_name = table_name.replace('.', '_')
query = f'select id, type, new, alive, request, resolve, url, ' \
@@ -240,11 +240,11 @@ class Database(object):
elif alive:
where = f' where alive = 1'
query += where
logger.log('TRACE', f'获取{table_name}表中的数据')
logger.log('TRACE', f'Get the data from {table_name} table')
return self.query(query)
def close(self):
"""
关闭数据库连接
Close the database connection
"""
self.conn.close()
common/domain.py
+9 -8
View File
@@ -5,10 +5,11 @@ from config import setting
class Domain(object):
"""
域名处理类
Processing domain class
:param str string: 传入的字符串
:param str string: input string
"""
def __init__(self, string):
self.string = str(string)
self.regexp = r'\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b'
@@ -16,9 +17,9 @@ class Domain(object):
def match(self):
"""
域名匹配
match domain
:return: 匹配结果
:return : result
"""
result = re.search(self.regexp, self.string, re.I)
if result:
@@ -28,14 +29,14 @@ class Domain(object):
def extract(self):
"""
域名导出
extract domain
>>> d = Domain('www.example.com')
<domain.Domain object>
>>> d.extract()
ExtractResult(subdomain='www', domain='example', suffix='com')
:return: 导出结果
:return: extracted domain results
"""
data_storage_dir = setting.data_storage_dir
extract_cache_file = data_storage_dir.joinpath('public_suffix_list.dat')
@@ -48,14 +49,14 @@ class Domain(object):
def registered(self):
"""
获取注册域名
registered domain
>>> d = Domain('www.example.com')
<domain.Domain object>
>>> d.registered()
example.com
:return: 注册域名
:return: registered domain result
"""
result = self.extract()
if result:
common/lookup.py
+4 -3
View File
@@ -4,15 +4,16 @@ from common import utils
class Lookup(Module):
"""
DNS查询基类
DNS query base class
"""
def __init__(self):
Module.__init__(self)
def query(self):
"""
查询域名的TXT记录
:return: 查询结果
Query the TXT record of domain
:return: query result
"""
answer = utils.dns_query(self.domain, self.type)
if answer is None:
common/module.py
+58 -59
View File
@@ -1,6 +1,6 @@
# coding=utf-8
"""
模块基类
Module base class
"""
import json
@@ -39,43 +39,43 @@ class Module(object):
def check(self, *apis):
"""
简单检查是否配置了api信息
Simply check whether the api information configure or not
:param apis: api信息元组
:return: 检查结果
:param apis: apis set
:return bool: check result
"""
if not all(apis):
logger.log('ALERT', f'{self.source}模块没有配置API跳过执行')
logger.log('ALERT', f'{self.source} module is not configured, skip')
return False
return True
def begin(self):
"""
输出模块开始信息
begin log
"""
logger.log('DEBUG', f'开始执行{self.source}模块收集{self.domain}的子域')
logger.log('DEBUG', f'Start {self.source} module to collect subdomains of {self.domain}')
def finish(self):
"""
输出模块结束信息
finish log
"""
self.end = time.time()
self.elapse = round(self.end - self.start, 1)
logger.log('DEBUG', f'结束执行{self.source}模块收集{self.domain}的子域')
logger.log('INFOR', f'{self.source}模块耗时{self.elapse}秒发现子域'
f'{len(self.subdomains)}个')
logger.log('DEBUG', f'{self.source}模块发现{self.domain}的子域\n'
logger.log('DEBUG', f'Finished {self.source} module to collect {self.domain}\'s subdomains')
logger.log('INFOR', f'The {self.source} module took {self.elapse} seconds, '
f'found {len(self.subdomains)} subdomains')
logger.log('DEBUG', f'{self.source} module found subdomains of {self.domain}\n'
f'{self.subdomains}')
def head(self, url, params=None, check=True, **kwargs):
"""
自定义head请求
Custom head request
:param str url: 请求地址
:param dict params: 请求参数
:param bool check: 检查响应
:param kwargs: 其他参数
:return: requests响应对象
:param str url: request url
:param dict params: request parameters
:param bool check: check response
:param kwargs: other params
:return: requests's response object
"""
try:
resp = requests.head(url,
@@ -97,13 +97,13 @@ class Module(object):
def get(self, url, params=None, check=True, **kwargs):
"""
自定义get请求
Custom get request
:param str url: 请求地址
:param dict params: 请求参数
:param bool check: 检查响应
:param kwargs: 其他参数
:return: requests响应对象
:param str url: request url
:param dict params: request parameters
:param bool check: check response
:param kwargs: other params
:return: requests's response object
"""
try:
resp = requests.get(url,
@@ -125,13 +125,13 @@ class Module(object):
def post(self, url, data=None, check=True, **kwargs):
"""
自定义post请求
Custom post request
:param str url: 请求地址
:param dict data: 请求数据
:param bool check: 检查响应
:param kwargs: 其他参数
:return: requests响应对象
:param str url: request url
:param dict params: request parameters
:param bool check: check response
:param kwargs: other params
:return: requests's response object
"""
try:
resp = requests.post(url,
@@ -153,11 +153,11 @@ class Module(object):
def get_header(self):
"""
获取请求头
Get request header
:return: 请求头
:return: header
"""
# logger.log('DEBUG', f'获取请求头')
# logger.log('DEBUG', f'Get request header')
if setting.enable_fake_header:
return utils.gen_fake_header()
else:
@@ -165,36 +165,35 @@ class Module(object):
def get_proxy(self, module):
"""
获取代理
Get proxy
:param str module: 模块名
:return: 代理字典
:param str module: module name
:return: proxy
"""
if not setting.enable_proxy:
logger.log('TRACE', f'所有模块不使用代理')
logger.log('TRACE', f'All modules do not use proxy')
return self.proxy
if setting.proxy_all_module:
logger.log('TRACE', f'{module}模块使用代理')
logger.log('TRACE', f'{module} module uses proxy')
return utils.get_random_proxy()
if module in setting.proxy_partial_module:
logger.log('TRACE', f'{module}模块使用代理')
logger.log('TRACE', f'{module} module uses proxy')
return utils.get_random_proxy()
else:
logger.log('TRACE', f'{module}模块不使用代理')
logger.log('TRACE', f'{module} module does not use proxy')
return self.proxy
@staticmethod
def match(domain, html, distinct=True):
"""
正则匹配出子域
Use regexp to match subdomains
:param str domain: 域名
:param str html: 要匹配的html响应体
:param bool distinct: 匹配结果去除
:return: 匹配出的子域集合或列表
:rtype: set or list
:param str domain: domain
:param str html: response html text
:param bool distinct: deduplicate results or not (default True)
:return set/list: result set or list
"""
logger.log('TRACE', f'正则匹配响应体中的子域')
logger.log('TRACE', f'Use regexp to match subdomains in the response body')
regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.')
@@ -211,22 +210,22 @@ class Module(object):
@staticmethod
def register(domain):
"""
获取注册域名
Get registered domain
:param str domain: 域名
:return: 注册域名
:param str domain: domain
:return: registered domain
"""
return Domain(domain).registered()
def save_json(self):
"""
将各模块结果保存为json文件
Save the results of each module as a json file
:return： 是否保存成功
:return bool: whether saved successfully
"""
if not setting.save_module_result:
return False
logger.log('TRACE', f'将{self.source}模块发现的子域结果保存为json文件')
logger.log('TRACE', f'Save the subdomain results found by {self.source} module as a json file')
path = setting.result_save_dir.joinpath(self.domain, self.module)
path.mkdir(parents=True, exist_ok=True)
name = self.source + '.json'
@@ -244,7 +243,7 @@ class Module(object):
def gen_record(self, subdomains, record):
"""
生成记录字典
Generate record dictionary
"""
item = dict()
item['content'] = record
@@ -253,11 +252,11 @@ class Module(object):
def gen_result(self, find=0, brute=None, valid=0):
"""
生成结果
Generate results
"""
logger.log('DEBUG', f'正在生成最终结果')
logger.log('DEBUG', f'Generating final results...')
if not len(self.subdomains): # 该模块一个子域都没有发现的情况
logger.log('DEBUG', f'{self.source}模块收集结果为空')
logger.log('DEBUG', f'{self.source} module result is empty')
result = {'id': None,
'type': self.type,
'alive': None,
@@ -347,10 +346,10 @@ class Module(object):
def save_db(self):
"""
将模块结果存入数据库中
Save module results into the database
"""
logger.log('DEBUG', f'正在将结果存入到数据库')
logger.log('DEBUG', f'Saving results to database')
lock.acquire()
db = Database()
db.create_table(self.domain)
common/query.py
+2 -1
View File
@@ -3,7 +3,8 @@ from .module import Module
class Query(Module):
"""
查询基类
Query base class
"""
def __init__(self):
Module.__init__(self)
common/request.py
+17 -18
View File
@@ -22,7 +22,7 @@ def get_limit_conn():
def get_ports(port):
logger.log('DEBUG', f'正在获取请求端口范围')
logger.log('DEBUG', f'Getting port range...')
ports = set()
if isinstance(port, (set, list, tuple)):
ports = port
@@ -30,17 +30,17 @@ def get_ports(port):
if 0 <= port <= 65535:
ports = {port}
elif port in {'default', 'small', 'large'}:
logger.log('DEBUG', f'请求{port}等端口范围')
logger.log('DEBUG', f'{port} port range')
ports = setting.ports.get(port)
if not ports: # 意外情况
logger.log('ERROR', f'指定请求端口范围有误')
logger.log('ERROR', f'The specified request port range is incorrect')
ports = {80}
logger.log('INFOR', f'请求端口范围：{ports}')
logger.log('INFOR', f'Port range:{ports}')
return set(ports)
def gen_req_data(data, ports):
logger.log('INFOR', f'正在生成请求地址')
logger.log('INFOR', f'Generating request urls...')
new_data = []
for data in data:
resolve = data.get('resolve')
@@ -202,8 +202,8 @@ async def bulk_request(data, port):
no_req_data = utils.get_filtered_data(data)
to_req_data = gen_req_data(data, ports)
method = setting.request_method
logger.log('INFOR', f'请求使用{method}方法')
logger.log('INFOR', f'正在进行异步子域请求')
logger.log('INFOR', f'Use {method} method to request')
logger.log('INFOR', f'Async subdomains request in progress...')
connector = get_connector()
header = get_header()
async with ClientSession(connector=connector, headers=header) as session:
@@ -238,15 +238,14 @@ def set_loop_policy():
def run_request(domain, data, port):
"""
调用子域请求入口函数
HTTP request entrance
:param str domain: 待请求的主域
:param list data: 待请求的子域数据
:param str port: 待请求的端口范围
:return: 请求后得到的结果列表
:rtype: list
:param str domain: domain to be requested
:param list data: subdomains data to be requested
:param str port: range of ports to be requested
:return list: result
"""
logger.log('INFOR', f'开始执行子域请求模块')
logger.log('INFOR', f'Start subdomain request module')
set_loop_policy()
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
@@ -256,16 +255,16 @@ def run_request(domain, data, port):
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
count = utils.count_alive(data)
logger.log('INFOR', f'经验证{domain}存活子域{count}个')
logger.log('INFOR', f'After verify, found {domain} have {count} alive subdomains')
return data
def save_data(name, data):
"""
保存请求结果到数据库
Save request results to database
:param str name: 保存表名
:param list data: 待保存的数据
:param str name: table name
:param list data: data to be saved
"""
db = Database()
db.drop_table(name)
common/resolve.py
+12 -12
View File
@@ -14,7 +14,7 @@ def filter_subdomain(data):
:param list data: 待过滤的数据列表
:return: 符合条件的子域列表
"""
logger.log('DEBUG', f'正在过滤出待解析的子域')
logger.log('DEBUG', f'Filtering subdomains to be resolved...')
subdomains = []
for data in data:
if not data.get('content'):
@@ -31,9 +31,9 @@ def update_data(data, records):
:param dict records: 解析结果字典
:return: 更新后的数据列表
"""
logger.log('DEBUG', f'正在更新解析结果')
logger.log('DEBUG', f'Updating resolved results...')
if not records:
logger.log('ERROR', f'无有效解析结果')
logger.log('ERROR', f'No valid resolved result')
return data
for index, items in enumerate(data):
if not items.get('content'):
@@ -52,7 +52,7 @@ def save_data(name, data):
:param str name: 保存表名
:param list data: 待保存的数据
"""
logger.log('INFOR', f'正在保存解析结果')
logger.log('INFOR', f'Saving resolved results...')
db = Database()
db.drop_table(name)
db.create_table(name)
@@ -61,15 +61,15 @@ def save_data(name, data):
def save_subdomains(save_path, subdomain_list):
logger.log('DEBUG', f'正在保存待解析的子域')
logger.log('DEBUG', f'Saving resolved subdomain...')
subdomain_data = '\n'.join(subdomain_list)
if not utils.save_data(save_path, subdomain_data):
logger.log('FATAL', '保存待解析的子域出错')
logger.log('FATAL', 'Save resolved subdomain error')
exit(1)
def deal_output(output_path):
logger.log('INFOR', f'正在处理解析结果')
logger.log('INFOR', f'Processing resolved results...')
records = dict() # 用来记录所有域名解析数据
with open(output_path) as fd:
for line in fd:
@@ -78,11 +78,11 @@ def deal_output(output_path):
items = json.loads(line)
except Exception as e:
logger.log('ERROR', e.args)
logger.log('ERROR', f'解析行{line}出错跳过解析该行')
logger.log('ERROR', f'Error resolve line {line}, skip this line')
continue
record = dict()
record['resolver'] = items.get('resolver')
qname = items.get('name')[:-1] # 去出最右边的`.`点号
qname = items.get('name')[:-1] # 去除最右边的`.`点号
status = items.get('status')
if status != 'NOERROR':
record['alive'] = 0
@@ -106,7 +106,7 @@ def deal_output(output_path):
for answer in answers:
if answer.get('type') == 'A':
flag = True
cname.append(answer.get('name')[:-1]) # 去出最右边的`.`点号
cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号
ip = answer.get('data')
ips.append(ip)
ttl = answer.get('ttl')
@@ -137,7 +137,7 @@ def run_resolve(domain, data):
:return: 解析得到的结果列表
:rtype: list
"""
logger.log('INFOR', f'开始解析{domain}的子域')
logger.log('INFOR', f'Start resolve subdomains of {domain}')
subdomains = filter_subdomain(data)
if not subdomains:
return data
@@ -166,5 +166,5 @@ def run_resolve(domain, data):
records = deal_output(output_path)
data = update_data(data, records)
logger.log('INFOR', f'结束解析{domain}的子域')
logger.log('INFOR', f'Finished resolve subdomains of {domain}')
return data
common/search.py
+1 -1
View File
@@ -5,7 +5,7 @@ from . import utils
class Search(Module):
"""
搜索基类
Search base class
"""
def __init__(self):
Module.__init__(self)
common/utils.py
+46 -47
View File
@@ -34,13 +34,12 @@ user_agents = [
def match_subdomain(domain, text, distinct=True):
"""
匹配text中domain的子域名
Use regexp to match subdomains in text
:param str domain: 域名
:param str text: 响应文本
:param bool distinct: 结果去重
:return: 匹配结果
:rtype: set or list
:param str domain: domain
:param str text: response text
:param bool distinct: deduplicate results
:return set/list: match result
"""
regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.')
@@ -56,7 +55,7 @@ def match_subdomain(domain, text, distinct=True):
def gen_random_ip():
"""
生成随机的点分十进制的IP字符串
Generate random decimal IP string
"""
while True:
ip = IPv4Address(random.randint(0, 2 ** 32 - 1))
@@ -66,7 +65,7 @@ def gen_random_ip():
def gen_fake_header():
"""
生成伪造请求头
Generate fake request headers
"""
ua = random.choice(user_agents)
ip = gen_random_ip()
@@ -89,7 +88,7 @@ def gen_fake_header():
def get_random_proxy():
"""
获取随机代理
Get random proxy
"""
try:
return random.choice(setting.proxy_pool)
@@ -99,11 +98,11 @@ def get_random_proxy():
def split_list(ls, size):
"""
将ls列表按size大小划分并返回新的划分结果列表
Split list
:param list ls: 要划分的列表
:param int size: 划分大小
:return 划分结果
:param list ls: list
:param int size: size
:return list: result
>>> split_list([1, 2, 3, 4], 3)
[[1, 2, 3], [4]]
@@ -115,13 +114,13 @@ def split_list(ls, size):
def get_domains(target):
"""
获取域名
Get domains
:param set or str target:
:return: 域名集合
:return list: domain list
"""
domains = list()
logger.log('DEBUG', f'正在获取域名')
logger.log('DEBUG', f'Getting domains...')
if isinstance(target, (set, tuple)):
domains = list(target)
elif isinstance(target, list):
@@ -142,9 +141,9 @@ def get_domains(target):
domains.append(domain)
count = len(domains)
if count == 0:
logger.log('FATAL', f'获取到{count}个域名')
logger.log('FATAL', f'Get {count} domains')
exit(1)
logger.log('INFOR', f'获取到{count}个域名')
logger.log('INFOR', f'Get {count} domains')
return domains
@@ -165,7 +164,7 @@ def get_semaphore():
def check_dir(dir_path):
if not dir_path.exists():
logger.log('INFOR', f'不存在{dir_path}目录将会新建')
logger.log('INFOR', f'{dir_path} does not exist, directory will be created')
dir_path.mkdir(parents=True, exist_ok=True)
@@ -190,10 +189,10 @@ def check_path(path, name, format):
path = path.joinpath(filename)
parent_dir = path.parent
if not parent_dir.exists():
logger.log('ALERT', f'不存在{parent_dir}目录将会新建')
logger.log('ALERT', f'{parent_dir} does not exist, directory will be created')
parent_dir.mkdir(parents=True, exist_ok=True)
if path.exists():
logger.log('ALERT', f'存在{path}文件将会覆盖')
logger.log('ALERT', f'{path} exists, file will be overwritten')
return path
@@ -208,14 +207,14 @@ def check_format(format, count):
formats = ['rst', 'csv', 'tsv', 'json', 'yaml', 'html',
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods']
if format == 'xls' and count > 65000:
logger.log('ALERT', 'xls文件限制为最多65000行')
logger.log('ALERT', '使用xlsx格式导出')
logger.log('ALERT', '\'xls\' file is limited to 65000 lines')
logger.log('ALERT', 'So use xlsx format replace')
return 'xlsx'
if format in formats:
return format
else:
logger.log('ALERT', f'不支持{format}格式导出')
logger.log('ALERT', '默认使用csv格式导出')
logger.log('ALERT', f'Does not support {format} format')
logger.log('ALERT', 'So use csv format by default')
return 'csv'
@@ -311,7 +310,7 @@ def check_value(values):
def export_all_results(path, name, format, datas):
path = check_path(path, name, format)
logger.log('INFOR', f'所有主域的子域结果 {path}')
logger.log('ALERT', f'Subdomain result for all domains: {path}')
row_list = list()
for row in datas:
if 'header' in row:
@@ -330,7 +329,7 @@ def export_all_results(path, name, format, datas):
def export_all_subdomains(alive, path, name, datas):
path = check_path(path, name, 'txt')
logger.log('INFOR', f'所有主域的纯子域结果 {path}')
logger.log('ALERT', f'Txt subdomain result for all main domains {path}')
subdomains = set()
for row in datas:
subdomain = row.get('subdomain')
@@ -379,16 +378,16 @@ def dns_query(qname, qtype):
:param str qtype: 查询类型
:return: 查询结果
"""
logger.log('TRACE', f'尝试查询{qname}的{qtype}记录')
logger.log('TRACE', f'Try to query {qtype} record of {qname}')
resolver = dns_resolver()
try:
answer = resolver.query(qname, qtype)
except Exception as e:
logger.log('TRACE', e.args)
logger.log('TRACE', f'查询{qname}的{qtype}记录失败')
logger.log('TRACE', f'Query {qtype} record of {qname} failed')
return None
else:
logger.log('TRACE', f'查询{qname}的{qtype}记录成功')
logger.log('TRACE', f'Query {qtype} record of {qname} succeeded')
return answer
@@ -509,56 +508,56 @@ def delete_file(*paths):
@tenacity.retry(stop=tenacity.stop_after_attempt(3))
def check_net():
logger.log('INFOR', '正在检查网络环境')
logger.log('INFOR', 'Checking Internet environment...')
urls = ['http://www.example.com', 'http://www.baidu.com',
'http://www.bing.com', 'http://www.taobao.com',
'http://www.linkedin.com', 'http://www.msn.com',
'http://www.apple.com', 'http://microsoft.com']
url = random.choice(urls)
logger.log('INFOR', f'正在尝试访问 {url}')
logger.log('INFOR', f'Trying to access {url}')
try:
rsp = requests.get(url)
except Exception as e:
logger.log('ERROR', e.args)
logger.log('ALERT', '访问外网出错 重新检查中')
logger.log('ALERT', 'Can not access Internet, retrying...')
raise tenacity.TryAgain
if rsp.status_code != 200:
logger.log('ALERT', f'{rsp.request.method} {rsp.request.url} '
f'{rsp.status_code} {rsp.reason}')
logger.log('ALERT', '不能正常访问外网 重新检查中')
logger.log('ALERT', 'Can not access Internet normally, retrying...')
raise tenacity.TryAgain
logger.log('INFOR', '能正常访问外网')
logger.log('INFOR', 'Access to Internet OK')
def check_pre():
logger.log('INFOR', '正在检查依赖环境')
logger.log('INFOR', 'Checking dependent environment...')
system = platform.system()
implementation = platform.python_implementation()
version = platform.python_version()
if implementation != 'CPython':
logger.log('FATAL', f'OneForAll只在CPython下测试通过')
logger.log('FATAL', f'OneForAll only passed the test under CPython')
exit(1)
if version < '3.6':
logger.log('FATAL', 'OneForAll需要Python 3.6以上版本')
logger.log('FATAL', 'OneForAll requires Python 3.6 or higher')
exit(1)
if system == 'Windows' and implementation == 'CPython':
if version < '3.8':
logger.log('FATAL', 'OneForAll在Windows系统运行时需要Python 3.8以上版本')
logger.log('FATAL', 'OneForAll requires Python 3.8 or higher when running on Windows')
exit(1)
if system in {"Linux", "Darwin"}:
try:
import uvloop
except ImportError:
logger.log('ALERT', f'请手动安装uvloop的Python库加速子域请求')
logger.log('ALERT', f'Please install the uvloop manually Python library to accelerate subdomain requests')
def check_env():
logger.log('INFOR', '正在检查运行环境')
logger.log('INFOR', 'Checking the environment...')
try:
check_net()
except Exception as e:
logger.log('DEBUG', e.args)
logger.log('FATAL', '不能正常访问外网')
logger.log('FATAL', 'Can not access Internet')
exit(1)
check_pre()
@@ -570,7 +569,7 @@ def get_maindomain(domain):
def call_massdns(massdns_path, dict_path, ns_path, output_path, log_path,
query_type='A', process_num=1, concurrent_num=10000,
quiet_mode=False):
logger.log('INFOR', f'开始执行massdns')
logger.log('INFOR', f'Start running massdns')
quiet = ''
if quiet_mode:
quiet = '--quiet'
@@ -583,9 +582,9 @@ def call_massdns(massdns_path, dict_path, ns_path, output_path, log_path,
f'--resolve-count {resolve_num} --type {query_type} ' \
f'--flush --output J --outfile {output_path} ' \
f'--root --error-log {log_path} {dict_path}'
logger.log('INFOR', f'执行命令 {cmd}')
logger.log('INFOR', f'Run command {cmd}')
subprocess.run(args=cmd, shell=True)
logger.log('INFOR', f'结束执行massdns')
logger.log('INFOR', f'Finished massdns')
def get_massdns_path(massdns_dir):
@@ -604,7 +603,7 @@ def get_massdns_path(massdns_dir):
path = massdns_dir.joinpath(name)
path.chmod(S_IXUSR)
if not path.exists():
logger.log('FATAL', '暂无该系统平台及架构的massdns')
logger.log('INFOR', '请尝试自行编译massdns并在配置里指定路径')
logger.log('FATAL', 'There is no massdns for this platform or architecture')
logger.log('INFOR', 'Please try to compile massdns yourself and specify the path in the configuration')
exit(0)
return path
config/log.py
+4 -2
View File
@@ -26,10 +26,12 @@ logger.remove()
logger.level(name='TRACE', no=5, color='<cyan><bold>', icon='✏️')
logger.level(name='DEBUG', no=10, color='<blue><bold>', icon='🐞 ')
logger.level(name='INFOR', no=20, color='<green><bold>', icon='ℹ️')
logger.level(name='QUITE', no=25, color='<green><bold>', icon='🤫 ')
logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️')
logger.level(name='ERROR', no=40, color='<red><bold>', icon='❌️')
logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')
logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True)
# 如果你想在命令终端静默运行OneForAll，可以将以下一行中的level设置为QUITE
logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True) # 命令终端日志级别默认为INFOR
logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True,
encoding='utf-8')
encoding='utf-8') # 日志文件默认为级别为DEBUG
oneforall.py
+1 -1
View File
@@ -234,7 +234,7 @@ class OneForAll(object):
utils.check_env()
logger.log('DEBUG', 'Python ' + utils.python_version())
logger.log('DEBUG', 'OneForAll ' + version)
logger.log('INFOR', f'Start Running OneForAll')
logger.log('INFOR', f'Start running OneForAll')
self.config()
self.domains = utils.get_domains(self.target)
if self.domains:
takeover.py
+3 -3
View File
@@ -78,7 +78,7 @@ class Takeover(Module):
self.results = Dataset()
def save(self):
logger.log('DEBUG', 'Saving Results...')
logger.log('DEBUG', 'Saving results...')
if self.format == 'txt':
data = str(self.results)
else:
@@ -158,8 +158,8 @@ class Takeover(Module):
logger.log('FATAL', f'Failed to obtain domain')
end = time.time()
elapse = round(end - start, 1)
logger.log('INFOR', f'{self.source} module spends {elapse} seconds'
f'There are {len(self.results)} subdomains exit takeover')
logger.log('INFOR', f'{self.source} module spends {elapse} seconds, '
f'There are {len(self.results)} subdomains exists takeover')
logger.log('INFOR', f'Subdomain takeover results: {self.path}')
logger.log('INFOR', f'Finished {self.source} module')