translate

This commit is contained in:
JrD
2020-05-11 18:36:50 +08:00
parent 8e0daa48a1
commit 038b91b52f
14 changed files with 230 additions and 227 deletions
+8 -8
View File
@@ -132,7 +132,7 @@ def gen_word_subdomains(expression, path):
def query_domain_ns_a(ns_list): def query_domain_ns_a(ns_list):
logger.log('INFOR', f'Querying authoritative name server {ns_list} A record') logger.log('INFOR', f'Querying A record from authoritative name server: {ns_list} ')
if not isinstance(ns_list, list): if not isinstance(ns_list, list):
return list() return list()
ns_ip_list = [] ns_ip_list = []
@@ -147,7 +147,7 @@ def query_domain_ns_a(ns_list):
if answer: if answer:
for item in answer: for item in answer:
ns_ip_list.append(item.address) ns_ip_list.append(item.address)
logger.log('INFOR', f'Authoritative name server A record: {ns_ip_list}') logger.log('INFOR', f'Authoritative name server A record result: {ns_ip_list}')
return ns_ip_list return ns_ip_list
@@ -162,7 +162,7 @@ def query_domain_ns(domain):
logger.log('ERROR', f'Querying NS records of {domain} error') logger.log('ERROR', f'Querying NS records of {domain} error')
return list() return list()
ns = [item.to_text() for item in answer] ns = [item.to_text() for item in answer]
logger.log('INFOR', f'{domain}\'s authoritative name server: {ns}') logger.log('INFOR', f'{domain}\'s authoritative name server is {ns}')
return ns return ns
@@ -186,7 +186,7 @@ def get_wildcard_record(domain, resolver):
exit(1) exit(1)
else: else:
if answer.rrset is None: if answer.rrset is None:
logger.log('DEBUG', f'No record of query results') logger.log('DEBUG', f'No record of query result')
return None, None return None, None
name = answer.name name = answer.name
ip = {item.address for item in answer} ip = {item.address for item in answer}
@@ -282,7 +282,7 @@ def gen_records(items, records, subdomains, ip_times, wc_ips, wc_ttl):
have_a_record = True have_a_record = True
ttl = answer.get('ttl') ttl = answer.get('ttl')
ttls.append(ttl) ttls.append(ttl)
cname.append(answer.get('name')[:-1]) # 去最右边的`.`点号 cname.append(answer.get('name')[:-1]) # 去最右边的`.`点号
ip = answer.get('data') ip = answer.get('data')
ips.append(ip) ips.append(ip)
public.append(utils.ip_is_public(ip)) public.append(utils.ip_is_public(ip))
@@ -339,7 +339,7 @@ def stat_ip_times(result_paths):
def deal_output(output_paths, ip_times, wildcard_ips, wildcard_ttl): def deal_output(output_paths, ip_times, wildcard_ips, wildcard_ttl):
logger.log('INFOR', f'Processing results...') logger.log('INFOR', f'Processing result...')
records = dict() # 用来记录所有域名解析数据 records = dict() # 用来记录所有域名解析数据
subdomains = list() # 用来保存所有通过有效性检查的子域 subdomains = list() # 用来保存所有通过有效性检查的子域
for output_path in output_paths: for output_path in output_paths:
@@ -598,8 +598,8 @@ class Brute(Module):
delete_file(dict_path, output_paths) delete_file(dict_path, output_paths)
end = time.time() end = time.time()
self.elapse = round(end - start, 1) self.elapse = round(end - start, 1)
logger.log('INFOR', f'{self.source} module spends {self.elapse} seconds' logger.log('INFOR', f'{self.source} module spends {self.elapse} seconds, '
f'Found {len(self.subdomains)} subdomains of {domain}') f'found {len(self.subdomains)} subdomains of {domain}')
logger.log('DEBUG', f'{self.source} module found subdomains of {domain}:\n' logger.log('DEBUG', f'{self.source} module found subdomains of {domain}:\n'
f'{self.subdomains}') f'{self.subdomains}')
self.gen_result(brute=dict_len, valid=len(self.subdomains)) self.gen_result(brute=dict_len, valid=len(self.subdomains))
+2 -1
View File
@@ -3,7 +3,8 @@ from .module import Module
class Crawl(Module): class Crawl(Module):
""" """
爬虫基类 Crawl base class
""" """
def __init__(self): def __init__(self):
Module.__init__(self) Module.__init__(self)
+51 -51
View File
@@ -2,7 +2,7 @@
# coding=utf-8 # coding=utf-8
""" """
SQLite数据库初始化和操作 SQLite database initialization and operation
""" """
import records import records
@@ -19,12 +19,12 @@ class Database(object):
@staticmethod @staticmethod
def get_conn(db_path): def get_conn(db_path):
""" """
获取数据库对象 Get database connection
:param db_path: 数据库连接或路径 :param db_path: Database path
:return: SQLite数据库 :return: db_conn: SQLite database connection
""" """
logger.log('TRACE', f'正在获取数据库连接') logger.log('TRACE', f'Establishing database connection...')
if isinstance(db_path, Connection): if isinstance(db_path, Connection):
return db_path return db_path
protocol = 'sqlite:///' protocol = 'sqlite:///'
@@ -33,7 +33,7 @@ class Database(object):
else: else:
db_path = protocol + db_path db_path = protocol + db_path
db = records.Database(db_path) # 不存在数据库时会新建一个数据库 db = records.Database(db_path) # 不存在数据库时会新建一个数据库
logger.log('TRACE', f'使用数据库: {db_path}') logger.log('TRACE', f'Use the database: {db_path}')
return db.get_connection() return db.get_connection()
def query(self, sql): def query(self, sql):
@@ -46,15 +46,15 @@ class Database(object):
def create_table(self, table_name): def create_table(self, table_name):
""" """
创建表结构 Create table
:param str table_name: 要创建的表名 :param str table_name: table name
""" """
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
if self.exist_table(table_name): if self.exist_table(table_name):
logger.log('TRACE', f'已经存在{table_name}') logger.log('TRACE', f'{table_name} table already exists')
return return
logger.log('TRACE', f'正在创建{table_name}') logger.log('TRACE', f'Creating {table_name} table')
self.query(f'create table "{table_name}" (' self.query(f'create table "{table_name}" ('
f'id integer primary key,' f'id integer primary key,'
f'type text,' f'type text,'
@@ -87,14 +87,14 @@ class Database(object):
def save_db(self, table_name, results, module_name=None): def save_db(self, table_name, results, module_name=None):
""" """
将各模块结果存入数据库 Save the results of each module in the database
:param str table_name: 表名 :param str table_name: table name
:param list results: 结果列表 :param list results: results list
:param str module_name: 模块名 :param str module_name: mo
""" """
logger.log('TRACE', f'正在将{module_name}模块发现{table_name}的子域' logger.log('TRACE',
'结果存入数据库') f'Saving the subdomain results of {table_name} found by module {module_name} into database...')
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
if results: if results:
try: try:
@@ -114,13 +114,13 @@ class Database(object):
def exist_table(self, table_name): def exist_table(self, table_name):
""" """
判断是否存在某表 Determine table exists
:param str table_name: 表名 :param str table_name: table name
:return: 是否存在某表 :return bool: Whether table exists
""" """
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在查询是否存在{table_name}') logger.log('TRACE', f'Determining whether the {table_name} table exists')
results = self.query(f'select count() from sqlite_master ' results = self.query(f'select count() from sqlite_master '
f'where type = "table" and ' f'where type = "table" and '
f'name = "{table_name}"') f'name = "{table_name}"')
@@ -131,80 +131,80 @@ class Database(object):
def copy_table(self, table_name, bak_table_name): def copy_table(self, table_name, bak_table_name):
""" """
复制表创建备份 Copy table to create backup
:param str table_name: 表名 :param str table_name: table name
:param str bak_table_name: 新表名 :param str bak_table_name: new table name
""" """
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
bak_table_name = bak_table_name.replace('.', '_') bak_table_name = bak_table_name.replace('.', '_')
logger.log('TRACE', f'正在将{table_name}表复制到{bak_table_name}新表') logger.log('TRACE', f'Copying {table_name} table to {bak_table_name} new table')
self.query(f'drop table if exists "{bak_table_name}"') self.query(f'drop table if exists "{bak_table_name}"')
self.query(f'create table "{bak_table_name}" ' self.query(f'create table "{bak_table_name}" '
f'as select * from "{table_name}"') f'as select * from "{table_name}"')
def clear_table(self, table_name): def clear_table(self, table_name):
""" """
清空表中数据 Clear the table
:param str table_name: 表名 :param str table_name: 表名
""" """
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在清空{table_name}表中的数据') logger.log('TRACE', f'Clearing data in table {table_name}')
self.query(f'delete from "{table_name}"') self.query(f'delete from "{table_name}"')
def drop_table(self, table_name): def drop_table(self, table_name):
""" """
删除表 Delete table
:param str table_name: 表名 :param str table_name: table name
""" """
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在删除{table_name}') logger.log('TRACE', f'Deleting {table_name} table')
self.query(f'drop table if exists "{table_name}"') self.query(f'drop table if exists "{table_name}"')
def rename_table(self, table_name, new_table_name): def rename_table(self, table_name, new_table_name):
""" """
重命名表名 Rename table name
:param str table_name: 表名 :param str table_name: old table name
:param str new_table_name: 新表名 :param str new_table_name: new table name
""" """
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
new_table_name = new_table_name.replace('.', '_') new_table_name = new_table_name.replace('.', '_')
logger.log('TRACE', f'正在将{table_name}表重命名为{table_name}') logger.log('TRACE', f'Renaming {table_name} table to {new_table_name} table')
self.query(f'alter table "{table_name}" ' self.query(f'alter table "{table_name}" '
f'rename to "{new_table_name}"') f'rename to "{new_table_name}"')
def deduplicate_subdomain(self, table_name): def deduplicate_subdomain(self, table_name):
""" """
去重表中的子域 Deduplicates of subdomains in the table
:param str table_name: 表名 :param str table_name: table name
""" """
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在去重{table_name}表中的子域') logger.log('TRACE', f'Deduplicating subdomains in {table_name} table')
self.query(f'delete from "{table_name}" where ' self.query(f'delete from "{table_name}" where '
f'id not in (select min(id) ' f'id not in (select min(id) '
f'from "{table_name}" group by subdomain)') f'from "{table_name}" group by subdomain)')
def remove_invalid(self, table_name): def remove_invalid(self, table_name):
""" """
去除表中的空值或无效子域 Remove nulls or invalid subdomains in the table
:param str table_name: 表名 :param str table_name: table name
""" """
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在去除{table_name}表中的无效子域') logger.log('TRACE', f'Removing invalid subdomains in {table_name} table')
self.query(f'delete from "{table_name}" where ' self.query(f'delete from "{table_name}" where '
f'subdomain is null or resolve == 0') f'subdomain is null or resolve == 0')
def deal_table(self, deal_table_name, backup_table_name): def deal_table(self, deal_table_name, backup_table_name):
""" """
收集任务完成时对表进行处理 Process the table when the collection task is complete
:param str deal_table_name: 待处理的表名 :param str deal_table_name: Pending table name
:param str backup_table_name: 备份的表名 :param str backup_table_name: Table name for backup
""" """
self.copy_table(deal_table_name, backup_table_name) self.copy_table(deal_table_name, backup_table_name)
self.remove_invalid(deal_table_name) self.remove_invalid(deal_table_name)
@@ -212,21 +212,21 @@ class Database(object):
def get_data(self, table_name): def get_data(self, table_name):
""" """
获取表中的所有数据 Get all the data in the table
:param str table_name: 表名 :param str table_name: table name
""" """
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
logger.log('TRACE', f'获取{table_name}表中的所有数据') logger.log('TRACE', f'Get all the data from {table_name} table')
return self.query(f'select * from "{table_name}"') return self.query(f'select * from "{table_name}"')
def export_data(self, table_name, alive, limit): def export_data(self, table_name, alive, limit):
""" """
获取表中的部分数据 Get part of the data in the table
:param str table_name: 表名 :param str table_name: table name
:param any alive: 存活 :param any alive: alive flag
:param str limit: 限制字段 :param str limit: limit value
""" """
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
query = f'select id, type, new, alive, request, resolve, url, ' \ query = f'select id, type, new, alive, request, resolve, url, ' \
@@ -240,11 +240,11 @@ class Database(object):
elif alive: elif alive:
where = f' where alive = 1' where = f' where alive = 1'
query += where query += where
logger.log('TRACE', f'获取{table_name}表中的数据') logger.log('TRACE', f'Get the data from {table_name} table')
return self.query(query) return self.query(query)
def close(self): def close(self):
""" """
关闭数据库连接 Close the database connection
""" """
self.conn.close() self.conn.close()
+9 -8
View File
@@ -5,10 +5,11 @@ from config import setting
class Domain(object): class Domain(object):
""" """
域名处理类 Processing domain class
:param str string: 传入的字符串 :param str string: input string
""" """
def __init__(self, string): def __init__(self, string):
self.string = str(string) self.string = str(string)
self.regexp = r'\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b' self.regexp = r'\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b'
@@ -16,9 +17,9 @@ class Domain(object):
def match(self): def match(self):
""" """
域名匹配 match domain
:return: 匹配结果 :return : result
""" """
result = re.search(self.regexp, self.string, re.I) result = re.search(self.regexp, self.string, re.I)
if result: if result:
@@ -28,14 +29,14 @@ class Domain(object):
def extract(self): def extract(self):
""" """
域名导出 extract domain
>>> d = Domain('www.example.com') >>> d = Domain('www.example.com')
<domain.Domain object> <domain.Domain object>
>>> d.extract() >>> d.extract()
ExtractResult(subdomain='www', domain='example', suffix='com') ExtractResult(subdomain='www', domain='example', suffix='com')
:return: 导出结果 :return: extracted domain results
""" """
data_storage_dir = setting.data_storage_dir data_storage_dir = setting.data_storage_dir
extract_cache_file = data_storage_dir.joinpath('public_suffix_list.dat') extract_cache_file = data_storage_dir.joinpath('public_suffix_list.dat')
@@ -48,14 +49,14 @@ class Domain(object):
def registered(self): def registered(self):
""" """
获取注册域名 registered domain
>>> d = Domain('www.example.com') >>> d = Domain('www.example.com')
<domain.Domain object> <domain.Domain object>
>>> d.registered() >>> d.registered()
example.com example.com
:return: 注册域名 :return: registered domain result
""" """
result = self.extract() result = self.extract()
if result: if result:
+4 -3
View File
@@ -4,15 +4,16 @@ from common import utils
class Lookup(Module): class Lookup(Module):
""" """
DNS查询基类 DNS query base class
""" """
def __init__(self): def __init__(self):
Module.__init__(self) Module.__init__(self)
def query(self): def query(self):
""" """
查询域名的TXT记录 Query the TXT record of domain
:return: 查询结果 :return: query result
""" """
answer = utils.dns_query(self.domain, self.type) answer = utils.dns_query(self.domain, self.type)
if answer is None: if answer is None:
+58 -59
View File
@@ -1,6 +1,6 @@
# coding=utf-8 # coding=utf-8
""" """
模块基类 Module base class
""" """
import json import json
@@ -39,43 +39,43 @@ class Module(object):
def check(self, *apis): def check(self, *apis):
""" """
简单检查是否配置了api信息 Simply check whether the api information configure or not
:param apis: api信息元组 :param apis: apis set
:return: 检查结果 :return bool: check result
""" """
if not all(apis): if not all(apis):
logger.log('ALERT', f'{self.source}模块没有配置API跳过执行') logger.log('ALERT', f'{self.source} module is not configured, skip')
return False return False
return True return True
def begin(self): def begin(self):
""" """
输出模块开始信息 begin log
""" """
logger.log('DEBUG', f'开始执行{self.source}模块收集{self.domain}的子域') logger.log('DEBUG', f'Start {self.source} module to collect subdomains of {self.domain}')
def finish(self): def finish(self):
""" """
输出模块结束信息 finish log
""" """
self.end = time.time() self.end = time.time()
self.elapse = round(self.end - self.start, 1) self.elapse = round(self.end - self.start, 1)
logger.log('DEBUG', f'结束执行{self.source}模块收集{self.domain}的子域') logger.log('DEBUG', f'Finished {self.source} module to collect {self.domain}\'s subdomains')
logger.log('INFOR', f'{self.source}模块耗时{self.elapse}秒发现子域' logger.log('INFOR', f'The {self.source} module took {self.elapse} seconds, '
f'{len(self.subdomains)}') f'found {len(self.subdomains)} subdomains')
logger.log('DEBUG', f'{self.source}模块发现{self.domain}的子域\n' logger.log('DEBUG', f'{self.source} module found subdomains of {self.domain}\n'
f'{self.subdomains}') f'{self.subdomains}')
def head(self, url, params=None, check=True, **kwargs): def head(self, url, params=None, check=True, **kwargs):
""" """
自定义head请求 Custom head request
:param str url: 请求地址 :param str url: request url
:param dict params: 请求参数 :param dict params: request parameters
:param bool check: 检查响应 :param bool check: check response
:param kwargs: 其他参数 :param kwargs: other params
:return: requests响应对象 :return: requests's response object
""" """
try: try:
resp = requests.head(url, resp = requests.head(url,
@@ -97,13 +97,13 @@ class Module(object):
def get(self, url, params=None, check=True, **kwargs): def get(self, url, params=None, check=True, **kwargs):
""" """
自定义get请求 Custom get request
:param str url: 请求地址 :param str url: request url
:param dict params: 请求参数 :param dict params: request parameters
:param bool check: 检查响应 :param bool check: check response
:param kwargs: 其他参数 :param kwargs: other params
:return: requests响应对象 :return: requests's response object
""" """
try: try:
resp = requests.get(url, resp = requests.get(url,
@@ -125,13 +125,13 @@ class Module(object):
def post(self, url, data=None, check=True, **kwargs): def post(self, url, data=None, check=True, **kwargs):
""" """
自定义post请求 Custom post request
:param str url: 请求地址 :param str url: request url
:param dict data: 请求数据 :param dict params: request parameters
:param bool check: 检查响应 :param bool check: check response
:param kwargs: 其他参数 :param kwargs: other params
:return: requests响应对象 :return: requests's response object
""" """
try: try:
resp = requests.post(url, resp = requests.post(url,
@@ -153,11 +153,11 @@ class Module(object):
def get_header(self): def get_header(self):
""" """
获取请求头 Get request header
:return: 请求头 :return: header
""" """
# logger.log('DEBUG', f'获取请求头') # logger.log('DEBUG', f'Get request header')
if setting.enable_fake_header: if setting.enable_fake_header:
return utils.gen_fake_header() return utils.gen_fake_header()
else: else:
@@ -165,36 +165,35 @@ class Module(object):
def get_proxy(self, module): def get_proxy(self, module):
""" """
获取代理 Get proxy
:param str module: 模块名 :param str module: module name
:return: 代理字典 :return: proxy
""" """
if not setting.enable_proxy: if not setting.enable_proxy:
logger.log('TRACE', f'所有模块不使用代理') logger.log('TRACE', f'All modules do not use proxy')
return self.proxy return self.proxy
if setting.proxy_all_module: if setting.proxy_all_module:
logger.log('TRACE', f'{module}模块使用代理') logger.log('TRACE', f'{module} module uses proxy')
return utils.get_random_proxy() return utils.get_random_proxy()
if module in setting.proxy_partial_module: if module in setting.proxy_partial_module:
logger.log('TRACE', f'{module}模块使用代理') logger.log('TRACE', f'{module} module uses proxy')
return utils.get_random_proxy() return utils.get_random_proxy()
else: else:
logger.log('TRACE', f'{module}模块不使用代理') logger.log('TRACE', f'{module} module does not use proxy')
return self.proxy return self.proxy
@staticmethod @staticmethod
def match(domain, html, distinct=True): def match(domain, html, distinct=True):
""" """
正则匹配出子域 Use regexp to match subdomains
:param str domain: 域名 :param str domain: domain
:param str html: 要匹配的html响应体 :param str html: response html text
:param bool distinct: 匹配结果去除 :param bool distinct: deduplicate results or not (default True)
:return: 匹配出的子域集合或列表 :return set/list: result set or list
:rtype: set or list
""" """
logger.log('TRACE', f'正则匹配响应体中的子域') logger.log('TRACE', f'Use regexp to match subdomains in the response body')
regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \ regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.') + domain.replace('.', r'\.')
@@ -211,22 +210,22 @@ class Module(object):
@staticmethod @staticmethod
def register(domain): def register(domain):
""" """
获取注册域名 Get registered domain
:param str domain: 域名 :param str domain: domain
:return: 注册域名 :return: registered domain
""" """
return Domain(domain).registered() return Domain(domain).registered()
def save_json(self): def save_json(self):
""" """
将各模块结果保存为json文件 Save the results of each module as a json file
:return 是否保存成功 :return bool: whether saved successfully
""" """
if not setting.save_module_result: if not setting.save_module_result:
return False return False
logger.log('TRACE', f'{self.source}模块发现的子域结果保存为json文件') logger.log('TRACE', f'Save the subdomain results found by {self.source} module as a json file')
path = setting.result_save_dir.joinpath(self.domain, self.module) path = setting.result_save_dir.joinpath(self.domain, self.module)
path.mkdir(parents=True, exist_ok=True) path.mkdir(parents=True, exist_ok=True)
name = self.source + '.json' name = self.source + '.json'
@@ -244,7 +243,7 @@ class Module(object):
def gen_record(self, subdomains, record): def gen_record(self, subdomains, record):
""" """
生成记录字典 Generate record dictionary
""" """
item = dict() item = dict()
item['content'] = record item['content'] = record
@@ -253,11 +252,11 @@ class Module(object):
def gen_result(self, find=0, brute=None, valid=0): def gen_result(self, find=0, brute=None, valid=0):
""" """
生成结果 Generate results
""" """
logger.log('DEBUG', f'正在生成最终结果') logger.log('DEBUG', f'Generating final results...')
if not len(self.subdomains): # 该模块一个子域都没有发现的情况 if not len(self.subdomains): # 该模块一个子域都没有发现的情况
logger.log('DEBUG', f'{self.source}模块收集结果为空') logger.log('DEBUG', f'{self.source} module result is empty')
result = {'id': None, result = {'id': None,
'type': self.type, 'type': self.type,
'alive': None, 'alive': None,
@@ -347,10 +346,10 @@ class Module(object):
def save_db(self): def save_db(self):
""" """
将模块结果存入数据库中 Save module results into the database
""" """
logger.log('DEBUG', f'正在将结果存入到数据库') logger.log('DEBUG', f'Saving results to database')
lock.acquire() lock.acquire()
db = Database() db = Database()
db.create_table(self.domain) db.create_table(self.domain)
+2 -1
View File
@@ -3,7 +3,8 @@ from .module import Module
class Query(Module): class Query(Module):
""" """
查询基类 Query base class
""" """
def __init__(self): def __init__(self):
Module.__init__(self) Module.__init__(self)
+17 -18
View File
@@ -22,7 +22,7 @@ def get_limit_conn():
def get_ports(port): def get_ports(port):
logger.log('DEBUG', f'正在获取请求端口范围') logger.log('DEBUG', f'Getting port range...')
ports = set() ports = set()
if isinstance(port, (set, list, tuple)): if isinstance(port, (set, list, tuple)):
ports = port ports = port
@@ -30,17 +30,17 @@ def get_ports(port):
if 0 <= port <= 65535: if 0 <= port <= 65535:
ports = {port} ports = {port}
elif port in {'default', 'small', 'large'}: elif port in {'default', 'small', 'large'}:
logger.log('DEBUG', f'请求{port}等端口范围') logger.log('DEBUG', f'{port} port range')
ports = setting.ports.get(port) ports = setting.ports.get(port)
if not ports: # 意外情况 if not ports: # 意外情况
logger.log('ERROR', f'指定请求端口范围有误') logger.log('ERROR', f'The specified request port range is incorrect')
ports = {80} ports = {80}
logger.log('INFOR', f'请求端口范围:{ports}') logger.log('INFOR', f'Port range:{ports}')
return set(ports) return set(ports)
def gen_req_data(data, ports): def gen_req_data(data, ports):
logger.log('INFOR', f'正在生成请求地址') logger.log('INFOR', f'Generating request urls...')
new_data = [] new_data = []
for data in data: for data in data:
resolve = data.get('resolve') resolve = data.get('resolve')
@@ -202,8 +202,8 @@ async def bulk_request(data, port):
no_req_data = utils.get_filtered_data(data) no_req_data = utils.get_filtered_data(data)
to_req_data = gen_req_data(data, ports) to_req_data = gen_req_data(data, ports)
method = setting.request_method method = setting.request_method
logger.log('INFOR', f'请求使用{method}方法') logger.log('INFOR', f'Use {method} method to request')
logger.log('INFOR', f'正在进行异步子域请求') logger.log('INFOR', f'Async subdomains request in progress...')
connector = get_connector() connector = get_connector()
header = get_header() header = get_header()
async with ClientSession(connector=connector, headers=header) as session: async with ClientSession(connector=connector, headers=header) as session:
@@ -238,15 +238,14 @@ def set_loop_policy():
def run_request(domain, data, port): def run_request(domain, data, port):
""" """
调用子域请求入口函数 HTTP request entrance
:param str domain: 待请求的主域 :param str domain: domain to be requested
:param list data: 待请求的子域数据 :param list data: subdomains data to be requested
:param str port: 待请求的端口范围 :param str port: range of ports to be requested
:return: 请求后得到的结果列表 :return list: result
:rtype: list
""" """
logger.log('INFOR', f'开始执行子域请求模块') logger.log('INFOR', f'Start subdomain request module')
set_loop_policy() set_loop_policy()
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop) asyncio.set_event_loop(loop)
@@ -256,16 +255,16 @@ def run_request(domain, data, port):
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间 # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25)) loop.run_until_complete(asyncio.sleep(0.25))
count = utils.count_alive(data) count = utils.count_alive(data)
logger.log('INFOR', f'经验证{domain}存活子域{count}') logger.log('INFOR', f'After verify, found {domain} have {count} alive subdomains')
return data return data
def save_data(name, data): def save_data(name, data):
""" """
保存请求结果到数据库 Save request results to database
:param str name: 保存表名 :param str name: table name
:param list data: 待保存的数据 :param list data: data to be saved
""" """
db = Database() db = Database()
db.drop_table(name) db.drop_table(name)
+12 -12
View File
@@ -14,7 +14,7 @@ def filter_subdomain(data):
:param list data: 待过滤的数据列表 :param list data: 待过滤的数据列表
:return: 符合条件的子域列表 :return: 符合条件的子域列表
""" """
logger.log('DEBUG', f'正在过滤出待解析的子域') logger.log('DEBUG', f'Filtering subdomains to be resolved...')
subdomains = [] subdomains = []
for data in data: for data in data:
if not data.get('content'): if not data.get('content'):
@@ -31,9 +31,9 @@ def update_data(data, records):
:param dict records: 解析结果字典 :param dict records: 解析结果字典
:return: 更新后的数据列表 :return: 更新后的数据列表
""" """
logger.log('DEBUG', f'正在更新解析结果') logger.log('DEBUG', f'Updating resolved results...')
if not records: if not records:
logger.log('ERROR', f'无有效解析结果') logger.log('ERROR', f'No valid resolved result')
return data return data
for index, items in enumerate(data): for index, items in enumerate(data):
if not items.get('content'): if not items.get('content'):
@@ -52,7 +52,7 @@ def save_data(name, data):
:param str name: 保存表名 :param str name: 保存表名
:param list data: 待保存的数据 :param list data: 待保存的数据
""" """
logger.log('INFOR', f'正在保存解析结果') logger.log('INFOR', f'Saving resolved results...')
db = Database() db = Database()
db.drop_table(name) db.drop_table(name)
db.create_table(name) db.create_table(name)
@@ -61,15 +61,15 @@ def save_data(name, data):
def save_subdomains(save_path, subdomain_list): def save_subdomains(save_path, subdomain_list):
logger.log('DEBUG', f'正在保存待解析的子域') logger.log('DEBUG', f'Saving resolved subdomain...')
subdomain_data = '\n'.join(subdomain_list) subdomain_data = '\n'.join(subdomain_list)
if not utils.save_data(save_path, subdomain_data): if not utils.save_data(save_path, subdomain_data):
logger.log('FATAL', '保存待解析的子域出错') logger.log('FATAL', 'Save resolved subdomain error')
exit(1) exit(1)
def deal_output(output_path): def deal_output(output_path):
logger.log('INFOR', f'正在处理解析结果') logger.log('INFOR', f'Processing resolved results...')
records = dict() # 用来记录所有域名解析数据 records = dict() # 用来记录所有域名解析数据
with open(output_path) as fd: with open(output_path) as fd:
for line in fd: for line in fd:
@@ -78,11 +78,11 @@ def deal_output(output_path):
items = json.loads(line) items = json.loads(line)
except Exception as e: except Exception as e:
logger.log('ERROR', e.args) logger.log('ERROR', e.args)
logger.log('ERROR', f'解析行{line}出错跳过解析该行') logger.log('ERROR', f'Error resolve line {line}, skip this line')
continue continue
record = dict() record = dict()
record['resolver'] = items.get('resolver') record['resolver'] = items.get('resolver')
qname = items.get('name')[:-1] # 去最右边的`.`点号 qname = items.get('name')[:-1] # 去最右边的`.`点号
status = items.get('status') status = items.get('status')
if status != 'NOERROR': if status != 'NOERROR':
record['alive'] = 0 record['alive'] = 0
@@ -106,7 +106,7 @@ def deal_output(output_path):
for answer in answers: for answer in answers:
if answer.get('type') == 'A': if answer.get('type') == 'A':
flag = True flag = True
cname.append(answer.get('name')[:-1]) # 去最右边的`.`点号 cname.append(answer.get('name')[:-1]) # 去最右边的`.`点号
ip = answer.get('data') ip = answer.get('data')
ips.append(ip) ips.append(ip)
ttl = answer.get('ttl') ttl = answer.get('ttl')
@@ -137,7 +137,7 @@ def run_resolve(domain, data):
:return: 解析得到的结果列表 :return: 解析得到的结果列表
:rtype: list :rtype: list
""" """
logger.log('INFOR', f'开始解析{domain}的子域') logger.log('INFOR', f'Start resolve subdomains of {domain}')
subdomains = filter_subdomain(data) subdomains = filter_subdomain(data)
if not subdomains: if not subdomains:
return data return data
@@ -166,5 +166,5 @@ def run_resolve(domain, data):
records = deal_output(output_path) records = deal_output(output_path)
data = update_data(data, records) data = update_data(data, records)
logger.log('INFOR', f'结束解析{domain}的子域') logger.log('INFOR', f'Finished resolve subdomains of {domain}')
return data return data
+1 -1
View File
@@ -5,7 +5,7 @@ from . import utils
class Search(Module): class Search(Module):
""" """
搜索基类 Search base class
""" """
def __init__(self): def __init__(self):
Module.__init__(self) Module.__init__(self)
+46 -47
View File
@@ -34,13 +34,12 @@ user_agents = [
def match_subdomain(domain, text, distinct=True): def match_subdomain(domain, text, distinct=True):
""" """
匹配text中domain的子域名 Use regexp to match subdomains in text
:param str domain: 域名 :param str domain: domain
:param str text: 响应文本 :param str text: response text
:param bool distinct: 结果去重 :param bool distinct: deduplicate results
:return: 匹配结果 :return set/list: match result
:rtype: set or list
""" """
regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.') + domain.replace('.', r'\.')
@@ -56,7 +55,7 @@ def match_subdomain(domain, text, distinct=True):
def gen_random_ip(): def gen_random_ip():
""" """
生成随机的点分十进制的IP字符串 Generate random decimal IP string
""" """
while True: while True:
ip = IPv4Address(random.randint(0, 2 ** 32 - 1)) ip = IPv4Address(random.randint(0, 2 ** 32 - 1))
@@ -66,7 +65,7 @@ def gen_random_ip():
def gen_fake_header(): def gen_fake_header():
""" """
生成伪造请求头 Generate fake request headers
""" """
ua = random.choice(user_agents) ua = random.choice(user_agents)
ip = gen_random_ip() ip = gen_random_ip()
@@ -89,7 +88,7 @@ def gen_fake_header():
def get_random_proxy(): def get_random_proxy():
""" """
获取随机代理 Get random proxy
""" """
try: try:
return random.choice(setting.proxy_pool) return random.choice(setting.proxy_pool)
@@ -99,11 +98,11 @@ def get_random_proxy():
def split_list(ls, size): def split_list(ls, size):
""" """
将ls列表按size大小划分并返回新的划分结果列表 Split list
:param list ls: 要划分的列表 :param list ls: list
:param int size: 划分大小 :param int size: size
:return 划分结果 :return list: result
>>> split_list([1, 2, 3, 4], 3) >>> split_list([1, 2, 3, 4], 3)
[[1, 2, 3], [4]] [[1, 2, 3], [4]]
@@ -115,13 +114,13 @@ def split_list(ls, size):
def get_domains(target): def get_domains(target):
""" """
获取域名 Get domains
:param set or str target: :param set or str target:
:return: 域名集合 :return list: domain list
""" """
domains = list() domains = list()
logger.log('DEBUG', f'正在获取域名') logger.log('DEBUG', f'Getting domains...')
if isinstance(target, (set, tuple)): if isinstance(target, (set, tuple)):
domains = list(target) domains = list(target)
elif isinstance(target, list): elif isinstance(target, list):
@@ -142,9 +141,9 @@ def get_domains(target):
domains.append(domain) domains.append(domain)
count = len(domains) count = len(domains)
if count == 0: if count == 0:
logger.log('FATAL', f'获取到{count}个域名') logger.log('FATAL', f'Get {count} domains')
exit(1) exit(1)
logger.log('INFOR', f'获取到{count}个域名') logger.log('INFOR', f'Get {count} domains')
return domains return domains
@@ -165,7 +164,7 @@ def get_semaphore():
def check_dir(dir_path): def check_dir(dir_path):
if not dir_path.exists(): if not dir_path.exists():
logger.log('INFOR', f'不存在{dir_path}目录将会新建') logger.log('INFOR', f'{dir_path} does not exist, directory will be created')
dir_path.mkdir(parents=True, exist_ok=True) dir_path.mkdir(parents=True, exist_ok=True)
@@ -190,10 +189,10 @@ def check_path(path, name, format):
path = path.joinpath(filename) path = path.joinpath(filename)
parent_dir = path.parent parent_dir = path.parent
if not parent_dir.exists(): if not parent_dir.exists():
logger.log('ALERT', f'不存在{parent_dir}目录将会新建') logger.log('ALERT', f'{parent_dir} does not exist, directory will be created')
parent_dir.mkdir(parents=True, exist_ok=True) parent_dir.mkdir(parents=True, exist_ok=True)
if path.exists(): if path.exists():
logger.log('ALERT', f'存在{path}文件将会覆盖') logger.log('ALERT', f'{path} exists, file will be overwritten')
return path return path
@@ -208,14 +207,14 @@ def check_format(format, count):
formats = ['rst', 'csv', 'tsv', 'json', 'yaml', 'html', formats = ['rst', 'csv', 'tsv', 'json', 'yaml', 'html',
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'] 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods']
if format == 'xls' and count > 65000: if format == 'xls' and count > 65000:
logger.log('ALERT', 'xls文件限制为最多65000行') logger.log('ALERT', '\'xls\' file is limited to 65000 lines')
logger.log('ALERT', '使用xlsx格式导出') logger.log('ALERT', 'So use xlsx format replace')
return 'xlsx' return 'xlsx'
if format in formats: if format in formats:
return format return format
else: else:
logger.log('ALERT', f'不支持{format}格式导出') logger.log('ALERT', f'Does not support {format} format')
logger.log('ALERT', '默认使用csv格式导出') logger.log('ALERT', 'So use csv format by default')
return 'csv' return 'csv'
@@ -311,7 +310,7 @@ def check_value(values):
def export_all_results(path, name, format, datas): def export_all_results(path, name, format, datas):
path = check_path(path, name, format) path = check_path(path, name, format)
logger.log('INFOR', f'所有主域的子域结果 {path}') logger.log('ALERT', f'Subdomain result for all domains: {path}')
row_list = list() row_list = list()
for row in datas: for row in datas:
if 'header' in row: if 'header' in row:
@@ -330,7 +329,7 @@ def export_all_results(path, name, format, datas):
def export_all_subdomains(alive, path, name, datas): def export_all_subdomains(alive, path, name, datas):
path = check_path(path, name, 'txt') path = check_path(path, name, 'txt')
logger.log('INFOR', f'所有主域的纯子域结果 {path}') logger.log('ALERT', f'Txt subdomain result for all main domains {path}')
subdomains = set() subdomains = set()
for row in datas: for row in datas:
subdomain = row.get('subdomain') subdomain = row.get('subdomain')
@@ -379,16 +378,16 @@ def dns_query(qname, qtype):
:param str qtype: 查询类型 :param str qtype: 查询类型
:return: 查询结果 :return: 查询结果
""" """
logger.log('TRACE', f'尝试查询{qname}{qtype}记录') logger.log('TRACE', f'Try to query {qtype} record of {qname}')
resolver = dns_resolver() resolver = dns_resolver()
try: try:
answer = resolver.query(qname, qtype) answer = resolver.query(qname, qtype)
except Exception as e: except Exception as e:
logger.log('TRACE', e.args) logger.log('TRACE', e.args)
logger.log('TRACE', f'查询{qname}{qtype}记录失败') logger.log('TRACE', f'Query {qtype} record of {qname} failed')
return None return None
else: else:
logger.log('TRACE', f'查询{qname}{qtype}记录成功') logger.log('TRACE', f'Query {qtype} record of {qname} succeeded')
return answer return answer
@@ -509,56 +508,56 @@ def delete_file(*paths):
@tenacity.retry(stop=tenacity.stop_after_attempt(3)) @tenacity.retry(stop=tenacity.stop_after_attempt(3))
def check_net(): def check_net():
logger.log('INFOR', '正在检查网络环境') logger.log('INFOR', 'Checking Internet environment...')
urls = ['http://www.example.com', 'http://www.baidu.com', urls = ['http://www.example.com', 'http://www.baidu.com',
'http://www.bing.com', 'http://www.taobao.com', 'http://www.bing.com', 'http://www.taobao.com',
'http://www.linkedin.com', 'http://www.msn.com', 'http://www.linkedin.com', 'http://www.msn.com',
'http://www.apple.com', 'http://microsoft.com'] 'http://www.apple.com', 'http://microsoft.com']
url = random.choice(urls) url = random.choice(urls)
logger.log('INFOR', f'正在尝试访问 {url}') logger.log('INFOR', f'Trying to access {url}')
try: try:
rsp = requests.get(url) rsp = requests.get(url)
except Exception as e: except Exception as e:
logger.log('ERROR', e.args) logger.log('ERROR', e.args)
logger.log('ALERT', '访问外网出错 重新检查中') logger.log('ALERT', 'Can not access Internet, retrying...')
raise tenacity.TryAgain raise tenacity.TryAgain
if rsp.status_code != 200: if rsp.status_code != 200:
logger.log('ALERT', f'{rsp.request.method} {rsp.request.url} ' logger.log('ALERT', f'{rsp.request.method} {rsp.request.url} '
f'{rsp.status_code} {rsp.reason}') f'{rsp.status_code} {rsp.reason}')
logger.log('ALERT', '不能正常访问外网 重新检查中') logger.log('ALERT', 'Can not access Internet normally, retrying...')
raise tenacity.TryAgain raise tenacity.TryAgain
logger.log('INFOR', '能正常访问外网') logger.log('INFOR', 'Access to Internet OK')
def check_pre(): def check_pre():
logger.log('INFOR', '正在检查依赖环境') logger.log('INFOR', 'Checking dependent environment...')
system = platform.system() system = platform.system()
implementation = platform.python_implementation() implementation = platform.python_implementation()
version = platform.python_version() version = platform.python_version()
if implementation != 'CPython': if implementation != 'CPython':
logger.log('FATAL', f'OneForAll只在CPython下测试通过') logger.log('FATAL', f'OneForAll only passed the test under CPython')
exit(1) exit(1)
if version < '3.6': if version < '3.6':
logger.log('FATAL', 'OneForAll需要Python 3.6以上版本') logger.log('FATAL', 'OneForAll requires Python 3.6 or higher')
exit(1) exit(1)
if system == 'Windows' and implementation == 'CPython': if system == 'Windows' and implementation == 'CPython':
if version < '3.8': if version < '3.8':
logger.log('FATAL', 'OneForAll在Windows系统运行时需要Python 3.8以上版本') logger.log('FATAL', 'OneForAll requires Python 3.8 or higher when running on Windows')
exit(1) exit(1)
if system in {"Linux", "Darwin"}: if system in {"Linux", "Darwin"}:
try: try:
import uvloop import uvloop
except ImportError: except ImportError:
logger.log('ALERT', f'请手动安装uvloop的Python库加速子域请求') logger.log('ALERT', f'Please install the uvloop manually Python library to accelerate subdomain requests')
def check_env(): def check_env():
logger.log('INFOR', '正在检查运行环境') logger.log('INFOR', 'Checking the environment...')
try: try:
check_net() check_net()
except Exception as e: except Exception as e:
logger.log('DEBUG', e.args) logger.log('DEBUG', e.args)
logger.log('FATAL', '不能正常访问外网') logger.log('FATAL', 'Can not access Internet')
exit(1) exit(1)
check_pre() check_pre()
@@ -570,7 +569,7 @@ def get_maindomain(domain):
def call_massdns(massdns_path, dict_path, ns_path, output_path, log_path, def call_massdns(massdns_path, dict_path, ns_path, output_path, log_path,
query_type='A', process_num=1, concurrent_num=10000, query_type='A', process_num=1, concurrent_num=10000,
quiet_mode=False): quiet_mode=False):
logger.log('INFOR', f'开始执行massdns') logger.log('INFOR', f'Start running massdns')
quiet = '' quiet = ''
if quiet_mode: if quiet_mode:
quiet = '--quiet' quiet = '--quiet'
@@ -583,9 +582,9 @@ def call_massdns(massdns_path, dict_path, ns_path, output_path, log_path,
f'--resolve-count {resolve_num} --type {query_type} ' \ f'--resolve-count {resolve_num} --type {query_type} ' \
f'--flush --output J --outfile {output_path} ' \ f'--flush --output J --outfile {output_path} ' \
f'--root --error-log {log_path} {dict_path}' f'--root --error-log {log_path} {dict_path}'
logger.log('INFOR', f'执行命令 {cmd}') logger.log('INFOR', f'Run command {cmd}')
subprocess.run(args=cmd, shell=True) subprocess.run(args=cmd, shell=True)
logger.log('INFOR', f'结束执行massdns') logger.log('INFOR', f'Finished massdns')
def get_massdns_path(massdns_dir): def get_massdns_path(massdns_dir):
@@ -604,7 +603,7 @@ def get_massdns_path(massdns_dir):
path = massdns_dir.joinpath(name) path = massdns_dir.joinpath(name)
path.chmod(S_IXUSR) path.chmod(S_IXUSR)
if not path.exists(): if not path.exists():
logger.log('FATAL', '暂无该系统平台及架构的massdns') logger.log('FATAL', 'There is no massdns for this platform or architecture')
logger.log('INFOR', '请尝试自行编译massdns并在配置里指定路径') logger.log('INFOR', 'Please try to compile massdns yourself and specify the path in the configuration')
exit(0) exit(0)
return path return path
+4 -2
View File
@@ -26,10 +26,12 @@ logger.remove()
logger.level(name='TRACE', no=5, color='<cyan><bold>', icon='✏️') logger.level(name='TRACE', no=5, color='<cyan><bold>', icon='✏️')
logger.level(name='DEBUG', no=10, color='<blue><bold>', icon='🐞 ') logger.level(name='DEBUG', no=10, color='<blue><bold>', icon='🐞 ')
logger.level(name='INFOR', no=20, color='<green><bold>', icon='') logger.level(name='INFOR', no=20, color='<green><bold>', icon='')
logger.level(name='QUITE', no=25, color='<green><bold>', icon='🤫 ')
logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️') logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️')
logger.level(name='ERROR', no=40, color='<red><bold>', icon='❌️') logger.level(name='ERROR', no=40, color='<red><bold>', icon='❌️')
logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️') logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')
logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True) # 如果你想在命令终端静默运行OneForAll,可以将以下一行中的level设置为QUITE
logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True) # 命令终端日志级别默认为INFOR
logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True, logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True,
encoding='utf-8') encoding='utf-8') # 日志文件默认为级别为DEBUG
+1 -1
View File
@@ -234,7 +234,7 @@ class OneForAll(object):
utils.check_env() utils.check_env()
logger.log('DEBUG', 'Python ' + utils.python_version()) logger.log('DEBUG', 'Python ' + utils.python_version())
logger.log('DEBUG', 'OneForAll ' + version) logger.log('DEBUG', 'OneForAll ' + version)
logger.log('INFOR', f'Start Running OneForAll') logger.log('INFOR', f'Start running OneForAll')
self.config() self.config()
self.domains = utils.get_domains(self.target) self.domains = utils.get_domains(self.target)
if self.domains: if self.domains:
+3 -3
View File
@@ -78,7 +78,7 @@ class Takeover(Module):
self.results = Dataset() self.results = Dataset()
def save(self): def save(self):
logger.log('DEBUG', 'Saving Results...') logger.log('DEBUG', 'Saving results...')
if self.format == 'txt': if self.format == 'txt':
data = str(self.results) data = str(self.results)
else: else:
@@ -158,8 +158,8 @@ class Takeover(Module):
logger.log('FATAL', f'Failed to obtain domain') logger.log('FATAL', f'Failed to obtain domain')
end = time.time() end = time.time()
elapse = round(end - start, 1) elapse = round(end - start, 1)
logger.log('INFOR', f'{self.source} module spends {elapse} seconds' logger.log('INFOR', f'{self.source} module spends {elapse} seconds, '
f'There are {len(self.results)} subdomains exit takeover') f'There are {len(self.results)} subdomains exists takeover')
logger.log('INFOR', f'Subdomain takeover results: {self.path}') logger.log('INFOR', f'Subdomain takeover results: {self.path}')
logger.log('INFOR', f'Finished {self.source} module') logger.log('INFOR', f'Finished {self.source} module')