diff --git a/brute.py b/brute.py index 428195b..cd3e1f3 100644 --- a/brute.py +++ b/brute.py @@ -132,7 +132,7 @@ def gen_word_subdomains(expression, path): def query_domain_ns_a(ns_list): - logger.log('INFOR', f'Querying authoritative name server {ns_list} A record') + logger.log('INFOR', f'Querying A record from authoritative name server: {ns_list} ') if not isinstance(ns_list, list): return list() ns_ip_list = [] @@ -147,7 +147,7 @@ def query_domain_ns_a(ns_list): if answer: for item in answer: ns_ip_list.append(item.address) - logger.log('INFOR', f'Authoritative name server A record: {ns_ip_list}') + logger.log('INFOR', f'Authoritative name server A record result: {ns_ip_list}') return ns_ip_list @@ -162,7 +162,7 @@ def query_domain_ns(domain): logger.log('ERROR', f'Querying NS records of {domain} error') return list() ns = [item.to_text() for item in answer] - logger.log('INFOR', f'{domain}\'s authoritative name server: {ns}') + logger.log('INFOR', f'{domain}\'s authoritative name server is {ns}') return ns @@ -186,7 +186,7 @@ def get_wildcard_record(domain, resolver): exit(1) else: if answer.rrset is None: - logger.log('DEBUG', f'No record of query results') + logger.log('DEBUG', f'No record of query result') return None, None name = answer.name ip = {item.address for item in answer} @@ -282,7 +282,7 @@ def gen_records(items, records, subdomains, ip_times, wc_ips, wc_ttl): have_a_record = True ttl = answer.get('ttl') ttls.append(ttl) - cname.append(answer.get('name')[:-1]) # 去出最右边的`.`点号 + cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号 ip = answer.get('data') ips.append(ip) public.append(utils.ip_is_public(ip)) @@ -339,7 +339,7 @@ def stat_ip_times(result_paths): def deal_output(output_paths, ip_times, wildcard_ips, wildcard_ttl): - logger.log('INFOR', f'Processing results...') + logger.log('INFOR', f'Processing result...') records = dict() # 用来记录所有域名解析数据 subdomains = list() # 用来保存所有通过有效性检查的子域 for output_path in output_paths: @@ -598,8 +598,8 @@ class Brute(Module): delete_file(dict_path, output_paths) end = time.time() self.elapse = round(end - start, 1) - logger.log('INFOR', f'{self.source} module spends {self.elapse} seconds' - f'Found {len(self.subdomains)} subdomains of {domain}') + logger.log('INFOR', f'{self.source} module spends {self.elapse} seconds, ' + f'found {len(self.subdomains)} subdomains of {domain}') logger.log('DEBUG', f'{self.source} module found subdomains of {domain}:\n' f'{self.subdomains}') self.gen_result(brute=dict_len, valid=len(self.subdomains)) diff --git a/common/crawl.py b/common/crawl.py index 4ee4ce0..6f0d35f 100644 --- a/common/crawl.py +++ b/common/crawl.py @@ -3,7 +3,8 @@ from .module import Module class Crawl(Module): """ - 爬虫基类 + Crawl base class """ + def __init__(self): Module.__init__(self) diff --git a/common/database.py b/common/database.py index 0436c28..7f9a968 100644 --- a/common/database.py +++ b/common/database.py @@ -2,7 +2,7 @@ # coding=utf-8 """ -SQLite数据库初始化和操作 +SQLite database initialization and operation """ import records @@ -19,12 +19,12 @@ class Database(object): @staticmethod def get_conn(db_path): """ - 获取数据库对象 + Get database connection - :param db_path: 数据库连接或路径 - :return: SQLite数据库 + :param db_path: Database path + :return: db_conn: SQLite database connection """ - logger.log('TRACE', f'正在获取数据库连接') + logger.log('TRACE', f'Establishing database connection...') if isinstance(db_path, Connection): return db_path protocol = 'sqlite:///' @@ -33,7 +33,7 @@ class Database(object): else: db_path = protocol + db_path db = records.Database(db_path) # 不存在数据库时会新建一个数据库 - logger.log('TRACE', f'使用数据库: {db_path}') + logger.log('TRACE', f'Use the database: {db_path}') return db.get_connection() def query(self, sql): @@ -46,15 +46,15 @@ class Database(object): def create_table(self, table_name): """ - 创建表结构 + Create table - :param str table_name: 要创建的表名 + :param str table_name: table name """ table_name = table_name.replace('.', '_') if self.exist_table(table_name): - logger.log('TRACE', f'已经存在{table_name}表') + logger.log('TRACE', f'{table_name} table already exists') return - logger.log('TRACE', f'正在创建{table_name}表') + logger.log('TRACE', f'Creating {table_name} table') self.query(f'create table "{table_name}" (' f'id integer primary key,' f'type text,' @@ -87,14 +87,14 @@ class Database(object): def save_db(self, table_name, results, module_name=None): """ - 将各模块结果存入数据库 + Save the results of each module in the database - :param str table_name: 表名 - :param list results: 结果列表 - :param str module_name: 模块名 + :param str table_name: table name + :param list results: results list + :param str module_name: mo """ - logger.log('TRACE', f'正在将{module_name}模块发现{table_name}的子域' - '结果存入数据库') + logger.log('TRACE', + f'Saving the subdomain results of {table_name} found by module {module_name} into database...') table_name = table_name.replace('.', '_') if results: try: @@ -114,13 +114,13 @@ class Database(object): def exist_table(self, table_name): """ - 判断是否存在某表 + Determine table exists - :param str table_name: 表名 - :return: 是否存在某表 + :param str table_name: table name + :return bool: Whether table exists """ table_name = table_name.replace('.', '_') - logger.log('TRACE', f'正在查询是否存在{table_name}表') + logger.log('TRACE', f'Determining whether the {table_name} table exists') results = self.query(f'select count() from sqlite_master ' f'where type = "table" and ' f'name = "{table_name}"') @@ -131,80 +131,80 @@ class Database(object): def copy_table(self, table_name, bak_table_name): """ - 复制表创建备份 + Copy table to create backup - :param str table_name: 表名 - :param str bak_table_name: 新表名 + :param str table_name: table name + :param str bak_table_name: new table name """ table_name = table_name.replace('.', '_') bak_table_name = bak_table_name.replace('.', '_') - logger.log('TRACE', f'正在将{table_name}表复制到{bak_table_name}新表') + logger.log('TRACE', f'Copying {table_name} table to {bak_table_name} new table') self.query(f'drop table if exists "{bak_table_name}"') self.query(f'create table "{bak_table_name}" ' f'as select * from "{table_name}"') def clear_table(self, table_name): """ - 清空表中数据 + Clear the table :param str table_name: 表名 """ table_name = table_name.replace('.', '_') - logger.log('TRACE', f'正在清空{table_name}表中的数据') + logger.log('TRACE', f'Clearing data in table {table_name}') self.query(f'delete from "{table_name}"') def drop_table(self, table_name): """ - 删除表 + Delete table - :param str table_name: 表名 + :param str table_name: table name """ table_name = table_name.replace('.', '_') - logger.log('TRACE', f'正在删除{table_name}表') + logger.log('TRACE', f'Deleting {table_name} table') self.query(f'drop table if exists "{table_name}"') def rename_table(self, table_name, new_table_name): """ - 重命名表名 + Rename table name - :param str table_name: 表名 - :param str new_table_name: 新表名 + :param str table_name: old table name + :param str new_table_name: new table name """ table_name = table_name.replace('.', '_') new_table_name = new_table_name.replace('.', '_') - logger.log('TRACE', f'正在将{table_name}表重命名为{table_name}表') + logger.log('TRACE', f'Renaming {table_name} table to {new_table_name} table') self.query(f'alter table "{table_name}" ' f'rename to "{new_table_name}"') def deduplicate_subdomain(self, table_name): """ - 去重表中的子域 + Deduplicates of subdomains in the table - :param str table_name: 表名 + :param str table_name: table name """ table_name = table_name.replace('.', '_') - logger.log('TRACE', f'正在去重{table_name}表中的子域') + logger.log('TRACE', f'Deduplicating subdomains in {table_name} table') self.query(f'delete from "{table_name}" where ' f'id not in (select min(id) ' f'from "{table_name}" group by subdomain)') def remove_invalid(self, table_name): """ - 去除表中的空值或无效子域 + Remove nulls or invalid subdomains in the table - :param str table_name: 表名 + :param str table_name: table name """ table_name = table_name.replace('.', '_') - logger.log('TRACE', f'正在去除{table_name}表中的无效子域') + logger.log('TRACE', f'Removing invalid subdomains in {table_name} table') self.query(f'delete from "{table_name}" where ' f'subdomain is null or resolve == 0') def deal_table(self, deal_table_name, backup_table_name): """ - 收集任务完成时对表进行处理 + Process the table when the collection task is complete - :param str deal_table_name: 待处理的表名 - :param str backup_table_name: 备份的表名 + :param str deal_table_name: Pending table name + :param str backup_table_name: Table name for backup """ self.copy_table(deal_table_name, backup_table_name) self.remove_invalid(deal_table_name) @@ -212,27 +212,27 @@ class Database(object): def get_data(self, table_name): """ - 获取表中的所有数据 + Get all the data in the table - :param str table_name: 表名 + :param str table_name: table name """ table_name = table_name.replace('.', '_') - logger.log('TRACE', f'获取{table_name}表中的所有数据') + logger.log('TRACE', f'Get all the data from {table_name} table') return self.query(f'select * from "{table_name}"') def export_data(self, table_name, alive, limit): """ - 获取表中的部分数据 + Get part of the data in the table - :param str table_name: 表名 - :param any alive: 存活 - :param str limit: 限制字段 + :param str table_name: table name + :param any alive: alive flag + :param str limit: limit value """ table_name = table_name.replace('.', '_') query = f'select id, type, new, alive, request, resolve, url, ' \ - f'subdomain, level, cname, content, public, port, status, ' \ - f'reason, title, banner, times, ttl, resolver, module, ' \ - f'source, elapse, find, brute, valid from "{table_name}"' + f'subdomain, level, cname, content, public, port, status, ' \ + f'reason, title, banner, times, ttl, resolver, module, ' \ + f'source, elapse, find, brute, valid from "{table_name}"' if alive and limit: if limit in ['resolve', 'request']: where = f' where {limit} = 1' @@ -240,11 +240,11 @@ class Database(object): elif alive: where = f' where alive = 1' query += where - logger.log('TRACE', f'获取{table_name}表中的数据') + logger.log('TRACE', f'Get the data from {table_name} table') return self.query(query) def close(self): """ - 关闭数据库连接 + Close the database connection """ self.conn.close() diff --git a/common/domain.py b/common/domain.py index f5985c0..55dcab4 100644 --- a/common/domain.py +++ b/common/domain.py @@ -5,10 +5,11 @@ from config import setting class Domain(object): """ - 域名处理类 + Processing domain class - :param str string: 传入的字符串 + :param str string: input string """ + def __init__(self, string): self.string = str(string) self.regexp = r'\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b' @@ -16,9 +17,9 @@ class Domain(object): def match(self): """ - 域名匹配 + match domain - :return: 匹配结果 + :return : result """ result = re.search(self.regexp, self.string, re.I) if result: @@ -28,14 +29,14 @@ class Domain(object): def extract(self): """ - 域名导出 + extract domain >>> d = Domain('www.example.com') >>> d.extract() ExtractResult(subdomain='www', domain='example', suffix='com') - :return: 导出结果 + :return: extracted domain results """ data_storage_dir = setting.data_storage_dir extract_cache_file = data_storage_dir.joinpath('public_suffix_list.dat') @@ -48,14 +49,14 @@ class Domain(object): def registered(self): """ - 获取注册域名 + registered domain >>> d = Domain('www.example.com') >>> d.registered() example.com - :return: 注册域名 + :return: registered domain result """ result = self.extract() if result: diff --git a/common/lookup.py b/common/lookup.py index 113ce0d..cf72d81 100644 --- a/common/lookup.py +++ b/common/lookup.py @@ -4,15 +4,16 @@ from common import utils class Lookup(Module): """ - DNS查询基类 + DNS query base class """ + def __init__(self): Module.__init__(self) def query(self): """ - 查询域名的TXT记录 - :return: 查询结果 + Query the TXT record of domain + :return: query result """ answer = utils.dns_query(self.domain, self.type) if answer is None: diff --git a/common/module.py b/common/module.py index e557859..3b8946f 100644 --- a/common/module.py +++ b/common/module.py @@ -1,6 +1,6 @@ # coding=utf-8 """ -模块基类 +Module base class """ import json @@ -39,43 +39,43 @@ class Module(object): def check(self, *apis): """ - 简单检查是否配置了api信息 + Simply check whether the api information configure or not - :param apis: api信息元组 - :return: 检查结果 + :param apis: apis set + :return bool: check result """ if not all(apis): - logger.log('ALERT', f'{self.source}模块没有配置API跳过执行') + logger.log('ALERT', f'{self.source} module is not configured, skip') return False return True def begin(self): """ - 输出模块开始信息 + begin log """ - logger.log('DEBUG', f'开始执行{self.source}模块收集{self.domain}的子域') + logger.log('DEBUG', f'Start {self.source} module to collect subdomains of {self.domain}') def finish(self): """ - 输出模块结束信息 + finish log """ self.end = time.time() self.elapse = round(self.end - self.start, 1) - logger.log('DEBUG', f'结束执行{self.source}模块收集{self.domain}的子域') - logger.log('INFOR', f'{self.source}模块耗时{self.elapse}秒发现子域' - f'{len(self.subdomains)}个') - logger.log('DEBUG', f'{self.source}模块发现{self.domain}的子域\n' - f'{self.subdomains}') + logger.log('DEBUG', f'Finished {self.source} module to collect {self.domain}\'s subdomains') + logger.log('INFOR', f'The {self.source} module took {self.elapse} seconds, ' + f'found {len(self.subdomains)} subdomains') + logger.log('DEBUG', f'{self.source} module found subdomains of {self.domain}\n' + f'{self.subdomains}') def head(self, url, params=None, check=True, **kwargs): """ - 自定义head请求 + Custom head request - :param str url: 请求地址 - :param dict params: 请求参数 - :param bool check: 检查响应 - :param kwargs: 其他参数 - :return: requests响应对象 + :param str url: request url + :param dict params: request parameters + :param bool check: check response + :param kwargs: other params + :return: requests's response object """ try: resp = requests.head(url, @@ -97,13 +97,13 @@ class Module(object): def get(self, url, params=None, check=True, **kwargs): """ - 自定义get请求 + Custom get request - :param str url: 请求地址 - :param dict params: 请求参数 - :param bool check: 检查响应 - :param kwargs: 其他参数 - :return: requests响应对象 + :param str url: request url + :param dict params: request parameters + :param bool check: check response + :param kwargs: other params + :return: requests's response object """ try: resp = requests.get(url, @@ -125,13 +125,13 @@ class Module(object): def post(self, url, data=None, check=True, **kwargs): """ - 自定义post请求 + Custom post request - :param str url: 请求地址 - :param dict data: 请求数据 - :param bool check: 检查响应 - :param kwargs: 其他参数 - :return: requests响应对象 + :param str url: request url + :param dict params: request parameters + :param bool check: check response + :param kwargs: other params + :return: requests's response object """ try: resp = requests.post(url, @@ -153,11 +153,11 @@ class Module(object): def get_header(self): """ - 获取请求头 + Get request header - :return: 请求头 + :return: header """ - # logger.log('DEBUG', f'获取请求头') + # logger.log('DEBUG', f'Get request header') if setting.enable_fake_header: return utils.gen_fake_header() else: @@ -165,36 +165,35 @@ class Module(object): def get_proxy(self, module): """ - 获取代理 + Get proxy - :param str module: 模块名 - :return: 代理字典 + :param str module: module name + :return: proxy """ if not setting.enable_proxy: - logger.log('TRACE', f'所有模块不使用代理') + logger.log('TRACE', f'All modules do not use proxy') return self.proxy if setting.proxy_all_module: - logger.log('TRACE', f'{module}模块使用代理') + logger.log('TRACE', f'{module} module uses proxy') return utils.get_random_proxy() if module in setting.proxy_partial_module: - logger.log('TRACE', f'{module}模块使用代理') + logger.log('TRACE', f'{module} module uses proxy') return utils.get_random_proxy() else: - logger.log('TRACE', f'{module}模块不使用代理') + logger.log('TRACE', f'{module} module does not use proxy') return self.proxy @staticmethod def match(domain, html, distinct=True): """ - 正则匹配出子域 + Use regexp to match subdomains - :param str domain: 域名 - :param str html: 要匹配的html响应体 - :param bool distinct: 匹配结果去除 - :return: 匹配出的子域集合或列表 - :rtype: set or list + :param str domain: domain + :param str html: response html text + :param bool distinct: deduplicate results or not (default True) + :return set/list: result set or list """ - logger.log('TRACE', f'正则匹配响应体中的子域') + logger.log('TRACE', f'Use regexp to match subdomains in the response body') regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \ r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + domain.replace('.', r'\.') @@ -211,22 +210,22 @@ class Module(object): @staticmethod def register(domain): """ - 获取注册域名 + Get registered domain - :param str domain: 域名 - :return: 注册域名 + :param str domain: domain + :return: registered domain """ return Domain(domain).registered() def save_json(self): """ - 将各模块结果保存为json文件 + Save the results of each module as a json file - :return： 是否保存成功 + :return bool: whether saved successfully """ if not setting.save_module_result: return False - logger.log('TRACE', f'将{self.source}模块发现的子域结果保存为json文件') + logger.log('TRACE', f'Save the subdomain results found by {self.source} module as a json file') path = setting.result_save_dir.joinpath(self.domain, self.module) path.mkdir(parents=True, exist_ok=True) name = self.source + '.json' @@ -244,7 +243,7 @@ class Module(object): def gen_record(self, subdomains, record): """ - 生成记录字典 + Generate record dictionary """ item = dict() item['content'] = record @@ -253,11 +252,11 @@ class Module(object): def gen_result(self, find=0, brute=None, valid=0): """ - 生成结果 + Generate results """ - logger.log('DEBUG', f'正在生成最终结果') + logger.log('DEBUG', f'Generating final results...') if not len(self.subdomains): # 该模块一个子域都没有发现的情况 - logger.log('DEBUG', f'{self.source}模块收集结果为空') + logger.log('DEBUG', f'{self.source} module result is empty') result = {'id': None, 'type': self.type, 'alive': None, @@ -347,10 +346,10 @@ class Module(object): def save_db(self): """ - 将模块结果存入数据库中 + Save module results into the database """ - logger.log('DEBUG', f'正在将结果存入到数据库') + logger.log('DEBUG', f'Saving results to database') lock.acquire() db = Database() db.create_table(self.domain) diff --git a/common/query.py b/common/query.py index be3b78b..0ab4dda 100644 --- a/common/query.py +++ b/common/query.py @@ -3,7 +3,8 @@ from .module import Module class Query(Module): """ - 查询基类 + Query base class """ + def __init__(self): Module.__init__(self) diff --git a/common/request.py b/common/request.py index 0583839..12294ed 100644 --- a/common/request.py +++ b/common/request.py @@ -22,7 +22,7 @@ def get_limit_conn(): def get_ports(port): - logger.log('DEBUG', f'正在获取请求端口范围') + logger.log('DEBUG', f'Getting port range...') ports = set() if isinstance(port, (set, list, tuple)): ports = port @@ -30,17 +30,17 @@ def get_ports(port): if 0 <= port <= 65535: ports = {port} elif port in {'default', 'small', 'large'}: - logger.log('DEBUG', f'请求{port}等端口范围') + logger.log('DEBUG', f'{port} port range') ports = setting.ports.get(port) if not ports: # 意外情况 - logger.log('ERROR', f'指定请求端口范围有误') + logger.log('ERROR', f'The specified request port range is incorrect') ports = {80} - logger.log('INFOR', f'请求端口范围：{ports}') + logger.log('INFOR', f'Port range:{ports}') return set(ports) def gen_req_data(data, ports): - logger.log('INFOR', f'正在生成请求地址') + logger.log('INFOR', f'Generating request urls...') new_data = [] for data in data: resolve = data.get('resolve') @@ -202,8 +202,8 @@ async def bulk_request(data, port): no_req_data = utils.get_filtered_data(data) to_req_data = gen_req_data(data, ports) method = setting.request_method - logger.log('INFOR', f'请求使用{method}方法') - logger.log('INFOR', f'正在进行异步子域请求') + logger.log('INFOR', f'Use {method} method to request') + logger.log('INFOR', f'Async subdomains request in progress...') connector = get_connector() header = get_header() async with ClientSession(connector=connector, headers=header) as session: @@ -238,15 +238,14 @@ def set_loop_policy(): def run_request(domain, data, port): """ - 调用子域请求入口函数 + HTTP request entrance - :param str domain: 待请求的主域 - :param list data: 待请求的子域数据 - :param str port: 待请求的端口范围 - :return: 请求后得到的结果列表 - :rtype: list + :param str domain: domain to be requested + :param list data: subdomains data to be requested + :param str port: range of ports to be requested + :return list: result """ - logger.log('INFOR', f'开始执行子域请求模块') + logger.log('INFOR', f'Start subdomain request module') set_loop_policy() loop = asyncio.get_event_loop() asyncio.set_event_loop(loop) @@ -256,16 +255,16 @@ def run_request(domain, data, port): # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间 loop.run_until_complete(asyncio.sleep(0.25)) count = utils.count_alive(data) - logger.log('INFOR', f'经验证{domain}存活子域{count}个') + logger.log('INFOR', f'After verify, found {domain} have {count} alive subdomains') return data def save_data(name, data): """ - 保存请求结果到数据库 + Save request results to database - :param str name: 保存表名 - :param list data: 待保存的数据 + :param str name: table name + :param list data: data to be saved """ db = Database() db.drop_table(name) diff --git a/common/resolve.py b/common/resolve.py index d8d4404..dcd5a02 100644 --- a/common/resolve.py +++ b/common/resolve.py @@ -14,7 +14,7 @@ def filter_subdomain(data): :param list data: 待过滤的数据列表 :return: 符合条件的子域列表 """ - logger.log('DEBUG', f'正在过滤出待解析的子域') + logger.log('DEBUG', f'Filtering subdomains to be resolved...') subdomains = [] for data in data: if not data.get('content'): @@ -31,9 +31,9 @@ def update_data(data, records): :param dict records: 解析结果字典 :return: 更新后的数据列表 """ - logger.log('DEBUG', f'正在更新解析结果') + logger.log('DEBUG', f'Updating resolved results...') if not records: - logger.log('ERROR', f'无有效解析结果') + logger.log('ERROR', f'No valid resolved result') return data for index, items in enumerate(data): if not items.get('content'): @@ -52,7 +52,7 @@ def save_data(name, data): :param str name: 保存表名 :param list data: 待保存的数据 """ - logger.log('INFOR', f'正在保存解析结果') + logger.log('INFOR', f'Saving resolved results...') db = Database() db.drop_table(name) db.create_table(name) @@ -61,15 +61,15 @@ def save_data(name, data): def save_subdomains(save_path, subdomain_list): - logger.log('DEBUG', f'正在保存待解析的子域') + logger.log('DEBUG', f'Saving resolved subdomain...') subdomain_data = '\n'.join(subdomain_list) if not utils.save_data(save_path, subdomain_data): - logger.log('FATAL', '保存待解析的子域出错') + logger.log('FATAL', 'Save resolved subdomain error') exit(1) def deal_output(output_path): - logger.log('INFOR', f'正在处理解析结果') + logger.log('INFOR', f'Processing resolved results...') records = dict() # 用来记录所有域名解析数据 with open(output_path) as fd: for line in fd: @@ -78,11 +78,11 @@ def deal_output(output_path): items = json.loads(line) except Exception as e: logger.log('ERROR', e.args) - logger.log('ERROR', f'解析行{line}出错跳过解析该行') + logger.log('ERROR', f'Error resolve line {line}, skip this line') continue record = dict() record['resolver'] = items.get('resolver') - qname = items.get('name')[:-1] # 去出最右边的`.`点号 + qname = items.get('name')[:-1] # 去除最右边的`.`点号 status = items.get('status') if status != 'NOERROR': record['alive'] = 0 @@ -106,7 +106,7 @@ def deal_output(output_path): for answer in answers: if answer.get('type') == 'A': flag = True - cname.append(answer.get('name')[:-1]) # 去出最右边的`.`点号 + cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号 ip = answer.get('data') ips.append(ip) ttl = answer.get('ttl') @@ -137,7 +137,7 @@ def run_resolve(domain, data): :return: 解析得到的结果列表 :rtype: list """ - logger.log('INFOR', f'开始解析{domain}的子域') + logger.log('INFOR', f'Start resolve subdomains of {domain}') subdomains = filter_subdomain(data) if not subdomains: return data @@ -166,5 +166,5 @@ def run_resolve(domain, data): records = deal_output(output_path) data = update_data(data, records) - logger.log('INFOR', f'结束解析{domain}的子域') + logger.log('INFOR', f'Finished resolve subdomains of {domain}') return data diff --git a/common/search.py b/common/search.py index 450136a..16c12a3 100644 --- a/common/search.py +++ b/common/search.py @@ -5,7 +5,7 @@ from . import utils class Search(Module): """ - 搜索基类 + Search base class """ def __init__(self): Module.__init__(self) diff --git a/common/utils.py b/common/utils.py index ca381e4..f99f7b3 100644 --- a/common/utils.py +++ b/common/utils.py @@ -34,13 +34,12 @@ user_agents = [ def match_subdomain(domain, text, distinct=True): """ - 匹配text中domain的子域名 + Use regexp to match subdomains in text - :param str domain: 域名 - :param str text: 响应文本 - :param bool distinct: 结果去重 - :return: 匹配结果 - :rtype: set or list + :param str domain: domain + :param str text: response text + :param bool distinct: deduplicate results + :return set/list: match result """ regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + domain.replace('.', r'\.') @@ -56,7 +55,7 @@ def match_subdomain(domain, text, distinct=True): def gen_random_ip(): """ - 生成随机的点分十进制的IP字符串 + Generate random decimal IP string """ while True: ip = IPv4Address(random.randint(0, 2 ** 32 - 1)) @@ -66,7 +65,7 @@ def gen_random_ip(): def gen_fake_header(): """ - 生成伪造请求头 + Generate fake request headers """ ua = random.choice(user_agents) ip = gen_random_ip() @@ -89,7 +88,7 @@ def gen_fake_header(): def get_random_proxy(): """ - 获取随机代理 + Get random proxy """ try: return random.choice(setting.proxy_pool) @@ -99,11 +98,11 @@ def get_random_proxy(): def split_list(ls, size): """ - 将ls列表按size大小划分并返回新的划分结果列表 + Split list - :param list ls: 要划分的列表 - :param int size: 划分大小 - :return 划分结果 + :param list ls: list + :param int size: size + :return list: result >>> split_list([1, 2, 3, 4], 3) [[1, 2, 3], [4]] @@ -115,13 +114,13 @@ def split_list(ls, size): def get_domains(target): """ - 获取域名 + Get domains - :param set or str target: - :return: 域名集合 + :param set or str target: + :return list: domain list """ domains = list() - logger.log('DEBUG', f'正在获取域名') + logger.log('DEBUG', f'Getting domains...') if isinstance(target, (set, tuple)): domains = list(target) elif isinstance(target, list): @@ -142,9 +141,9 @@ def get_domains(target): domains.append(domain) count = len(domains) if count == 0: - logger.log('FATAL', f'获取到{count}个域名') + logger.log('FATAL', f'Get {count} domains') exit(1) - logger.log('INFOR', f'获取到{count}个域名') + logger.log('INFOR', f'Get {count} domains') return domains @@ -165,7 +164,7 @@ def get_semaphore(): def check_dir(dir_path): if not dir_path.exists(): - logger.log('INFOR', f'不存在{dir_path}目录将会新建') + logger.log('INFOR', f'{dir_path} does not exist, directory will be created') dir_path.mkdir(parents=True, exist_ok=True) @@ -190,10 +189,10 @@ def check_path(path, name, format): path = path.joinpath(filename) parent_dir = path.parent if not parent_dir.exists(): - logger.log('ALERT', f'不存在{parent_dir}目录将会新建') + logger.log('ALERT', f'{parent_dir} does not exist, directory will be created') parent_dir.mkdir(parents=True, exist_ok=True) if path.exists(): - logger.log('ALERT', f'存在{path}文件将会覆盖') + logger.log('ALERT', f'{path} exists, file will be overwritten') return path @@ -208,14 +207,14 @@ def check_format(format, count): formats = ['rst', 'csv', 'tsv', 'json', 'yaml', 'html', 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'] if format == 'xls' and count > 65000: - logger.log('ALERT', 'xls文件限制为最多65000行') - logger.log('ALERT', '使用xlsx格式导出') + logger.log('ALERT', '\'xls\' file is limited to 65000 lines') + logger.log('ALERT', 'So use xlsx format replace') return 'xlsx' if format in formats: return format else: - logger.log('ALERT', f'不支持{format}格式导出') - logger.log('ALERT', '默认使用csv格式导出') + logger.log('ALERT', f'Does not support {format} format') + logger.log('ALERT', 'So use csv format by default') return 'csv' @@ -252,7 +251,7 @@ def check_response(method, resp): if resp.status_code == 200 and resp.content: return True logger.log('ALERT', f'{method} {resp.url} {resp.status_code} - ' - f'{resp.reason} {len(resp.content)}') + f'{resp.reason} {len(resp.content)}') content_type = resp.headers.get('Content-Type') if content_type and 'json' in content_type and resp.content: try: @@ -311,7 +310,7 @@ def check_value(values): def export_all_results(path, name, format, datas): path = check_path(path, name, format) - logger.log('INFOR', f'所有主域的子域结果 {path}') + logger.log('ALERT', f'Subdomain result for all domains: {path}') row_list = list() for row in datas: if 'header' in row: @@ -330,7 +329,7 @@ def export_all_results(path, name, format, datas): def export_all_subdomains(alive, path, name, datas): path = check_path(path, name, 'txt') - logger.log('INFOR', f'所有主域的纯子域结果 {path}') + logger.log('ALERT', f'Txt subdomain result for all main domains {path}') subdomains = set() for row in datas: subdomain = row.get('subdomain') @@ -379,16 +378,16 @@ def dns_query(qname, qtype): :param str qtype: 查询类型 :return: 查询结果 """ - logger.log('TRACE', f'尝试查询{qname}的{qtype}记录') + logger.log('TRACE', f'Try to query {qtype} record of {qname}') resolver = dns_resolver() try: answer = resolver.query(qname, qtype) except Exception as e: logger.log('TRACE', e.args) - logger.log('TRACE', f'查询{qname}的{qtype}记录失败') + logger.log('TRACE', f'Query {qtype} record of {qname} failed') return None else: - logger.log('TRACE', f'查询{qname}的{qtype}记录成功') + logger.log('TRACE', f'Query {qtype} record of {qname} succeeded') return answer @@ -509,56 +508,56 @@ def delete_file(*paths): @tenacity.retry(stop=tenacity.stop_after_attempt(3)) def check_net(): - logger.log('INFOR', '正在检查网络环境') + logger.log('INFOR', 'Checking Internet environment...') urls = ['http://www.example.com', 'http://www.baidu.com', 'http://www.bing.com', 'http://www.taobao.com', 'http://www.linkedin.com', 'http://www.msn.com', 'http://www.apple.com', 'http://microsoft.com'] url = random.choice(urls) - logger.log('INFOR', f'正在尝试访问 {url}') + logger.log('INFOR', f'Trying to access {url}') try: rsp = requests.get(url) except Exception as e: logger.log('ERROR', e.args) - logger.log('ALERT', '访问外网出错 重新检查中') + logger.log('ALERT', 'Can not access Internet, retrying...') raise tenacity.TryAgain if rsp.status_code != 200: logger.log('ALERT', f'{rsp.request.method} {rsp.request.url} ' - f'{rsp.status_code} {rsp.reason}') - logger.log('ALERT', '不能正常访问外网 重新检查中') + f'{rsp.status_code} {rsp.reason}') + logger.log('ALERT', 'Can not access Internet normally, retrying...') raise tenacity.TryAgain - logger.log('INFOR', '能正常访问外网') + logger.log('INFOR', 'Access to Internet OK') def check_pre(): - logger.log('INFOR', '正在检查依赖环境') + logger.log('INFOR', 'Checking dependent environment...') system = platform.system() implementation = platform.python_implementation() version = platform.python_version() if implementation != 'CPython': - logger.log('FATAL', f'OneForAll只在CPython下测试通过') + logger.log('FATAL', f'OneForAll only passed the test under CPython') exit(1) if version < '3.6': - logger.log('FATAL', 'OneForAll需要Python 3.6以上版本') + logger.log('FATAL', 'OneForAll requires Python 3.6 or higher') exit(1) if system == 'Windows' and implementation == 'CPython': if version < '3.8': - logger.log('FATAL', 'OneForAll在Windows系统运行时需要Python 3.8以上版本') + logger.log('FATAL', 'OneForAll requires Python 3.8 or higher when running on Windows') exit(1) if system in {"Linux", "Darwin"}: try: import uvloop except ImportError: - logger.log('ALERT', f'请手动安装uvloop的Python库加速子域请求') + logger.log('ALERT', f'Please install the uvloop manually Python library to accelerate subdomain requests') def check_env(): - logger.log('INFOR', '正在检查运行环境') + logger.log('INFOR', 'Checking the environment...') try: check_net() except Exception as e: logger.log('DEBUG', e.args) - logger.log('FATAL', '不能正常访问外网') + logger.log('FATAL', 'Can not access Internet') exit(1) check_pre() @@ -570,7 +569,7 @@ def get_maindomain(domain): def call_massdns(massdns_path, dict_path, ns_path, output_path, log_path, query_type='A', process_num=1, concurrent_num=10000, quiet_mode=False): - logger.log('INFOR', f'开始执行massdns') + logger.log('INFOR', f'Start running massdns') quiet = '' if quiet_mode: quiet = '--quiet' @@ -578,14 +577,14 @@ def call_massdns(massdns_path, dict_path, ns_path, output_path, log_path, socket_num = setting.brute_socket_num resolve_num = setting.brute_resolve_num cmd = f'{massdns_path} {quiet} --status-format {status_format} ' \ - f'--processes {process_num} --socket-count {socket_num} ' \ - f'--hashmap-size {concurrent_num} --resolvers {ns_path} ' \ - f'--resolve-count {resolve_num} --type {query_type} ' \ - f'--flush --output J --outfile {output_path} ' \ - f'--root --error-log {log_path} {dict_path}' - logger.log('INFOR', f'执行命令 {cmd}') + f'--processes {process_num} --socket-count {socket_num} ' \ + f'--hashmap-size {concurrent_num} --resolvers {ns_path} ' \ + f'--resolve-count {resolve_num} --type {query_type} ' \ + f'--flush --output J --outfile {output_path} ' \ + f'--root --error-log {log_path} {dict_path}' + logger.log('INFOR', f'Run command {cmd}') subprocess.run(args=cmd, shell=True) - logger.log('INFOR', f'结束执行massdns') + logger.log('INFOR', f'Finished massdns') def get_massdns_path(massdns_dir): @@ -604,7 +603,7 @@ def get_massdns_path(massdns_dir): path = massdns_dir.joinpath(name) path.chmod(S_IXUSR) if not path.exists(): - logger.log('FATAL', '暂无该系统平台及架构的massdns') - logger.log('INFOR', '请尝试自行编译massdns并在配置里指定路径') + logger.log('FATAL', 'There is no massdns for this platform or architecture') + logger.log('INFOR', 'Please try to compile massdns yourself and specify the path in the configuration') exit(0) return path diff --git a/config/log.py b/config/log.py index 730aa37..32f2d83 100644 --- a/config/log.py +++ b/config/log.py @@ -26,10 +26,12 @@ logger.remove() logger.level(name='TRACE', no=5, color='', icon='✏️') logger.level(name='DEBUG', no=10, color='', icon='🐞 ') logger.level(name='INFOR', no=20, color='', icon='ℹ️') +logger.level(name='QUITE', no=25, color='', icon='🤫 ') logger.level(name='ALERT', no=30, color='', icon='⚠️') logger.level(name='ERROR', no=40, color='', icon='❌️') logger.level(name='FATAL', no=50, color='', icon='☠️') -logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True) +# 如果你想在命令终端静默运行OneForAll，可以将以下一行中的level设置为QUITE +logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True) # 命令终端日志级别默认为INFOR logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True, - encoding='utf-8') + encoding='utf-8') # 日志文件默认为级别为DEBUG \ No newline at end of file diff --git a/oneforall.py b/oneforall.py index 94c83aa..5f9cdc1 100644 --- a/oneforall.py +++ b/oneforall.py @@ -234,7 +234,7 @@ class OneForAll(object): utils.check_env() logger.log('DEBUG', 'Python ' + utils.python_version()) logger.log('DEBUG', 'OneForAll ' + version) - logger.log('INFOR', f'Start Running OneForAll') + logger.log('INFOR', f'Start running OneForAll') self.config() self.domains = utils.get_domains(self.target) if self.domains: diff --git a/takeover.py b/takeover.py index 356d88d..3f8cd97 100644 --- a/takeover.py +++ b/takeover.py @@ -78,7 +78,7 @@ class Takeover(Module): self.results = Dataset() def save(self): - logger.log('DEBUG', 'Saving Results...') + logger.log('DEBUG', 'Saving results...') if self.format == 'txt': data = str(self.results) else: @@ -158,8 +158,8 @@ class Takeover(Module): logger.log('FATAL', f'Failed to obtain domain') end = time.time() elapse = round(end - start, 1) - logger.log('INFOR', f'{self.source} module spends {elapse} seconds' - f'There are {len(self.results)} subdomains exit takeover') + logger.log('INFOR', f'{self.source} module spends {elapse} seconds, ' + f'There are {len(self.results)} subdomains exists takeover') logger.log('INFOR', f'Subdomain takeover results: {self.path}') logger.log('INFOR', f'Finished {self.source} module')