river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-26 12:57:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

重构子域爆破模块

Browse Source
This commit is contained in:
Jing Ling
2020-04-04 04:22:22 +08:00
parent 4b1aafa719
commit aee7ce727d
14 changed files with 1903973 additions and 36133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
oneforall/common/database.py
+34 -18
View File
@@ -58,24 +58,32 @@ class Database(object):
self.query(f'create table "{table_name}" ('
f'id integer primary key,'
f'type text,'
f'valid int,'
f'alive int,'
f'request int,'
f'resolve int,'
f'new int,'
f'url text,'
f'subdomain text,'
f'port int,'
f'level int,'
f'cname text,'
f'content text,'
f'public int,'
f'port int,'
f'status int,'
f'reason text,'
f'title text,'
f'banner text,'
f'header text,'
f'response text,'
f'times text,'
f'ttl text,'
f'resolver text,'
f'module text,'
f'source text,'
f'elapse float,'
f'count int)')
f'find int,'
f'brute int,'
f'valid int)')
def save_db(self, table_name, results, module_name=None):
"""
@@ -92,15 +100,17 @@ class Database(object):
try:
self.conn.bulk_query(
f'insert into "{table_name}" ('
f'id, type, valid, new, url, subdomain, port, level, content,'
f'public, status, reason, title, banner, header, response,'
f'module, source, elapse, count)'
f'values (:id, :type, :valid, :new, :url, :subdomain,'
f':port, :level, :content, :public, :status, :reason,'
f':title, :banner, :header, :response, :module, :source,'
f':elapse, :count)', results)
f'id, type, alive, resolve, request, new, url, subdomain,'
f'port, level, cname, content, public, status, reason,'
f'title, banner, header, response, times, ttl, resolver,'
f'module, source, elapse, find, brute, valid) '
f'values (:id, :type, :alive, :resolve, :request, :new,'
f':url, :subdomain, :port, :level, :cname, :content,'
f':public, :status, :reason, :title, :banner, :header,'
f':response, :times, :ttl, :resolver, :module, :source,'
f':elapse, :find, :brute, :valid)', results)
except Exception as e:
logger.log('ERROR', e.args)
logger.log('ERROR', e)
def exist_table(self, table_name):
"""
@@ -187,7 +197,7 @@ class Database(object):
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'正在去除{table_name}表中的无效子域')
self.query(f'delete from "{table_name}" where '
f'subdomain is null or valid == 0')
f'subdomain is null or resolve == 0')
def deal_table(self, deal_table_name, backup_table_name):
"""
@@ -210,19 +220,25 @@ class Database(object):
logger.log('TRACE', f'获取{table_name}表中的所有数据')
return self.query(f'select * from "{table_name}"')
def export_data(self, table_name, valid):
def export_data(self, table_name, valid, limit):
"""
获取表中的部分数据
:param str table_name: 表名
:param any valid: 有效性
:param str limit: 限制字段
"""
table_name = table_name.replace('.', '_')
query = f'select id, type, valid, new, url, subdomain, level, ' \
f'content, public, port, status, reason, title, banner ' \
f'from "{table_name}"'
if valid:
where = f' where valid = 1'
query = f'select id, type, new, alive, request, resolve, url, ' \
f'subdomain, level, cname, content, public, port, status, ' \
f'reason, title, banner, times, ttl, resolver, module, ' \
f'source, elapse, find, brute, valid from "{table_name}"'
if valid and limit:
if limit in ['resolve', 'request']:
where = f' where {limit} = 1'
query += where
else:
where = f' where alive = 1'
query += where
logger.log('TRACE', f'获取{table_name}表中的所有数据')
return self.query(query)
oneforall/common/lookup.py
+1 -1
View File
@@ -18,7 +18,7 @@ class Lookup(Module):
if answer is None:
return None
for item in answer:
record = str(item)
record = item.to_text()
subdomains = utils.match_subdomain(self.domain, record)
self.subdomains = self.subdomains.union(subdomains)
self.gen_record(subdomains, record)
oneforall/common/module.py
+55 -15
View File
@@ -236,7 +236,7 @@ class Module(object):
'name': self.module,
'source': self.source,
'elapse': self.elapse,
'count': len(self.subdomains),
'find': len(self.subdomains),
'subdomains': list(self.subdomains),
'records': self.records}
json.dump(result, file, ensure_ascii=False, indent=4)
@@ -246,21 +246,27 @@ class Module(object):
"""
生成记录字典
"""
item = dict()
item['content'] = record
for subdomain in subdomains:
self.records[subdomain] = record
self.records[subdomain] = item
def gen_result(self):
def gen_result(self, find=0, brute=None, valid=0):
"""
生成结果
"""
logger.log('DEBUG', f'正在生成最终结果')
if not len(self.subdomains): # 该模块一个子域都没有发现的情况
result = {'id': None,
'type': self.type,
'valid': None,
'alive': None,
'request': None,
'resolve': None,
'new': None,
'url': None,
'subdomain': None,
'level': None,
'cname': None,
'content': None,
'public': None,
'port': None,
@@ -270,39 +276,72 @@ class Module(object):
'banner': None,
'header': None,
'response': None,
'times': None,
'ttl': None,
'resolver': None,
'module': self.module,
'source': self.source,
'elapse': self.elapse,
'count': 0}
'find': find,
'brute': brute,
'valid': valid}
self.results.append(result)
else:
for subdomain in self.subdomains:
valid = None
if self.type != 'A': # 不是利用的DNS记录的A记录查询子域默认都有效
valid = 1
url = 'http://' + subdomain
level = subdomain.count('.') - self.domain.count('.')
content = self.records.get(subdomain)
record = self.records.get(subdomain)
if record is None:
record = dict()
resolve = record.get('resolve')
request = record.get('request')
alive = record.get('alive')
if self.type != 'A': # 不是利用的DNS记录的A记录查询子域默认都有效
resolve = 1
request = 1
alive = 1
reason = record.get('reason')
resolver = record.get('resolver')
cname = record.get('cname')
content = record.get('content')
times = record.get('times')
ttl = record.get('ttl')
public = record.get('public')
if isinstance(cname, list):
cname = ','.join(cname)
content = ','.join(content)
times = ','.join([str(num) for num in times])
ttl = ','.join([str(num) for num in ttl])
public = ','.join([str(num) for num in public])
result = {'id': None,
'type': self.type,
'valid': valid,
'alive': alive,
'request': request,
'resolve': resolve,
'new': None,
'url': url,
'subdomain': subdomain,
'level': level,
'cname': cname,
'content': content,
'public': None,
'port': None,
'public': public,
'port': 80,
'status': None,
'reason': None,
'reason': reason,
'title': None,
'banner': None,
'module': self.module,
'header': None,
'response': None,
'times': times,
'ttl': ttl,
'resolver': resolver,
'module': self.module,
'source': self.source,
'elapse': self.elapse,
'count': len(self.subdomains)}
'find': find,
'brute': brute,
'valid': valid,
}
self.results.append(result)
def save_db(self):
@@ -310,6 +349,7 @@ class Module(object):
将模块结果存入数据库中
"""
logger.log('DEBUG', f'正在将结果存入到数据库')
lock.acquire()
db = Database()
db.create_table(self.domain)
oneforall/common/request.py
+11 -9
View File
@@ -43,10 +43,9 @@ def gen_req_data(data, ports):
logger.log('INFOR', f'正在生成请求地址')
new_data = []
for data in data:
valid = data.get('valid')
# 无效(0)和有效子域(1)不进行http请求探测
# 有效性待确认(None)的子域才进行http请求探测
if valid == 0 or valid == 1:
resolve = data.get('resolve')
# 解析失败(0)的子域不进行http请求探测
if resolve == 0:
continue
subdomain = data.get('subdomain')
for port in ports:
@@ -162,15 +161,18 @@ def request_callback(future, index, datas):
logger.log('TRACE', result.args)
name = utils.get_classname(result)
datas[index]['reason'] = name + ' ' + str(result)
datas[index]['valid'] = 0
datas[index]['request'] = 0
datas[index]['alive'] = 0
elif isinstance(result, tuple):
resp, text = result
datas[index]['reason'] = resp.reason
datas[index]['status'] = resp.status
if resp.status == 400 or resp.status >= 500:
datas[index]['valid'] = 0
datas[index]['request'] = 0
datas[index]['alive'] = 0
else:
datas[index]['valid'] = 1
datas[index]['request'] = 1
datas[index]['alive'] = 1
headers = resp.headers
datas[index]['banner'] = utils.get_sample_banner(headers)
datas[index]['header'] = str(dict(headers))[1:-1]
@@ -244,8 +246,8 @@ def run_request(domain, data, port):
data = loop.run_until_complete(request_coroutine)
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
loop.run_until_complete(asyncio.sleep(0.25))
count = utils.count_valid(data)
logger.log('INFOR', f'经验证{domain}有效子域{count}个')
count = utils.count_alive(data)
logger.log('INFOR', f'经验证{domain}存活子域{count}个')
return data
oneforall/common/resolve.py
+2
View File
@@ -214,6 +214,8 @@ def run_resolve(data):
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
need_resolve_subdomains = filter_subdomain(data)
if not need_resolve_subdomains:
return data
resolve_coroutine = run_aio_resolve(need_resolve_subdomains)
results_list = loop.run_until_complete(resolve_coroutine)
results_dict = convert_results(results_list)
oneforall/common/utils.py
+45 -16
View File
@@ -135,7 +135,11 @@ def get_domains(target):
domains.append(domain)
elif Domain(target).match():
domains = [target]
logger.log('INFOR', f'获取到{len(domains)}个域名')
count = len(domains)
if count == 0:
logger.log('FATAL', f'获取到{count}个域名')
exit(1)
logger.log('INFOR', f'获取到{count}个域名')
return domains
@@ -154,6 +158,12 @@ def get_semaphore():
return 800
def check_dir(dir_path):
if not dir_path.exists():
logger.log('INFOR', f'不存在{dir_path}目录将会新建')
dir_path.mkdir(parents=True, exist_ok=True)
def check_path(path, name, format):
"""
检查结果输出目录路径
@@ -206,7 +216,7 @@ def check_format(format, count):
def save_data(path, data):
"""
保存结果数据到文件
保存数据到文件
:param path: 保存路径
:param data: 待存数据
@@ -283,12 +293,14 @@ def remove_invalid_string(string):
def check_value(values):
for i, value in enumerate(values):
if not isinstance(values, dict):
return values
for key, value in values.items():
if value is None:
continue
if isinstance(value, str) and len(value) > 32767:
# Excel文件中单元格值长度不能超过32767
values[i] = value[:32767]
values[key] = value[:32767]
return values
@@ -301,18 +313,12 @@ def export_all(format, path, datas):
:param list datas: 待导出的结果数据
"""
format = check_format(format, len(datas))
timestamp = get_timestamp()
timestamp = get_timestring()
name = f'all_subdomain_result_{timestamp}'
path = check_path(path, name, format)
logger.log('INFOR', f'所有主域的子域结果 {path}')
row_list = list()
for row in datas:
row.pop('header')
row.pop('response')
row.pop('module')
row.pop('source')
row.pop('elapse')
row.pop('count')
keys = row.keys()
values = row.values()
if format in {'xls', 'xlsx'}:
@@ -348,6 +354,10 @@ def get_timestamp():
return int(time.time())
def get_timestring():
return time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time()))
def get_classname(classobj):
return classobj.__class__.__name__
@@ -356,8 +366,8 @@ def python_version():
return sys.version
def count_valid(data):
return len(list(filter(lambda item: item.get('valid') == 1, data)))
def count_alive(data):
return len(list(filter(lambda item: item.get('alive') == 1, data)))
def get_subdomains(data):
@@ -404,16 +414,23 @@ def check_ip_public(ip_list):
return 1
def ip_is_public(ip_str):
ip = ip_address(ip_str)
if not ip.is_global:
return 0
return 1
def get_process_num():
process_num = config.brute_process_num
if isinstance(process_num, int):
return max(1, process_num)
return min(os.cpu_count(), process_num)
else:
return os.cpu_count()
return 1
def get_coroutine_num():
coroutine_num = config.brute_coroutine_num
coroutine_num = config.resolve_coroutine_num
if isinstance(coroutine_num, int):
return max(64, coroutine_num)
elif coroutine_num is None:
@@ -434,3 +451,15 @@ def get_coroutine_num():
return 2048
else:
return 64
def uniq_dict_list(dict_list):
return list(filter(lambda name: dict_list.count(name) == 1, dict_list))
def delete_file(*paths):
for path in paths:
try:
path.unlink()
except Exception as e:
logger.log('ERROR', e.args)