river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-25 20:37:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

1.重构请求，边请求边存入数据库，解决内存占用过大问题。

Browse Source 
2.format参数改为fmt。
3.添加信息富化模块。
4.优化iscdn模块代码。
5.移除数据库new字段。
This commit is contained in:
Jing Ling
2020-11-09 09:01:06 +08:00
parent 2428017d29
commit 3fbbd1dc25
17 changed files with 473 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3 -4
View File
@@ -170,7 +170,7 @@ DESCRIPTION
python3 oneforall.py --target example.com --valid None run
python3 oneforall.py --target example.com --brute True run
python3 oneforall.py --target example.com --port small run
python3 oneforall.py --target example.com --format csv run
python3 oneforall.py --target example.com --fmt csv run
python3 oneforall.py --target example.com --dns False run
python3 oneforall.py --target example.com --req False run
python3 oneforall.py --target example.com --takeover False run
@@ -179,7 +179,7 @@ DESCRIPTION
Note:
参数alive可选值True，False分别表示导出存活，全部子域结果
参数port可选值有'default', 'small', 'large', 详见config.py配置
参数format可选格式有 'csv','json'
参数fmt可选格式有 'csv','json'
参数path默认None使用OneForAll结果目录生成路径
ARGUMENTS
@@ -199,7 +199,7 @@ FLAGS
请求验证子域的端口范围(默认只探测80端口)
--valid=VALID
只导出存活的子域结果(默认False)
--format=FORMAT
--fmt=FMT
结果保存格式(默认csv)
--path=PATH
结果保存路径(默认None)
@@ -269,7 +269,6 @@ FLAGS
## ⌛后续计划
- [ ] 各模块持续优化和完善
- [x] 子域监控（标记每次新发现的子域）
- [x] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
- [ ] 操作强大交互人性的前端界面实现
brute.py
+6 -9
View File
@@ -384,7 +384,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
info = dict()
cnames = list()
ips = list()
public = list()
times = list()
ttls = list()
is_valid_flags = list()
@@ -401,7 +400,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
cnames.append(cname) # 去除最右边的`.`点号
ip = answer.get('data')
ips.append(ip)
public.append(utils.ip_is_public(ip))
num = ip_times.get(ip)
times.append(num)
isvalid, reason = is_valid_subdomain(ip, ttl, num, wc_ips, wc_ttl, cname)
@@ -416,7 +414,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
info['ttl'] = ttls
info['cname'] = cnames
info['ip'] = ips
info['public'] = public
info['times'] = times
info['resolver'] = resolver
infos[qname] = info
@@ -559,7 +556,7 @@ class Brute(Module):
brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
Note:
--format csv/json (result format)
--fmt csv/json (result fmt)
--path Result path (default None, automatically generated)
@@ -578,13 +575,13 @@ class Brute(Module):
:param str rule: Specify the regexp rules used in fuzz mode (required if use fuzz mode)
:param str fuzzlist: Dictionary path used in fuzz mode (default use ./config/default.py)
:param bool export: Export the results (default True)
:param str format: Result format (default csv)
:param str fmt: Result format (default csv)
:param str path: Result directory (default None)
"""
def __init__(self, target=None, targets=None, process=None, concurrent=None,
word=False, wordlist=None, recursive=False, depth=None, nextlist=None,
fuzz=False, place=None, rule=None, fuzzlist=None, export=True,
alive=True, format='csv', path=None):
alive=True, fmt='csv', path=None):
Module.__init__(self)
self.module = 'Brute'
self.source = 'Brute'
@@ -603,7 +600,7 @@ class Brute(Module):
self.fuzzlist = fuzzlist or settings.fuzz_list
self.export = export
self.alive = alive
self.format = format
self.fmt = fmt
self.path = path
self.bulk = False # 是否是批量爆破场景
self.domains = list() # 待爆破的所有域名集合
@@ -761,7 +758,7 @@ class Brute(Module):
logger.log('INFOR', f'Finished {self.source} module to brute {self.domain}')
if not self.path:
name = f'{self.domain}_brute_result.{self.format}'
name = f'{self.domain}_brute_result.{self.fmt}'
self.path = settings.result_save_dir.joinpath(name)
# 数据库导出
if self.export:
@@ -770,7 +767,7 @@ class Brute(Module):
alive=self.alive,
limit='resolve',
path=self.path,
format=self.format)
fmt=self.fmt)
if __name__ == '__main__':
common/database.py
+48 -22
View File
@@ -57,7 +57,6 @@ class Database(object):
f'alive int,'
f'request int,'
f'resolve int,'
f'new int,'
f'url text,'
f'subdomain text,'
f'port int,'
@@ -86,6 +85,20 @@ class Database(object):
f'elapse float,'
f'find int)')
def insert_table(self, table_name, result):
table_name = table_name.replace('.', '_')
self.conn.query(
f'insert into "{table_name}" '
f'(id, alive, resolve, request, url, subdomain, port, level,'
f'cname, ip, public, cdn, status, reason, title, banner, header,'
f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,'
f'module, source, elapse, find) '
f'values (:id, :alive, :resolve, :request, :url,'
f':subdomain, :port, :level, :cname, :ip, :public, :cdn,'
f':status, :reason, :title, :banner, :header, :history, :response,'
f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,'
f':source, :elapse, :find)', **result)
def save_db(self, table_name, results, module_name=None):
"""
Save the results of each module in the database
@@ -101,11 +114,11 @@ class Database(object):
try:
self.conn.bulk_query(
f'insert into "{table_name}" '
f'(id, alive, resolve, request, new, url, subdomain, port, level,'
f'(id, alive, resolve, request, url, subdomain, port, level,'
f'cname, ip, public, cdn, status, reason, title, banner, header,'
f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,'
f'module, source, elapse, find) '
f'values (:id, :alive, :resolve, :request, :new, :url,'
f'values (:id, :alive, :resolve, :request, :url,'
f':subdomain, :port, :level, :cname, :ip, :public, :cdn,'
f':status, :reason, :title, :banner, :header, :history, :response,'
f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,'
@@ -178,7 +191,7 @@ class Database(object):
def deduplicate_subdomain(self, table_name):
"""
Deduplicates of subdomains in the table
Deduplicate subdomains in the table
:param str table_name: table name
"""
@@ -199,17 +212,6 @@ class Database(object):
self.query(f'delete from "{table_name}" where '
f'subdomain is null or resolve == 0')
def deal_table(self, deal_table_name, backup_table_name):
"""
Process the table when the collection task is complete
:param str deal_table_name: Pending table name
:param str backup_table_name: Table name for backup
"""
self.copy_table(deal_table_name, backup_table_name)
self.remove_invalid(deal_table_name)
self.deduplicate_subdomain(deal_table_name)
def get_data(self, table_name):
"""
Get all the data in the table
@@ -218,7 +220,7 @@ class Database(object):
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'Get all the data from {table_name} table')
return self.query(f'select * from "{table_name}"')
return self.query(f'select * from {table_name}')
def export_data(self, table_name, alive, limit):
"""
@@ -229,18 +231,42 @@ class Database(object):
:param str limit: limit value
"""
table_name = table_name.replace('.', '_')
query = f'select id, new, alive, request, resolve, url, subdomain, level,' \
f'cname, ip, public, cdn, port, status, reason, title, banner,' \
f'cidr, asn, org, addr, isp, source from "{table_name}"'
sql = f'select id, alive, request, resolve, url, subdomain, level,' \
f'cname, ip, public, cdn, port, status, reason, title, banner,' \
f'cidr, asn, org, addr, isp, source from {table_name} order by subdomain'
if alive and limit:
if limit in ['resolve', 'request']:
where = f' where {limit} = 1'
query += where
sql += where
elif alive:
where = f' where alive = 1'
query += where
sql += where
logger.log('TRACE', f'Get the data from {table_name} table')
return self.query(query)
return self.query(sql)
def count_alive(self, table_name):
table_name = table_name.replace('.', '_')
sql = f'select count() from {table_name} where alive = 1'
return self.query(sql)
def get_resp_by_url(self, table_name, url):
table_name = table_name.replace('.', '_')
sql = f'select response from {table_name} where url = "{url}"'
logger.log('TRACE', f'Get response data from {url}')
return self.query(sql).scalar()
def get_data_by_fields(self, table_name, fields):
table_name = table_name.replace('.', '_')
field_str = ', '.join(fields)
sql = f"select {field_str} from {table_name}"
logger.log('TRACE', f'Get specified field data {fields} from {table_name} table')
return self.query(sql)
def update_data_by_url(self, table_name, info, url):
table_name = table_name.replace('.', '_')
field_str = ', '.join(map(lambda kv: f'{kv[0]} = "{kv[1]}"', info.items()))
sql = f'update {table_name} set {field_str} where url = "{url}"'
return self.query(sql)
def close(self):
"""
common/module.py
+1 -5
View File
@@ -275,7 +275,6 @@ class Module(object):
'alive': None,
'request': None,
'resolve': None,
'new': None,
'url': None,
'subdomain': None,
'port': None,
@@ -315,25 +314,22 @@ class Module(object):
ip = info.get('ip')
times = info.get('times')
ttl = info.get('ttl')
public = info.get('public')
if isinstance(cname, list):
cname = ','.join(cname)
ip = ','.join(ip)
times = ','.join([str(num) for num in times])
ttl = ','.join([str(num) for num in ttl])
public = ','.join([str(num) for num in public])
result = {'id': None,
'alive': info.get('alive'),
'request': info.get('request'),
'resolve': info.get('resolve'),
'new': None,
'url': url,
'subdomain': subdomain,
'port': 80,
'level': level,
'cname': cname,
'ip': ip,
'public': public,
'public': info.get('public'),
'cdn': info.get('cdn'),
'status': None,
'reason': info.get('reason'),
common/request.py
+57 -54
View File
@@ -1,7 +1,6 @@
import json
from threading import Thread
from queue import Queue
from operator import attrgetter
import tqdm
import requests
@@ -9,6 +8,7 @@ from bs4 import BeautifulSoup
from common import utils
from config.log import logger
from common.database import Database
from config import settings
@@ -127,7 +127,7 @@ def get_progress_bar(total):
return bar
def get(url, resp_list, session):
def get_resp(url, session):
timeout = settings.request_timeout_second
redirect = settings.request_allow_redirect
proxy = utils.get_proxy()
@@ -136,13 +136,14 @@ def get(url, resp_list, session):
except Exception as e:
logger.log('DEBUG', e.args)
resp = e
resp_list.append((url, resp))
return resp
def request(urls_queue, resp_list, session):
def request(urls_queue, resp_queue, session):
while not urls_queue.empty():
url = urls_queue.get()
get(url, resp_list, session)
index, url = urls_queue.get()
resp = get_resp(url, session)
resp_queue.put((index, resp))
urls_queue.task_done()
@@ -168,31 +169,6 @@ def get_session():
return session
def bulk_request(urls):
logger.log('INFOR', 'Requesting urls in bulk')
resp_list = list()
urls_queue = Queue()
for url in urls:
urls_queue.put(url)
total = len(urls)
session = get_session()
thread_count = req_thread_count()
bar = get_progress_bar(total)
progress_thread = Thread(target=progress, name='ProgressThread',
args=(bar, total, urls_queue), daemon=True)
progress_thread.start()
for i in range(thread_count):
request_thread = Thread(target=request, name=f'RequestThread-{i}',
args=(urls_queue, resp_list, session), daemon=True)
request_thread.start()
urls_queue.join()
return resp_list
def gen_new_info(info, resp):
if isinstance(resp, Exception):
info['reason'] = str(resp.args)
@@ -220,13 +196,54 @@ def gen_new_info(info, resp):
return info
def gen_new_data(data, resp_list):
new_data = list()
for url, resp in resp_list:
for info in data:
if info.get('url') == url:
new_data.append(gen_new_info(info, resp))
return new_data
def save(name, total, req_data, resp_queue):
db = Database()
db.create_table(name)
i = 0
while True:
if not resp_queue.empty():
i += 1
index, resp = resp_queue.get()
old_info = req_data[index]
new_info = gen_new_info(old_info, resp)
db.insert_table(name, new_info)
resp_queue.task_done()
if i >= total:
break
db.close()
def bulk_request(domain, req_data, ret=False):
logger.log('INFOR', 'Requesting urls in bulk')
resp_queue = Queue()
urls_queue = Queue()
total = len(req_data)
for index, info in enumerate(req_data):
try:
url = info.get('url')
except Exception:
pass
urls_queue.put((index, url))
session = get_session()
thread_count = req_thread_count()
bar = get_progress_bar(total)
progress_thread = Thread(target=progress, name='ProgressThread',
args=(bar, total, urls_queue), daemon=True)
progress_thread.start()
for i in range(thread_count):
request_thread = Thread(target=request, name=f'RequestThread-{i}',
args=(urls_queue, resp_queue, session), daemon=True)
request_thread.start()
if ret:
urls_queue.join()
return resp_queue
save_thread = Thread(target=save, name=f'SaveThread',
args=(domain, total, req_data, resp_queue), daemon=True)
save_thread.start()
urls_queue.join()
save_thread.join()
def run_request(domain, data, port):
@@ -242,20 +259,6 @@ def run_request(domain, data, port):
data = utils.set_id_none(data)
ports = get_port_seq(port)
req_data, req_urls = gen_req_data(data, ports)
resp_list = bulk_request(req_urls)
new_data = gen_new_data(req_data, resp_list)
count = utils.count_alive(new_data)
bulk_request(domain, req_data)
count = utils.count_alive(domain)
logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
sorted_data = utils.sort_by_subdomain(new_data)
return sorted_data
def save_db(name, data):
"""
Save request results to database
:param str name: table name
:param list data: data to be saved
"""
logger.log('INFOR', f'Saving requested results')
utils.save_db(name, data, 'request')
common/resolve.py
-26
View File
@@ -4,12 +4,6 @@ import json
from config.log import logger
from config import settings
from common import utils
from common.ipasn import IPAsnInfo
from common.ipreg import IpRegData
ip_asn = IPAsnInfo()
ip_reg = IpRegData()
def filter_subdomain(data):
@@ -79,13 +73,7 @@ def gen_infos(data, qname, info, infos):
flag = False
cname = list()
ips = list()
public = list()
ttl = list()
cidr = list()
asn = list()
org = list()
addr = list()
isp = list()
answers = data.get('answers')
for answer in answers:
if answer.get('type') == 'A':
@@ -94,25 +82,11 @@ def gen_infos(data, qname, info, infos):
ip = answer.get('data')
ips.append(ip)
ttl.append(str(answer.get('ttl')))
public.append(str(utils.ip_is_public(ip)))
asn_info = ip_asn.find(ip)
cidr.append(asn_info.get('cidr'))
asn.append(asn_info.get('asn'))
org.append(asn_info.get('org'))
ip_info = ip_reg.query(ip)
addr.append(ip_info.get('addr'))
isp.append(ip_info.get('isp'))
info['resolve'] = 1
info['reason'] = 'OK'
info['cname'] = ','.join(cname)
info['ip'] = ','.join(ips)
info['public'] = ','.join(public)
info['ttl'] = ','.join(ttl)
info['cidr'] = ','.join(cidr)
info['asn'] = ','.join(asn)
info['org'] = ','.join(org)
info['addr'] = ','.join(addr)
info['isp'] = ','.join(isp)
infos[qname] = info
if not flag:
info['alive'] = 0
common/utils.py
+44 -17
View File
@@ -176,16 +176,16 @@ def check_dir(dir_path):
dir_path.mkdir(parents=True, exist_ok=True)
def check_path(path, name, format):
def check_path(path, name, fmt):
"""
检查结果输出目录路径
:param path: 保存路径
:param name: 导出名字
:param format: 保存格式
:param fmt: 保存格式
:return: 保存路径
"""
filename = f'{name}.{format}'
filename = f'{name}.{fmt}'
default_path = settings.result_save_dir.joinpath(filename)
if isinstance(path, str):
path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠
@@ -204,19 +204,18 @@ def check_path(path, name, format):
return path
def check_format(format, count):
def check_format(fmt):
"""
检查导出格式
:param format: 传入的导出格式
:param count: 数量
:param fmt: 传入的导出格式
:return: 导出格式
"""
formats = ['csv', 'json', ]
if format in formats:
return format
if fmt in formats:
return fmt
else:
logger.log('ALERT', f'Does not support {format} format')
logger.log('ALERT', f'Does not support {fmt} format')
logger.log('ALERT', 'So use csv format by default')
return 'csv'
@@ -333,8 +332,8 @@ def remove_invalid_string(string):
return re.sub(r'[\000-\010]|[\013-\014]|[\016-\037]', r'', string)
def export_all_results(path, name, format, datas):
path = check_path(path, name, format)
def export_all_results(path, name, fmt, datas):
path = check_path(path, name, fmt)
logger.log('ALERT', f'The subdomain result for all main domains: {path}')
row_list = list()
for row in datas:
@@ -346,7 +345,7 @@ def export_all_results(path, name, format, datas):
values = row.values()
row_list.append(Record(keys, values))
rows = RecordCollection(iter(row_list))
content = rows.export(format)
content = rows.export(fmt)
save_data(path, content)
@@ -366,19 +365,19 @@ def export_all_subdomains(alive, path, name, datas):
save_data(path, data)
def export_all(alive, format, path, datas):
def export_all(alive, fmt, path, datas):
"""
将所有结果数据导出
:param bool alive: 只导出存活子域结果
:param str format: 导出文件格式
:param str fmt: 导出文件格式
:param str path: 导出文件路径
:param list datas: 待导出的结果数据
"""
format = check_format(format, len(datas))
fmt = check_format(fmt)
timestamp = get_timestring()
name = f'all_subdomain_result_{timestamp}'
export_all_results(path, name, format, datas)
export_all_results(path, name, fmt, datas)
export_all_subdomains(alive, path, name, datas)
@@ -430,10 +429,18 @@ def python_version():
return sys.version
def count_alive(data):
def calc_alive(data):
return len(list(filter(lambda item: item.get('alive') == 1, data)))
def count_alive(name):
db = Database()
result = db.count_alive(name)
count = result.scalar()
db.close()
return count
def get_subdomains(data):
return set(map(lambda item: item.get('subdomain'), data))
@@ -833,3 +840,23 @@ def looks_like_ip(maybe_ip):
return True
except socket.error:
return False
def deal_data(domain):
db = Database()
db.remove_invalid(domain)
db.deduplicate_subdomain(domain)
db.close()
def get_data(domain):
db = Database()
data = db.get_data(domain).as_dict()
db.close()
return data
def clear_data(domain):
db = Database()
db.drop_table(domain)
db.close()
config/default.py
+1 -1
View File
@@ -24,7 +24,7 @@ enable_dns_resolve = True # 使用DNS解析子域(默认True)
enable_http_request = True # 使用HTTP请求子域(默认True)
enable_finder_module = True # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True)
enable_altdns_module = False # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
enable_cdn_check = True # 开启cdn检查模块(默认True)
enable_enrich_module = True # 开启enrich模块，开启会富化出信息，如ip的cdn，cidr，asn，org，addr和isp等信息
enable_banner_identify = True # 开启WEB指纹识别模块(默认True)
enable_takeover_check = False # 开启子域接管风险检查(默认False)
# 参数可选值有'small', 'medium', 'large'
data/cdn_asn_list.json
+133 -133
View File
@@ -1,134 +1,134 @@
[
"10576",
"10762",
"11748",
"131099",
"132601",
"133496",
"134409",
"135295",
"136764",
"137187",
"13777",
"13890",
"14103",
"14520",
"17132",
"199251",
"200013",
"200325",
"200856",
"201263",
"202294",
"203075",
"203139",
"204248",
"204286",
"204545",
"206227",
"206734",
"206848",
"206986",
"207158",
"208559",
"209403",
"21030",
"21257",
"23327",
"23393",
"23637",
"23794",
"24997",
"26492",
"268843",
"28709",
"29264",
"30282",
"30637",
"328126",
"36408",
"38107",
"397192",
"40366",
"43303",
"44907",
"46071",
"46177",
"47542",
"49287",
"49689",
"51286",
"55082",
"55254",
"56636",
"57363",
"58127",
"59730",
"59776",
"60068",
"60626",
"60922",
"61107",
"61159",
"62026",
"62229",
"63062",
"64232",
"8868",
"9053",
"55770",
"49846",
"49249",
"48163",
"45700",
"43639",
"39836",
"393560",
"393234",
"36183",
"35994",
"35993",
"35204",
"34850",
"34164",
"33905",
"32787",
"31377",
"31110",
"31109",
"31108",
"31107",
"30675",
"24319",
"23903",
"23455",
"23454",
"22207",
"21399",
"21357",
"21342",
"20940",
"20189",
"18717",
"18680",
"17334",
"16702",
"16625",
"12222",
"209101",
"201585",
"135429",
"395747",
"394536",
"209242",
"203898",
"202623",
"14789",
"133877",
"13335",
"132892",
"21859",
"6185",
"47823",
"4134"
]
"AS10576",
"AS10762",
"AS11748",
"AS131099",
"AS132601",
"AS133496",
"AS134409",
"AS135295",
"AS136764",
"AS137187",
"AS13777",
"AS13890",
"AS14103",
"AS14520",
"AS17132",
"AS199251",
"AS200013",
"AS200325",
"AS200856",
"AS201263",
"AS202294",
"AS203075",
"AS203139",
"AS204248",
"AS204286",
"AS204545",
"AS206227",
"AS206734",
"AS206848",
"AS206986",
"AS207158",
"AS208559",
"AS209403",
"AS21030",
"AS21257",
"AS23327",
"AS23393",
"AS23637",
"AS23794",
"AS24997",
"AS26492",
"AS268843",
"AS28709",
"AS29264",
"AS30282",
"AS30637",
"AS328126",
"AS36408",
"AS38107",
"AS397192",
"AS40366",
"AS43303",
"AS44907",
"AS46071",
"AS46177",
"AS47542",
"AS49287",
"AS49689",
"AS51286",
"AS55082",
"AS55254",
"AS56636",
"AS57363",
"AS58127",
"AS59730",
"AS59776",
"AS60068",
"AS60626",
"AS60922",
"AS61107",
"AS61159",
"AS62026",
"AS62229",
"AS63062",
"AS64232",
"AS8868",
"AS9053",
"AS55770",
"AS49846",
"AS49249",
"AS48163",
"AS45700",
"AS43639",
"AS39836",
"AS393560",
"AS393234",
"AS36183",
"AS35994",
"AS35993",
"AS35204",
"AS34850",
"AS34164",
"AS33905",
"AS32787",
"AS31377",
"AS31110",
"AS31109",
"AS31108",
"AS31107",
"AS30675",
"AS24319",
"AS23903",
"AS23455",
"AS23454",
"AS22207",
"AS21399",
"AS21357",
"AS21342",
"AS20940",
"AS20189",
"AS18717",
"AS18680",
"AS17334",
"AS16702",
"AS16625",
"AS12222",
"AS209101",
"AS201585",
"AS135429",
"AS395747",
"AS394536",
"AS209242",
"AS203898",
"AS202623",
"AS14789",
"AS133877",
"AS13335",
"AS132892",
"AS21859",
"AS6185",
"AS47823",
"AS4134"
]
dbexport.py
+12 -12
View File
@@ -15,17 +15,17 @@ from common.database import Database
from config.log import logger
def export(target, type='target', db=None, alive=False, limit=None, path=None, format='csv', show=False):
def export(target, type='target', db=None, alive=False, limit=None, path=None, fmt='csv', show=False):
"""
OneForAll export from database module
Example:
python3 dbexport.py --target name --format csv --dir= ./result.csv
python3 dbexport.py --target name --fmt csv --dir= ./result.csv
python3 dbexport.py --db result.db --target name --show False
python3 dbexport.py --target table_name --tb True --show False
Note:
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
--fmt csv/json (result format)
--path Result directory (default directory is ./results)
:param str target: Table to be exported
@@ -33,7 +33,7 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f
:param str db: Database path to be exported (default ./results/result.sqlite3)
:param bool alive: Only export the results of alive subdomains (default False)
:param str limit: Export limit (default None)
:param str format: Result format (default csv)
:param str fmt: Result format (default csv)
:param str path: Result directory (default None)
:param bool show: Displays the exported data in terminal (default False)
"""
@@ -48,29 +48,29 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f
rows = database.export_data(table_name, alive, limit)
if rows is None:
continue
data = export_data(format, path, rows, show, table_name, target)
data = export_data(fmt, path, rows, show, table_name, target)
datas.extend(data)
database.close()
if len(domains) > 1:
utils.export_all(alive, format, path, datas)
utils.export_all(alive, fmt, path, datas)
elif type == 'table':
database = Database(db)
rows = database.export_data(target, alive, limit)
data = export_data(format, path, rows, show, target, target)
data, _, _ = export_data(fmt, path, rows, show, target, target)
database.close()
return data
def export_data(format, path, rows, show, table_name, target):
format = utils.check_format(format, len(rows))
path = utils.check_path(path, target, format)
def export_data(fmt, path, rows, show, table_name, target):
fmt = utils.check_format(fmt)
path = utils.check_path(path, target, fmt)
if show:
print(rows.dataset)
data = rows.export(format)
data = rows.export(fmt)
utils.save_data(path, data)
logger.log('ALERT', f'The subdomain result for {table_name}: {path}')
data = rows.as_dict()
return data, format, path
return data, fmt, path
def domain_to_table(table):
docs/en-us/README.md
+3 -4
View File
@@ -164,7 +164,7 @@ DESCRIPTION
python3 oneforall.py --target example.com --alive False run
python3 oneforall.py --target example.com --brute True run
python3 oneforall.py --target example.com --port medium run
python3 oneforall.py --target example.com --format csv run
python3 oneforall.py --target example.com --fmt csv run
python3 oneforall.py --target example.com --dns False run
python3 oneforall.py --target example.com --req False run
python3 oneforall.py --target example.com --takeover False run
@@ -173,7 +173,7 @@ DESCRIPTION
Note:
--alive True/False Only export alive subdomains or not (default False)
--port default/small/large See details in ./config/setting.py(default port 80)
--format csv/json (result format)
--fmt csv/json (result format)
--path Result directory (default directory is ./results)
ARGUMENTS
@@ -193,7 +193,7 @@ FLAGS
The port range request to the subdomains (default port 80)
--alive=ALIVE
Only export alive subdomains (default False)
--format=FORMAT
--fmt=FMT
Result format (default csv)
--path=PATH
Result directory (default None)
@@ -270,7 +270,6 @@ The project uses [SemVer](https://semver.org/) for version management, and you c
## ⌛Follow-up plan
- [ ] Continuous optimize and improve of each module
- [x] Subdomain monitoring (mark newly discovered subdomain)
- [x] Subdomain collection crawler (collect subdomains from static files such as JS)
- [ ] Implementation of front-end interface for powerful interaction
modules/altdns.py
+2 -4
View File
@@ -193,9 +193,7 @@ class Altdns(Module):
self.end = time.time()
self.elapse = round(self.end - self.start, 1)
self.gen_result()
temp_data = resolve.run_resolve(self.domain, self.results)
fina_data = request.run_request(self.domain, temp_data, port)
data = data + fina_data
resolved_data = resolve.run_resolve(self.domain, self.results)
request.run_request(self.domain, resolved_data, port)
logger.log('INFOR', f'Saving altdns results')
utils.save_db(self.domain, data, 'altdns')
return data
modules/enrich.py
+72
View File
@@ -0,0 +1,72 @@
from modules import iscdn
from common import utils
from common.database import Database
from common.ipasn import IPAsnInfo
from common.ipreg import IpRegData
def get_ips(info):
ip = info.get('ip')
if not ip:
return None
ips = ip.split(',')
return ips
def enrich_info(data):
ip_asn = IPAsnInfo()
ip_reg = IpRegData()
for index, info in enumerate(data):
ips = get_ips(info)
if not ips:
continue
public = list()
cidr = list()
asn = list()
org = list()
addr = list()
isp = list()
for ip in ips:
public.append(str(utils.ip_is_public(ip)))
asn_info = ip_asn.find(ip)
cidr.append(asn_info.get('cidr'))
asn.append(asn_info.get('asn'))
org.append(asn_info.get('org'))
ip_info = ip_reg.query(ip)
addr.append(ip_info.get('addr'))
isp.append(ip_info.get('isp'))
data[index]['public'] = ','.join(public)
data[index]['cidr'] = ','.join(cidr)
data[index]['asn'] = ','.join(asn)
data[index]['org'] = ','.join(org)
data[index]['addr'] = ','.join(addr)
data[index]['isp'] = ','.join(isp)
return data
class Enrich(object):
def __init__(self, domain):
self.domain = domain
def get_data(self):
db = Database()
fields = ['url', 'cname', 'ip', 'public', 'cdn', 'header',
'cidr', 'asn', 'org', 'addr', 'isp']
results = db.get_data_by_fields(self.domain, fields)
return results.as_dict()
def save_db(self, data):
db = Database()
for info in data:
url = info.pop('url')
info.pop('cname')
info.pop('ip')
info.pop('header')
db.update_data_by_url(self.domain, info, url)
db.close()
def run(self):
data = self.get_data()
data = enrich_info(data)
data = iscdn.do_check(data)
self.save_db(data)
modules/finder.py
+15 -10
View File
@@ -7,6 +7,7 @@ from common import utils
from common import resolve
from common import request
from common.module import Module
from common.database import Database
from config import settings
from config.log import logger
@@ -25,16 +26,11 @@ class Finder(Module):
new_subdomains = found_subdomains - existing_subdomains
if not len(new_subdomains):
self.finish() # 未发现新的子域就直接返回
return data
self.subdomains = new_subdomains
self.finish()
self.gen_result()
temp_data = resolve.run_resolve(domain, self.results)
fina_data = request.run_request(domain, temp_data, port)
data = data + fina_data
logger.log('INFOR', f'Saving finder results')
utils.save_db(domain, data, 'finder')
return data
resolved_data = resolve.run_resolve(domain, self.results)
request.run_request(domain, resolved_data, port)
file_path = settings.data_storage_dir.joinpath('common_js_library.json')
@@ -177,22 +173,31 @@ def find_js_urls(domain, req_url, rsp_html):
return js_urls
def convert_to_dict(url_list):
url_dict = []
for url in url_list:
url_dict.append({'url': url})
return url_dict
def find_subdomains(domain, data):
subdomains = set()
js_urls = set()
db = Database()
for infos in data:
jump_history = infos.get('history')
req_url = infos.get('url')
subdomains.update(find_in_history(domain, req_url, jump_history))
rsp_html = infos.get('response')
rsp_html = db.get_resp_by_url(domain, req_url)
if not rsp_html:
logger.log('DEBUG', f'an abnormal response occurred in the request {req_url}')
continue
subdomains.update(find_in_resp(domain, req_url, rsp_html))
js_urls.update(find_js_urls(domain, req_url, rsp_html))
resp_data = request.bulk_request(js_urls)
for _, resp in resp_data:
req_data = convert_to_dict(js_urls)
resp_data = request.bulk_request(domain, req_data, ret=True)
while not resp_data.empty():
_, resp = resp_data.get()
if not isinstance(resp, Response):
continue
text = utils.decode_resp_text(resp)
modules/iscdn.py
+35 -33
View File
@@ -3,6 +3,7 @@ import ipaddress
from config import settings
from common import utils
from common.database import Database
from config.log import logger
data_dir = settings.data_storage_dir
@@ -18,6 +19,8 @@ cdn_header_key = utils.load_json(data_dir.joinpath('cdn_header_keys.json'))
def check_cname_keyword(cname):
if not cname:
return False
names = cname.lower().split(',')
for name in names:
for keyword in cdn_cname_keyword.keys():
@@ -26,14 +29,22 @@ def check_cname_keyword(cname):
def check_header_key(header):
header = set(map(lambda x: x.lower(), header.keys()))
for key in cdn_header_key:
if key in header:
return True
if isinstance(header, str):
header = json.loads(header)
if isinstance(header, dict):
header = set(map(lambda x: x.lower(), header.keys()))
for key in cdn_header_key:
if key in header:
return True
else:
return False
def check_cdn_cidr(content):
ips = set(content.split(','))
def check_cdn_cidr(ips):
if isinstance(ips, str):
ips = set(ips.split(','))
else:
return False
for ip in ips:
try:
ip = ipaddress.ip_address(ip)
@@ -46,39 +57,30 @@ def check_cdn_cidr(content):
def check_cdn_asn(asn):
if str(asn) in cdn_asn_list:
return True
if isinstance(asn, str):
if asn in cdn_asn_list:
return True
return False
def check_cdn(data):
logger.log('DEBUG', f'Start cdn check module')
def do_check(data):
logger.log('DEBUG', f'Checking cdn')
for index, item in enumerate(data):
cname = item.get('cname')
if cname:
if check_cname_keyword(cname):
data[index]['cdn'] = 1
continue
if check_cname_keyword(cname):
data[index]['cdn'] = 1
continue
header = item.get('header')
if header:
header = json.loads(header)
if check_header_key(header):
data[index]['cdn'] = 1
continue
if check_header_key(header):
data[index]['cdn'] = 1
continue
ip = item.get('ip')
if ip:
if check_cdn_cidr(ip):
data[index]['cdn'] = 1
continue
if check_cdn_cidr(ip):
data[index]['cdn'] = 1
continue
asn = item.get('asn')
if asn:
asn = asn[2:] # 去除AS
if check_cdn_asn(asn):
data[index]['cdn'] = 1
continue
if check_cdn_asn(asn):
data[index]['cdn'] = 1
continue
data[index]['cdn'] = 0
return data
def save_db(name, data):
logger.log('DEBUG', f'Saving cdn check results')
utils.save_db(name, data, 'cdn')
oneforall.py
+32 -81
View File
@@ -15,12 +15,11 @@ from datetime import datetime
import dbexport
from brute import Brute
from common import utils, resolve, request
from common.database import Database
from modules.collect import Collect
from modules.srv import BruteSRV
from modules.finder import Finder
from modules.altdns import Altdns
from modules import iscdn
from modules.enrich import Enrich
from config import settings
from config.log import logger
from takeover import Takeover
@@ -60,7 +59,7 @@ class OneForAll(object):
python3 oneforall.py --target example.com --alive False run
python3 oneforall.py --target example.com --brute False run
python3 oneforall.py --target example.com --port medium run
python3 oneforall.py --target example.com --format csv run
python3 oneforall.py --target example.com --fmt csv run
python3 oneforall.py --target example.com --dns False run
python3 oneforall.py --target example.com --req False run
python3 oneforall.py --target example.com --takeover False run
@@ -68,22 +67,22 @@ class OneForAll(object):
Note:
--port small/medium/large See details in ./config/setting.py(default small)
--format csv/json (result format)
--fmt csv/json (result format)
--path Result path (default None, automatically generated)
:param str target: One domain (target or targets must be provided)
:param str targets: File path of one domain per line
:param bool brute: Use brute module (default True)
:param bool dns: Use DNS resolution (default True)
:param bool req: HTTP request subdomains (default True)
:param bool brute: Use brute module (default True)
:param bool dns: Use DNS resolution (default True)
:param bool req: HTTP request subdomains (default True)
:param str port: The port range to request (default small port is 80,443)
:param bool alive: Only export alive subdomains (default False)
:param str format: Result format (default csv)
:param bool alive: Only export alive subdomains (default False)
:param str fmt: Result format (default csv)
:param str path: Result path (default None, automatically generated)
:param bool takeover: Scan subdomain takeover (default False)
:param bool takeover: Scan subdomain takeover (default False)
"""
def __init__(self, target=None, targets=None, brute=None, dns=None, req=None,
port=None, alive=None, format=None, path=None, takeover=None):
port=None, alive=None, fmt=None, path=None, takeover=None):
self.target = target
self.targets = targets
self.brute = brute
@@ -91,17 +90,13 @@ class OneForAll(object):
self.req = req
self.port = port
self.alive = alive
self.format = format
self.fmt = fmt
self.path = path
self.takeover = takeover
self.domain = str() # The domain currently being collected
self.domains = set() # All domains that are to be collected
self.data = list() # The subdomain results of the current domain
self.datas = list() # All subdomain results of the domain
self.old_table = str() # The table name of the last result
self.new_table = str() # The table name of the current result
self.origin_table = str() # The table name of the origin result
self.resolve_table = str() # The table name of the resolute result
def config_param(self):
"""
@@ -119,8 +114,8 @@ class OneForAll(object):
self.port = settings.http_request_port
if self.alive is None:
self.alive = bool(settings.result_export_alive)
if self.format is None:
self.format = settings.result_save_format
if self.fmt is None:
self.fmt = settings.result_save_format
if self.path is None:
self.path = settings.result_save_path
@@ -134,47 +129,13 @@ class OneForAll(object):
def export(self, table):
"""
Export data from the database and do some follow-up processing
Export data from the database
:param table: table name
:return: export data
:rtype: list
"""
db = Database()
data = dbexport.export(table, type='table', alive=self.alive, format=self.format)
db.drop_table(self.new_table)
db.rename_table(self.domain, self.new_table)
db.close()
return data
def deal_db(self):
"""
Process the data when the collection task is completed
"""
db = Database()
db.deal_table(self.domain, self.origin_table)
db.close()
def mark(self):
"""
Mark the new discovered subdomain
:return: marked data
:rtype: list
"""
db = Database()
old_data = list()
now_data = db.get_data(self.domain).as_dict()
# Database pre-processing when it is not the first time to collect this subdomain
if db.exist_table(self.new_table):
# If there is the last collection result table, delete it first
db.drop_table(self.old_table)
# Rename the new table to the old table
db.rename_table(self.new_table, self.old_table)
old_data = db.get_data(self.old_table).as_dict()
db.close()
marked_data = utils.mark_subdomain(old_data, now_data)
return marked_data
return dbexport.export(table, type='table', alive=self.alive, fmt=self.fmt)
def main(self):
"""
@@ -183,11 +144,6 @@ class OneForAll(object):
:return: subdomain results
:rtype: list
"""
self.old_table = self.domain + '_old_result'
self.new_table = self.domain + '_now_result'
self.origin_table = self.domain + '_origin_result'
self.resolve_table = self.domain + '_resolve_result'
collect = Collect(self.domain)
collect.run()
@@ -202,49 +158,44 @@ class OneForAll(object):
brute.quite = True
brute.run()
# Database processing
self.deal_db()
# Mark the new discovered subdomain
self.data = self.mark()
utils.deal_data(self.domain)
# Export results without resolve
if not self.dns:
return self.export(self.domain)
self.data = utils.get_data(self.domain)
# Resolve subdomains
utils.clear_data(self.domain)
self.data = resolve.run_resolve(self.domain, self.data)
# Save resolve results
resolve.save_db(self.resolve_table, self.data)
resolve.save_db(self.domain, self.data)
# Export results without HTTP request
if not self.req:
return self.export(self.resolve_table)
return self.export(self.domain)
# HTTP request
self.data = request.run_request(self.domain, self.data, self.port)
# Save HTTP request result
request.save_db(self.domain, self.data)
utils.clear_data(self.domain)
request.run_request(self.domain, self.data, self.port)
# Finder module
if settings.enable_finder_module:
finder = Finder()
self.data = finder.run(self.domain, self.data, self.port)
finder.run(self.domain, self.data, self.port)
# altdns module
if settings.enable_altdns_module:
finder = Altdns(self.domain)
self.data = finder.run(self.data, self.port)
altdns = Altdns(self.domain)
altdns.run(self.data, self.port)
# check cdn module
if settings.enable_cdn_check:
self.data = iscdn.check_cdn(self.data)
iscdn.save_db(self.domain, self.data)
# Add the final result list to the total data list
self.datas.extend(self.data)
# Information enrichment module
if settings.enable_enrich_module:
enrich = Enrich(self.domain)
enrich.run()
# Export
self.export(self.domain)
self.datas.extend(self.export(self.domain))
# Scan subdomain takeover
if self.takeover:
@@ -277,7 +228,7 @@ class OneForAll(object):
for domain in self.domains:
self.domain = utils.get_main_domain(domain)
self.main()
utils.export_all(self.alive, self.format, self.path, self.datas)
utils.export_all(self.alive, self.fmt, self.path, self.datas)
else:
logger.log('FATAL', 'Failed to obtain domain')
logger.log('INFOR', 'Finished OneForAll')
takeover.py
+9 -9
View File
@@ -45,21 +45,21 @@ class Takeover(Module):
OneForAll subdomain takeover module
Example:
python3 takeover.py --target www.example.com --format csv run
python3 takeover.py --target www.example.com --fmt csv run
python3 takeover.py --targets ./subdomains.txt --thread 10 run
Note:
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
--fmt txt/csv/json (result format)
--path Result directory (default directory is ./results)
:param str target: One domain (target or targets must be provided)
:param str targets: File path of one domain per line
:param int thread: threads number (default 20)
:param str format: Result format (default csv)
:param str fmt: Result format (default csv)
:param str path: Result directory (default None)
"""
def __init__(self, target=None, targets=None, thread=20, path=None, format='csv'):
def __init__(self, target=None, targets=None, thread=20, path=None, fmt='csv'):
Module.__init__(self)
self.subdomains = set()
self.module = 'Check'
@@ -68,7 +68,7 @@ class Takeover(Module):
self.targets = targets
self.thread = thread
self.path = path
self.format = format
self.fmt = fmt
self.fingerprints = None
self.subdomainq = Queue()
self.cnames = list()
@@ -76,10 +76,10 @@ class Takeover(Module):
def save(self):
logger.log('DEBUG', 'Saving results')
if self.format == 'txt':
if self.fmt == 'txt':
data = str(self.results)
else:
data = self.results.export(self.format)
data = self.results.export(self.fmt)
utils.save_data(self.path, data)
def compare(self, subdomain, cname, responses):
@@ -131,10 +131,10 @@ class Takeover(Module):
self.subdomains = self.targets
else:
self.subdomains = utils.get_domains(self.target, self.targets)
self.format = utils.check_format(self.format, len(self.subdomains))
self.fmt = utils.check_format(self.fmt)
timestamp = utils.get_timestamp()
name = f'takeover_check_result_{timestamp}'
self.path = utils.check_path(self.path, name, self.format)
self.path = utils.check_path(self.path, name, self.fmt)
if self.subdomains:
logger.log('INFOR', f'Checking subdomain takeover')
self.fingerprints = get_fingerprint()