mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-10-25 00:56:14 +08:00
1.重构请求，边请求边存入数据库，解决内存占用过大问题。
2.format参数改为fmt。 3.添加信息富化模块。 4.优化iscdn模块代码。 5.移除数据库new字段。
This commit is contained in:
Jing Ling
+32
-81
@@ -15,12 +15,11 @@ from datetime import datetime
|
||||
import dbexport
|
||||
from brute import Brute
|
||||
from common import utils, resolve, request
|
||||
from common.database import Database
|
||||
from modules.collect import Collect
|
||||
from modules.srv import BruteSRV
|
||||
from modules.finder import Finder
|
||||
from modules.altdns import Altdns
|
||||
from modules import iscdn
|
||||
from modules.enrich import Enrich
|
||||
from config import settings
|
||||
from config.log import logger
|
||||
from takeover import Takeover
|
||||
@@ -60,7 +59,7 @@ class OneForAll(object):
|
||||
python3 oneforall.py --target example.com --alive False run
|
||||
python3 oneforall.py --target example.com --brute False run
|
||||
python3 oneforall.py --target example.com --port medium run
|
||||
python3 oneforall.py --target example.com --format csv run
|
||||
python3 oneforall.py --target example.com --fmt csv run
|
||||
python3 oneforall.py --target example.com --dns False run
|
||||
python3 oneforall.py --target example.com --req False run
|
||||
python3 oneforall.py --target example.com --takeover False run
|
||||
@@ -68,22 +67,22 @@ class OneForAll(object):
|
||||
|
||||
Note:
|
||||
--port small/medium/large See details in ./config/setting.py(default small)
|
||||
--format csv/json (result format)
|
||||
--fmt csv/json (result format)
|
||||
--path Result path (default None, automatically generated)
|
||||
|
||||
:param str target: One domain (target or targets must be provided)
|
||||
:param str targets: File path of one domain per line
|
||||
:param bool brute: Use brute module (default True)
|
||||
:param bool dns: Use DNS resolution (default True)
|
||||
:param bool req: HTTP request subdomains (default True)
|
||||
:param bool brute: Use brute module (default True)
|
||||
:param bool dns: Use DNS resolution (default True)
|
||||
:param bool req: HTTP request subdomains (default True)
|
||||
:param str port: The port range to request (default small port is 80,443)
|
||||
:param bool alive: Only export alive subdomains (default False)
|
||||
:param str format: Result format (default csv)
|
||||
:param bool alive: Only export alive subdomains (default False)
|
||||
:param str fmt: Result format (default csv)
|
||||
:param str path: Result path (default None, automatically generated)
|
||||
:param bool takeover: Scan subdomain takeover (default False)
|
||||
:param bool takeover: Scan subdomain takeover (default False)
|
||||
"""
|
||||
def __init__(self, target=None, targets=None, brute=None, dns=None, req=None,
|
||||
port=None, alive=None, format=None, path=None, takeover=None):
|
||||
port=None, alive=None, fmt=None, path=None, takeover=None):
|
||||
self.target = target
|
||||
self.targets = targets
|
||||
self.brute = brute
|
||||
@@ -91,17 +90,13 @@ class OneForAll(object):
|
||||
self.req = req
|
||||
self.port = port
|
||||
self.alive = alive
|
||||
self.format = format
|
||||
self.fmt = fmt
|
||||
self.path = path
|
||||
self.takeover = takeover
|
||||
self.domain = str() # The domain currently being collected
|
||||
self.domains = set() # All domains that are to be collected
|
||||
self.data = list() # The subdomain results of the current domain
|
||||
self.datas = list() # All subdomain results of the domain
|
||||
self.old_table = str() # The table name of the last result
|
||||
self.new_table = str() # The table name of the current result
|
||||
self.origin_table = str() # The table name of the origin result
|
||||
self.resolve_table = str() # The table name of the resolute result
|
||||
|
||||
def config_param(self):
|
||||
"""
|
||||
@@ -119,8 +114,8 @@ class OneForAll(object):
|
||||
self.port = settings.http_request_port
|
||||
if self.alive is None:
|
||||
self.alive = bool(settings.result_export_alive)
|
||||
if self.format is None:
|
||||
self.format = settings.result_save_format
|
||||
if self.fmt is None:
|
||||
self.fmt = settings.result_save_format
|
||||
if self.path is None:
|
||||
self.path = settings.result_save_path
|
||||
|
||||
@@ -134,47 +129,13 @@ class OneForAll(object):
|
||||
|
||||
def export(self, table):
|
||||
"""
|
||||
Export data from the database and do some follow-up processing
|
||||
Export data from the database
|
||||
|
||||
:param table: table name
|
||||
:return: export data
|
||||
:rtype: list
|
||||
"""
|
||||
db = Database()
|
||||
data = dbexport.export(table, type='table', alive=self.alive, format=self.format)
|
||||
db.drop_table(self.new_table)
|
||||
db.rename_table(self.domain, self.new_table)
|
||||
db.close()
|
||||
return data
|
||||
|
||||
def deal_db(self):
|
||||
"""
|
||||
Process the data when the collection task is completed
|
||||
"""
|
||||
db = Database()
|
||||
db.deal_table(self.domain, self.origin_table)
|
||||
db.close()
|
||||
|
||||
def mark(self):
|
||||
"""
|
||||
Mark the new discovered subdomain
|
||||
|
||||
:return: marked data
|
||||
:rtype: list
|
||||
"""
|
||||
db = Database()
|
||||
old_data = list()
|
||||
now_data = db.get_data(self.domain).as_dict()
|
||||
# Database pre-processing when it is not the first time to collect this subdomain
|
||||
if db.exist_table(self.new_table):
|
||||
# If there is the last collection result table, delete it first
|
||||
db.drop_table(self.old_table)
|
||||
# Rename the new table to the old table
|
||||
db.rename_table(self.new_table, self.old_table)
|
||||
old_data = db.get_data(self.old_table).as_dict()
|
||||
db.close()
|
||||
marked_data = utils.mark_subdomain(old_data, now_data)
|
||||
return marked_data
|
||||
return dbexport.export(table, type='table', alive=self.alive, fmt=self.fmt)
|
||||
|
||||
def main(self):
|
||||
"""
|
||||
@@ -183,11 +144,6 @@ class OneForAll(object):
|
||||
:return: subdomain results
|
||||
:rtype: list
|
||||
"""
|
||||
self.old_table = self.domain + '_old_result'
|
||||
self.new_table = self.domain + '_now_result'
|
||||
self.origin_table = self.domain + '_origin_result'
|
||||
self.resolve_table = self.domain + '_resolve_result'
|
||||
|
||||
collect = Collect(self.domain)
|
||||
collect.run()
|
||||
|
||||
@@ -202,49 +158,44 @@ class OneForAll(object):
|
||||
brute.quite = True
|
||||
brute.run()
|
||||
|
||||
# Database processing
|
||||
self.deal_db()
|
||||
# Mark the new discovered subdomain
|
||||
self.data = self.mark()
|
||||
|
||||
utils.deal_data(self.domain)
|
||||
# Export results without resolve
|
||||
if not self.dns:
|
||||
return self.export(self.domain)
|
||||
|
||||
self.data = utils.get_data(self.domain)
|
||||
|
||||
# Resolve subdomains
|
||||
utils.clear_data(self.domain)
|
||||
self.data = resolve.run_resolve(self.domain, self.data)
|
||||
# Save resolve results
|
||||
resolve.save_db(self.resolve_table, self.data)
|
||||
resolve.save_db(self.domain, self.data)
|
||||
|
||||
# Export results without HTTP request
|
||||
if not self.req:
|
||||
return self.export(self.resolve_table)
|
||||
return self.export(self.domain)
|
||||
|
||||
# HTTP request
|
||||
self.data = request.run_request(self.domain, self.data, self.port)
|
||||
# Save HTTP request result
|
||||
request.save_db(self.domain, self.data)
|
||||
utils.clear_data(self.domain)
|
||||
request.run_request(self.domain, self.data, self.port)
|
||||
|
||||
# Finder module
|
||||
if settings.enable_finder_module:
|
||||
finder = Finder()
|
||||
self.data = finder.run(self.domain, self.data, self.port)
|
||||
finder.run(self.domain, self.data, self.port)
|
||||
|
||||
# altdns module
|
||||
if settings.enable_altdns_module:
|
||||
finder = Altdns(self.domain)
|
||||
self.data = finder.run(self.data, self.port)
|
||||
altdns = Altdns(self.domain)
|
||||
altdns.run(self.data, self.port)
|
||||
|
||||
# check cdn module
|
||||
if settings.enable_cdn_check:
|
||||
self.data = iscdn.check_cdn(self.data)
|
||||
iscdn.save_db(self.domain, self.data)
|
||||
|
||||
# Add the final result list to the total data list
|
||||
self.datas.extend(self.data)
|
||||
# Information enrichment module
|
||||
if settings.enable_enrich_module:
|
||||
enrich = Enrich(self.domain)
|
||||
enrich.run()
|
||||
|
||||
# Export
|
||||
self.export(self.domain)
|
||||
self.datas.extend(self.export(self.domain))
|
||||
|
||||
# Scan subdomain takeover
|
||||
if self.takeover:
|
||||
@@ -277,7 +228,7 @@ class OneForAll(object):
|
||||
for domain in self.domains:
|
||||
self.domain = utils.get_main_domain(domain)
|
||||
self.main()
|
||||
utils.export_all(self.alive, self.format, self.path, self.datas)
|
||||
utils.export_all(self.alive, self.fmt, self.path, self.datas)
|
||||
else:
|
||||
logger.log('FATAL', 'Failed to obtain domain')
|
||||
logger.log('INFOR', 'Finished OneForAll')
|
||||
|
||||
Reference in New Issue
Block a user