river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-10-27 19:25:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

实现配置文件插拔式设计

Browse Source
This commit is contained in:
Jing Ling
2020-08-07 20:06:16 +08:00
parent d7f7c54b1a
commit 69ce692ff1
14 changed files with 103 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/database.py
+2 -2
View File
@@ -9,7 +9,7 @@ import records
from records import Connection
from config.log import logger
from config import setting
from config import settings
class Database(object):
@@ -29,7 +29,7 @@ class Database(object):
return db_path
protocol = 'sqlite:///'
if not db_path: # 数据库路径为空连接默认数据库
db_path = f'{protocol}{setting.result_save_dir}/result.sqlite3'
db_path = f'{protocol}{settings.result_save_dir}/result.sqlite3'
else:
db_path = protocol + db_path
db = records.Database(db_path) # 不存在数据库时会新建一个数据库
common/domain.py
+2 -2
View File
@@ -1,6 +1,6 @@
import re
import tldextract
from config import setting
from config import settings
class Domain(object):
@@ -38,7 +38,7 @@ class Domain(object):
:return: extracted domain results
"""
data_storage_dir = setting.data_storage_dir
data_storage_dir = settings.data_storage_dir
extract_cache_file = data_storage_dir.joinpath('public_suffix_list.dat')
tldext = tldextract.TLDExtract(extract_cache_file, None)
result = self.match()
common/module.py
+10 -10
View File
@@ -9,7 +9,7 @@ import time
import requests
from config.log import logger
from config import setting
from config import settings
from common import utils
from common.domain import Domain
from common.database import Database
@@ -24,9 +24,9 @@ class Module(object):
self.cookie = None
self.header = dict()
self.proxy = None
self.delay = setting.request_delay # 请求睡眠时延
self.timeout = setting.request_timeout # 请求超时时间
self.verify = setting.request_verify # 请求SSL验证
self.delay = settings.request_delay # 请求睡眠时延
self.timeout = settings.request_timeout # 请求超时时间
self.verify = settings.request_verify # 请求SSL验证
self.domain = str() # 当前进行子域名收集的主域
self.type = 'A' # 对主域进行子域收集时利用的DNS记录查询类型(默认利用A记录)
self.subdomains = set() # 存放发现的子域
@@ -185,7 +185,7 @@ class Module(object):
:return: header
"""
# logger.log('DEBUG', f'Get request header')
if setting.enable_fake_header:
if settings.enable_fake_header:
return utils.gen_fake_header()
else:
return self.header
@@ -197,13 +197,13 @@ class Module(object):
:param str module: module name
:return: proxy
"""
if not setting.enable_proxy:
if not settings.enable_proxy:
logger.log('TRACE', f'All modules do not use proxy')
return self.proxy
if setting.proxy_all_module:
if settings.proxy_all_module:
logger.log('TRACE', f'{module} module uses proxy')
return utils.get_random_proxy()
if module in setting.proxy_partial_module:
if module in settings.proxy_partial_module:
logger.log('TRACE', f'{module} module uses proxy')
return utils.get_random_proxy()
else:
@@ -219,11 +219,11 @@ class Module(object):
:return bool: whether saved successfully
"""
if not setting.save_module_result:
if not settings.save_module_result:
return False
logger.log('TRACE', f'Save the subdomain results found by '
f'{self.source} module as a json file')
path = setting.result_save_dir.joinpath(self.domain, self.module)
path = settings.result_save_dir.joinpath(self.domain, self.module)
path.mkdir(parents=True, exist_ok=True)
name = self.source + '.json'
path = path.joinpath(name)
common/request.py
+14 -14
View File
@@ -9,11 +9,11 @@ from bs4 import BeautifulSoup
from common import utils
from config.log import logger
from config import setting
from config import settings
def get_limit_conn():
limit_open_conn = setting.limit_open_conn
limit_open_conn = settings.limit_open_conn
if limit_open_conn is None: # 默认情况
limit_open_conn = utils.get_semaphore()
elif not isinstance(limit_open_conn, int): # 如果传入不是数字的情况
@@ -31,7 +31,7 @@ def get_ports(port):
ports = {port}
elif port in {'small', 'medium', 'large'}:
logger.log('DEBUG', f'{port} port range')
ports = setting.ports.get(port)
ports = settings.ports.get(port)
if not ports: # 意外情况
logger.log('ERROR', 'The specified request port range is incorrect')
ports = {80}
@@ -81,22 +81,22 @@ async def fetch(session, method, url):
"""
timeout = aiohttp.ClientTimeout(total=None,
connect=None,
sock_read=setting.sockread_timeout,
sock_connect=setting.sockconn_timeout)
sock_read=settings.sockread_timeout,
sock_connect=settings.sockconn_timeout)
try:
if method == 'HEAD':
async with session.head(url,
ssl=setting.verify_ssl,
allow_redirects=setting.allow_redirects,
ssl=settings.verify_ssl,
allow_redirects=settings.allow_redirects,
timeout=timeout,
proxy=setting.aiohttp_proxy) as resp:
proxy=settings.aiohttp_proxy) as resp:
text = await resp.text()
else:
async with session.get(url,
ssl=setting.verify_ssl,
allow_redirects=setting.allow_redirects,
ssl=settings.verify_ssl,
allow_redirects=settings.allow_redirects,
timeout=timeout,
proxy=setting.aiohttp_proxy) as resp:
proxy=settings.aiohttp_proxy) as resp:
try:
# 先尝试用utf-8解码
@@ -182,9 +182,9 @@ def request_callback(future, index, datas):
def get_connector():
limit_open_conn = get_limit_conn()
return aiohttp.TCPConnector(ttl_dns_cache=300,
ssl=setting.verify_ssl,
ssl=settings.verify_ssl,
limit=limit_open_conn,
limit_per_host=setting.limit_per_host)
limit_per_host=settings.limit_per_host)
async def async_request(urls):
@@ -211,7 +211,7 @@ async def bulk_request(data, port):
ports = get_ports(port)
no_req_data = utils.get_filtered_data(data)
to_req_data = gen_req_data(data, ports)
method = setting.request_method.upper()
method = settings.request_method.upper()
logger.log('INFOR', f'Use {method} method to request')
logger.log('INFOR', 'Async subdomains request in progress')
connector = get_connector()
common/resolve.py
+5 -5
View File
@@ -2,7 +2,7 @@ import gc
import json
from config.log import logger
from config import setting
from config import settings
from common import utils
from common.database import Database
from common.ipasn import IPAsnInfo
@@ -72,7 +72,7 @@ def deal_output(output_path):
records = dict() # 用来记录所有域名解析数据
ip_asn = IPAsnInfo()
ip_geo = IpGeoInfo
db_path = setting.data_storage_dir.joinpath('ip2region.db')
db_path = settings.data_storage_dir.joinpath('ip2region.db')
ip_reg = IpRegInfo(db_path)
with open(output_path) as fd:
for line in fd:
@@ -165,8 +165,8 @@ def run_resolve(domain, data):
if not subdomains:
return data
massdns_dir = setting.third_party_dir.joinpath('massdns')
result_dir = setting.result_save_dir
massdns_dir = settings.third_party_dir.joinpath('massdns')
result_dir = settings.result_save_dir
temp_dir = result_dir.joinpath('temp')
utils.check_dir(temp_dir)
massdns_path = utils.get_massdns_path(massdns_dir)
@@ -182,7 +182,7 @@ def run_resolve(domain, data):
output_path = temp_dir.joinpath(output_name)
log_path = result_dir.joinpath('massdns.log')
ns_path = setting.brute_nameservers_path
ns_path = settings.brute_nameservers_path
logger.log('INFOR', f'Running massdns to resolve subdomains')
utils.call_massdns(massdns_path, save_path, ns_path,
common/search.py
+4 -4
View File
@@ -1,6 +1,6 @@
import re
from config import setting
from config import settings
from config.log import logger
from common.module import Module
@@ -13,8 +13,8 @@ class Search(Module):
Module.__init__(self)
self.page_num = 0 # 要显示搜索起始条数
self.per_page_num = 50 # 每页显示搜索条数
self.recursive_search = setting.enable_recursive_search
self.recursive_times = setting.search_recursive_times
self.recursive_search = settings.enable_recursive_search
self.recursive_times = settings.search_recursive_times
@staticmethod
def filter(domain, subdomain):
@@ -29,7 +29,7 @@ class Search(Module):
"""
statements_list = []
subdomains_temp = set(map(lambda x: x + '.' + domain,
setting.subdomains_common))
settings.subdomains_common))
subdomains_temp = list(subdomain.intersection(subdomains_temp))
for i in range(0, len(subdomains_temp), 2): # 同时排除2个子域
statements_list.append(''.join(set(map(lambda s: ' -site:' + s,
common/utils.py
+16 -16
View File
@@ -19,7 +19,7 @@ from dns.resolver import Resolver
from common.domain import Domain
from common.database import Database
from config import setting
from config import settings
from config.log import logger
user_agents = [
@@ -70,7 +70,7 @@ def get_random_header():
Get random proxy
"""
header = None
if setting.fake_header:
if settings.fake_header:
header = gen_fake_header()
return header
@@ -80,7 +80,7 @@ def get_random_proxy():
Get random proxy
"""
try:
return random.choice(setting.proxy_pool)
return random.choice(settings.proxy_pool)
except IndexError:
return None
@@ -89,7 +89,7 @@ def get_proxy():
"""
Get proxy
"""
if setting.enable_proxy:
if settings.enable_proxy:
return get_random_proxy()
return None
@@ -176,7 +176,7 @@ def check_path(path, name, format):
:return: 保存路径
"""
filename = f'{name}.{format}'
default_path = setting.result_save_dir.joinpath(filename)
default_path = settings.result_save_dir.joinpath(filename)
if isinstance(path, str):
path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠
path = path.replace('\'', '') # 去除多余的转义
@@ -396,9 +396,9 @@ def dns_resolver():
dns解析器
"""
resolver = Resolver()
resolver.nameservers = setting.resolver_nameservers
resolver.timeout = setting.resolver_timeout
resolver.lifetime = setting.resolver_lifetime
resolver.nameservers = settings.resolver_nameservers
resolver.timeout = settings.resolver_timeout
resolver.lifetime = settings.resolver_lifetime
return resolver
@@ -495,7 +495,7 @@ def ip_is_public(ip_str):
def get_process_num():
process_num = setting.brute_process_num
process_num = settings.brute_process_num
if isinstance(process_num, int):
return min(os.cpu_count(), process_num)
else:
@@ -503,7 +503,7 @@ def get_process_num():
def get_coroutine_num():
coroutine_num = setting.resolve_coroutine_num
coroutine_num = settings.resolve_coroutine_num
if isinstance(coroutine_num, int):
return max(64, coroutine_num)
elif coroutine_num is None:
@@ -601,8 +601,8 @@ def check_version(local):
api = 'https://api.github.com/repos/shmilylty/OneForAll/releases/latest'
header = get_random_header()
proxy = get_proxy()
timeout = setting.request_timeout
verify = setting.request_verify
timeout = settings.request_timeout
verify = settings.request_verify
try:
resp = requests.get(url=api, headers=header, proxies=proxy,
timeout=timeout, verify=verify)
@@ -633,9 +633,9 @@ def call_massdns(massdns_path, dict_path, ns_path, output_path, log_path,
quiet = ''
if quiet_mode:
quiet = '--quiet'
status_format = setting.brute_status_format
socket_num = setting.brute_socket_num
resolve_num = setting.brute_resolve_num
status_format = settings.brute_status_format
socket_num = settings.brute_socket_num
resolve_num = settings.brute_resolve_num
cmd = f'{massdns_path} {quiet} --status-format {status_format} ' \
f'--processes {process_num} --socket-count {socket_num} ' \
f'--hashmap-size {concurrent_num} --resolvers {ns_path} ' \
@@ -648,7 +648,7 @@ def call_massdns(massdns_path, dict_path, ns_path, output_path, log_path,
def get_massdns_path(massdns_dir):
path = setting.brute_massdns_path
path = settings.brute_massdns_path
if path:
return path
system = platform.system().lower()