river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-26 04:47:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

实现finder模块（从响应体和JS文件中再次发现新子域）

Browse Source
This commit is contained in:
Jing Ling
2020-07-24 01:43:59 +08:00
parent 3a935c201d
commit 62d8b76c21
12 changed files with 797 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/database.py
+1 -1
View File
@@ -95,7 +95,7 @@ class Database(object):
:param str table_name: table name
:param list results: results list
:param str module_name: mo
:param str module_name: module
"""
logger.log('TRACE', f'Saving the subdomain results of {table_name} '
f'found by module {module_name} into database')
common/module.py
+3 -35
View File
@@ -212,39 +212,7 @@ class Module(object):
return self.proxy
def match_subdomains(self, html, distinct=True, fuzzy=True):
"""
Use regexp to match subdomains
:param str html: response html text
:param bool distinct: deduplicate results or not (default True)
:param bool fuzzy: fuzzy match subdomain or not (default True)
:return set/list: result set or list
"""
logger.log('TRACE', f'Use regexp to match subdomains in the response body')
if fuzzy:
regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ self.domain.replace('.', r'\.')
result = re.findall(regexp, html, re.I)
if not result:
return set()
deal = map(lambda s: s.lower(), result)
if distinct:
return set(deal)
else:
return list(deal)
else:
regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ self.domain.replace('.', r'\.')
result = re.findall(regexp, html, re.I)
if not result:
return set()
regexp = r'(?:http://|https://)'
deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
if distinct:
return set(deal)
else:
return list(deal)
return utils.match_subdomains(self.domain, html, distinct, fuzzy)
@staticmethod
def get_maindomain(domain):
@@ -305,11 +273,11 @@ class Module(object):
'new': None,
'url': None,
'subdomain': None,
'port': None,
'level': None,
'cname': None,
'content': None,
'public': None,
'port': None,
'status': None,
'reason': None,
'title': None,
@@ -369,11 +337,11 @@ class Module(object):
'new': None,
'url': url,
'subdomain': subdomain,
'port': 80,
'level': level,
'cname': cname,
'content': content,
'public': public,
'port': 80,
'status': None,
'reason': reason,
'title': None,
common/request.py
+14 -19
View File
@@ -9,7 +9,6 @@ from bs4 import BeautifulSoup
from common import utils
from config.log import logger
from config import setting
from common.database import Database
def get_limit_conn():
@@ -35,7 +34,7 @@ def get_ports(port):
if not ports: # 意外情况
logger.log('ERROR', 'The specified request port range is incorrect')
ports = {80}
logger.log('INFOR', 'Port range:{ports}')
logger.log('INFOR', f'Port range:{ports}')
return set(ports)
@@ -110,7 +109,7 @@ async def fetch(session, method, url):
text = await resp.text(encoding=None, errors='ignore')
return resp, text
except Exception as e:
return e
return e, None
def get_title(markup):
@@ -154,15 +153,15 @@ def get_title(markup):
def request_callback(future, index, datas):
result = future.result()
if isinstance(result, BaseException):
logger.log('TRACE', result.args)
name = utils.get_classname(result)
datas[index]['reason'] = name + ' ' + str(result)
resp, text = future.result()
if isinstance(resp, BaseException):
exception = resp
logger.log('TRACE', exception.args)
name = utils.get_classname(exception)
datas[index]['reason'] = name + ' ' + str(exception)
datas[index]['request'] = 0
datas[index]['alive'] = 0
elif isinstance(result, tuple):
resp, text = result
else:
datas[index]['reason'] = resp.reason
datas[index]['status'] = resp.status
datas[index]['request'] = 1
@@ -257,17 +256,17 @@ def run_request(domain, data, port):
:param str domain: domain to be requested
:param list data: subdomains data to be requested
:param str port: range of ports to be requested
:param any port: range of ports to be requested
:return list: result
"""
logger.log('INFOR', 'Start subdomain request module')
logger.log('INFOR', f'Start requesting subdomains of {domain}')
loop = set_loop()
data = utils.set_id_none(data)
request_coroutine = bulk_request(data, port)
data = loop.run_until_complete(request_coroutine)
loop.run_until_complete(asyncio.sleep(0.25))
count = utils.count_alive(data)
logger.log('INFOR', f'Request module found {domain} have {count} alive subdomains')
logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
return data
@@ -277,7 +276,6 @@ def urls_request(urls):
request_coroutine = async_request(urls)
data = loop.run_until_complete(request_coroutine)
loop.run_until_complete(asyncio.sleep(0.25))
logger.log('INFOR', 'End urls request module')
return data
@@ -288,8 +286,5 @@ def save_db(name, data):
:param str name: table name
:param list data: data to be saved
"""
db = Database()
db.drop_table(name)
db.create_table(name)
db.save_db(name, data, 'request')
db.close()
logger.log('INFOR', f'Saving requested results')
utils.save_db(name, data, 'request')
common/resolve.py
+5 -9
View File
@@ -50,17 +50,13 @@ def update_data(data, records):
def save_db(name, data):
"""
保存解析结果到数据库
Save resolved results to database
:param str name: 保存表名
:param list data: 待保存的数据
:param str name: table name
:param list data: data to be saved
"""
logger.log('INFOR', f'Saving resolved results')
db = Database()
db.drop_table(name)
db.create_table(name)
db.save_db(name, data, 'resolve')
db.close()
utils.save_db(name, data, 'resolve')
def save_subdomains(save_path, subdomain_list):
@@ -161,7 +157,7 @@ def run_resolve(domain, data):
:return: 解析得到的结果列表
:rtype: list
"""
logger.log('INFOR', f'Start resolve subdomains of {domain}')
logger.log('INFOR', f'Start resolving subdomains of {domain}')
subdomains = filter_subdomain(data)
if not subdomains:
return data
common/utils.py
+53
View File
@@ -17,6 +17,7 @@ from records import Record, RecordCollection
from dns.resolver import Resolver
from common.domain import Domain
from common.database import Database
from config import setting
from config.log import logger
@@ -214,6 +215,21 @@ def check_format(format, count):
return 'csv'
def save_db(name, data, module):
"""
Save request results to database
:param str name: table name
:param list data: data to be saved
:param str module: module name
"""
db = Database()
db.drop_table(name)
db.create_table(name)
db.save_db(name, data, module)
db.close()
def save_data(path, data):
"""
保存数据到文件
@@ -663,3 +679,40 @@ def ip_to_int(ip):
logger.log('ERROR', e.args)
return None
return int(ipv4)
def match_subdomains(domain, html, distinct=True, fuzzy=True):
"""
Use regexp to match subdomains
:param str domain: main domain
:param str html: response html text
:param bool distinct: deduplicate results or not (default True)
:param bool fuzzy: fuzzy match subdomain or not (default True)
:return set/list: result set or list
"""
logger.log('TRACE', f'Use regexp to match subdomains in the response body')
if fuzzy:
regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.')
result = re.findall(regexp, html, re.I)
if not result:
return set()
deal = map(lambda s: s.lower(), result)
if distinct:
return set(deal)
else:
return list(deal)
else:
regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.')
result = re.findall(regexp, html, re.I)
if not result:
return set()
regexp = r'(?:http://|https://)'
deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
if distinct:
return set(deal)
else:
return list(deal)