mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-26 12:57:50 +08:00
v0.0.1
This commit is contained in:
Jing Ling
@@ -0,0 +1 @@
|
||||
# coding=utf-8
|
||||
@@ -0,0 +1,87 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class CensysAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Certificate'
|
||||
self.source = "CensysAPIQuery"
|
||||
self.addr = 'https://www.censys.io/api/v1/search/certificates'
|
||||
self.id = config.censys_api_id
|
||||
self.secret = config.censys_api_secret
|
||||
self.delay = 3.0 # Censys 接口查询速率限制 最快2.5秒查1次
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
data = {
|
||||
'query': 'parsed.names: owasp.org',
|
||||
'page': 1,
|
||||
'fields': ['parsed.subject_dn'],
|
||||
'flatten': True}
|
||||
|
||||
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
|
||||
if not resp:
|
||||
return
|
||||
resp_json = resp.json()
|
||||
status = resp_json.get('status')
|
||||
if status != 'ok':
|
||||
logger.log('ALERT', status)
|
||||
return
|
||||
subdomains_find = self.match(self.domain, str(resp_json))
|
||||
self.subdomains = self.subdomains.union(subdomains_find)
|
||||
pages = resp_json.get('metadata').get('pages')
|
||||
for page in range(2, pages+1):
|
||||
time.sleep(self.delay)
|
||||
data['page'] = page
|
||||
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, str(resp.json()))
|
||||
self.subdomains = self.subdomains.union(subdomains_find)
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not (self.id and self.secret):
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = CensysAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,74 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class CertDBAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Certificate'
|
||||
self.source = 'CertDBQuery'
|
||||
self.addr = 'https://api.spyse.com/v1/subdomains'
|
||||
self.token = config.certdb_api_token
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
page_num = 1
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'domain': self.domain, 'api_token': self.token, 'page': page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
resp_json = resp.json()
|
||||
subdomains_find = utils.match_subdomain(self.domain, str(resp_json))
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
page_num += 1
|
||||
if resp_json.get('count') < 30: # 默认每次查询最多返回30条 当前条数小于30条说明已经查完
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.token:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = CertDBAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,62 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class CertSpotter(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Certificate'
|
||||
self.source = 'CertSpotterQuery'
|
||||
self.addr = 'https://api.certspotter.com/v1/issuances'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'domain': self.domain, 'include_subdomains': 'true', 'expand': 'dns_names'}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = CertSpotter(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,62 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class Crtsh(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Certificate'
|
||||
self.source = 'CrtshQuery'
|
||||
self.addr = 'https://crt.sh/'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'q': f'%.{self.domain}', 'output': 'json'}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
|
||||
self.subdomains = self.subdomains.union(subdomains_find)
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = Crtsh(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,62 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class Entrust(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Certificate'
|
||||
self.source = 'EntrustQuery'
|
||||
self.addr = 'https://ctsearch.entrust.com/api/v1/certificates'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'fields': 'subjectDN', 'domain': self.domain, 'includeExpired': 'true'}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
|
||||
self.subdomains = self.subdomains.union(subdomains_find)
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = Entrust(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,62 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class Google(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Certificate'
|
||||
self.source = 'GoogleQuery'
|
||||
self.addr = 'https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'include_expired': 'true', 'include_subdomains': 'true', 'domain': self.domain}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = utils.match_subdomain(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = Google(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,98 @@
|
||||
# coding=utf-8
|
||||
"""
|
||||
查询域名的NS记录（域名服务器记录，记录该域名由哪台域名服务器解析），
|
||||
检查查出的域名服务器是否开启DNS域传送，如果开启且没做访问控制和身份验证便加以利用获取域名的所有记录
|
||||
|
||||
DNS域传送（DNS zone transfer）指的是一台备用域名服务器使用来自主域名服务器的数据刷新自己的域数据库，
|
||||
目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
|
||||
当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获取某个域的所有记录。
|
||||
"""
|
||||
import time
|
||||
import queue
|
||||
import dns.resolver
|
||||
import dns.zone
|
||||
from config import logger
|
||||
from common import utils, resolve
|
||||
from common.module import Module
|
||||
|
||||
|
||||
class CheckAXFR(Module):
|
||||
"""
|
||||
DNS域传送漏洞检查类
|
||||
"""
|
||||
def __init__(self, domain: str):
|
||||
Module.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Check'
|
||||
self.source = 'AXFRCheck'
|
||||
self.nsservers = []
|
||||
self.results = []
|
||||
|
||||
def check(self):
|
||||
"""
|
||||
正则匹配响应头中的内容安全策略字段以发现子域名
|
||||
:return: None
|
||||
"""
|
||||
resolver = resolve.dns_resolver()
|
||||
try:
|
||||
answers = resolver.query(self.domain, "NS")
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
return
|
||||
self.nsservers = [str(answer) for answer in answers]
|
||||
if not len(self.nsservers):
|
||||
logger.log('ALERT', f'没有找到{self.domain}的NS域名服务器记录')
|
||||
return
|
||||
for nsserver in self.nsservers:
|
||||
logger.log('DEBUG', f'正在尝试对{self.domain}的域名服务器{nsserver}进行域传送')
|
||||
try:
|
||||
xfr = dns.query.xfr(nsserver, self.domain)
|
||||
zone = dns.zone.from_xfr(xfr)
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', str(e))
|
||||
logger.log('INFOR', f'对{self.domain}的域名服务器{nsserver}进行域传送失败')
|
||||
continue
|
||||
else:
|
||||
names = zone.nodes.keys()
|
||||
for name in names:
|
||||
subdomain = utils.match_subdomain(self.domain, str(name)+'.'+self.domain)
|
||||
self.subdomains = self.subdomains.union(subdomain)
|
||||
record = zone[name].to_text(name)
|
||||
self.results.append(record)
|
||||
if self.results:
|
||||
logger.log('INFOR', f'发现{self.domain}在{nsserver}上的域传送记录')
|
||||
logger.log('DEBUG', '\n'.join(self.results))
|
||||
self.results = []
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}的域传送漏洞')
|
||||
start = time.time()
|
||||
self.check()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}的域传送漏洞')
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
check = CheckAXFR(domain)
|
||||
check.run(rx_queue)
|
||||
logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
|
||||
logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# do('ZoneTransfer.me')
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,64 @@
|
||||
# coding=utf-8
|
||||
"""
|
||||
检查crossdomain.xml文件收集子域名
|
||||
"""
|
||||
import time
|
||||
import queue
|
||||
from config import logger
|
||||
from common.utils import match_subdomain
|
||||
from common.module import Module
|
||||
|
||||
|
||||
class CheckCDX(Module):
|
||||
"""
|
||||
检查crossdomain.xml文件收集子域名
|
||||
"""
|
||||
def __init__(self, domain: str):
|
||||
Module.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Check'
|
||||
self.source = "CrossDomainXml"
|
||||
|
||||
def check(self):
|
||||
"""
|
||||
检查crossdomain.xml收集子域名
|
||||
:return:
|
||||
"""
|
||||
url = f'http://{self.domain}/crossdomain.xml'
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
self.subdomains = match_subdomain(self.domain, resp.text)
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的crossdomain.xml')
|
||||
start = time.time()
|
||||
self.check()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的crossdomain.xml')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
:param domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
check = CheckCDX(domain)
|
||||
check.run(rx_queue)
|
||||
logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
|
||||
logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
do('163.com')
|
||||
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env python3
|
||||
# coding=utf-8
|
||||
|
||||
"""
|
||||
检查域名证书收集子域名
|
||||
"""
|
||||
import ssl
|
||||
import time
|
||||
import queue
|
||||
import socket
|
||||
from config import logger
|
||||
from common import utils
|
||||
from common.module import Module
|
||||
|
||||
|
||||
class CheckCert(Module):
|
||||
def __init__(self, domain):
|
||||
Module.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.port = 443 # ssl port
|
||||
self.module = 'Check'
|
||||
self.source = 'CertInfo'
|
||||
|
||||
def check(self):
|
||||
"""
|
||||
获取域名证书并匹配证书中的子域名
|
||||
"""
|
||||
ctx = ssl.create_default_context()
|
||||
sock = ctx.wrap_socket(socket.socket(), server_hostname=self.domain)
|
||||
try:
|
||||
sock.connect((self.domain, self.port))
|
||||
cert_dict = sock.getpeercert()
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
return
|
||||
subdomains_find = utils.match_subdomain(self.domain, str(cert_dict))
|
||||
self.subdomains = self.subdomains.union(subdomains_find)
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的证书中的子域')
|
||||
start = time.time()
|
||||
self.check()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的证书中的子域')
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
check = CheckCert(domain)
|
||||
check.run(rx_queue)
|
||||
logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
|
||||
logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,75 @@
|
||||
# coding=utf-8
|
||||
"""
|
||||
检查内容安全策略收集子域名收集子域名
|
||||
"""
|
||||
import time
|
||||
import queue
|
||||
from config import logger
|
||||
from common import utils
|
||||
from common.module import Module
|
||||
|
||||
|
||||
class CheckCSP(Module):
|
||||
"""
|
||||
检查内容安全策略收集子域名
|
||||
"""
|
||||
def __init__(self, domain, header):
|
||||
Module.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Check'
|
||||
self.source = 'Content-Security-Policy'
|
||||
self.header = header
|
||||
|
||||
def check(self):
|
||||
"""
|
||||
正则匹配响应头中的内容安全策略字段以发现子域名
|
||||
"""
|
||||
if not self.header:
|
||||
url = f'http://www.{self.domain}'
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
self.header = resp.headers
|
||||
csp = self.header.get('Content-Security-Policy')
|
||||
if not csp:
|
||||
logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段')
|
||||
return
|
||||
logger.log('DEBUG', f'{self.domain}域的响应头存在内容安全策略字段')
|
||||
self.subdomains = utils.match_subdomain(self.domain, csp)
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
|
||||
start = time.time()
|
||||
self.check()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
|
||||
|
||||
def do(domain, rx_queue, header=None): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
:param dict or None header: 响应头
|
||||
"""
|
||||
check = CheckCSP(domain, header)
|
||||
check.run(rx_queue)
|
||||
logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
|
||||
logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import requests
|
||||
# resp = requests.get('https://content-security-policy.com/')
|
||||
result_queue = queue.Queue()
|
||||
resp = requests.get('https://www.baidu.com/')
|
||||
do('google-analytics.com', result_queue, resp.headers)
|
||||
@@ -0,0 +1,71 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import cdx_toolkit
|
||||
from common.crawl import Crawl
|
||||
from config import logger
|
||||
|
||||
|
||||
class ArchiveCrawl(Crawl):
|
||||
def __init__(self, domain):
|
||||
Crawl.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Crawl'
|
||||
self.source = 'ArchiveCrawl'
|
||||
|
||||
def crawl(self, domain, limit):
|
||||
"""
|
||||
|
||||
:param domain:
|
||||
:param limit:
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
cdx = cdx_toolkit.CDXFetcher(source='ia')
|
||||
url = f'*.{domain}/*'
|
||||
size = cdx.get_size_estimate(url)
|
||||
logger.log('DEBUG', f'{url} ArchiveCrawl size estimate {size}')
|
||||
|
||||
for resp in cdx.iter(url, limit=limit):
|
||||
if resp.data.get('status') not in ['301', '302']:
|
||||
url = resp.data.get('url')
|
||||
subdomains_find = self.match(self.register(domain), url + resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.crawl(self.domain, 50)
|
||||
|
||||
# 爬取已发现的子域以发现新的子域
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain != self.domain:
|
||||
self.crawl(subdomain, 10)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
crawl = ArchiveCrawl(domain)
|
||||
crawl.run(result)
|
||||
logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
|
||||
logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,71 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from tqdm import tqdm
|
||||
import cdx_toolkit
|
||||
from common.crawl import Crawl
|
||||
from config import logger
|
||||
|
||||
|
||||
class CommonCrawl(Crawl):
|
||||
def __init__(self, domain):
|
||||
Crawl.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Crawl'
|
||||
self.source = 'CommonCrawl'
|
||||
|
||||
def crawl(self, domain, limit):
|
||||
"""
|
||||
|
||||
:param domain:
|
||||
:param limit:
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
cdx = cdx_toolkit.CDXFetcher()
|
||||
url = f'*.{domain}/*'
|
||||
size = cdx.get_size_estimate(url)
|
||||
print(url, 'CommonCrawl size estimate', size)
|
||||
|
||||
for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
|
||||
if resp.data.get('status') not in ['301', '302']:
|
||||
subdomains_find = self.match(self.register(domain), resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.crawl(self.domain, 50)
|
||||
|
||||
# 爬取已发现的子域以发现新的子域
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain != self.domain:
|
||||
self.crawl(subdomain, 10)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
crawl = CommonCrawl(domain)
|
||||
crawl.run(result)
|
||||
logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
|
||||
logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,68 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class BinaryEdgeAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'BinaryEdgeAPIQuery'
|
||||
self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/'
|
||||
self.api = config.binaryedge_api
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.header.update({'X-Key': self.api})
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
url = self.addr + self.domain
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, str(resp.json()))
|
||||
self.subdomains = self.subdomains.union(subdomains_find)
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.api:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = BinaryEdgeAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,61 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class BufferOver(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'BufferOverQuery'
|
||||
self.addr = 'https://dns.bufferover.run/dns'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'q': self.domain}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = BufferOver(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,61 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class Chinaz(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'ChinazQuery'
|
||||
self.addr = 'https://alexa.chinaz.com/'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
self.addr = self.addr + self.domain
|
||||
resp = self.get(self.addr)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = Chinaz(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,67 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class ChinazAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'ChinazAPIQuery'
|
||||
self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa'
|
||||
self.api = config.chinaz_api
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'key': self.api, 'domainName': self.domain}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, str(resp.json()))
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.api:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = ChinazAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,64 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class CirclAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'CirclAPIQuery'
|
||||
self.addr = 'https://www.circl.lu/pdns/query/'
|
||||
self.user = config.circl_api_username
|
||||
self.pwd = config.circl_api_password
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, str(resp.json()))
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
if self.user and self.pwd:
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = CirclAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,76 @@
|
||||
# coding=utf-8
|
||||
import random
|
||||
import time
|
||||
import queue
|
||||
from bs4 import BeautifulSoup
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class DNSdb(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'DNSdbQuery'
|
||||
self.addr = 'https://www.dnsdb.org/'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.header.update({'Referer': self.addr})
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
url = self.addr + self.domain + '/'
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
if resp.status_code == 200:
|
||||
if 'index' in resp.text:
|
||||
soup = BeautifulSoup(resp.text, features='lxml')
|
||||
index_urls = set(map(lambda x: self.addr + self.domain + x.text, soup.find_all('a')))
|
||||
for url in index_urls:
|
||||
self.delay = random.randint(2, 5) # 休眠绕过CloudFlare的DDoS保护
|
||||
time.sleep(self.delay)
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
else:
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = DNSdb(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,70 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class DNSdbAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'DNSdbAPIQuery'
|
||||
self.addr = 'https://api.dnsdb.info/lookup/rrset/name/'
|
||||
self.api = config.dnsdb_api_key
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.header.update({'X-API-Key': self.api})
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
url = f'{self.addr}*.{self.domain}'
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
if resp.status_code == 200:
|
||||
subdomains_find = utils.match_subdomain(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.api:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = DNSdbAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,71 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class DNSdumpster(Query):
|
||||
"""
|
||||
|
||||
"""
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = "DNSdumpsterQuery"
|
||||
self.addr = 'https://dnsdumpster.com/'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.header.update({'Referer': 'https://dnsdumpster.com'})
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(self.addr)
|
||||
if not resp:
|
||||
return
|
||||
self.cookie = resp.cookies
|
||||
data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'), 'targetip': self.domain}
|
||||
resp = self.post(self.addr, data)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = utils.match_subdomain(self.domain, resp.text)
|
||||
if subdomains_find:
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = DNSdumpster(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,63 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class HackerTarget(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = "HackerTargetQuery"
|
||||
self.addr = 'https://api.hackertarget.com/hostsearch/'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'q': self.domain}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
if resp.status_code == 200:
|
||||
subdomains_find = utils.match_subdomain(self.domain, resp.text)
|
||||
if subdomains_find:
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = HackerTarget(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,75 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class IPv4InfoAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'IPv4InfoAPIQuery'
|
||||
self.addr = ' http://ipv4info.com/api_v1/'
|
||||
self.api = config.ipv4info_api_key
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
page = 0
|
||||
while True:
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'type': 'SUBDOMAINS', 'key': self.api, 'value': self.domain, 'page': page}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
if resp.status_code != 200:
|
||||
break # 请求不正常通常网络是有问题，不再继续请求下去
|
||||
resp_json = resp.json()
|
||||
subdomains_find = self.match(self.domain, str(resp_json))
|
||||
if not subdomains_find:
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
subdomains = resp_json.get('Subdomains') # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
|
||||
if len(subdomains) < 300: # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
|
||||
break
|
||||
page += 1
|
||||
if page >= 50: # ipv4info子域查询接口最多允许查询50页
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = IPv4InfoAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,86 @@
|
||||
# coding=utf-8
|
||||
import re
|
||||
import time
|
||||
import queue
|
||||
import hashlib
|
||||
from urllib import parse
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class NetCraft(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'NetCraftQuery'
|
||||
self.init = 'https://searchdns.netcraft.com/'
|
||||
self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains'
|
||||
self.page_num = 1
|
||||
self.per_page_num = 20
|
||||
|
||||
def bypass_verification(self):
|
||||
"""
|
||||
绕过NetCraft的JS验证
|
||||
"""
|
||||
self.header = self.get_header() # Netcraft会检查User-Agent
|
||||
self.cookie = self.get(self.init).cookies
|
||||
cookie_value = self.cookie['netcraft_js_verification_challenge']
|
||||
verify_taken = hashlib.sha1(parse.unquote(cookie_value).encode('utf-8')).hexdigest()
|
||||
self.cookie['netcraft_js_verification_response'] = verify_taken
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.bypass_verification()
|
||||
last = ''
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'host': '*.' + self.domain, 'from': self.page_num}
|
||||
resp = self.get(self.addr + last, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
if 'Next page' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
last = re.search(r'&last=.*' + self.domain, resp.text).group(0)
|
||||
self.page_num += self.per_page_num
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = NetCraft(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,65 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import random
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class PTRArchive(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = "PTRArchiveQuery"
|
||||
self.addr = 'http://ptrarchive.com/tools/search3.htm'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
self.cookie = {'pa_id': str(random.randint(0, 1000000000))} # 绕过主页前端JS验证
|
||||
params = {'label': self.domain, 'date': 'ALL'}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
if resp.status_code == 200:
|
||||
subdomains_find = utils.match_subdomain(self.domain, resp.text)
|
||||
if subdomains_find:
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = PTRArchive(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,61 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class Riddler(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'RiddlerQuery'
|
||||
self.addr = 'https://riddler.io/search'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'q': 'pld:' + self.domain}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = Riddler(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,72 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import json
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class Robtex(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = "RobtexQuery"
|
||||
self.addr = 'https://freeapi.robtex.com/pdns/'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
url = self.addr + 'forward/' + self.domain
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
text_list = resp.text.splitlines()
|
||||
text_json = list(map(lambda x: json.loads(x), text_list))
|
||||
for record in text_json:
|
||||
if record.get('rrtype') in ['A', 'AAAA']:
|
||||
time.sleep(self.delay) # Robtex有查询频率限制
|
||||
ip = record.get('rrdata')
|
||||
url = self.addr + 'reverse/' + ip
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
if subdomains_find:
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = Robtex(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,71 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class SecurityTrailsAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'SecurityTrailsQuery'
|
||||
self.addr = 'https://api.securitytrails.com/v1/domain/'
|
||||
self.api = config.securitytrails_api
|
||||
self.delay = 2 # SecurityTrails查询时延至少2秒
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'apikey': self.api}
|
||||
url = f'{self.addr}{self.domain}/subdomains'
|
||||
resp = self.get(url, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_prefix = resp.json()['subdomains']
|
||||
subdomains_find = [f'{prefix}.{self.domain}' for prefix in subdomains_prefix]
|
||||
if subdomains_find:
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.api:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = SecurityTrailsAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,70 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class SiteDossier(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'SiteDossierQuery'
|
||||
self.addr = 'http://www.sitedossier.com/parentdomain/'
|
||||
self.delay = 2
|
||||
self.page_num = 1
|
||||
self.per_page_num = 100
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
url = f'{self.addr}{self.domain}/{self.page_num}'
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
if 'Show next 100 items' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
self.page_num += self.per_page_num
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = SiteDossier(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
# coding=utf-8
|
||||
|
||||
"""
|
||||
通过枚举域名常见的SRV记录并做查询来发现子域
|
||||
"""
|
||||
|
||||
import time
|
||||
import queue
|
||||
import json
|
||||
import asyncio
|
||||
import aiodns
|
||||
from common.module import Module
|
||||
from common import utils
|
||||
from config import logger, data_storage_path, resolver_nameservers
|
||||
|
||||
|
||||
class BruteSRV(Module):
|
||||
def __init__(self, domain: str):
|
||||
Module.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'dnsquery'
|
||||
self.source = "BruteSRV"
|
||||
self.loop = asyncio.new_event_loop()
|
||||
self.nameservers = resolver_nameservers
|
||||
self.resolver = aiodns.DNSResolver(self.nameservers, self.loop)
|
||||
|
||||
async def query(self, name):
|
||||
"""
|
||||
查询域名的SRV记录
|
||||
:param str name: SRV记录
|
||||
:return: 查询结果
|
||||
"""
|
||||
logger.log('TRACE', f'尝试查询{name}的SRV记录')
|
||||
try:
|
||||
answers = await self.resolver.query(name, 'SRV')
|
||||
except Exception as e:
|
||||
logger.log('TRACE', e)
|
||||
logger.log('TRACE', f'查询{name}的SRV记录失败')
|
||||
return None
|
||||
else:
|
||||
logger.log('TRACE', f'查询{name}的SRV记录成功')
|
||||
return answers
|
||||
|
||||
def brute(self):
|
||||
"""
|
||||
枚举域名的SRV记录
|
||||
"""
|
||||
names_path = data_storage_path.joinpath('srv_names.json')
|
||||
with open(names_path) as fp:
|
||||
names_dict = json.load(fp)
|
||||
query_map = map(lambda name: name + self.domain, names_dict)
|
||||
|
||||
tasks = []
|
||||
for query in query_map:
|
||||
tasks.append(self.query(query))
|
||||
task_group = asyncio.gather(*tasks, loop=self.loop)
|
||||
self.loop.run_until_complete(asyncio.gather(task_group))
|
||||
self.loop.close()
|
||||
results = task_group.result()
|
||||
for result in results:
|
||||
if result:
|
||||
for answer in result:
|
||||
subdomain = utils.match_subdomain(self.domain, answer.host)
|
||||
if subdomain:
|
||||
self.subdomains = self.subdomains.union(subdomain)
|
||||
else:
|
||||
logger.log('DEBUG', f'{answer.host}不是{self.domain}的子域')
|
||||
if not len(self.subdomains):
|
||||
logger.log('DEBUG', f'没有找到{self.domain}的SRV记录')
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始枚举{self.domain}域的SRV记录')
|
||||
start = time.time()
|
||||
self.brute()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束枚举{self.domain}域的SRV记录')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
brute = BruteSRV(domain)
|
||||
brute.run(rx_queue)
|
||||
logger.log('INFOR', f'{brute.source}模块耗时{brute.elapsed}秒发现{brute.domain}的子域{len(brute.subdomains)}个')
|
||||
logger.log('DEBUG', f'{brute.source}模块发现{brute.domain}的子域 {brute.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,66 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class RiskIQ(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Intelligence'
|
||||
self.source = 'RiskIQQuery'
|
||||
self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
|
||||
self.username = config.riskiq_api_username
|
||||
self.key = config.riskiq_api_key
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'query': self.domain}
|
||||
resp = self.get(url=self.addr, params=params, auth=(self.username, self.key))
|
||||
if not resp:
|
||||
return
|
||||
resp_json = resp.json()
|
||||
subdomains_find = resp_json.get('subdomains')
|
||||
if subdomains_find:
|
||||
self.subdomains = set(map(lambda x: x + '.' + self.domain, subdomains_find))
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
return
|
||||
query = RiskIQ(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,66 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class ThreatBookAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Intelligence'
|
||||
self.source = 'ThreatBookAPIQuery'
|
||||
self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
|
||||
self.key = config.threatbook_api_key
|
||||
|
||||
def query(self, domain):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'apikey': self.key, 'domain': domain, 'field': 'sub_domains'}
|
||||
resp = self.post(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(self.domain, str(resp.json()))
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.key:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query(self.domain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = ThreatBookAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,61 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class ThreatMiner(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Intelligence'
|
||||
self.source = 'ThreatMinerQuery'
|
||||
self.addr = 'https://www.threatminer.org/getData.php'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'e': 'subdomains_container', 'q': self.domain, 't': 0, 'rt': 10}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = ThreatMiner(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,85 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
'''
|
||||
最多查询100条
|
||||
'''
|
||||
|
||||
|
||||
class VirusTotal(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.source = 'VirusTotalQuery'
|
||||
self.module = 'Intelligence'
|
||||
self.addr = 'https://www.virustotal.com/ui/domains/{}/subdomains'
|
||||
self.domain = self.register(domain)
|
||||
|
||||
def query(self, domain):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
next_cursor = ''
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.header.update({'Referer': 'https://www.virustotal.com/',
|
||||
'TE': 'Trailers'})
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'limit': '40', 'cursor': next_cursor}
|
||||
resp = self.get(url=self.addr.format(domain), params=params)
|
||||
if not resp:
|
||||
return
|
||||
resp_json = resp.json()
|
||||
subdomain_find = set()
|
||||
datas = resp_json.get('data')
|
||||
|
||||
if datas:
|
||||
for data in datas:
|
||||
subdomain = data.get('id')
|
||||
if subdomain:
|
||||
subdomain_find.add(subdomain)
|
||||
else:
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
meta = resp_json.get('meta')
|
||||
if meta:
|
||||
next_cursor = meta.get('cursor')
|
||||
else:
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query(self.domain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = VirusTotal(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,68 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class VirusTotalAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Intelligence'
|
||||
self.source = 'VirusTotalAPIQuery'
|
||||
self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'
|
||||
self.key = config.virustotal_api_key
|
||||
|
||||
def query(self, domain):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'apikey': self.key, 'domain': domain}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
resp_json = resp.json()
|
||||
subdomain_find = set(resp_json.get('subdomains'))
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.key:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query(self.domain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
return
|
||||
query = VirusTotalAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,89 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Ask(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'AskSearch'
|
||||
self.addr = 'https://www.search.ask.com/web'
|
||||
self.limit_num = 200 # 限制搜索条数
|
||||
self.per_page_num = 10 # 默认每页显示10页
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 1
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'q': query, 'page': self.page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, resp.text)
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
self.page_num += 1
|
||||
if '>Next<' not in resp.text:
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Ask(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,110 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from bs4 import BeautifulSoup
|
||||
from config import logger
|
||||
|
||||
|
||||
class Baidu(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.module = 'Search'
|
||||
self.source = 'BaiduSearch'
|
||||
self.init = 'https://www.baidu.com/'
|
||||
self.addr = 'https://www.baidu.com/s'
|
||||
self.domain = domain
|
||||
self.limit_num = 750 # 限制搜索条数
|
||||
|
||||
def redirect_match(self, domain, html):
|
||||
"""
|
||||
|
||||
:param domain:
|
||||
:param html:
|
||||
:return:
|
||||
"""
|
||||
bs = BeautifulSoup(html, features='lxml')
|
||||
subdomains_all = set()
|
||||
for find_res in bs.find_all('a', {'class': 'c-showurl'}): # 获取搜索结果中所有的跳转URL地址
|
||||
url = find_res.get('href')
|
||||
subdomain = self.match_location(domain, url)
|
||||
subdomains_all = subdomains_all.union(subdomain)
|
||||
return subdomains_all
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 0 # 二次搜索重新置0
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'wd': query, 'pn': self.page_num, 'rn': self.per_page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
if len(domain) > 12: # 解决百度搜索结果中域名过长会显示不全的问题
|
||||
subdomains_find = self.redirect_match(domain, resp.text) # 获取百度跳转URL响应头的Location字段获取直链
|
||||
else:
|
||||
subdomains_find = self.match(domain, resp.text)
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
if not full_search:
|
||||
if subdomains_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
self.page_num += self.per_page_num
|
||||
if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
if self.page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Baidu(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,96 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Bing(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'BingSearch'
|
||||
self.init = 'https://www.bing.com/'
|
||||
self.addr = 'https://www.bing.com/search'
|
||||
self.limit_num = 1000 # 限制搜索条数
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 0 # 二次搜索重新置0
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(self.init)
|
||||
if not resp:
|
||||
return
|
||||
self.cookie = resp.cookies # 获取cookie bing在搜索时需要带上cookie
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'q': query, 'first': self.page_num, 'count': self.per_page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(domain, resp.text)
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
if not full_search:
|
||||
if subdomains_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
if '<div class="sw_next>' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
self.page_num += self.per_page_num
|
||||
if self.page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Bing(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,98 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class BingAPI(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'BingCustomSearch'
|
||||
self.addr = 'https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search'
|
||||
self.id = config.bing_api_id
|
||||
self.key = config.bing_api_key
|
||||
self.limit_num = 1000 # 必应同一个搜索关键词限制搜索条数
|
||||
self.delay = 1 # 必应自定义搜索限制时延1秒
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 0 # 二次搜索重新置0
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.header = {'Ocp-Apim-Subscription-Key': self.key}
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'q': query, 'customconfig': self.id, 'safesearch': 'Off',
|
||||
'count': self.per_page_num, 'offset': self.page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(domain, str(resp.json()))
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
if not full_search:
|
||||
if subdomains_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
self.page_num += self.per_page_num
|
||||
if self.page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not (self.id and self.key):
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = BingAPI(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,93 @@
|
||||
# coding=utf-8
|
||||
import re
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class DuckDuckGO(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'DuckDuckGoSearch'
|
||||
self.addr = 'https://duckduckgo.com/html/'
|
||||
self.header = self.get_header()
|
||||
self.delay = 2
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
data = {'q': query, 'kl': 'us-en', 'v': 'l'}
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.post(self.addr, data)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, resp.text)
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
try:
|
||||
s = re.findall(r'name="s" value="(\d.*)"', resp.text)[-1]
|
||||
dc = re.findall(r'name="dc" value="(\d.*)"', resp.text)
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
break
|
||||
data.update({'s': s, 'nextParams': '', 'o': 'json', 'dc': dc, 'api': '/d.js'})
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
# return # 暂时还有点问题
|
||||
search = DuckDuckGO(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,93 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import random
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Exalead(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = "ExaleadSearch"
|
||||
self.addr = "http://www.exalead.com/search/web/results/"
|
||||
self.per_page_num = 30
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 0
|
||||
while True:
|
||||
self.delay = random.randint(1, 5)
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'q': query, 'elements_per_page': '30', "start_index": self.page_num}
|
||||
resp = self.get(url=self.addr, params=params)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, resp.text)
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains):
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
self.page_num += self.per_page_num
|
||||
if self.page_num > 1999:
|
||||
break
|
||||
if 'title="Go to the next page"' not in resp.text:
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
statement = statement.replace('-site', 'and -site')
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Exalead(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,71 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import json
|
||||
import base64
|
||||
import config
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class FoFa(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'FoFaSearch'
|
||||
self.addr = 'https://fofa.so/api/v1/search/all'
|
||||
self.delay = 1
|
||||
self.email = config.fofa_api_email
|
||||
self.key = config.fofa_api_key
|
||||
|
||||
def search(self):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
"""
|
||||
self.page_num = 1
|
||||
query_base64 = base64.b64encode(f'domain={self.domain}'.encode('utf-8'))
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = {'email': self.email, 'key': self.key, 'qbase64': query_base64, 'page': self.page_num}
|
||||
resp = self.get(self.addr, query)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
self.page_num += 1
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = FoFa(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,101 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import random
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Google(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'GoogleSearch'
|
||||
self.init = 'https://www.google.com/'
|
||||
self.addr = 'https://www.google.com/search'
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
page_num = 1
|
||||
per_page_num = 50
|
||||
self.header = self.get_header()
|
||||
self.header.update({'User-Agent': 'Googlebot',
|
||||
'Referer': 'https://www.google.com'})
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(self.init)
|
||||
if not resp:
|
||||
return
|
||||
self.cookie = resp.cookies
|
||||
while True:
|
||||
self.delay = random.randint(1, 5)
|
||||
time.sleep(self.delay)
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
word = 'site:' + domain + filtered_subdomain
|
||||
payload = {'q': word, 'start': page_num, 'num': per_page_num,
|
||||
'filter': '0', 'btnG': 'Search', 'gbv': '1', 'hl': 'en'}
|
||||
resp = self.get(url=self.addr, params=payload)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, resp.text)
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains):
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
page_num += per_page_num
|
||||
if 'start='+str(page_num) not in resp.text:
|
||||
break
|
||||
if '302 Moved' in resp.text:
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Google(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,97 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class GoogleAPI(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'GoogleAPISearch'
|
||||
self.addr = 'https://www.googleapis.com/customsearch/v1'
|
||||
self.delay = 1
|
||||
self.key = config.google_api_key
|
||||
self.cx = config.google_api_cx
|
||||
self.per_page_num = 10 # 每次只能请求10个结果
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 1
|
||||
while True:
|
||||
word = 'site:' + domain + filtered_subdomain
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'key': self.key, 'cx': self.cx, 'q': word, 'fields': 'items/link',
|
||||
'start': self.page_num, 'num': self.per_page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, str(resp.json()))
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains):
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
self.page_num += self.per_page_num
|
||||
if self.page_num > 100: # 免费的API只能查询前100条结果
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not (self.cx and self.key):
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = GoogleAPI(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,73 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import queue
|
||||
import config
|
||||
# from shodan import Shodan
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class ShodanAPI(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Search'
|
||||
self.source = 'ShodanSearch'
|
||||
self.addr = 'https://api.shodan.io/shodan/host/search'
|
||||
self.key = config.shodan_api_key
|
||||
|
||||
def search(self):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'hostname:.' + self.domain
|
||||
page = 1
|
||||
while True:
|
||||
params = {'key': self.key, 'page': page, 'query': query, 'minify': True, 'facets': {'hostnames'}}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(self.domain, resp.text)
|
||||
if subdomain_find:
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
page += 1
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.key:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = ShodanAPI(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
results = queue.Queue()
|
||||
do('qq.com', results)
|
||||
@@ -0,0 +1,91 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class So(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'SoSearch'
|
||||
self.addr = 'http://www.so.com/s'
|
||||
self.limit_num = 640 # 限制搜索条数
|
||||
self.per_page_num = 10 # 默认每页显示10页
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
page_num = 1
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
word = 'site:' + domain + filtered_subdomain
|
||||
payload = {'q': word, 'pn': page_num}
|
||||
resp = self.get(url=self.addr, params=payload)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, resp.text)
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
page_num += 1
|
||||
if '<a id="snext"' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
if self.page_num * self.per_page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = So(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,89 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Sogou(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'SogouSearch'
|
||||
self.addr = 'https://www.sogou.com/web'
|
||||
self.limit_num = 1000 # 限制搜索条数
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 1
|
||||
while True:
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
word = 'site:' + domain + filtered_subdomain
|
||||
payload = {'query': word, 'page': self.page_num, "num": self.per_page_num}
|
||||
resp = self.get(self.addr, payload)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, resp.text)
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
self.page_num += 1
|
||||
if '<a id="sogou_next"' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
if self.page_num * self.per_page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Sogou(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,97 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Yahoo(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'YahooSearch'
|
||||
self.init = 'https://hk.search.yahoo.com/'
|
||||
self.addr = 'https://hk.search.yahoo.com/search'
|
||||
self.limit_num = 1000 # 限制搜索条数
|
||||
self.delay = 5
|
||||
self.per_page_num = 40
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 0
|
||||
resp = self.get(self.init)
|
||||
if not resp:
|
||||
return
|
||||
self.cookie = resp.cookies # 获取cookie Yahoo在搜索时需要带上cookie
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'q': query, 'b': self.page_num, 'n': self.per_page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(domain, resp.text)
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
if not full_search:
|
||||
if subdomains_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
if '>Next</a>' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
self.page_num += self.per_page_num
|
||||
if self.page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Yahoo(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,93 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Yandex(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'YandexSearch'
|
||||
self.init = 'https://yandex.com/'
|
||||
self.addr = 'https://yandex.com/search'
|
||||
self.limit_num = 1000 # 限制搜索条数
|
||||
self.delay = 5
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 0 # 二次搜索重新置0
|
||||
self.cookie = self.get(self.init).cookies # 获取cookie bing在搜索时需要带上cookie
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'text': query, 'p': self.page_num, 'numdoc': self.per_page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(domain, resp.text)
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
if not full_search:
|
||||
if subdomains_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
if '>next</a>' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
self.page_num += 1
|
||||
if self.page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Yandex(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,97 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class ZoomEyeAPI(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'ZoomEyeAPISearch'
|
||||
self.addr = 'https://api.zoomeye.org/web/search'
|
||||
self.delay = 2
|
||||
self.user = config.zoomeye_api_username
|
||||
self.pwd = config.zoomeye_api_password
|
||||
|
||||
def login(self):
|
||||
"""
|
||||
登陆获取查询taken
|
||||
:return:
|
||||
"""
|
||||
url = 'https://api.zoomeye.org/user/login'
|
||||
data = {'username': self.user, 'password': self.pwd}
|
||||
resp = self.post(url=url, json=data)
|
||||
if not resp:
|
||||
logger.log('FETAL', f'登录失败无法获取{self.source}的访问token')
|
||||
return
|
||||
resp_json = resp.json()
|
||||
if resp.status_code == 200:
|
||||
# print('登陆成功')
|
||||
return resp_json.get('access_token')
|
||||
else:
|
||||
logger.log('ALERT', resp_json.get('message'))
|
||||
exit(1)
|
||||
|
||||
def search(self):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
"""
|
||||
page_num = 1
|
||||
access_token = self.login()
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
self.header.update({'Authorization': 'JWT ' + access_token})
|
||||
params = {'query': 'hostname:' + self.domain, 'page': page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
page_num += 1
|
||||
if page_num > 500:
|
||||
break
|
||||
if resp.status_code == 403:
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not (self.user and self.pwd):
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = ZoomEyeAPI(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
Reference in New Issue
Block a user