river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-26 04:47:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

重构项目目录结构

Browse Source
This commit is contained in:
Jing Ling
2020-05-08 18:37:07 +08:00
parent e09c8b9dfc
commit 5d64f69348
115 changed files with 5443 additions and 5349 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/autotake/github.py
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
# coding=utf-8
"""
github自动接管
"""
import json
import base64
import requests
from config import api
HEADERS = {
"Accept": "application/json, text/javascript, */*; q=0.01",
"Accept-Language": "zh-CN,zh;q=0.9",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
}
def github_takeover(url):
# 读取config配置文件
repo_name = url
print('[*]正在读取配置文件...')
user = api.github_api_user
token = api.github_api_token
CHECK_HEADERS = {
"Authorization": 'token ' + token,
"Accept": "application/vnd.github.switcheroo-preview+json"
}
repos_url = 'https://api.github.com/repos/' + user + '/' + repo_name
repos_r = requests.get(url=repos_url, headers=CHECK_HEADERS)
# 验证token是否正确
if 'message' in repos_r.json():
if repos_r.json()['message'] == 'Bad credentials':
print('[*]请检查Token是否正确')
elif repos_r.json()['message'] == 'Not Found':
print('[*]正在生成接管库...') # 生成接管库
creat_repo_dict = {
"name": repo_name,
"description": "This is a subdomain takeover Repository",
}
creat_repo_url = 'https://api.github.com/user/repos'
creat_repo_r = requests.post(url=creat_repo_url,
headers=CHECK_HEADERS,
data=json.dumps(creat_repo_dict))
creat_repo_status = creat_repo_r.status_code
if creat_repo_status == 201:
print('[*]创建接管库' + repo_name + '成功，正在进行自动接管...')
# 接管文件生成
# index.html文件
html = b'''
<html>
<p>Subdomain Takerover Test!</>
</html>
'''
html64 = base64.b64encode(html).decode('utf-8')
html_dict = {
"message": "my commit message",
"committer": {
"name": "user", # 提交id，非必改项
"email": "user@163.com" # 同上
},
"content": html64
}
# CNAME文件
cname_url = bytes(url, encoding='utf-8')
cname_url64 = base64.b64encode(cname_url).decode('utf-8')
url_dict = {
"message": "my commit message",
"committer": {
"name": "user",
"email": "user@163.com"
},
"content": cname_url64
}
html_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/index.html'
url_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/CNAME'
html_r = requests.put(url=html_url, data=json.dumps(html_dict),
headers=CHECK_HEADERS) # 上传index.html
cname_r = requests.put(url=url_url, data=json.dumps(url_dict),
headers=CHECK_HEADERS) # 上传CNAME
rs = cname_r.status_code
if rs == 201:
print('[*]生成接管库成功，正在开启Github pages...')
page_url = "https://api.github.com/repos/" + user + "/" + url + "/pages"
page_dict = {
"source": {
"branch": "master"
}
}
page_r = requests.post(url=page_url,
data=json.dumps(page_dict),
headers=CHECK_HEADERS) # 开启page
if page_r.status_code == 201:
print('[+]自动接管成功，请稍后访问http://' + str(url) + '查看结果')
else:
print('[+]开启Github pages失败，请检查网络或稍后重试...')
else:
print('[+]生成接管库失败，请检查网络或稍后重试...')
elif url in repos_r.json()['name']:
print('[*]生成接管库失败，请检查https://github.com/' + user +
'?tab=repositories是否存在同名接管库...')
modules/certificates/censys_api.py
+72
View File
@@ -0,0 +1,72 @@
from config import api
from common.query import Query
from config.log import logger
class CensysAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Certificate'
self.source = "CensysAPIQuery"
self.addr = 'https://www.censys.io/api/v1/search/certificates'
self.id = api.censys_api_id
self.secret = api.censys_api_secret
self.delay = 3.0 # Censys 接口查询速率限制 最快2.5秒查1次
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
data = {
'query': f'parsed.names: {self.domain}',
'page': 1,
'fields': ['parsed.subject_dn', 'parsed.names'],
'flatten': True}
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
if not resp:
return
json = resp.json()
status = json.get('status')
if status != 'ok':
logger.log('ALERT', status)
return
subdomains = self.match(self.domain, str(json))
self.subdomains = self.subdomains.union(subdomains)
pages = json.get('metadata').get('pages')
for page in range(2, pages + 1):
data['page'] = page
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
if not resp:
return
subdomains = self.match(self.domain, str(resp.json()))
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
if not self.check(self.id, self.secret):
return
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = CensysAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/certificates/certspotter.py
+53
View File
@@ -0,0 +1,53 @@
from common import utils
from common.query import Query
class CertSpotter(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = domain
self.module = 'Certificate'
self.source = 'CertSpotterQuery'
self.addr = 'https://api.certspotter.com/v1/issuances'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'domain': self.domain,
'include_subdomains': 'true',
'expand': 'dns_names'}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = utils.match_subdomain(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = CertSpotter(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/certificates/crtsh.py
+49
View File
@@ -0,0 +1,49 @@
from common import utils
from common.query import Query
class Crtsh(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Certificate'
self.source = 'CrtshQuery'
self.addr = 'https://crt.sh/'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'q': f'%.{self.domain}', 'output': 'json'}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = utils.match_subdomain(self.domain, str(resp.json()))
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = Crtsh(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/certificates/entrust.py
+51
View File
@@ -0,0 +1,51 @@
from common import utils
from common.query import Query
class Entrust(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Certificate'
self.source = 'EntrustQuery'
self.addr = 'https://ctsearch.entrust.com/api/v1/certificates'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'fields': 'subjectDN',
'domain': self.domain,
'includeExpired': 'true'}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = utils.match_subdomain(self.domain, str(resp.json()))
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = Entrust(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/certificates/google.py
+53
View File
@@ -0,0 +1,53 @@
from common import utils
from common.query import Query
class Google(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Certificate'
self.source = 'GoogleQuery'
self.addr = 'https://transparencyreport.google.com/' \
'transparencyreport/api/v3/httpsreport/ct/certsearch'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'include_expired': 'true',
'include_subdomains': 'true',
'domain': self.domain}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = utils.match_subdomain(self.domain, resp.text)
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = Google(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/certificates/spyse_api.py
+65
View File
@@ -0,0 +1,65 @@
from config import api
from common import utils
from common.query import Query
class SpyseAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = domain
self.module = 'Certificate'
self.source = 'CertDBAPIQuery'
self.addr = 'https://api.spyse.com/v1/subdomains'
self.token = api.spyse_api_token
def query(self):
"""
向接口查询子域并做子域匹配
"""
page_num = 1
while True:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'domain': self.domain,
'api_token': self.token,
'page': page_num}
resp = self.get(self.addr, params)
if not resp:
return
json = resp.json()
subdomains = utils.match_subdomain(self.domain, str(json))
if not subdomains: # 搜索没有发现子域名则停止搜索
break
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
page_num += 1
# 默认每次查询最多返回30条 当前条数小于30条说明已经查完
if json.get('count') < 30:
break
def run(self):
"""
类执行入口
"""
if not self.check(self.token):
return
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = SpyseAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/check/axfr.py
+97
View File
@@ -0,0 +1,97 @@
"""
查询域名的NS记录(域名服务器记录，记录该域名由哪台域名服务器解析)，检查查出的域名服务器是
否开启DNS域传送，如果开启且没做访问控制和身份验证便加以利用获取域名的所有记录。
DNS域传送(DNS zone transfer)指的是一台备用域名服务器使用来自主域名服务器的数据刷新自己
的域数据库，目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获
取某个域的所有记录。
"""
import dns.resolver
import dns.zone
from common import utils
from common.module import Module
from config.log import logger
class CheckAXFR(Module):
"""
DNS域传送漏洞检查类
"""
def __init__(self, domain: str):
Module.__init__(self)
self.domain = self.register(domain)
self.module = 'Check'
self.source = 'AXFRCheck'
self.results = []
def axfr(self, server):
"""
执行域传送
:param server: 域名服务器
"""
logger.log('DEBUG', f'尝试对{self.domain}的域名服务器{server}进行域传送')
try:
xfr = dns.query.xfr(where=server, zone=self.domain,
timeout=5.0, lifetime=10.0)
zone = dns.zone.from_xfr(xfr)
except Exception as e:
logger.log('DEBUG', e.args)
logger.log('DEBUG', f'对{self.domain}的域名服务器{server}进行域传送失败')
return
names = zone.nodes.keys()
for name in names:
full_domain = str(name) + '.' + self.domain
subdomain = utils.match_subdomain(self.domain, full_domain)
self.subdomains = self.subdomains.union(subdomain)
record = zone[name].to_text(name)
self.results.append(record)
if self.results:
logger.log('DEBUG', f'发现{self.domain}在{server}上的域传送记录')
logger.log('DEBUG', '\n'.join(self.results))
self.results = []
def check(self):
"""
正则匹配响应头中的内容安全策略字段以发现子域名
"""
resolver = utils.dns_resolver()
try:
answers = resolver.query(self.domain, "NS")
except Exception as e:
logger.log('ERROR', e.args)
return
nsservers = [str(answer) for answer in answers]
if not len(nsservers):
logger.log('ALERT', f'没有找到{self.domain}的NS域名服务器记录')
return
for nsserver in nsservers:
self.axfr(nsserver)
def run(self):
"""
类执行入口
"""
self.begin()
self.check()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
check = CheckAXFR(domain)
check.run()
if __name__ == '__main__':
do('ZoneTransfer.me')
# do('example.com')
modules/check/cdx.py
+60
View File
@@ -0,0 +1,60 @@
"""
检查crossdomain.xml文件收集子域名
"""
from common.module import Module
from common import utils
class CheckCDX(Module):
"""
检查crossdomain.xml文件收集子域名
"""
def __init__(self, domain: str):
Module.__init__(self)
self.domain = self.register(domain)
self.module = 'Check'
self.source = "CrossDomainXml"
def check(self):
"""
检查crossdomain.xml收集子域名
"""
urls = [f'http://{self.domain}/crossdomain.xml',
f'https://{self.domain}/crossdomain.xml',
f'http://www.{self.domain}/crossdomain.xml',
f'https://www.{self.domain}/crossdomain.xml']
for url in urls:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
response = self.get(url, check=False)
if not response:
return
if response and len(response.content):
self.subdomains = utils.match_subdomain(self.domain,
response.text)
def run(self):
"""
类执行入口
"""
self.begin()
self.check()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param domain: 域名
"""
check = CheckCDX(domain)
check.run()
if __name__ == '__main__':
do('example.com')
modules/check/cert.py
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env python3
"""
检查域名证书收集子域名
"""
import socket
import ssl
from common import utils
from common.module import Module
from config.log import logger
class CheckCert(Module):
def __init__(self, domain):
Module.__init__(self)
self.domain = self.register(domain)
self.port = 443 # ssl port
self.module = 'Check'
self.source = 'CertInfo'
def check(self):
"""
获取域名证书并匹配证书中的子域名
"""
try:
ctx = ssl.create_default_context()
sock = ctx.wrap_socket(socket.socket(),
server_hostname=self.domain)
sock.connect((self.domain, self.port))
cert_dict = sock.getpeercert()
except Exception as e:
logger.log('DEBUG', e.args)
return
subdomains = utils.match_subdomain(self.domain, str(cert_dict))
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.check()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
check = CheckCert(domain)
check.run()
if __name__ == '__main__':
do('example.com')
modules/check/csp.py
+82
View File
@@ -0,0 +1,82 @@
"""
检查内容安全策略收集子域名收集子域名
"""
import requests
from common import utils
from common.module import Module
from config.log import logger
class CheckCSP(Module):
"""
检查内容安全策略收集子域名
"""
def __init__(self, domain, header):
Module.__init__(self)
self.domain = self.register(domain)
self.module = 'Check'
self.source = 'ContentSecurityPolicy'
self.csp_header = header
def grab_header(self):
"""
抓取请求头
:return: 请求头
"""
csp_header = dict()
urls = [f'http://{self.domain}',
f'https://{self.domain}',
f'http://www.{self.domain}',
f'https://www.{self.domain}']
for url in urls:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
response = self.get(url, check=False)
if response:
csp_header = response.headers
break
return csp_header
def check(self):
"""
正则匹配响应头中的内容安全策略字段以发现子域名
"""
if not self.csp_header:
self.csp_header = self.grab_header()
csp = self.header.get('Content-Security-Policy')
if not self.csp_header:
logger.log('DEBUG', f'获取{self.domain}域的请求头失败')
return
if not csp:
logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段')
return
self.subdomains = utils.match_subdomain(self.domain, csp)
def run(self):
"""
类执行入口
"""
self.begin()
self.check()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain, header=None): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
:param dict or None header: 响应头
"""
check = CheckCSP(domain, header)
check.run()
if __name__ == '__main__':
resp = requests.get('https://content-security-policy.com/')
do('google-analytics.com', resp.headers)
modules/check/robots.py
+59
View File
@@ -0,0 +1,59 @@
"""
检查内容安全策略收集子域名收集子域名
"""
from common.module import Module
from common import utils
class CheckRobots(Module):
"""
检查robots.txt收集子域名
"""
def __init__(self, domain):
Module.__init__(self)
self.domain = self.register(domain)
self.module = 'Check'
self.source = 'Robots'
def check(self):
"""
正则匹配域名的robots.txt文件中的子域
"""
urls = [f'http://{self.domain}/robots.txt',
f'https://{self.domain}/robots.txt',
f'http://www.{self.domain}/robots.txt',
f'https://www.{self.domain}/robots.txt']
for url in urls:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
response = self.get(url, check=False, allow_redirects=False)
if not response:
return
if response and len(response.content):
self.subdomains = utils.match_subdomain(self.domain,
response.text)
def run(self):
"""
类执行入口
"""
self.begin()
self.check()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
check = CheckRobots(domain)
check.run()
if __name__ == '__main__':
do('qq.com')
modules/check/sitemap.py
+73
View File
@@ -0,0 +1,73 @@
"""
检查内容安全策略收集子域名收集子域名
"""
from common.module import Module
from common import utils
class CheckRobots(Module):
"""
检查sitemap收集子域名
"""
def __init__(self, domain):
Module.__init__(self)
self.domain = self.register(domain)
self.module = 'Check'
self.source = 'Sitemap'
def check(self):
"""
正则匹配域名的sitemap文件中的子域
"""
urls = [f'http://{self.domain}/sitemap.xml',
f'https://{self.domain}/sitemap.xml',
f'http://www.{self.domain}/sitemap.xml',
f'https://www.{self.domain}/sitemap.xml',
f'http://{self.domain}/sitemap.txt',
f'https://{self.domain}/sitemap.txt',
f'http://www.{self.domain}/sitemap.txt',
f'https://www.{self.domain}/sitemap.txt',
f'http://{self.domain}/sitemap.html',
f'https://{self.domain}/sitemap.html',
f'http://www.{self.domain}/sitemap.html',
f'https://www.{self.domain}/sitemap.html',
f'http://{self.domain}/sitemap_index.xml',
f'https://{self.domain}/sitemap_index.xml',
f'http://www.{self.domain}/sitemap_index.xml',
f'https://www.{self.domain}/sitemap_index.xml']
for url in urls:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
self.timeout = 10
response = self.get(url, check=False, allow_redirects=False)
if not response:
return
if response and len(response.content):
self.subdomains = utils.match_subdomain(self.domain,
response.text)
def run(self):
"""
类执行入口
"""
self.begin()
self.check()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
check = CheckRobots(domain)
check.run()
if __name__ == '__main__':
do('qq.com')
modules/crawl/archivecrawl.py
+61
View File
@@ -0,0 +1,61 @@
import cdx_toolkit
from common.crawl import Crawl
from config.log import logger
class ArchiveCrawl(Crawl):
def __init__(self, domain):
Crawl.__init__(self)
self.domain = domain
self.module = 'Crawl'
self.source = 'ArchiveCrawl'
def crawl(self, domain, limit):
"""
:param domain:
:param limit:
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
cdx = cdx_toolkit.CDXFetcher(source='ia')
url = f'*.{domain}/*'
size = cdx.get_size_estimate(url)
logger.log('DEBUG', f'{url} ArchiveCrawl size estimate {size}')
for resp in cdx.iter(url, limit=limit):
if resp.data.get('status') not in ['301', '302']:
url = resp.data.get('url')
subdomains = self.match(self.register(domain),
url + resp.text)
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.crawl(self.domain, 50)
# 爬取已发现的子域以发现新的子域
for subdomain in self.subdomains:
if subdomain != self.domain:
self.crawl(subdomain, 10)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
crawl = ArchiveCrawl(domain)
crawl.run()
if __name__ == '__main__':
do('example.com')
modules/crawl/commoncrawl.py
+60
View File
@@ -0,0 +1,60 @@
import cdx_toolkit
from tqdm import tqdm
from common.crawl import Crawl
class CommonCrawl(Crawl):
def __init__(self, domain):
Crawl.__init__(self)
self.domain = domain
self.module = 'Crawl'
self.source = 'CommonCrawl'
def crawl(self, domain, limit):
"""
:param domain:
:param limit:
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
cdx = cdx_toolkit.CDXFetcher()
url = f'*.{domain}/*'
size = cdx.get_size_estimate(url)
print(url, 'CommonCrawl size estimate', size)
for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
if resp.data.get('status') not in ['301', '302']:
subdomains = self.match(self.register(domain), resp.text)
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.crawl(self.domain, 50)
# 爬取已发现的子域以发现新的子域
for subdomain in self.subdomains:
if subdomain != self.domain:
self.crawl(subdomain, 10)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
crawl = CommonCrawl(domain)
crawl.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/binaryedge_api.py
+53
View File
@@ -0,0 +1,53 @@
from config import api
from common.query import Query
class BinaryEdgeAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'BinaryEdgeAPIQuery'
self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/'
self.api = api.binaryedge_api
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.header.update({'X-Key': self.api})
self.proxy = self.get_proxy(self.source)
url = self.addr + self.domain
resp = self.get(url)
if not resp:
return
subdomains = self.match(self.domain, str(resp.json()))
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
if not self.check(self.api):
return
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = BinaryEdgeAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/bufferover.py
+57
View File
@@ -0,0 +1,57 @@
import cloudscraper
from common.query import Query
from config.log import logger
class BufferOver(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'BufferOverQuery'
self.addr = 'https://dns.bufferover.run/dns?q='
def query(self):
"""
向接口查询子域并做子域匹配
"""
# 绕过cloudFlare验证
scraper = cloudscraper.create_scraper()
scraper.interpreter = 'js2py'
scraper.proxies = self.get_proxy(self.source)
url = self.addr + self.domain
try:
resp = scraper.get(url, timeout=self.timeout)
except Exception as e:
logger.log('ERROR', e.args)
return
if resp.status_code != 200:
return
subdomains = self.match(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = BufferOver(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/cebaidu.py
+49
View File
@@ -0,0 +1,49 @@
from common.query import Query
class CeBaidu(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'CeBaiduQuery'
self.addr = 'https://ce.baidu.com/index/getRelatedSites'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'site_address': self.domain}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = CeBaidu(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/chinaz.py
+49
View File
@@ -0,0 +1,49 @@
from common.query import Query
class Chinaz(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'ChinazQuery'
self.addr = 'https://alexa.chinaz.com/'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
self.addr = self.addr + self.domain
resp = self.get(self.addr)
if not resp:
return
subdomains = self.match(self.domain, resp.text)
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = Chinaz(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/chinaz_api.py
+53
View File
@@ -0,0 +1,53 @@
from config import api
from common.query import Query
class ChinazAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'ChinazAPIQuery'
self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa'
self.api = api.chinaz_api
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'key': self.api, 'domainName': self.domain}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
if not self.check(self.api):
return
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = ChinazAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/circl_api.py
+53
View File
@@ -0,0 +1,53 @@
from config import api
from common.query import Query
class CirclAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'CirclAPIQuery'
self.addr = 'https://www.circl.lu/pdns/query/'
self.user = api.circl_api_username
self.pwd = api.circl_api_password
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
if not resp:
return
subdomains = self.match(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
if not self.check(self.user, self.pwd):
return
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = CirclAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/dnsdb_api.py
+56
View File
@@ -0,0 +1,56 @@
from config import api
from common import utils
from common.query import Query
class DNSdbAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'DNSdbAPIQuery'
self.addr = 'https://api.dnsdb.info/lookup/rrset/name/'
self.api = api.dnsdb_api_key
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.header.update({'X-API-Key': self.api})
self.proxy = self.get_proxy(self.source)
url = f'{self.addr}*.{self.domain}'
resp = self.get(url)
if not resp:
return
subdomains = utils.match_subdomain(self.domain, resp.text)
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
if not self.check(self.api):
return
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = DNSdbAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/dnsdumpster.py
+58
View File
@@ -0,0 +1,58 @@
from common import utils
from common.query import Query
class DNSdumpster(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = "DNSdumpsterQuery"
self.addr = 'https://dnsdumpster.com/'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.header.update({'Referer': 'https://dnsdumpster.com'})
self.proxy = self.get_proxy(self.source)
resp = self.get(self.addr)
if not resp:
return
self.cookie = resp.cookies
data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'),
'targetip': self.domain}
resp = self.post(self.addr, data)
if not resp:
return
subdomains = utils.match_subdomain(self.domain, resp.text)
if subdomains:
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = DNSdumpster(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/hackertarget.py
+52
View File
@@ -0,0 +1,52 @@
from common import utils
from common.query import Query
class HackerTarget(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = "HackerTargetQuery"
self.addr = 'https://api.hackertarget.com/hostsearch/'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'q': self.domain}
resp = self.get(self.addr, params)
if not resp:
return
if resp.status_code == 200:
subdomains = utils.match_subdomain(self.domain, resp.text)
if subdomains:
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = HackerTarget(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/ip138.py
+49
View File
@@ -0,0 +1,49 @@
from common.query import Query
class IP138(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'IP138Query'
self.addr = 'https://site.ip138.com/{domain}/domain.htm'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
self.addr = self.addr.format(domain=self.domain)
resp = self.get(self.addr)
if not resp:
return
subdomains = self.match(self.domain, resp.text)
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = IP138(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/ipv4info_api.py
+75
View File
@@ -0,0 +1,75 @@
from config import api
from common.query import Query
from config.log import logger
class IPv4InfoAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'IPv4InfoAPIQuery'
self.addr = ' http://ipv4info.com/api_v1/'
self.api = api.ipv4info_api_key
def query(self):
"""
向接口查询子域并做子域匹配
"""
page = 0
while True:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'type': 'SUBDOMAINS', 'key': self.api,
'value': self.domain, 'page': page}
resp = self.get(self.addr, params)
if not resp:
return
if resp.status_code != 200:
break # 请求不正常通常网络是有问题，不再继续请求下去
try:
json = resp.json()
except Exception as e:
logger.log('DEBUG', e.args)
break
subdomains = self.match(self.domain, str(json))
if not subdomains:
break
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
# 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
subdomains = json.get('Subdomains')
if subdomains:
# ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
if len(subdomains) < 300:
break
page += 1
if page >= 50: # ipv4info子域查询接口最多允许查询50页
break
def run(self):
"""
类执行入口
"""
if not self.check(self.api):
return
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = IPv4InfoAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/netcraft.py
+83
View File
@@ -0,0 +1,83 @@
import hashlib
import re
import time
from urllib import parse
from common.query import Query
class NetCraft(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'NetCraftQuery'
self.init = 'https://searchdns.netcraft.com/'
self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains'
self.page_num = 1
self.per_page_num = 20
def bypass_verification(self):
"""
绕过NetCraft的JS验证
"""
self.header = self.get_header() # Netcraft会检查User-Agent
resp = self.get(self.init)
if not resp:
return False
self.cookie = resp.cookies
cookie_value = self.cookie['netcraft_js_verification_challenge']
cookie_encode = parse.unquote(cookie_value).encode('utf-8')
verify_taken = hashlib.sha1(cookie_encode).hexdigest()
self.cookie['netcraft_js_verification_response'] = verify_taken
return True
def query(self):
"""
向接口查询子域并做子域匹配
"""
if not self.bypass_verification():
return
last = ''
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'host': '*.' + self.domain, 'from': self.page_num}
resp = self.get(self.addr + last, params)
if not resp:
return
subdomains = self.match(self.domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
if 'Next page' not in resp.text: # 搜索页面没有出现下一页时停止搜索
break
last = re.search(r'&last=.*' + self.domain, resp.text).group(0)
self.page_num += self.per_page_num
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = NetCraft(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/passivedns_api.py
+54
View File
@@ -0,0 +1,54 @@
from config import api
from common.query import Query
class PassiveDnsAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'PassiveDnsQuery'
self.addr = api.passivedns_api_addr or 'http://api.passivedns.cn'
self.token = api.passivedns_api_token
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.header.update({'X-AuthToken': self.token})
self.proxy = self.get_proxy(self.source)
url = self.addr + '/flint/rrset/*.' + self.domain
resp = self.get(url)
if not resp:
return
subdomains = self.match(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
if not self.check(self.addr):
return
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = PassiveDnsAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/ptrarchive.py
+56
View File
@@ -0,0 +1,56 @@
import random
from common import utils
from common.query import Query
class PTRArchive(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = "PTRArchiveQuery"
self.addr = 'http://ptrarchive.com/tools/search4.htm'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
# 绕过主页前端JS验证
self.cookie = {'pa_id': str(random.randint(0, 1000000000))}
params = {'label': self.domain, 'date': 'ALL'}
resp = self.get(self.addr, params)
if not resp:
return
if resp.status_code == 200:
subdomains = utils.match_subdomain(self.domain, resp.text)
if subdomains:
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = PTRArchive(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/qianxun.py
+61
View File
@@ -0,0 +1,61 @@
from common.query import Query
class QianXun(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = domain
self.module = 'Query'
self.source = 'QianXunQuery'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
num = 1
while True:
data = {'ecmsfrom': '',
'show': '',
'num': '',
'classid': '0',
'keywords': self.domain}
url = f'https://www.dnsscan.cn/dns.html?' \
f'keywords={self.domain}&page={num}'
resp = self.post(url, data)
if not resp:
break
subdomains = self.match(self.domain, resp.text)
self.subdomains = self.subdomains.union(subdomains)
if '<div id="page" class="pagelist">' not in resp.text:
break
if '<li class="disabled"><span>&raquo;</span></li>' in resp.text:
break
num += 1
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = QianXun(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/rapiddns.py
+50
View File
@@ -0,0 +1,50 @@
from common import utils
from common.query import Query
class RapidDNS(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'RapidDNSQuery'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
url = f'http://rapiddns.io/subdomain/{self.domain}'
params = {'full': '1'}
resp = self.get(url, params)
if not resp:
return
subdomains = utils.match_subdomain(self.domain, resp.text)
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = RapidDNS(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/riddler.py
+49
View File
@@ -0,0 +1,49 @@
from common.query import Query
class Riddler(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'RiddlerQuery'
self.addr = 'https://riddler.io/search'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'q': 'pld:' + self.domain}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match(self.domain, resp.text)
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = Riddler(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/robtex.py
+63
View File
@@ -0,0 +1,63 @@
import json
import time
from common.query import Query
class Robtex(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = "RobtexQuery"
self.addr = 'https://freeapi.robtex.com/pdns/'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
url = self.addr + 'forward/' + self.domain
resp = self.get(url)
if not resp:
return
text_list = resp.text.splitlines()
text_json = list(map(lambda x: json.loads(x), text_list))
for record in text_json:
if record.get('rrtype') in ['A', 'AAAA']:
time.sleep(self.delay) # Robtex有查询频率限制
ip = record.get('rrdata')
url = self.addr + 'reverse/' + ip
resp = self.get(url)
if not resp:
return
subdomains = self.match(self.domain, resp.text)
if subdomains:
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = Robtex(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/securitytrails_api.py
+57
View File
@@ -0,0 +1,57 @@
from config import api
from common.query import Query
class SecurityTrailsAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'SecurityTrailsAPIQuery'
self.addr = 'https://api.securitytrails.com/v1/domain/'
self.api = api.securitytrails_api
self.delay = 2 # SecurityTrails查询时延至少2秒
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'apikey': self.api}
url = f'{self.addr}{self.domain}/subdomains'
resp = self.get(url, params)
if not resp:
return
prefixs = resp.json()['subdomains']
subdomains = [f'{prefix}.{self.domain}' for prefix in prefixs]
if subdomains:
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
if not self.check(self.api):
return
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = SecurityTrailsAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/sitedossier.py
+60
View File
@@ -0,0 +1,60 @@
from common.query import Query
class SiteDossier(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'SiteDossierQuery'
self.addr = 'http://www.sitedossier.com/parentdomain/'
self.delay = 2
self.page_num = 1
self.per_page_num = 100
def query(self):
"""
向接口查询子域并做子域匹配
"""
while True:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
url = f'{self.addr}{self.domain}/{self.page_num}'
resp = self.get(url)
if not resp:
return
subdomains = self.match(self.domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
# 搜索页面没有出现下一页时停止搜索
if 'Show next 100 items' not in resp.text:
break
self.page_num += self.per_page_num
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = SiteDossier(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/datasets/threatcrowd.py
+56
View File
@@ -0,0 +1,56 @@
import cloudscraper
from common.query import Query
from config.log import logger
class ThreatCrowd(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'ThreatCrowdQuery'
self.addr = 'https://www.threatcrowd.org/searchApi' \
'/v2/domain/report?domain='
def query(self):
# 绕过cloudFlare验证
scraper = cloudscraper.create_scraper()
scraper.interpreter = 'js2py'
scraper.proxies = self.get_proxy(self.source)
url = self.addr + self.domain
try:
resp = scraper.get(url, timeout=self.timeout)
except Exception as e:
logger.log('ERROR', e.args)
return
if resp.status_code != 200:
return
subdomains = self.match(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = ThreatCrowd(domain)
query.run()
if __name__ == '__main__':
do('mi.com')
modules/datasets/wzpc.py
+66
View File
@@ -0,0 +1,66 @@
import time
from config.log import logger
from common.query import Query
class WZPCQuery(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'WZPCQuery'
def query(self):
"""
向接口查询子域并做子域匹配
"""
base_addr = 'http://114.55.181.28/check_web/' \
'databaseInfo_mainSearch.action'
page_num = 1
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'isSearch': 'true', 'searchType': 'url',
'term': self.domain, 'pageNo': page_num}
try:
resp = self.get(base_addr, params)
except Exception as e:
logger.log('ERROR', e.args)
break
if not resp:
break
subdomains = self.match(self.domain, resp.text)
self.subdomains = self.subdomains.union(subdomains)
if not subdomains:
break
if page_num > 10:
break
page_num += 1
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain):
"""
类统一调用入口
:param str domain: 域名
"""
query = WZPCQuery(domain)
query.run()
if __name__ == '__main__':
do('sc.gov.cn')
do('bkzy.org')
modules/datasets/ximcx.py
+50
View File
@@ -0,0 +1,50 @@
from common.query import Query
class Ximcx(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Dataset'
self.source = 'XimcxQuery'
self.addr = 'http://sbd.ximcx.cn/DomainServlet'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
data = {'domain': self.domain}
resp = self.post(self.addr, data=data)
if not resp:
return
json = resp.json()
subdomains = self.match(self.domain, str(json))
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = Ximcx(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/dnsquery/mx.py
+35
View File
@@ -0,0 +1,35 @@
from common.lookup import Lookup
class QueryMX(Lookup):
def __init__(self, domain):
Lookup.__init__(self)
self.domain = self.register(domain)
self.module = 'dnsquery'
self.source = "QueryMX"
self.type = 'MX' # 利用的DNS记录的MX记录收集子域
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
brute = QueryMX(domain)
brute.run()
if __name__ == '__main__':
do('cuit.edu.cn')
modules/dnsquery/ns.py
+35
View File
@@ -0,0 +1,35 @@
from common.lookup import Lookup
class QueryNS(Lookup):
def __init__(self, domain):
Lookup.__init__(self)
self.domain = self.register(domain)
self.module = 'dnsquery'
self.source = "QueryNS"
self.type = 'NS' # 利用的DNS记录的NS记录收集子域
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
brute = QueryNS(domain)
brute.run()
if __name__ == '__main__':
do('cuit.edu.cn')
modules/dnsquery/soa.py
+35
View File
@@ -0,0 +1,35 @@
from common.lookup import Lookup
class QuerySOA(Lookup):
def __init__(self, domain):
Lookup.__init__(self)
self.domain = self.register(domain)
self.module = 'dnsquery'
self.source = "QuerySOA"
self.type = 'SOA' # 利用的DNS记录的SOA记录收集子域
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
brute = QuerySOA(domain)
brute.run()
if __name__ == '__main__':
do('cuit.edu.cn')
modules/dnsquery/srv.py
+95
View File
@@ -0,0 +1,95 @@
"""
通过枚举域名常见的SRV记录并做查询来发现子域
"""
import json
import queue
import threading
from common import utils
from common.module import Module
from config.setting import data_storage_dir
class BruteSRV(Module):
def __init__(self, domain):
Module.__init__(self)
self.domain = self.register(domain)
self.module = 'dnsquery'
self.source = "BruteSRV"
self.type = 'SRV' # 利用的DNS记录的SRV记录查询子域
self.thread_num = 10
self.names_que = queue.Queue()
self.answers_que = queue.Queue()
def gen_names(self):
path = data_storage_dir.joinpath('srv_prefixes.json')
with open(path, encoding='utf-8', errors='ignore') as file:
prefixes = json.load(file)
names = map(lambda prefix: prefix + self.domain, prefixes)
for name in names:
self.names_que.put(name)
def brute(self):
"""
枚举域名的SRV记录
"""
self.gen_names()
for i in range(self.thread_num):
thread = BruteThread(self.names_que, self.answers_que)
thread.daemon = True
thread.start()
self.names_que.join()
while not self.answers_que.empty():
answer = self.answers_que.get()
if answer is None:
continue
for item in answer:
record = str(item)
subdomains = utils.match_subdomain(self.domain, record)
self.subdomains = self.subdomains.union(subdomains)
self.gen_record(subdomains, record)
def run(self):
"""
类执行入口
"""
self.begin()
self.brute()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
class BruteThread(threading.Thread):
def __init__(self, names_que, answers_que):
threading.Thread.__init__(self)
self.names_que = names_que
self.answers_que = answers_que
def run(self):
while True:
name = self.names_que.get()
answer = utils.dns_query(name, 'SRV')
self.answers_que.put(answer)
self.names_que.task_done()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
brute = BruteSRV(domain)
brute.run()
if __name__ == '__main__':
do('zonetransfer.me')
# do('example.com')
modules/dnsquery/txt.py
+35
View File
@@ -0,0 +1,35 @@
from common.lookup import Lookup
class QueryTXT(Lookup):
def __init__(self, domain):
Lookup.__init__(self)
self.domain = self.register(domain)
self.module = 'dnsquery'
self.source = "QueryTXT"
self.type = 'TXT' # 利用的DNS记录的TXT记录收集子域
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
brute = QueryTXT(domain)
brute.run()
if __name__ == '__main__':
do('cuit.edu.cn')
modules/intelligence/alienvault.py
+58
View File
@@ -0,0 +1,58 @@
from common.query import Query
class AlienVault(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Intelligence'
self.source = 'AlienVaultQuery'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
base = 'https://otx.alienvault.com/api/v1/indicators/domain'
dns = f'{base}/{self.domain}/passive_dns'
resp = self.get(dns)
if not resp:
return
json = resp.json()
subdomains = self.match(self.domain, str(json))
self.subdomains = self.subdomains.union(subdomains)
url = f'{base}/{self.domain}/url_list'
resp = self.get(url)
if not resp:
return
json = resp.json()
subdomains = self.match(self.domain, str(json))
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = AlienVault(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/intelligence/riskiq_api.py
+56
View File
@@ -0,0 +1,56 @@
from config import api
from common.query import Query
class RiskIQ(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Intelligence'
self.source = 'RiskIQAPIQuery'
self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
self.user = api.riskiq_api_username
self.key = api.riskiq_api_key
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'query': self.domain}
resp = self.get(url=self.addr,
params=params,
auth=(self.user, self.key))
if not resp:
return
data = resp.json()
names = data.get('subdomains')
self.subdomains = set(map(lambda sub: f'{sub}.{self.domain}', names))
def run(self):
"""
类执行入口
"""
if not self.check(self.user, self.key):
return
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = RiskIQ(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/intelligence/threatbook_api.py
+54
View File
@@ -0,0 +1,54 @@
from config import api
from common.query import Query
class ThreatBookAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Intelligence'
self.source = 'ThreatBookAPIQuery'
self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
self.key = api.threatbook_api_key
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'apikey': self.key,
'domain': self.domain,
'field': 'sub_domains'}
resp = self.post(self.addr, params)
if not resp:
return
subdomains = self.match(self.domain, str(resp.json()))
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
if not self.check(self.key):
return
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = ThreatBookAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/intelligence/threatminer.py
+50
View File
@@ -0,0 +1,50 @@
from common.query import Query
class ThreatMiner(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Intelligence'
self.source = 'ThreatMinerQuery'
self.addr = 'https://www.threatminer.org/getData.php'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'e': 'subdomains_container',
'q': self.domain, 't': 0, 'rt': 10}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match(self.domain, resp.text)
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = ThreatMiner(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/intelligence/virustotal.py
+71
View File
@@ -0,0 +1,71 @@
from common.query import Query
'''
最多查询100条
'''
class VirusTotal(Query):
def __init__(self, domain):
Query.__init__(self)
self.source = 'VirusTotalQuery'
self.module = 'Intelligence'
self.addr = 'https://www.virustotal.com/ui/domains/{}/subdomains'
self.domain = self.register(domain)
def query(self):
"""
向接口查询子域并做子域匹配
"""
next_cursor = ''
while True:
self.header = self.get_header()
self.header.update({'Referer': 'https://www.virustotal.com/',
'TE': 'Trailers'})
self.proxy = self.get_proxy(self.source)
params = {'limit': '40', 'cursor': next_cursor}
resp = self.get(url=self.addr.format(self.domain), params=params)
if not resp:
return
data = resp.json()
subdomains = set()
datas = data.get('data')
if datas:
for data in datas:
subdomain = data.get('id')
if subdomain:
subdomains.add(subdomain)
else:
break
self.subdomains = self.subdomains.union(subdomains)
meta = data.get('meta')
if meta:
next_cursor = meta.get('cursor')
else:
break
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = VirusTotal(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/intelligence/virustotal_api.py
+55
View File
@@ -0,0 +1,55 @@
from config import api
from common.query import Query
class VirusTotalAPI(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = self.register(domain)
self.module = 'Intelligence'
self.source = 'VirusTotalAPIQuery'
self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'
self.key = api.virustotal_api_key
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'apikey': self.key, 'domain': self.domain}
resp = self.get(self.addr, params)
if not resp:
return
json = resp.json()
data = json.get('subdomains')
if data:
subdomains = set(data)
self.subdomains = self.subdomains.union(subdomains)
def run(self):
"""
类执行入口
"""
if not self.check(self.key):
return
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = VirusTotalAPI(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/search/ask.py
+81
View File
@@ -0,0 +1,81 @@
import time
from common.search import Search
class Ask(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = domain
self.module = 'Search'
self.source = 'AskSearch'
self.addr = 'https://www.search.ask.com/web'
self.limit_num = 200 # 限制搜索条数
self.per_page_num = 10 # 默认每页显示10页
def search(self, domain, filtered_subdomain='', full_search=False):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 1
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
query = 'site:.' + domain + filtered_subdomain
params = {'q': query, 'page': self.page_num}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match(domain, resp.text)
if not subdomains:
break
if not full_search:
# 搜索中发现搜索出的结果有完全重复的结果就停止搜索
if subdomains.issubset(self.subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
self.page_num += 1
if '>Next<' not in resp.text:
break
def run(self):
"""
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
self.search(subdomain)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = Ask(domain)
search.run()
if __name__ == '__main__':
do('example.com')
modules/search/baidu.py
+109
View File
@@ -0,0 +1,109 @@
import time
from bs4 import BeautifulSoup
from common.search import Search
class Baidu(Search):
def __init__(self, domain):
Search.__init__(self)
self.module = 'Search'
self.source = 'BaiduSearch'
self.init = 'https://www.baidu.com/'
self.addr = 'https://www.baidu.com/s'
self.domain = domain
self.limit_num = 750 # 限制搜索条数
def redirect_match(self, domain, html):
"""
获取跳转地址并传递地址进行跳转head请求
:param domain: 域名
:param html: 响应体
:return: 子域
"""
bs = BeautifulSoup(html, 'html.parser')
subdomains_all = set()
# 获取搜索结果中所有的跳转URL地址
for find_res in bs.find_all('a', {'class': 'c-showurl'}):
url = find_res.get('href')
subdomains = self.match_location(domain, url)
subdomains_all = subdomains_all.union(subdomains)
return subdomains_all
def search(self, domain, filtered_subdomain='', full_search=False):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 0 # 二次搜索重新置0
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
query = 'site:.' + domain + filtered_subdomain
params = {'wd': query,
'pn': self.page_num,
'rn': self.per_page_num}
resp = self.get(self.addr, params)
if not resp:
return
if len(domain) > 12: # 解决百度搜索结果中域名过长会显示不全的问题
# 获取百度跳转URL响应头的Location字段获取直链
subdomains = self.redirect_match(domain, resp.text)
else:
subdomains = self.match(domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
if not full_search:
# 搜索中发现搜索出的结果有完全重复的结果就停止搜索
if subdomains.issubset(self.subdomains):
break
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
self.page_num += self.per_page_num
# 搜索页面没有出现下一页时停止搜索
if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:
break
if self.page_num >= self.limit_num: # 搜索条数限制
break
def run(self):
"""
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
self.search(subdomain)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = Baidu(domain)
search.run()
if __name__ == '__main__':
do('huayunshuzi.com')
modules/search/bing.py
+92
View File
@@ -0,0 +1,92 @@
import time
from common.search import Search
class Bing(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = domain
self.module = 'Search'
self.source = 'BingSearch'
self.init = 'https://www.bing.com/'
self.addr = 'https://www.bing.com/search'
self.limit_num = 1000 # 限制搜索条数
def search(self, domain, filtered_subdomain='', full_search=False):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 0 # 二次搜索重新置0
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
resp = self.get(self.init)
if not resp:
return
self.cookie = resp.cookies # 获取cookie bing在搜索时需要带上cookie
while True:
time.sleep(self.delay)
self.proxy = self.get_proxy(self.source)
query = 'site:.' + domain + filtered_subdomain
params = {'q': query, 'first': self.page_num,
'count': self.per_page_num}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match(domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
if not full_search:
# 搜索中发现搜索出的结果有完全重复的结果就停止搜索
if subdomains.issubset(self.subdomains):
break
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
# 搜索页面没有出现下一页时停止搜索
if '<div class="sw_next">' not in resp.text:
break
self.page_num += self.per_page_num
if self.page_num >= self.limit_num: # 搜索条数限制
break
def run(self):
"""
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
self.search(subdomain)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = Bing(domain)
search.run()
if __name__ == '__main__':
do('example.com')
modules/search/bing_api.py
+92
View File
@@ -0,0 +1,92 @@
import time
from config import api
from common.search import Search
class BingAPI(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = domain
self.module = 'Search'
self.source = 'BingAPISearch'
self.addr = 'https://api.cognitive.microsoft.com/' \
'bing/v7.0/search'
self.id = api.bing_api_id
self.key = api.bing_api_key
self.limit_num = 1000 # 必应同一个搜索关键词限制搜索条数
self.delay = 1 # 必应自定义搜索限制时延1秒
def search(self, domain, filtered_subdomain='', full_search=False):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 0 # 二次搜索重新置0
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.header = {'Ocp-Apim-Subscription-Key': self.key}
self.proxy = self.get_proxy(self.source)
query = 'site:.' + domain + filtered_subdomain
params = {'q': query, 'safesearch': 'Off',
'count': self.per_page_num,
'offset': self.page_num}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match(domain, str(resp.json()))
if not subdomains: # 搜索没有发现子域名则停止搜索
break
if not full_search:
# 搜索中发现搜索出的结果有完全重复的结果就停止搜索
if subdomains.issubset(self.subdomains):
break
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
self.page_num += self.per_page_num
if self.page_num >= self.limit_num: # 搜索条数限制
break
def run(self):
"""
类执行入口
"""
if not self.check(self.id, self.key):
return
self.begin()
self.search(self.domain, full_search=True)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
self.search(subdomain)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = BingAPI(domain)
search.run()
if __name__ == '__main__':
do('example.com')
modules/search/exalead.py
+87
View File
@@ -0,0 +1,87 @@
import random
import time
from common.search import Search
class Exalead(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = domain
self.module = 'Search'
self.source = "ExaleadSearch"
self.addr = "http://www.exalead.com/search/web/results/"
self.per_page_num = 30
def search(self, domain, filtered_subdomain='', full_search=False):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 0
while True:
self.delay = random.randint(1, 5)
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
query = 'site:.' + domain + filtered_subdomain
params = {'q': query, 'elements_per_page': '30',
"start_index": self.page_num}
resp = self.get(url=self.addr, params=params)
if not resp:
return
subdomains = self.match(domain, resp.text)
if not subdomains:
break
if not full_search:
if subdomains.issubset(self.subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
self.page_num += self.per_page_num
if self.page_num > 1999:
break
if 'title="Go to the next page"' not in resp.text:
break
def run(self):
"""
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
statement = statement.replace('-site', 'and -site')
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
self.search(subdomain)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = Exalead(domain)
search.run()
if __name__ == '__main__':
do('example.com')
modules/search/fofa_api.py
+73
View File
@@ -0,0 +1,73 @@
import base64
import time
from config import api
from common.search import Search
class FoFa(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = domain
self.module = 'Search'
self.source = 'FoFaAPISearch'
self.addr = 'https://fofa.so/api/v1/search/all'
self.delay = 1
self.email = api.fofa_api_email
self.key = api.fofa_api_key
def search(self):
"""
发送搜索请求并做子域匹配
"""
self.page_num = 1
subdomain_encode = f'domain={self.domain}'.encode('utf-8')
query_data = base64.b64encode(subdomain_encode)
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
query = {'email': self.email,
'key': self.key,
'qbase64': query_data,
'page': self.page_num,
'size': 10000}
resp = self.get(self.addr, query)
if not resp:
return
resp_json = resp.json()
subdomains = self.match(self.domain, str(resp_json))
if not subdomains: # 搜索没有发现子域名则停止搜索
break
self.subdomains = self.subdomains.union(subdomains)
size = resp_json.get('size')
if size < 10000:
break
self.page_num += 1
def run(self):
"""
类执行入口
"""
if not self.check(self.email, self.key):
return
self.begin()
self.search()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = FoFa(domain)
search.run()
if __name__ == '__main__':
do('example.com')
modules/search/gitee.py
+74
View File
@@ -0,0 +1,74 @@
import time
from bs4 import BeautifulSoup
from common.search import Search
from config.log import logger
class Gitee(Search):
def __init__(self, domain):
Search.__init__(self)
self.source = 'GiteeSearch'
self.module = 'Search'
self.addr = 'https://search.gitee.com/'
self.domain = self.register(domain)
self.header = self.get_header()
def search(self, full_search=False):
"""
向接口查询子域并做子域匹配
"""
page_num = 1
while True:
time.sleep(self.delay)
params = {'pageno': page_num, 'q': self.domain, 'type': 'code'}
try:
resp = self.get(self.addr, params=params)
except Exception as e:
logger.log('ERROR', e.args)
break
if not resp:
break
if resp.status_code != 200:
logger.log('ERROR', f'{self.source}模块搜索出错')
break
if 'class="empty-box"' in resp.text:
break
soup = BeautifulSoup(resp.text, 'html.parser')
subdomains = self.match(self.domain, soup.text)
self.subdomains = self.subdomains.union(subdomains)
if not subdomains:
break
if not full_search:
# 搜索中发现搜索出的结果有完全重复的结果就停止搜索
if subdomains.issubset(self.subdomains):
break
if '<li class="disabled"><a href="###">' in resp.text:
break
if page_num > 100:
break
page_num += 1
def run(self):
"""
类执行入口
"""
self.begin()
self.search()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = Gitee(domain)
query.run()
if __name__ == '__main__':
do('example.com')
modules/search/github_api.py
+107
View File
@@ -0,0 +1,107 @@
import requests
from config import api
from common.utils import match_subdomain
from common.search import Search
from config.log import logger
class GithubAPI(Search):
def __init__(self, domain):
Search.__init__(self)
self.source = 'GithubAPISearch'
self.module = 'Search'
self.addr = 'https://api.github.com/search/code'
self.domain = self.register(domain)
self.session = requests.Session()
self.auth_url = 'https://api.github.com'
self.token = api.github_api_token
def auth_github(self):
"""
github api 认证
:return: 认证失败返回False 成功返回True
"""
self.session.headers.update({'Authorization': 'token ' + self.token})
try:
resp = self.session.get(self.auth_url)
except Exception as e:
logger.log('ERROR', e.args)
return False
if resp.status_code != 200:
resp_json = resp.json()
msg = resp_json.get('message')
logger.log('ERROR', msg)
return False
else:
return True
def search(self):
"""
向接口查询子域并做子域匹配
"""
self.session.headers = self.get_header()
self.session.proxies = self.get_proxy(self.source)
self.session.verify = self.verify
self.session.headers.update(
{'Accept': 'application/vnd.github.v3.text-match+json'})
if not self.auth_github():
logger.log('ERROR', f'{self.source}模块登录失败')
return
page = 1
while True:
params = {'q': self.domain, 'per_page': 100,
'page': page, 'sort': 'indexed'}
try:
resp = self.session.get(self.addr, params=params)
except Exception as e:
logger.log('ERROR', e.args)
break
if resp.status_code != 200:
logger.log('ERROR', f'{self.source}模块搜索出错')
break
subdomains = match_subdomain(self.domain, resp.text)
if not subdomains:
break
self.subdomains = self.subdomains.union(subdomains)
page += 1
try:
resp_json = resp.json()
except Exception as e:
logger.log('ERROR', e.args)
break
total_count = resp_json.get('total_count')
if not isinstance(total_count, int):
break
if page * 100 > total_count:
break
if page * 100 > 1000:
break
def run(self):
"""
类执行入口
"""
if not self.check(self.token):
return
self.begin()
self.search()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
query = GithubAPI(domain)
query.run()
if __name__ == '__main__':
do('exmaple.com')
modules/search/google.py
+94
View File
@@ -0,0 +1,94 @@
import random
import time
from common.search import Search
class Google(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = domain
self.module = 'Search'
self.source = 'GoogleSearch'
self.init = 'https://www.google.com/'
self.addr = 'https://www.google.com/search'
def search(self, domain, filtered_subdomain='', full_search=False):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
page_num = 1
per_page_num = 50
self.header = self.get_header()
self.header.update({'User-Agent': 'Googlebot',
'Referer': 'https://www.google.com'})
self.proxy = self.get_proxy(self.source)
resp = self.get(self.init)
if not resp:
return
self.cookie = resp.cookies
while True:
self.delay = random.randint(1, 5)
time.sleep(self.delay)
self.proxy = self.get_proxy(self.source)
word = 'site:.' + domain + filtered_subdomain
payload = {'q': word, 'start': page_num, 'num': per_page_num,
'filter': '0', 'btnG': 'Search', 'gbv': '1', 'hl': 'en'}
resp = self.get(url=self.addr, params=payload)
if not resp:
return
subdomains = self.match(domain, resp.text)
if not subdomains:
break
if not full_search:
if subdomains.issubset(self.subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
page_num += per_page_num
if 'start=' + str(page_num) not in resp.text:
break
if '302 Moved' in resp.text:
break
def run(self):
"""
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
self.search(subdomain)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = Google(domain)
search.run()
if __name__ == '__main__':
do('example.com')
modules/search/google_api.py
+88
View File
@@ -0,0 +1,88 @@
import time
from config import api
from common.search import Search
class GoogleAPI(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = domain
self.module = 'Search'
self.source = 'GoogleAPISearch'
self.addr = 'https://www.googleapis.com/customsearch/v1'
self.delay = 1
self.key = api.google_api_key
self.cx = api.google_api_cx
self.per_page_num = 10 # 每次只能请求10个结果
def search(self, domain, filtered_subdomain='', full_search=False):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 1
while True:
word = 'site:.' + domain + filtered_subdomain
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
params = {'key': self.key, 'cx': self.cx,
'q': word, 'fields': 'items/link',
'start': self.page_num, 'num': self.per_page_num}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match(domain, str(resp.json()))
if not subdomains:
break
if not full_search:
if subdomains.issubset(self.subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
self.page_num += self.per_page_num
if self.page_num > 100: # 免费的API只能查询前100条结果
break
def run(self):
"""
类执行入口
"""
if not self.check(self.cx, self.key):
return
self.begin()
self.search(self.domain, full_search=True)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
self.search(subdomain)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = GoogleAPI(domain)
search.run()
if __name__ == '__main__':
do('example.com')
modules/search/shodan_api.py
+60
View File
@@ -0,0 +1,60 @@
from config import api
from common.search import Search
class ShodanAPI(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = self.register(domain)
self.module = 'Search'
self.source = 'ShodanAPISearch'
self.addr = 'https://api.shodan.io/shodan/host/search'
self.key = api.shodan_api_key
def search(self):
"""
发送搜索请求并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
query = 'hostname:.' + self.domain
page = 1
while True:
params = {'key': self.key, 'page': page, 'query': query,
'minify': True, 'facets': {'hostnames'}}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match(self.domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
if subdomains:
self.subdomains = self.subdomains.union(subdomains)
page += 1
def run(self):
"""
类执行入口
"""
if not self.check(self.key):
return
self.begin()
self.search()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = ShodanAPI(domain)
search.run()
if __name__ == '__main__':
do('example.com')
modules/search/so.py
+87
View File
@@ -0,0 +1,87 @@
import time
from common.search import Search
class So(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = domain
self.module = 'Search'
self.source = 'SoSearch'
self.addr = 'https://www.so.com/s'
self.limit_num = 640 # 限制搜索条数
self.per_page_num = 10 # 默认每页显示10页
def search(self, domain, filtered_subdomain='', full_search=False):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
page_num = 1
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
word = 'site:.' + domain + filtered_subdomain
payload = {'q': word, 'pn': page_num}
resp = self.get(url=self.addr, params=payload)
if not resp:
return
subdomains = self.match(domain, resp.text)
if not subdomains:
break
if not full_search:
# 搜索中发现搜索出的结果有完全重复的结果就停止搜索
if subdomains.issubset(self.subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
page_num += 1
# 搜索页面没有出现下一页时停止搜索
if '<a id="snext"' not in resp.text:
break
# 搜索条数限制
if self.page_num * self.per_page_num >= self.limit_num:
break
def run(self):
"""
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
self.search(subdomain)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = So(domain)
search.run()
if __name__ == '__main__':
do('example.com')
modules/search/sogou.py
+85
View File
@@ -0,0 +1,85 @@
from common.search import Search
class Sogou(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = domain
self.module = 'Search'
self.source = 'SogouSearch'
self.addr = 'https://www.sogou.com/web'
self.limit_num = 1000 # 限制搜索条数
def search(self, domain, filtered_subdomain='', full_search=False):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 1
while True:
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
word = 'site:.' + domain + filtered_subdomain
payload = {'query': word, 'page': self.page_num,
"num": self.per_page_num}
resp = self.get(self.addr, payload)
if not resp:
return
subdomains = self.match(domain, resp.text)
if not subdomains:
break
if not full_search:
# 搜索中发现搜索出的结果有完全重复的结果就停止搜索
if subdomains.issubset(self.subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
self.page_num += 1
# 搜索页面没有出现下一页时停止搜索
if '<a id="sogou_next"' not in resp.text:
break
# 搜索条数限制
if self.page_num * self.per_page_num >= self.limit_num:
break
def run(self):
"""
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
self.search(subdomain)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = Sogou(domain)
search.run()
if __name__ == '__main__':
do('example.com')
modules/search/yahoo.py
+92
View File
@@ -0,0 +1,92 @@
import time
from common.search import Search
class Yahoo(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = domain
self.module = 'Search'
self.source = 'YahooSearch'
self.init = 'https://search.yahoo.com/'
self.addr = 'https://search.yahoo.com/search'
self.limit_num = 1000 # Yahoo限制搜索条数
self.delay = 2
self.per_page_num = 30 # Yahoo每次搜索最大条数
def search(self, domain, filtered_subdomain='', full_search=False):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
resp = self.get(self.init)
if not resp:
return
self.cookie = resp.cookies # 获取cookie Yahoo在搜索时需要带上cookie
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
query = 'site:.' + domain + filtered_subdomain
params = {'p': query, 'b': self.page_num, 'pz': self.per_page_num}
resp = self.get(self.addr, params)
if not resp:
return
text = resp.text.replace('<b>', '').replace('</b>', '')
subdomains = self.match(domain, text)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
if not full_search:
# 搜索中发现搜索出的结果有完全重复的结果就停止搜索
if subdomains.issubset(self.subdomains):
break
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
if '>Next</a>' not in resp.text: # 搜索页面没有出现下一页时停止搜索
break
self.page_num += self.per_page_num
if self.page_num >= self.limit_num: # 搜索条数限制
break
def run(self):
"""
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
self.search(subdomain)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = Yahoo(domain)
search.run()
if __name__ == '__main__':
do('example.com')
modules/search/yandex.py
+92
View File
@@ -0,0 +1,92 @@
import time
from common.search import Search
class Yandex(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = domain
self.module = 'Search'
self.source = 'YandexSearch'
self.init = 'https://yandex.com/'
self.addr = 'https://yandex.com/search'
self.limit_num = 1000 # 限制搜索条数
self.delay = 5
def search(self, domain, filtered_subdomain='', full_search=False):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 0 # 二次搜索重新置0
resp = self.get(self.init)
if not resp:
return
self.cookie = resp.cookies # 获取cookie
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
query = 'site:.' + domain + filtered_subdomain
params = {'text': query, 'p': self.page_num,
'numdoc': self.per_page_num}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match(domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
if not full_search:
# 搜索中发现搜索出的结果有完全重复的结果就停止搜索
if subdomains.issubset(self.subdomains):
break
# 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains)
if '>next</a>' not in resp.text: # 搜索页面没有出现下一页时停止搜索
break
self.page_num += 1
if self.page_num >= self.limit_num: # 搜索条数限制
break
def run(self):
"""
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
self.search(subdomain)
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = Yandex(domain)
search.run()
if __name__ == '__main__':
do('example.com')
modules/search/zoomeye_api.py
+86
View File
@@ -0,0 +1,86 @@
import time
from config import api
from common.search import Search
from config.log import logger
class ZoomEyeAPI(Search):
def __init__(self, domain):
Search.__init__(self)
self.domain = domain
self.module = 'Search'
self.source = 'ZoomEyeAPISearch'
self.addr = 'https://api.zoomeye.org/web/search'
self.delay = 2
self.user = api.zoomeye_api_usermail
self.pwd = api.zoomeye_api_password
def login(self):
"""
登陆获取查询taken
"""
url = 'https://api.zoomeye.org/user/login'
data = {'username': self.user, 'password': self.pwd}
resp = self.post(url=url, json=data)
if not resp:
logger.log('FATAL', f'登录失败无法获取{self.source}的访问token')
exit(1)
data = resp.json()
if resp.status_code == 200:
logger.log('DEBUG', f'{self.source}模块登录成功')
return data.get('access_token')
else:
logger.log('ALERT', data.get('message'))
exit(1)
def search(self):
"""
发送搜索请求并做子域匹配
"""
page_num = 1
access_token = self.login()
while True:
time.sleep(self.delay)
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
self.header.update({'Authorization': 'JWT ' + access_token})
params = {'query': 'hostname:' + self.domain, 'page': page_num}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match(self.domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
self.subdomains = self.subdomains.union(subdomains)
page_num += 1
if page_num > 500:
break
if resp.status_code == 403:
break
def run(self):
"""
类执行入口
"""
if not self.check(self.user, self.pwd):
return
self.begin()
self.search()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def do(domain): # 统一入口名字 方便多线程调用
"""
类统一调用入口
:param str domain: 域名
"""
search = ZoomEyeAPI(domain)
search.run()
if __name__ == '__main__':
do('mi.com')