mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-26 12:57:50 +08:00
v0.0.1
This commit is contained in:
Jing Ling
@@ -0,0 +1,68 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class BinaryEdgeAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'BinaryEdgeAPIQuery'
|
||||
self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/'
|
||||
self.api = config.binaryedge_api
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.header.update({'X-Key': self.api})
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
url = self.addr + self.domain
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, str(resp.json()))
|
||||
self.subdomains = self.subdomains.union(subdomains_find)
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.api:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = BinaryEdgeAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,61 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class BufferOver(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'BufferOverQuery'
|
||||
self.addr = 'https://dns.bufferover.run/dns'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'q': self.domain}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = BufferOver(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,61 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class Chinaz(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'ChinazQuery'
|
||||
self.addr = 'https://alexa.chinaz.com/'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
self.addr = self.addr + self.domain
|
||||
resp = self.get(self.addr)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = Chinaz(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,67 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class ChinazAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'ChinazAPIQuery'
|
||||
self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa'
|
||||
self.api = config.chinaz_api
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'key': self.api, 'domainName': self.domain}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, str(resp.json()))
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.api:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = ChinazAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,64 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class CirclAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'CirclAPIQuery'
|
||||
self.addr = 'https://www.circl.lu/pdns/query/'
|
||||
self.user = config.circl_api_username
|
||||
self.pwd = config.circl_api_password
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, str(resp.json()))
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
if self.user and self.pwd:
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = CirclAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,76 @@
|
||||
# coding=utf-8
|
||||
import random
|
||||
import time
|
||||
import queue
|
||||
from bs4 import BeautifulSoup
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class DNSdb(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'DNSdbQuery'
|
||||
self.addr = 'https://www.dnsdb.org/'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.header.update({'Referer': self.addr})
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
url = self.addr + self.domain + '/'
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
if resp.status_code == 200:
|
||||
if 'index' in resp.text:
|
||||
soup = BeautifulSoup(resp.text, features='lxml')
|
||||
index_urls = set(map(lambda x: self.addr + self.domain + x.text, soup.find_all('a')))
|
||||
for url in index_urls:
|
||||
self.delay = random.randint(2, 5) # 休眠绕过CloudFlare的DDoS保护
|
||||
time.sleep(self.delay)
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
else:
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = DNSdb(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,70 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class DNSdbAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'DNSdbAPIQuery'
|
||||
self.addr = 'https://api.dnsdb.info/lookup/rrset/name/'
|
||||
self.api = config.dnsdb_api_key
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.header.update({'X-API-Key': self.api})
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
url = f'{self.addr}*.{self.domain}'
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
if resp.status_code == 200:
|
||||
subdomains_find = utils.match_subdomain(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.api:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = DNSdbAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,71 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class DNSdumpster(Query):
|
||||
"""
|
||||
|
||||
"""
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = "DNSdumpsterQuery"
|
||||
self.addr = 'https://dnsdumpster.com/'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.header.update({'Referer': 'https://dnsdumpster.com'})
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(self.addr)
|
||||
if not resp:
|
||||
return
|
||||
self.cookie = resp.cookies
|
||||
data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'), 'targetip': self.domain}
|
||||
resp = self.post(self.addr, data)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = utils.match_subdomain(self.domain, resp.text)
|
||||
if subdomains_find:
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = DNSdumpster(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,63 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class HackerTarget(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = "HackerTargetQuery"
|
||||
self.addr = 'https://api.hackertarget.com/hostsearch/'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'q': self.domain}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
if resp.status_code == 200:
|
||||
subdomains_find = utils.match_subdomain(self.domain, resp.text)
|
||||
if subdomains_find:
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = HackerTarget(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,75 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class IPv4InfoAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'IPv4InfoAPIQuery'
|
||||
self.addr = ' http://ipv4info.com/api_v1/'
|
||||
self.api = config.ipv4info_api_key
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
page = 0
|
||||
while True:
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'type': 'SUBDOMAINS', 'key': self.api, 'value': self.domain, 'page': page}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
if resp.status_code != 200:
|
||||
break # 请求不正常通常网络是有问题，不再继续请求下去
|
||||
resp_json = resp.json()
|
||||
subdomains_find = self.match(self.domain, str(resp_json))
|
||||
if not subdomains_find:
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
subdomains = resp_json.get('Subdomains') # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
|
||||
if len(subdomains) < 300: # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
|
||||
break
|
||||
page += 1
|
||||
if page >= 50: # ipv4info子域查询接口最多允许查询50页
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = IPv4InfoAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,86 @@
|
||||
# coding=utf-8
|
||||
import re
|
||||
import time
|
||||
import queue
|
||||
import hashlib
|
||||
from urllib import parse
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class NetCraft(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'NetCraftQuery'
|
||||
self.init = 'https://searchdns.netcraft.com/'
|
||||
self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains'
|
||||
self.page_num = 1
|
||||
self.per_page_num = 20
|
||||
|
||||
def bypass_verification(self):
|
||||
"""
|
||||
绕过NetCraft的JS验证
|
||||
"""
|
||||
self.header = self.get_header() # Netcraft会检查User-Agent
|
||||
self.cookie = self.get(self.init).cookies
|
||||
cookie_value = self.cookie['netcraft_js_verification_challenge']
|
||||
verify_taken = hashlib.sha1(parse.unquote(cookie_value).encode('utf-8')).hexdigest()
|
||||
self.cookie['netcraft_js_verification_response'] = verify_taken
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.bypass_verification()
|
||||
last = ''
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'host': '*.' + self.domain, 'from': self.page_num}
|
||||
resp = self.get(self.addr + last, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
if 'Next page' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
last = re.search(r'&last=.*' + self.domain, resp.text).group(0)
|
||||
self.page_num += self.per_page_num
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = NetCraft(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,65 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import random
|
||||
from common.query import Query
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
class PTRArchive(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = "PTRArchiveQuery"
|
||||
self.addr = 'http://ptrarchive.com/tools/search3.htm'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
self.cookie = {'pa_id': str(random.randint(0, 1000000000))} # 绕过主页前端JS验证
|
||||
params = {'label': self.domain, 'date': 'ALL'}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
if resp.status_code == 200:
|
||||
subdomains_find = utils.match_subdomain(self.domain, resp.text)
|
||||
if subdomains_find:
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = PTRArchive(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,61 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class Riddler(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'RiddlerQuery'
|
||||
self.addr = 'https://riddler.io/search'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'q': 'pld:' + self.domain}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = Riddler(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,72 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import json
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class Robtex(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = "RobtexQuery"
|
||||
self.addr = 'https://freeapi.robtex.com/pdns/'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
url = self.addr + 'forward/' + self.domain
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
text_list = resp.text.splitlines()
|
||||
text_json = list(map(lambda x: json.loads(x), text_list))
|
||||
for record in text_json:
|
||||
if record.get('rrtype') in ['A', 'AAAA']:
|
||||
time.sleep(self.delay) # Robtex有查询频率限制
|
||||
ip = record.get('rrdata')
|
||||
url = self.addr + 'reverse/' + ip
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
if subdomains_find:
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = Robtex(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,71 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class SecurityTrailsAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'SecurityTrailsQuery'
|
||||
self.addr = 'https://api.securitytrails.com/v1/domain/'
|
||||
self.api = config.securitytrails_api
|
||||
self.delay = 2 # SecurityTrails查询时延至少2秒
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'apikey': self.api}
|
||||
url = f'{self.addr}{self.domain}/subdomains'
|
||||
resp = self.get(url, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_prefix = resp.json()['subdomains']
|
||||
subdomains_find = [f'{prefix}.{self.domain}' for prefix in subdomains_prefix]
|
||||
if subdomains_find:
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.api:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = SecurityTrailsAPI(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,70 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class SiteDossier(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'SiteDossierQuery'
|
||||
self.addr = 'http://www.sitedossier.com/parentdomain/'
|
||||
self.delay = 2
|
||||
self.page_num = 1
|
||||
self.per_page_num = 100
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
url = f'{self.addr}{self.domain}/{self.page_num}'
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(self.domain, resp.text)
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
if 'Show next 100 items' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
self.page_num += self.per_page_num
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.query()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
query = SiteDossier(domain)
|
||||
query.run(rx_queue)
|
||||
logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
|
||||
logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
Reference in New Issue
Block a user