mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-26 12:57:50 +08:00
v0.0.1
This commit is contained in:
Jing Ling
@@ -0,0 +1,89 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Ask(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'AskSearch'
|
||||
self.addr = 'https://www.search.ask.com/web'
|
||||
self.limit_num = 200 # 限制搜索条数
|
||||
self.per_page_num = 10 # 默认每页显示10页
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 1
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'q': query, 'page': self.page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, resp.text)
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
self.page_num += 1
|
||||
if '>Next<' not in resp.text:
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Ask(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,110 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from bs4 import BeautifulSoup
|
||||
from config import logger
|
||||
|
||||
|
||||
class Baidu(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.module = 'Search'
|
||||
self.source = 'BaiduSearch'
|
||||
self.init = 'https://www.baidu.com/'
|
||||
self.addr = 'https://www.baidu.com/s'
|
||||
self.domain = domain
|
||||
self.limit_num = 750 # 限制搜索条数
|
||||
|
||||
def redirect_match(self, domain, html):
|
||||
"""
|
||||
|
||||
:param domain:
|
||||
:param html:
|
||||
:return:
|
||||
"""
|
||||
bs = BeautifulSoup(html, features='lxml')
|
||||
subdomains_all = set()
|
||||
for find_res in bs.find_all('a', {'class': 'c-showurl'}): # 获取搜索结果中所有的跳转URL地址
|
||||
url = find_res.get('href')
|
||||
subdomain = self.match_location(domain, url)
|
||||
subdomains_all = subdomains_all.union(subdomain)
|
||||
return subdomains_all
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 0 # 二次搜索重新置0
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'wd': query, 'pn': self.page_num, 'rn': self.per_page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
if len(domain) > 12: # 解决百度搜索结果中域名过长会显示不全的问题
|
||||
subdomains_find = self.redirect_match(domain, resp.text) # 获取百度跳转URL响应头的Location字段获取直链
|
||||
else:
|
||||
subdomains_find = self.match(domain, resp.text)
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
if not full_search:
|
||||
if subdomains_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
self.page_num += self.per_page_num
|
||||
if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
if self.page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Baidu(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,96 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Bing(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'BingSearch'
|
||||
self.init = 'https://www.bing.com/'
|
||||
self.addr = 'https://www.bing.com/search'
|
||||
self.limit_num = 1000 # 限制搜索条数
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 0 # 二次搜索重新置0
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(self.init)
|
||||
if not resp:
|
||||
return
|
||||
self.cookie = resp.cookies # 获取cookie bing在搜索时需要带上cookie
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'q': query, 'first': self.page_num, 'count': self.per_page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(domain, resp.text)
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
if not full_search:
|
||||
if subdomains_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
if '<div class="sw_next>' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
self.page_num += self.per_page_num
|
||||
if self.page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Bing(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,98 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class BingAPI(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'BingCustomSearch'
|
||||
self.addr = 'https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search'
|
||||
self.id = config.bing_api_id
|
||||
self.key = config.bing_api_key
|
||||
self.limit_num = 1000 # 必应同一个搜索关键词限制搜索条数
|
||||
self.delay = 1 # 必应自定义搜索限制时延1秒
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 0 # 二次搜索重新置0
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.header = {'Ocp-Apim-Subscription-Key': self.key}
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'q': query, 'customconfig': self.id, 'safesearch': 'Off',
|
||||
'count': self.per_page_num, 'offset': self.page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(domain, str(resp.json()))
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
if not full_search:
|
||||
if subdomains_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
self.page_num += self.per_page_num
|
||||
if self.page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not (self.id and self.key):
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = BingAPI(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,93 @@
|
||||
# coding=utf-8
|
||||
import re
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class DuckDuckGO(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'DuckDuckGoSearch'
|
||||
self.addr = 'https://duckduckgo.com/html/'
|
||||
self.header = self.get_header()
|
||||
self.delay = 2
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
data = {'q': query, 'kl': 'us-en', 'v': 'l'}
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.post(self.addr, data)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, resp.text)
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
try:
|
||||
s = re.findall(r'name="s" value="(\d.*)"', resp.text)[-1]
|
||||
dc = re.findall(r'name="dc" value="(\d.*)"', resp.text)
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
break
|
||||
data.update({'s': s, 'nextParams': '', 'o': 'json', 'dc': dc, 'api': '/d.js'})
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
# return # 暂时还有点问题
|
||||
search = DuckDuckGO(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,93 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import random
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Exalead(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = "ExaleadSearch"
|
||||
self.addr = "http://www.exalead.com/search/web/results/"
|
||||
self.per_page_num = 30
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 0
|
||||
while True:
|
||||
self.delay = random.randint(1, 5)
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'q': query, 'elements_per_page': '30', "start_index": self.page_num}
|
||||
resp = self.get(url=self.addr, params=params)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, resp.text)
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains):
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
self.page_num += self.per_page_num
|
||||
if self.page_num > 1999:
|
||||
break
|
||||
if 'title="Go to the next page"' not in resp.text:
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
statement = statement.replace('-site', 'and -site')
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Exalead(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,71 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import json
|
||||
import base64
|
||||
import config
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class FoFa(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'FoFaSearch'
|
||||
self.addr = 'https://fofa.so/api/v1/search/all'
|
||||
self.delay = 1
|
||||
self.email = config.fofa_api_email
|
||||
self.key = config.fofa_api_key
|
||||
|
||||
def search(self):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
"""
|
||||
self.page_num = 1
|
||||
query_base64 = base64.b64encode(f'domain={self.domain}'.encode('utf-8'))
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = {'email': self.email, 'key': self.key, 'qbase64': query_base64, 'page': self.page_num}
|
||||
resp = self.get(self.addr, query)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
self.page_num += 1
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = FoFa(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,101 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import random
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Google(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'GoogleSearch'
|
||||
self.init = 'https://www.google.com/'
|
||||
self.addr = 'https://www.google.com/search'
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
page_num = 1
|
||||
per_page_num = 50
|
||||
self.header = self.get_header()
|
||||
self.header.update({'User-Agent': 'Googlebot',
|
||||
'Referer': 'https://www.google.com'})
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(self.init)
|
||||
if not resp:
|
||||
return
|
||||
self.cookie = resp.cookies
|
||||
while True:
|
||||
self.delay = random.randint(1, 5)
|
||||
time.sleep(self.delay)
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
word = 'site:' + domain + filtered_subdomain
|
||||
payload = {'q': word, 'start': page_num, 'num': per_page_num,
|
||||
'filter': '0', 'btnG': 'Search', 'gbv': '1', 'hl': 'en'}
|
||||
resp = self.get(url=self.addr, params=payload)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, resp.text)
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains):
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
page_num += per_page_num
|
||||
if 'start='+str(page_num) not in resp.text:
|
||||
break
|
||||
if '302 Moved' in resp.text:
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Google(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,97 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class GoogleAPI(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'GoogleAPISearch'
|
||||
self.addr = 'https://www.googleapis.com/customsearch/v1'
|
||||
self.delay = 1
|
||||
self.key = config.google_api_key
|
||||
self.cx = config.google_api_cx
|
||||
self.per_page_num = 10 # 每次只能请求10个结果
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 1
|
||||
while True:
|
||||
word = 'site:' + domain + filtered_subdomain
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'key': self.key, 'cx': self.cx, 'q': word, 'fields': 'items/link',
|
||||
'start': self.page_num, 'num': self.per_page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, str(resp.json()))
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains):
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
self.page_num += self.per_page_num
|
||||
if self.page_num > 100: # 免费的API只能查询前100条结果
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not (self.cx and self.key):
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = GoogleAPI(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,73 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import queue
|
||||
import config
|
||||
# from shodan import Shodan
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class ShodanAPI(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Search'
|
||||
self.source = 'ShodanSearch'
|
||||
self.addr = 'https://api.shodan.io/shodan/host/search'
|
||||
self.key = config.shodan_api_key
|
||||
|
||||
def search(self):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'hostname:.' + self.domain
|
||||
page = 1
|
||||
while True:
|
||||
params = {'key': self.key, 'page': page, 'query': query, 'minify': True, 'facets': {'hostnames'}}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(self.domain, resp.text)
|
||||
if subdomain_find:
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
page += 1
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.key:
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = ShodanAPI(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
results = queue.Queue()
|
||||
do('qq.com', results)
|
||||
@@ -0,0 +1,91 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class So(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'SoSearch'
|
||||
self.addr = 'http://www.so.com/s'
|
||||
self.limit_num = 640 # 限制搜索条数
|
||||
self.per_page_num = 10 # 默认每页显示10页
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
page_num = 1
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
word = 'site:' + domain + filtered_subdomain
|
||||
payload = {'q': word, 'pn': page_num}
|
||||
resp = self.get(url=self.addr, params=payload)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, resp.text)
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
page_num += 1
|
||||
if '<a id="snext"' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
if self.page_num * self.per_page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = So(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,89 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Sogou(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'SogouSearch'
|
||||
self.addr = 'https://www.sogou.com/web'
|
||||
self.limit_num = 1000 # 限制搜索条数
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 1
|
||||
while True:
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
word = 'site:' + domain + filtered_subdomain
|
||||
payload = {'query': word, 'page': self.page_num, "num": self.per_page_num}
|
||||
resp = self.get(self.addr, payload)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(domain, resp.text)
|
||||
if not subdomain_find:
|
||||
break
|
||||
if not full_search:
|
||||
if subdomain_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
self.page_num += 1
|
||||
if '<a id="sogou_next"' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
if self.page_num * self.per_page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Sogou(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,97 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Yahoo(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'YahooSearch'
|
||||
self.init = 'https://hk.search.yahoo.com/'
|
||||
self.addr = 'https://hk.search.yahoo.com/search'
|
||||
self.limit_num = 1000 # 限制搜索条数
|
||||
self.delay = 5
|
||||
self.per_page_num = 40
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 0
|
||||
resp = self.get(self.init)
|
||||
if not resp:
|
||||
return
|
||||
self.cookie = resp.cookies # 获取cookie Yahoo在搜索时需要带上cookie
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'q': query, 'b': self.page_num, 'n': self.per_page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(domain, resp.text)
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
if not full_search:
|
||||
if subdomains_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
if '>Next</a>' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
self.page_num += self.per_page_num
|
||||
if self.page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Yahoo(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,93 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class Yandex(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'YandexSearch'
|
||||
self.init = 'https://yandex.com/'
|
||||
self.addr = 'https://yandex.com/search'
|
||||
self.limit_num = 1000 # 限制搜索条数
|
||||
self.delay = 5
|
||||
|
||||
def search(self, domain, filtered_subdomain='', full_search=False):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
:param bool full_search: 全量搜索
|
||||
"""
|
||||
self.page_num = 0 # 二次搜索重新置0
|
||||
self.cookie = self.get(self.init).cookies # 获取cookie bing在搜索时需要带上cookie
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:' + domain + filtered_subdomain
|
||||
params = {'text': query, 'p': self.page_num, 'numdoc': self.per_page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains_find = self.match(domain, resp.text)
|
||||
if not subdomains_find: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
if not full_search:
|
||||
if subdomains_find.issubset(self.subdomains): # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
|
||||
break
|
||||
self.subdomains = self.subdomains.union(subdomains_find) # 合并搜索子域名搜索结果
|
||||
if '>next</a>' not in resp.text: # 搜索页面没有出现下一页时停止搜索
|
||||
break
|
||||
self.page_num += 1
|
||||
if self.page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search(self.domain, full_search=True)
|
||||
|
||||
# 排除同一子域搜索结果过多的子域以发现新的子域
|
||||
for statement in self.filter(self.domain, self.subdomains):
|
||||
self.search(self.domain, filtered_subdomain=statement)
|
||||
|
||||
# 递归搜索下一层的子域
|
||||
if self.recursive_search:
|
||||
for layer_num in range(1, self.recursive_times): # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
|
||||
for subdomain in self.subdomains:
|
||||
if subdomain.count('.') - self.domain.count('.') == layer_num: # 进行下一层子域搜索的限制条件
|
||||
self.search(subdomain)
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = Yandex(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
@@ -0,0 +1,97 @@
|
||||
# coding=utf-8
|
||||
import time
|
||||
import queue
|
||||
import config
|
||||
from common.search import Search
|
||||
from config import logger
|
||||
|
||||
|
||||
class ZoomEyeAPI(Search):
|
||||
def __init__(self, domain):
|
||||
Search.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Search'
|
||||
self.source = 'ZoomEyeAPISearch'
|
||||
self.addr = 'https://api.zoomeye.org/web/search'
|
||||
self.delay = 2
|
||||
self.user = config.zoomeye_api_username
|
||||
self.pwd = config.zoomeye_api_password
|
||||
|
||||
def login(self):
|
||||
"""
|
||||
登陆获取查询taken
|
||||
:return:
|
||||
"""
|
||||
url = 'https://api.zoomeye.org/user/login'
|
||||
data = {'username': self.user, 'password': self.pwd}
|
||||
resp = self.post(url=url, json=data)
|
||||
if not resp:
|
||||
logger.log('FETAL', f'登录失败无法获取{self.source}的访问token')
|
||||
return
|
||||
resp_json = resp.json()
|
||||
if resp.status_code == 200:
|
||||
# print('登陆成功')
|
||||
return resp_json.get('access_token')
|
||||
else:
|
||||
logger.log('ALERT', resp_json.get('message'))
|
||||
exit(1)
|
||||
|
||||
def search(self):
|
||||
"""
|
||||
发送搜索请求并做子域匹配
|
||||
"""
|
||||
page_num = 1
|
||||
access_token = self.login()
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
self.header.update({'Authorization': 'JWT ' + access_token})
|
||||
params = {'query': 'hostname:' + self.domain, 'page': page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomain_find = self.match(self.domain, resp.text)
|
||||
self.subdomains = self.subdomains.union(subdomain_find)
|
||||
page_num += 1
|
||||
if page_num > 500:
|
||||
break
|
||||
if resp.status_code == 403:
|
||||
break
|
||||
|
||||
def run(self, rx_queue):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not (self.user and self.pwd):
|
||||
logger.log('ERROR', f'{self.source}模块API配置错误')
|
||||
logger.log('ALERT', f'不执行{self.source}模块')
|
||||
return
|
||||
logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
|
||||
start = time.time()
|
||||
self.search()
|
||||
end = time.time()
|
||||
self.elapsed = round(end - start, 1)
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
rx_queue.put(self.results)
|
||||
logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
|
||||
|
||||
|
||||
def do(domain, rx_queue): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
:param rx_queue: 结果集队列
|
||||
"""
|
||||
search = ZoomEyeAPI(domain)
|
||||
search.run(rx_queue)
|
||||
logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
|
||||
logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
result_queue = queue.Queue()
|
||||
do('owasp.org', result_queue)
|
||||
Reference in New Issue
Block a user