模块优化

This commit is contained in:
Jing Ling
2020-08-11 14:50:30 +08:00
parent b7de3f8ea6
commit b7e9c7e781
17 changed files with 94 additions and 173 deletions
+26
View File
@@ -12,6 +12,7 @@ class Search(Module):
self.per_page_num = 50 # 每页显示搜索条数
self.recursive_search = settings.enable_recursive_search
self.recursive_times = settings.search_recursive_times
self.full_search = settings.enable_full_search
@staticmethod
def filter(domain, subdomain):
@@ -50,3 +51,28 @@ class Search(Module):
if not location:
return set()
return set(self.match_subdomains(location))
def check_subdomains(self, subdomains):
"""
检查搜索出的子域结果是否满足条件
:param subdomains: 子域结果
:return:
"""
if not subdomains:
# 搜索没有发现子域名则停止搜索
return False
if not self.full_search and subdomains.issubset(self.subdomains):
# 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
return False
return True
def recursive_subdomain(self):
# 递归搜索下一层的子域
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
yield subdomain
+4 -2
View File
@@ -102,8 +102,10 @@ request_verify = False # 请求SSL验证
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# 搜索模块设置
enable_recursive_search = False # 递归搜索子域
search_recursive_times = 2 # 递归搜索层数
# 开启全量搜索会尽量去获取搜索引擎搜索的全部结果,不过搜索耗时可能会过长
enable_full_search = False # 启用全量搜索(默认False)
enable_recursive_search = False # 递归搜索子域(默认False)
search_recursive_times = 2 # 递归搜索层数(默认2)
# DNS解析设置
resolve_coroutine_num = 64
+5 -16
View File
@@ -12,13 +12,12 @@ class Ask(Search):
self.limit_num = 200 # 限制搜索条数
self.per_page_num = 10 # 默认每页显示10页
def search(self, domain, filtered_subdomain='', full_search=False):
def search(self, domain, filtered_subdomain=''):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 1
while True:
@@ -28,13 +27,8 @@ class Ask(Search):
query = 'site:.' + domain + filtered_subdomain
params = {'q': query, 'page': self.page_num}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match_subdomains(resp.text, fuzzy=False)
if not subdomains:
break
if not full_search and subdomains.issubset(self.subdomains):
# 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
subdomains = self.match_subdomains(resp, fuzzy=False)
if not self.check_subdomains(subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
self.page_num += 1
@@ -46,19 +40,14 @@ class Ask(Search):
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
self.search(self.domain)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
for subdomain in self.recursive_subdomain():
self.search(subdomain)
self.finish()
self.save_json()
+5 -14
View File
@@ -28,13 +28,12 @@ class Baidu(Search):
subdomains_all = subdomains_all.union(subdomains)
return subdomains_all
def search(self, domain, filtered_subdomain='', full_search=False):
def search(self, domain, filtered_subdomain=''):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 0 # 二次搜索重新置0
while True:
@@ -52,11 +51,8 @@ class Baidu(Search):
# 获取百度跳转URL响应头的Location字段获取直链
subdomains = self.redirect_match(resp.text)
else:
subdomains = self.match_subdomains(resp.text, fuzzy=False)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
if not full_search and subdomains.issubset(self.subdomains):
# 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
subdomains = self.match_subdomains(resp, fuzzy=False)
if not self.check_subdomains(subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
self.page_num += self.per_page_num
@@ -71,19 +67,14 @@ class Baidu(Search):
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
self.search(self.domain)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
self.search(self.domain, filtered_subdomain=statement)
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
for subdomain in self.recursive_subdomain():
self.search(subdomain)
self.finish()
self.save_json()
+5 -16
View File
@@ -12,13 +12,12 @@ class Bing(Search):
self.addr = 'https://www.bing.com/search'
self.limit_num = 1000 # 限制搜索条数
def search(self, domain, filtered_subdomain='', full_search=False):
def search(self, domain, filtered_subdomain=''):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 0 # 二次搜索重新置0
self.header = self.get_header()
@@ -34,13 +33,8 @@ class Bing(Search):
params = {'q': query, 'first': self.page_num,
'count': self.per_page_num}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match_subdomains(resp.text, fuzzy=False)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
if not full_search and subdomains.issubset(self.subdomains):
# 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
subdomains = self.match_subdomains(resp, fuzzy=False)
if not self.check_subdomains(subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
# 搜索页面没有出现下一页时停止搜索
@@ -55,7 +49,7 @@ class Bing(Search):
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
self.search(self.domain)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
@@ -63,12 +57,7 @@ class Bing(Search):
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
for subdomain in self.recursive_subdomain():
self.search(subdomain)
self.finish()
self.save_json()
+4 -13
View File
@@ -15,13 +15,12 @@ class BingAPI(Search):
self.limit_num = 1000 # 必应同一个搜索关键词限制搜索条数
self.delay = 1 # 必应自定义搜索限制时延1秒
def search(self, domain, filtered_subdomain='', full_search=False):
def search(self, domain, filtered_subdomain=''):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 0 # 二次搜索重新置0
while True:
@@ -35,10 +34,7 @@ class BingAPI(Search):
'offset': self.page_num}
resp = self.get(self.addr, params)
subdomains = self.match_subdomains(resp)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
if not full_search and subdomains.issubset(self.subdomains):
# 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
if not self.check_subdomains(subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
self.page_num += self.per_page_num
@@ -52,7 +48,7 @@ class BingAPI(Search):
if not self.check(self.id, self.key):
return
self.begin()
self.search(self.domain, full_search=True)
self.search(self.domain)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
@@ -60,12 +56,7 @@ class BingAPI(Search):
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
for subdomain in self.recursive_subdomain():
self.search(subdomain)
self.finish()
self.save_json()
+1 -1
View File
@@ -37,7 +37,7 @@ class FoFa(Search):
if not resp:
return
resp_json = resp.json()
subdomains = self.match_subdomains(resp.text)
subdomains = self.match_subdomains(resp)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
self.subdomains = self.subdomains.union(subdomains)
+2 -5
View File
@@ -34,11 +34,8 @@ class Gitee(Search):
if 'class="empty-box"' in resp.text:
break
soup = BeautifulSoup(resp.text, 'html.parser')
subdomains = self.match_subdomains(soup.text, fuzzy=False)
if not subdomains:
break
if not full_search and subdomains.issubset(self.subdomains):
# 在全搜索中发现搜索出的结果有完全重复的结果就停止搜索
subdomains = self.match_subdomains(soup, fuzzy=False)
if not self.check_subdomains(subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
if '<li class="disabled"><a href="###">' in resp.text:
+1 -1
View File
@@ -60,7 +60,7 @@ class GithubAPI(Search):
if resp.status_code != 200:
logger.log('ERROR', f'{self.source} module query failed')
break
subdomains = self.match_subdomains(resp.text)
subdomains = self.match_subdomains(resp)
if not subdomains:
break
self.subdomains = self.subdomains.union(subdomains)
+5 -16
View File
@@ -12,13 +12,12 @@ class Google(Search):
self.init = 'https://www.google.com/'
self.addr = 'https://www.google.com/search'
def search(self, domain, filtered_subdomain='', full_search=False):
def search(self, domain, filtered_subdomain=''):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
page_num = 1
per_page_num = 50
@@ -38,13 +37,8 @@ class Google(Search):
payload = {'q': word, 'start': page_num, 'num': per_page_num,
'filter': '0', 'btnG': 'Search', 'gbv': '1', 'hl': 'en'}
resp = self.get(url=self.addr, params=payload)
if not resp:
return
subdomains = self.match_subdomains(resp.text, fuzzy=False)
if not subdomains:
break
if not full_search and subdomains.issubset(self.subdomains):
# 在全搜索中发现搜索出的结果有完全重复的结果就停止搜索
subdomains = self.match_subdomains(resp, fuzzy=False)
if not self.check_subdomains(subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
page_num += per_page_num
@@ -59,7 +53,7 @@ class Google(Search):
"""
self.begin()
self.search(self.domain, full_search=True)
self.search(self.domain)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
@@ -67,12 +61,7 @@ class Google(Search):
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
for subdomain in self.recursive_subdomain():
self.search(subdomain)
self.finish()
self.save_json()
+4 -13
View File
@@ -15,13 +15,12 @@ class GoogleAPI(Search):
self.id = api.google_api_id
self.per_page_num = 10 # 每次只能请求10个结果
def search(self, domain, filtered_subdomain='', full_search=False):
def search(self, domain, filtered_subdomain=''):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 1
while True:
@@ -34,10 +33,7 @@ class GoogleAPI(Search):
'start': self.page_num, 'num': self.per_page_num}
resp = self.get(self.addr, params)
subdomains = self.match_subdomains(resp)
if not subdomains:
break
if not full_search and subdomains.issubset(self.subdomains):
# 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
if not self.check_subdomains(subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
self.page_num += self.per_page_num
@@ -51,7 +47,7 @@ class GoogleAPI(Search):
if not self.check(self.id, self.key):
return
self.begin()
self.search(self.domain, full_search=True)
self.search(self.domain)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
@@ -59,12 +55,7 @@ class GoogleAPI(Search):
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
for subdomain in self.recursive_subdomain():
self.search(subdomain)
self.finish()
self.save_json()
-1
View File
@@ -26,7 +26,6 @@ class ShodanAPI(Search):
subdomains = self.match_subdomains(resp)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
if subdomains:
self.subdomains = self.subdomains.union(subdomains)
page += 1
+5 -16
View File
@@ -13,13 +13,12 @@ class So(Search):
self.limit_num = 640 # 限制搜索条数
self.per_page_num = 10 # 默认每页显示10页
def search(self, domain, filtered_subdomain='', full_search=False):
def search(self, domain, filtered_subdomain=''):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
page_num = 1
while True:
@@ -29,13 +28,8 @@ class So(Search):
word = 'site:.' + domain + filtered_subdomain
payload = {'q': word, 'pn': page_num}
resp = self.get(url=self.addr, params=payload)
if not resp:
return
subdomains = self.match_subdomains(resp.text, fuzzy=False)
if not subdomains:
break
if not full_search and subdomains.issubset(self.subdomains):
# 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
subdomains = self.match_subdomains(resp, fuzzy=False)
if not self.check_subdomains(subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
page_num += 1
@@ -51,7 +45,7 @@ class So(Search):
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
self.search(self.domain)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
@@ -59,12 +53,7 @@ class So(Search):
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
for subdomain in self.recursive_subdomain():
self.search(subdomain)
self.finish()
self.save_json()
+5 -17
View File
@@ -10,13 +10,12 @@ class Sogou(Search):
self.addr = 'https://www.sogou.com/web'
self.limit_num = 1000 # 限制搜索条数
def search(self, domain, filtered_subdomain='', full_search=False):
def search(self, domain, filtered_subdomain=''):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 1
while True:
@@ -26,13 +25,8 @@ class Sogou(Search):
payload = {'query': word, 'page': self.page_num,
"num": self.per_page_num}
resp = self.get(self.addr, payload)
if not resp:
return
subdomains = self.match_subdomains(resp.text, fuzzy=False)
if not subdomains:
break
if not full_search and subdomains.issubset(self.subdomains):
# 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
subdomains = self.match_subdomains(resp, fuzzy=False)
if not self.check_subdomains(subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
self.page_num += 1
@@ -48,8 +42,7 @@ class Sogou(Search):
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
self.search(self.domain)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
@@ -57,12 +50,7 @@ class Sogou(Search):
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
for subdomain in self.recursive_subdomain():
self.search(subdomain)
self.finish()
self.save_json()
+4 -14
View File
@@ -14,13 +14,12 @@ class Yahoo(Search):
self.delay = 2
self.per_page_num = 30 # Yahoo每次搜索最大条数
def search(self, domain, filtered_subdomain='', full_search=False):
def search(self, domain, filtered_subdomain=''):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
resp = self.get(self.init)
if not resp:
@@ -37,10 +36,7 @@ class Yahoo(Search):
return
text = resp.text.replace('<b>', '').replace('</b>', '')
subdomains = self.match_subdomains(text, fuzzy=False)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
if not full_search and subdomains.issubset(self.subdomains):
# 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
if not self.check_subdomains(subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
if '>Next</a>' not in resp.text: # 搜索页面没有出现下一页时停止搜索
@@ -54,8 +50,7 @@ class Yahoo(Search):
类执行入口
"""
self.begin()
self.search(self.domain, full_search=True)
self.search(self.domain)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
@@ -63,12 +58,7 @@ class Yahoo(Search):
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
for subdomain in self.recursive_subdomain():
self.search(subdomain)
self.finish()
self.save_json()
+5 -16
View File
@@ -13,13 +13,12 @@ class Yandex(Search):
self.limit_num = 1000 # 限制搜索条数
self.delay = 5
def search(self, domain, filtered_subdomain='', full_search=False):
def search(self, domain, filtered_subdomain=''):
"""
发送搜索请求并做子域匹配
:param str domain: 域名
:param str filtered_subdomain: 过滤的子域
:param bool full_search: 全量搜索
"""
self.page_num = 0 # 二次搜索重新置0
resp = self.get(self.init)
@@ -34,13 +33,8 @@ class Yandex(Search):
params = {'text': query, 'p': self.page_num,
'numdoc': self.per_page_num}
resp = self.get(self.addr, params)
if not resp:
return
subdomains = self.match_subdomains(resp.text, fuzzy=False)
if not subdomains: # 搜索没有发现子域名则停止搜索
break
if not full_search and subdomains.issubset(self.subdomains):
# 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
subdomains = self.match_subdomains(resp, fuzzy=False)
if not self.check_subdomains(subdomains):
break
self.subdomains = self.subdomains.union(subdomains)
if '>next</a>' not in resp.text: # 搜索页面没有出现下一页时停止搜索
@@ -55,7 +49,7 @@ class Yandex(Search):
"""
self.begin()
self.search(self.domain, full_search=True)
self.search(self.domain)
# 排除同一子域搜索结果过多的子域以发现新的子域
for statement in self.filter(self.domain, self.subdomains):
@@ -63,12 +57,7 @@ class Yandex(Search):
# 递归搜索下一层的子域
if self.recursive_search:
# 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
for layer_num in range(1, self.recursive_times):
for subdomain in self.subdomains:
# 进行下一层子域搜索的限制条件
count = subdomain.count('.') - self.domain.count('.')
if count == layer_num:
for subdomain in self.recursive_subdomain():
self.search(subdomain)
self.finish()
self.save_json()
+2 -1
View File
@@ -23,7 +23,8 @@ class ZoomEyeAPI(Search):
data = {'username': self.user, 'password': self.pwd}
resp = self.post(url=url, json=data)
if not resp:
logger.log('FATAL', f'{self.source} module login failed, can not get access token')
logger.log('FATAL', f'{self.source} module login failed,'
f' can not get access token')
exit(1)
data = resp.json()
if resp.status_code == 200: