统一调整子域匹配函数

This commit is contained in:
Jing Ling
2020-05-11 21:17:25 +08:00
parent ebd55faa9e
commit 74d5354ead
57 changed files with 94 additions and 93 deletions
+1 -1
View File
@@ -20,7 +20,7 @@ class Lookup(Module):
return None return None
for item in answer: for item in answer:
record = item.to_text() record = item.to_text()
subdomains = utils.match_subdomain(self.domain, record) subdomains = self.match_subdomains(self.domain, record)
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
self.gen_record(subdomains, record) self.gen_record(subdomains, record)
return self.subdomains return self.subdomains
+7 -11
View File
@@ -128,7 +128,7 @@ class Module(object):
Custom post request Custom post request
:param str url: request url :param str url: request url
:param dict params: request parameters :param dict data: request parameters
:param bool check: check response :param bool check: check response
:param kwargs: other params :param kwargs: other params
:return: requests's response object :return: requests's response object
@@ -184,24 +184,22 @@ class Module(object):
return self.proxy return self.proxy
@staticmethod @staticmethod
def match(domain, html, distinct=True): def match_subdomains(domain, text, distinct=True):
""" """
Use regexp to match subdomains Use regexp to match subdomains
:param str domain: domain :param str domain: domain
:param str html: response html text :param str text: text
:param bool distinct: deduplicate results or not (default True) :param bool distinct: deduplicate results or not (default True)
:return set/list: result set or list :return set/list: result set or list
""" """
logger.log('TRACE', f'Use regexp to match subdomains in the response body') logger.log('TRACE', f'Use regexp to match subdomains in the response body')
regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \ regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.') + domain.replace('.', r'\.')
result = re.findall(regexp, html, re.I) result = re.findall(regexp, text, re.I)
if not result: if not result:
return set() return set()
regexp = r'(?:http://|https://)' deal = map(lambda s: s.lower(), result)
deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
if distinct: if distinct:
return set(deal) return set(deal)
else: else:
@@ -340,14 +338,12 @@ class Module(object):
'elapse': self.elapse, 'elapse': self.elapse,
'find': find, 'find': find,
'brute': brute, 'brute': brute,
'valid': valid, 'valid': valid}
}
self.results.append(result) self.results.append(result)
def save_db(self): def save_db(self):
""" """
Save module results into the database Save module results into the database
""" """
logger.log('DEBUG', f'Saving results to database') logger.log('DEBUG', f'Saving results to database')
lock.acquire() lock.acquire()
+30 -3
View File
@@ -1,6 +1,9 @@
import re
from config import setting from config import setting
from .module import Module from config.log import logger
from . import utils from common.module import Module
from common import utils
class Search(Module): class Search(Module):
@@ -51,4 +54,28 @@ class Search(Module):
location = resp.headers.get('location') location = resp.headers.get('location')
if not location: if not location:
return set() return set()
return set(utils.match_subdomain(domain, location)) return set(self.match_subdomains(domain, location))
@staticmethod
def match_subdomains(domain, html, distinct=True):
"""
Use regexp to match subdomains
:param str domain: domain
:param str html: response html text
:param bool distinct: deduplicate results or not (default True)
:return set/list: result set or list
"""
logger.log('TRACE', f'Use regexp to match subdomains in the response body')
regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.')
result = re.findall(regexp, html, re.I)
if not result:
return set()
regexp = r'(?:http://|https://)'
deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
if distinct:
return set(deal)
else:
return list(deal)
-21
View File
@@ -32,27 +32,6 @@ user_agents = [
'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0'] 'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0']
def match_subdomain(domain, text, distinct=True):
"""
Use regexp to match subdomains in text
:param str domain: domain
:param str text: response text
:param bool distinct: deduplicate results
:return set/list: match result
"""
regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+ domain.replace('.', r'\.')
result = re.findall(regexp, text, re.I)
if not result:
return set()
deal = map(lambda s: s.lower(), result)
if distinct:
return set(deal)
else:
return list(deal)
def gen_random_ip(): def gen_random_ip():
""" """
Generate random decimal IP string Generate random decimal IP string
+2 -2
View File
@@ -33,7 +33,7 @@ class CensysAPI(Query):
if status != 'ok': if status != 'ok':
logger.log('ALERT', status) logger.log('ALERT', status)
return return
subdomains = self.match(self.domain, str(json)) subdomains = self.match_subdomains(self.domain, str(json))
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
pages = json.get('metadata').get('pages') pages = json.get('metadata').get('pages')
for page in range(2, pages + 1): for page in range(2, pages + 1):
@@ -41,7 +41,7 @@ class CensysAPI(Query):
resp = self.post(self.addr, json=data, auth=(self.id, self.secret)) resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
if not resp: if not resp:
return return
subdomains = self.match(self.domain, str(resp.json())) subdomains = self.match_subdomains(self.domain, str(resp.json()))
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
def run(self): def run(self):
+1 -1
View File
@@ -22,7 +22,7 @@ class CertSpotter(Query):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = utils.match_subdomain(self.domain, str(resp.json())) subdomains = self.match_subdomains(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -20,7 +20,7 @@ class Crtsh(Query):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = utils.match_subdomain(self.domain, str(resp.json())) subdomains = self.match_subdomains(self.domain, str(resp.json()))
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
def run(self): def run(self):
+1 -1
View File
@@ -23,7 +23,7 @@ class Google(Query):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = utils.match_subdomain(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -44,7 +44,7 @@ class CheckAXFR(Module):
names = zone.nodes.keys() names = zone.nodes.keys()
for name in names: for name in names:
full_domain = str(name) + '.' + self.domain full_domain = str(name) + '.' + self.domain
subdomain = utils.match_subdomain(self.domain, full_domain) subdomain = self.match_subdomains(self.domain, full_domain)
self.subdomains = self.subdomains.union(subdomain) self.subdomains = self.subdomains.union(subdomain)
record = zone[name].to_text(name) record = zone[name].to_text(name)
self.results.append(record) self.results.append(record)
+1 -1
View File
@@ -31,7 +31,7 @@ class CheckCDX(Module):
if not response: if not response:
return return
if response and len(response.content): if response and len(response.content):
self.subdomains = utils.match_subdomain(self.domain, self.subdomains = self.match_subdomains(self.domain,
response.text) response.text)
def run(self): def run(self):
+1 -1
View File
@@ -32,7 +32,7 @@ class CheckCert(Module):
except Exception as e: except Exception as e:
logger.log('DEBUG', e.args) logger.log('DEBUG', e.args)
return return
subdomains = utils.match_subdomain(self.domain, str(cert_dict)) subdomains = self.match_subdomains(self.domain, str(cert_dict))
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
def run(self): def run(self):
+1 -1
View File
@@ -52,7 +52,7 @@ class CheckCSP(Module):
if not csp: if not csp:
logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段') logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段')
return return
self.subdomains = utils.match_subdomain(self.domain, csp) self.subdomains = self.match_subdomains(self.domain, csp)
def run(self): def run(self):
""" """
+1 -1
View File
@@ -30,7 +30,7 @@ class CheckRobots(Module):
if not response: if not response:
return return
if response and len(response.content): if response and len(response.content):
self.subdomains = utils.match_subdomain(self.domain, self.subdomains = self.match_subdomains(self.domain,
response.text) response.text)
def run(self): def run(self):
+1 -1
View File
@@ -43,7 +43,7 @@ class CheckRobots(Module):
if not response: if not response:
return return
if response and len(response.content): if response and len(response.content):
self.subdomains = utils.match_subdomain(self.domain, self.subdomains = self.match_subdomains(self.domain,
response.text) response.text)
+1 -1
View File
@@ -26,7 +26,7 @@ class ArchiveCrawl(Crawl):
for resp in cdx.iter(url, limit=limit): for resp in cdx.iter(url, limit=limit):
if resp.data.get('status') not in ['301', '302']: if resp.data.get('status') not in ['301', '302']:
url = resp.data.get('url') url = resp.data.get('url')
subdomains = self.match(self.register(domain), subdomains = self.match_subdomains(self.register(domain),
url + resp.text) url + resp.text)
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -26,7 +26,7 @@ class CommonCrawl(Crawl):
for resp in tqdm(cdx.iter(url, limit=limit), total=limit): for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
if resp.data.get('status') not in ['301', '302']: if resp.data.get('status') not in ['301', '302']:
subdomains = self.match(self.register(domain), resp.text) subdomains = self.match_subdomains(self.register(domain), resp.text)
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -22,7 +22,7 @@ class BinaryEdgeAPI(Query):
resp = self.get(url) resp = self.get(url)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, str(resp.json())) subdomains = self.match_subdomains(self.domain, str(resp.json()))
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
def run(self): def run(self):
+1 -1
View File
@@ -26,7 +26,7 @@ class BufferOver(Query):
return return
if resp.status_code != 200: if resp.status_code != 200:
return return
subdomains = self.match(self.domain, str(resp.json())) subdomains = self.match_subdomains(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -19,7 +19,7 @@ class CeBaidu(Query):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, str(resp.json())) subdomains = self.match_subdomains(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -19,7 +19,7 @@ class Chinaz(Query):
resp = self.get(self.addr) resp = self.get(self.addr)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -21,7 +21,7 @@ class ChinazAPI(Query):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, str(resp.json())) subdomains = self.match_subdomains(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -21,7 +21,7 @@ class CirclAPI(Query):
resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd)) resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
if not resp: if not resp:
return return
subdomains = self.match(self.domain, str(resp.json())) subdomains = self.match_subdomains(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -23,7 +23,7 @@ class DNSdbAPI(Query):
resp = self.get(url) resp = self.get(url)
if not resp: if not resp:
return return
subdomains = utils.match_subdomain(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -26,7 +26,7 @@ class DNSdumpster(Query):
resp = self.post(self.addr, data) resp = self.post(self.addr, data)
if not resp: if not resp:
return return
subdomains = utils.match_subdomain(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
if subdomains: if subdomains:
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -21,7 +21,7 @@ class HackerTarget(Query):
if not resp: if not resp:
return return
if resp.status_code == 200: if resp.status_code == 200:
subdomains = utils.match_subdomain(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
if subdomains: if subdomains:
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -19,7 +19,7 @@ class IP138(Query):
resp = self.get(self.addr) resp = self.get(self.addr)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -32,7 +32,7 @@ class IPv4InfoAPI(Query):
except Exception as e: except Exception as e:
logger.log('DEBUG', e.args) logger.log('DEBUG', e.args)
break break
subdomains = self.match(self.domain, str(json)) subdomains = self.match_subdomains(self.domain, str(json))
if not subdomains: if not subdomains:
break break
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
+1 -1
View File
@@ -49,7 +49,7 @@ class NetCraft(Query):
resp = self.get(self.addr + last, params) resp = self.get(self.addr + last, params)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索 if not subdomains: # 搜索没有发现子域名则停止搜索
break break
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
+1 -1
View File
@@ -22,7 +22,7 @@ class PassiveDnsAPI(Query):
resp = self.get(url) resp = self.get(url)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, str(resp.json())) subdomains = self.match_subdomains(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -27,7 +27,7 @@ class QianXun(Query):
resp = self.post(url, data) resp = self.post(url, data)
if not resp: if not resp:
break break
subdomains = self.match(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
if '<div id="page" class="pagelist">' not in resp.text: if '<div id="page" class="pagelist">' not in resp.text:
break break
+1 -1
View File
@@ -20,7 +20,7 @@ class RapidDNS(Query):
resp = self.get(url, params) resp = self.get(url, params)
if not resp: if not resp:
return return
subdomains = utils.match_subdomain(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -19,7 +19,7 @@ class Riddler(Query):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -32,7 +32,7 @@ class Robtex(Query):
resp = self.get(url) resp = self.get(url)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
if subdomains: if subdomains:
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -24,7 +24,7 @@ class SiteDossier(Query):
resp = self.get(url) resp = self.get(url)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索 if not subdomains: # 搜索没有发现子域名则停止搜索
break break
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
+1 -1
View File
@@ -27,7 +27,7 @@ class SpyseAPI(Query):
if not resp: if not resp:
return return
json = resp.json() json = resp.json()
subdomains = utils.match_subdomain(self.domain, str(json)) subdomains = self.match_subdomains(self.domain, str(json))
if not subdomains: # 搜索没有发现子域名则停止搜索 if not subdomains: # 搜索没有发现子域名则停止搜索
break break
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
+1 -1
View File
@@ -25,7 +25,7 @@ class ThreatCrowd(Query):
return return
if resp.status_code != 200: if resp.status_code != 200:
return return
subdomains = self.match(self.domain, str(resp.json())) subdomains = self.match_subdomains(self.domain, str(resp.json()))
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -31,7 +31,7 @@ class WZPCQuery(Query):
break break
if not resp: if not resp:
break break
subdomains = self.match(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
if not subdomains: if not subdomains:
break break
+1 -1
View File
@@ -20,7 +20,7 @@ class Ximcx(Query):
if not resp: if not resp:
return return
json = resp.json() json = resp.json()
subdomains = self.match(self.domain, str(json)) subdomains = self.match_subdomains(self.domain, str(json))
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -50,7 +50,7 @@ class BruteSRV(Module):
continue continue
for item in answer: for item in answer:
record = str(item) record = str(item)
subdomains = utils.match_subdomain(self.domain, record) subdomains = self.match_subdomains(self.domain, record)
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
self.gen_record(subdomains, record) self.gen_record(subdomains, record)
+2 -2
View File
@@ -21,7 +21,7 @@ class AlienVault(Query):
if not resp: if not resp:
return return
json = resp.json() json = resp.json()
subdomains = self.match(self.domain, str(json)) subdomains = self.match_subdomains(self.domain, str(json))
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
url = f'{base}/{self.domain}/url_list' url = f'{base}/{self.domain}/url_list'
@@ -29,7 +29,7 @@ class AlienVault(Query):
if not resp: if not resp:
return return
json = resp.json() json = resp.json()
subdomains = self.match(self.domain, str(json)) subdomains = self.match_subdomains(self.domain, str(json))
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
def run(self): def run(self):
+1 -1
View File
@@ -23,7 +23,7 @@ class ThreatBookAPI(Query):
resp = self.post(self.addr, params) resp = self.post(self.addr, params)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, str(resp.json())) subdomains = self.match_subdomains(self.domain, str(resp.json()))
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
def run(self): def run(self):
+1 -1
View File
@@ -20,7 +20,7 @@ class ThreatMiner(Query):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
# 合并搜索子域名搜索结果 # 合并搜索子域名搜索结果
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -30,7 +30,7 @@ class Ask(Search):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = self.match(domain, resp.text) subdomains = self.match_subdomains(domain, resp.text)
if not subdomains: if not subdomains:
break break
if not full_search: if not full_search:
+1 -1
View File
@@ -54,7 +54,7 @@ class Baidu(Search):
# 获取百度跳转URL响应头的Location字段获取直链 # 获取百度跳转URL响应头的Location字段获取直链
subdomains = self.redirect_match(domain, resp.text) subdomains = self.redirect_match(domain, resp.text)
else: else:
subdomains = self.match(domain, resp.text) subdomains = self.match_subdomains(domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索 if not subdomains: # 搜索没有发现子域名则停止搜索
break break
if not full_search: if not full_search:
+1 -1
View File
@@ -36,7 +36,7 @@ class Bing(Search):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = self.match(domain, resp.text) subdomains = self.match_subdomains(domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索 if not subdomains: # 搜索没有发现子域名则停止搜索
break break
if not full_search: if not full_search:
+1 -1
View File
@@ -37,7 +37,7 @@ class BingAPI(Search):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = self.match(domain, str(resp.json())) subdomains = self.match_subdomains(domain, str(resp.json()))
if not subdomains: # 搜索没有发现子域名则停止搜索 if not subdomains: # 搜索没有发现子域名则停止搜索
break break
if not full_search: if not full_search:
+1 -1
View File
@@ -36,7 +36,7 @@ class FoFa(Search):
if not resp: if not resp:
return return
resp_json = resp.json() resp_json = resp.json()
subdomains = self.match(self.domain, str(resp_json)) subdomains = self.match_subdomains(self.domain, str(resp_json))
if not subdomains: # 搜索没有发现子域名则停止搜索 if not subdomains: # 搜索没有发现子域名则停止搜索
break break
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -34,7 +34,7 @@ class Gitee(Search):
if 'class="empty-box"' in resp.text: if 'class="empty-box"' in resp.text:
break break
soup = BeautifulSoup(resp.text, 'html.parser') soup = BeautifulSoup(resp.text, 'html.parser')
subdomains = self.match(self.domain, soup.text) subdomains = self.match_subdomains(self.domain, soup.text)
if not subdomains: if not subdomains:
break break
if not full_search: if not full_search:
+1 -2
View File
@@ -1,6 +1,5 @@
import requests import requests
from config import api from config import api
from common.utils import match_subdomain
from common.search import Search from common.search import Search
from config.log import logger from config.log import logger
@@ -61,7 +60,7 @@ class GithubAPI(Search):
if resp.status_code != 200: if resp.status_code != 200:
logger.log('ERROR', f'{self.source}模块搜索出错') logger.log('ERROR', f'{self.source}模块搜索出错')
break break
subdomains = match_subdomain(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
if not subdomains: if not subdomains:
break break
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)
+1 -1
View File
@@ -40,7 +40,7 @@ class Google(Search):
resp = self.get(url=self.addr, params=payload) resp = self.get(url=self.addr, params=payload)
if not resp: if not resp:
return return
subdomains = self.match(domain, resp.text) subdomains = self.match_subdomains(domain, resp.text)
if not subdomains: if not subdomains:
break break
if not full_search: if not full_search:
+1 -1
View File
@@ -35,7 +35,7 @@ class GoogleAPI(Search):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = self.match(domain, str(resp.json())) subdomains = self.match_subdomains(domain, str(resp.json()))
if not subdomains: if not subdomains:
break break
if not full_search: if not full_search:
+1 -1
View File
@@ -25,7 +25,7 @@ class ShodanAPI(Search):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索 if not subdomains: # 搜索没有发现子域名则停止搜索
break break
if subdomains: if subdomains:
+1 -1
View File
@@ -31,7 +31,7 @@ class So(Search):
resp = self.get(url=self.addr, params=payload) resp = self.get(url=self.addr, params=payload)
if not resp: if not resp:
return return
subdomains = self.match(domain, resp.text) subdomains = self.match_subdomains(domain, resp.text)
if not subdomains: if not subdomains:
break break
if not full_search: if not full_search:
+1 -1
View File
@@ -28,7 +28,7 @@ class Sogou(Search):
resp = self.get(self.addr, payload) resp = self.get(self.addr, payload)
if not resp: if not resp:
return return
subdomains = self.match(domain, resp.text) subdomains = self.match_subdomains(domain, resp.text)
if not subdomains: if not subdomains:
break break
if not full_search: if not full_search:
+1 -1
View File
@@ -36,7 +36,7 @@ class Yahoo(Search):
if not resp: if not resp:
return return
text = resp.text.replace('<b>', '').replace('</b>', '') text = resp.text.replace('<b>', '').replace('</b>', '')
subdomains = self.match(domain, text) subdomains = self.match_subdomains(domain, text)
if not subdomains: # 搜索没有发现子域名则停止搜索 if not subdomains: # 搜索没有发现子域名则停止搜索
break break
if not full_search: if not full_search:
+1 -1
View File
@@ -36,7 +36,7 @@ class Yandex(Search):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = self.match(domain, resp.text) subdomains = self.match_subdomains(domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索 if not subdomains: # 搜索没有发现子域名则停止搜索
break break
if not full_search: if not full_search:
+1 -1
View File
@@ -48,7 +48,7 @@ class ZoomEyeAPI(Search):
resp = self.get(self.addr, params) resp = self.get(self.addr, params)
if not resp: if not resp:
return return
subdomains = self.match(self.domain, resp.text) subdomains = self.match_subdomains(self.domain, resp.text)
if not subdomains: # 搜索没有发现子域名则停止搜索 if not subdomains: # 搜索没有发现子域名则停止搜索
break break
self.subdomains = self.subdomains.union(subdomains) self.subdomains = self.subdomains.union(subdomains)