mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-26 04:47:48 +08:00
统一调整子域匹配函数
This commit is contained in:
+1
-1
@@ -20,7 +20,7 @@ class Lookup(Module):
|
|||||||
return None
|
return None
|
||||||
for item in answer:
|
for item in answer:
|
||||||
record = item.to_text()
|
record = item.to_text()
|
||||||
subdomains = utils.match_subdomain(self.domain, record)
|
subdomains = self.match_subdomains(self.domain, record)
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
self.gen_record(subdomains, record)
|
self.gen_record(subdomains, record)
|
||||||
return self.subdomains
|
return self.subdomains
|
||||||
|
|||||||
+7
-11
@@ -128,7 +128,7 @@ class Module(object):
|
|||||||
Custom post request
|
Custom post request
|
||||||
|
|
||||||
:param str url: request url
|
:param str url: request url
|
||||||
:param dict params: request parameters
|
:param dict data: request parameters
|
||||||
:param bool check: check response
|
:param bool check: check response
|
||||||
:param kwargs: other params
|
:param kwargs: other params
|
||||||
:return: requests's response object
|
:return: requests's response object
|
||||||
@@ -184,24 +184,22 @@ class Module(object):
|
|||||||
return self.proxy
|
return self.proxy
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def match(domain, html, distinct=True):
|
def match_subdomains(domain, text, distinct=True):
|
||||||
"""
|
"""
|
||||||
Use regexp to match subdomains
|
Use regexp to match subdomains
|
||||||
|
|
||||||
:param str domain: domain
|
:param str domain: domain
|
||||||
:param str html: response html text
|
:param str text: text
|
||||||
:param bool distinct: deduplicate results or not (default True)
|
:param bool distinct: deduplicate results or not (default True)
|
||||||
:return set/list: result set or list
|
:return set/list: result set or list
|
||||||
"""
|
"""
|
||||||
logger.log('TRACE', f'Use regexp to match subdomains in the response body')
|
logger.log('TRACE', f'Use regexp to match subdomains in the response body')
|
||||||
regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
|
regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
|
||||||
r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
|
|
||||||
+ domain.replace('.', r'\.')
|
+ domain.replace('.', r'\.')
|
||||||
result = re.findall(regexp, html, re.I)
|
result = re.findall(regexp, text, re.I)
|
||||||
if not result:
|
if not result:
|
||||||
return set()
|
return set()
|
||||||
regexp = r'(?:http://|https://)'
|
deal = map(lambda s: s.lower(), result)
|
||||||
deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
|
|
||||||
if distinct:
|
if distinct:
|
||||||
return set(deal)
|
return set(deal)
|
||||||
else:
|
else:
|
||||||
@@ -340,14 +338,12 @@ class Module(object):
|
|||||||
'elapse': self.elapse,
|
'elapse': self.elapse,
|
||||||
'find': find,
|
'find': find,
|
||||||
'brute': brute,
|
'brute': brute,
|
||||||
'valid': valid,
|
'valid': valid}
|
||||||
}
|
|
||||||
self.results.append(result)
|
self.results.append(result)
|
||||||
|
|
||||||
def save_db(self):
|
def save_db(self):
|
||||||
"""
|
"""
|
||||||
Save module results into the database
|
Save module results into the database
|
||||||
|
|
||||||
"""
|
"""
|
||||||
logger.log('DEBUG', f'Saving results to database')
|
logger.log('DEBUG', f'Saving results to database')
|
||||||
lock.acquire()
|
lock.acquire()
|
||||||
|
|||||||
+30
-3
@@ -1,6 +1,9 @@
|
|||||||
|
import re
|
||||||
|
|
||||||
from config import setting
|
from config import setting
|
||||||
from .module import Module
|
from config.log import logger
|
||||||
from . import utils
|
from common.module import Module
|
||||||
|
from common import utils
|
||||||
|
|
||||||
|
|
||||||
class Search(Module):
|
class Search(Module):
|
||||||
@@ -51,4 +54,28 @@ class Search(Module):
|
|||||||
location = resp.headers.get('location')
|
location = resp.headers.get('location')
|
||||||
if not location:
|
if not location:
|
||||||
return set()
|
return set()
|
||||||
return set(utils.match_subdomain(domain, location))
|
return set(self.match_subdomains(domain, location))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def match_subdomains(domain, html, distinct=True):
|
||||||
|
"""
|
||||||
|
Use regexp to match subdomains
|
||||||
|
|
||||||
|
:param str domain: domain
|
||||||
|
:param str html: response html text
|
||||||
|
:param bool distinct: deduplicate results or not (default True)
|
||||||
|
:return set/list: result set or list
|
||||||
|
"""
|
||||||
|
logger.log('TRACE', f'Use regexp to match subdomains in the response body')
|
||||||
|
regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
|
||||||
|
r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
|
||||||
|
+ domain.replace('.', r'\.')
|
||||||
|
result = re.findall(regexp, html, re.I)
|
||||||
|
if not result:
|
||||||
|
return set()
|
||||||
|
regexp = r'(?:http://|https://)'
|
||||||
|
deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
|
||||||
|
if distinct:
|
||||||
|
return set(deal)
|
||||||
|
else:
|
||||||
|
return list(deal)
|
||||||
|
|||||||
@@ -32,27 +32,6 @@ user_agents = [
|
|||||||
'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0']
|
'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0']
|
||||||
|
|
||||||
|
|
||||||
def match_subdomain(domain, text, distinct=True):
|
|
||||||
"""
|
|
||||||
Use regexp to match subdomains in text
|
|
||||||
|
|
||||||
:param str domain: domain
|
|
||||||
:param str text: response text
|
|
||||||
:param bool distinct: deduplicate results
|
|
||||||
:return set/list: match result
|
|
||||||
"""
|
|
||||||
regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
|
|
||||||
+ domain.replace('.', r'\.')
|
|
||||||
result = re.findall(regexp, text, re.I)
|
|
||||||
if not result:
|
|
||||||
return set()
|
|
||||||
deal = map(lambda s: s.lower(), result)
|
|
||||||
if distinct:
|
|
||||||
return set(deal)
|
|
||||||
else:
|
|
||||||
return list(deal)
|
|
||||||
|
|
||||||
|
|
||||||
def gen_random_ip():
|
def gen_random_ip():
|
||||||
"""
|
"""
|
||||||
Generate random decimal IP string
|
Generate random decimal IP string
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ class CensysAPI(Query):
|
|||||||
if status != 'ok':
|
if status != 'ok':
|
||||||
logger.log('ALERT', status)
|
logger.log('ALERT', status)
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, str(json))
|
subdomains = self.match_subdomains(self.domain, str(json))
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
pages = json.get('metadata').get('pages')
|
pages = json.get('metadata').get('pages')
|
||||||
for page in range(2, pages + 1):
|
for page in range(2, pages + 1):
|
||||||
@@ -41,7 +41,7 @@ class CensysAPI(Query):
|
|||||||
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
|
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, str(resp.json()))
|
subdomains = self.match_subdomains(self.domain, str(resp.json()))
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ class CertSpotter(Query):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = utils.match_subdomain(self.domain, str(resp.json()))
|
subdomains = self.match_subdomains(self.domain, str(resp.json()))
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ class Crtsh(Query):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = utils.match_subdomain(self.domain, str(resp.json()))
|
subdomains = self.match_subdomains(self.domain, str(resp.json()))
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ class Google(Query):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = utils.match_subdomain(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ class CheckAXFR(Module):
|
|||||||
names = zone.nodes.keys()
|
names = zone.nodes.keys()
|
||||||
for name in names:
|
for name in names:
|
||||||
full_domain = str(name) + '.' + self.domain
|
full_domain = str(name) + '.' + self.domain
|
||||||
subdomain = utils.match_subdomain(self.domain, full_domain)
|
subdomain = self.match_subdomains(self.domain, full_domain)
|
||||||
self.subdomains = self.subdomains.union(subdomain)
|
self.subdomains = self.subdomains.union(subdomain)
|
||||||
record = zone[name].to_text(name)
|
record = zone[name].to_text(name)
|
||||||
self.results.append(record)
|
self.results.append(record)
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ class CheckCDX(Module):
|
|||||||
if not response:
|
if not response:
|
||||||
return
|
return
|
||||||
if response and len(response.content):
|
if response and len(response.content):
|
||||||
self.subdomains = utils.match_subdomain(self.domain,
|
self.subdomains = self.match_subdomains(self.domain,
|
||||||
response.text)
|
response.text)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ class CheckCert(Module):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.log('DEBUG', e.args)
|
logger.log('DEBUG', e.args)
|
||||||
return
|
return
|
||||||
subdomains = utils.match_subdomain(self.domain, str(cert_dict))
|
subdomains = self.match_subdomains(self.domain, str(cert_dict))
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ class CheckCSP(Module):
|
|||||||
if not csp:
|
if not csp:
|
||||||
logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段')
|
logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段')
|
||||||
return
|
return
|
||||||
self.subdomains = utils.match_subdomain(self.domain, csp)
|
self.subdomains = self.match_subdomains(self.domain, csp)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ class CheckRobots(Module):
|
|||||||
if not response:
|
if not response:
|
||||||
return
|
return
|
||||||
if response and len(response.content):
|
if response and len(response.content):
|
||||||
self.subdomains = utils.match_subdomain(self.domain,
|
self.subdomains = self.match_subdomains(self.domain,
|
||||||
response.text)
|
response.text)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ class CheckRobots(Module):
|
|||||||
if not response:
|
if not response:
|
||||||
return
|
return
|
||||||
if response and len(response.content):
|
if response and len(response.content):
|
||||||
self.subdomains = utils.match_subdomain(self.domain,
|
self.subdomains = self.match_subdomains(self.domain,
|
||||||
response.text)
|
response.text)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ class ArchiveCrawl(Crawl):
|
|||||||
for resp in cdx.iter(url, limit=limit):
|
for resp in cdx.iter(url, limit=limit):
|
||||||
if resp.data.get('status') not in ['301', '302']:
|
if resp.data.get('status') not in ['301', '302']:
|
||||||
url = resp.data.get('url')
|
url = resp.data.get('url')
|
||||||
subdomains = self.match(self.register(domain),
|
subdomains = self.match_subdomains(self.register(domain),
|
||||||
url + resp.text)
|
url + resp.text)
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ class CommonCrawl(Crawl):
|
|||||||
|
|
||||||
for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
|
for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
|
||||||
if resp.data.get('status') not in ['301', '302']:
|
if resp.data.get('status') not in ['301', '302']:
|
||||||
subdomains = self.match(self.register(domain), resp.text)
|
subdomains = self.match_subdomains(self.register(domain), resp.text)
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ class BinaryEdgeAPI(Query):
|
|||||||
resp = self.get(url)
|
resp = self.get(url)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, str(resp.json()))
|
subdomains = self.match_subdomains(self.domain, str(resp.json()))
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ class BufferOver(Query):
|
|||||||
return
|
return
|
||||||
if resp.status_code != 200:
|
if resp.status_code != 200:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, str(resp.json()))
|
subdomains = self.match_subdomains(self.domain, str(resp.json()))
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ class CeBaidu(Query):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, str(resp.json()))
|
subdomains = self.match_subdomains(self.domain, str(resp.json()))
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ class Chinaz(Query):
|
|||||||
resp = self.get(self.addr)
|
resp = self.get(self.addr)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ class ChinazAPI(Query):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, str(resp.json()))
|
subdomains = self.match_subdomains(self.domain, str(resp.json()))
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ class CirclAPI(Query):
|
|||||||
resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
|
resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, str(resp.json()))
|
subdomains = self.match_subdomains(self.domain, str(resp.json()))
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ class DNSdbAPI(Query):
|
|||||||
resp = self.get(url)
|
resp = self.get(url)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = utils.match_subdomain(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ class DNSdumpster(Query):
|
|||||||
resp = self.post(self.addr, data)
|
resp = self.post(self.addr, data)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = utils.match_subdomain(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
if subdomains:
|
if subdomains:
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ class HackerTarget(Query):
|
|||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
if resp.status_code == 200:
|
if resp.status_code == 200:
|
||||||
subdomains = utils.match_subdomain(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
if subdomains:
|
if subdomains:
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ class IP138(Query):
|
|||||||
resp = self.get(self.addr)
|
resp = self.get(self.addr)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ class IPv4InfoAPI(Query):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.log('DEBUG', e.args)
|
logger.log('DEBUG', e.args)
|
||||||
break
|
break
|
||||||
subdomains = self.match(self.domain, str(json))
|
subdomains = self.match_subdomains(self.domain, str(json))
|
||||||
if not subdomains:
|
if not subdomains:
|
||||||
break
|
break
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ class NetCraft(Query):
|
|||||||
resp = self.get(self.addr + last, params)
|
resp = self.get(self.addr + last, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||||
break
|
break
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ class PassiveDnsAPI(Query):
|
|||||||
resp = self.get(url)
|
resp = self.get(url)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, str(resp.json()))
|
subdomains = self.match_subdomains(self.domain, str(resp.json()))
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ class QianXun(Query):
|
|||||||
resp = self.post(url, data)
|
resp = self.post(url, data)
|
||||||
if not resp:
|
if not resp:
|
||||||
break
|
break
|
||||||
subdomains = self.match(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
if '<div id="page" class="pagelist">' not in resp.text:
|
if '<div id="page" class="pagelist">' not in resp.text:
|
||||||
break
|
break
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ class RapidDNS(Query):
|
|||||||
resp = self.get(url, params)
|
resp = self.get(url, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = utils.match_subdomain(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ class Riddler(Query):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ class Robtex(Query):
|
|||||||
resp = self.get(url)
|
resp = self.get(url)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
if subdomains:
|
if subdomains:
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ class SiteDossier(Query):
|
|||||||
resp = self.get(url)
|
resp = self.get(url)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||||
break
|
break
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ class SpyseAPI(Query):
|
|||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
json = resp.json()
|
json = resp.json()
|
||||||
subdomains = utils.match_subdomain(self.domain, str(json))
|
subdomains = self.match_subdomains(self.domain, str(json))
|
||||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||||
break
|
break
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ class ThreatCrowd(Query):
|
|||||||
return
|
return
|
||||||
if resp.status_code != 200:
|
if resp.status_code != 200:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, str(resp.json()))
|
subdomains = self.match_subdomains(self.domain, str(resp.json()))
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ class WZPCQuery(Query):
|
|||||||
break
|
break
|
||||||
if not resp:
|
if not resp:
|
||||||
break
|
break
|
||||||
subdomains = self.match(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
if not subdomains:
|
if not subdomains:
|
||||||
break
|
break
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ class Ximcx(Query):
|
|||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
json = resp.json()
|
json = resp.json()
|
||||||
subdomains = self.match(self.domain, str(json))
|
subdomains = self.match_subdomains(self.domain, str(json))
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ class BruteSRV(Module):
|
|||||||
continue
|
continue
|
||||||
for item in answer:
|
for item in answer:
|
||||||
record = str(item)
|
record = str(item)
|
||||||
subdomains = utils.match_subdomain(self.domain, record)
|
subdomains = self.match_subdomains(self.domain, record)
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
self.gen_record(subdomains, record)
|
self.gen_record(subdomains, record)
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ class AlienVault(Query):
|
|||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
json = resp.json()
|
json = resp.json()
|
||||||
subdomains = self.match(self.domain, str(json))
|
subdomains = self.match_subdomains(self.domain, str(json))
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
url = f'{base}/{self.domain}/url_list'
|
url = f'{base}/{self.domain}/url_list'
|
||||||
@@ -29,7 +29,7 @@ class AlienVault(Query):
|
|||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
json = resp.json()
|
json = resp.json()
|
||||||
subdomains = self.match(self.domain, str(json))
|
subdomains = self.match_subdomains(self.domain, str(json))
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ class ThreatBookAPI(Query):
|
|||||||
resp = self.post(self.addr, params)
|
resp = self.post(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, str(resp.json()))
|
subdomains = self.match_subdomains(self.domain, str(resp.json()))
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ class ThreatMiner(Query):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
# 合并搜索子域名搜索结果
|
# 合并搜索子域名搜索结果
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ class Ask(Search):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(domain, resp.text)
|
subdomains = self.match_subdomains(domain, resp.text)
|
||||||
if not subdomains:
|
if not subdomains:
|
||||||
break
|
break
|
||||||
if not full_search:
|
if not full_search:
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ class Baidu(Search):
|
|||||||
# 获取百度跳转URL响应头的Location字段获取直链
|
# 获取百度跳转URL响应头的Location字段获取直链
|
||||||
subdomains = self.redirect_match(domain, resp.text)
|
subdomains = self.redirect_match(domain, resp.text)
|
||||||
else:
|
else:
|
||||||
subdomains = self.match(domain, resp.text)
|
subdomains = self.match_subdomains(domain, resp.text)
|
||||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||||
break
|
break
|
||||||
if not full_search:
|
if not full_search:
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ class Bing(Search):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(domain, resp.text)
|
subdomains = self.match_subdomains(domain, resp.text)
|
||||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||||
break
|
break
|
||||||
if not full_search:
|
if not full_search:
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ class BingAPI(Search):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(domain, str(resp.json()))
|
subdomains = self.match_subdomains(domain, str(resp.json()))
|
||||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||||
break
|
break
|
||||||
if not full_search:
|
if not full_search:
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ class FoFa(Search):
|
|||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
resp_json = resp.json()
|
resp_json = resp.json()
|
||||||
subdomains = self.match(self.domain, str(resp_json))
|
subdomains = self.match_subdomains(self.domain, str(resp_json))
|
||||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||||
break
|
break
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ class Gitee(Search):
|
|||||||
if 'class="empty-box"' in resp.text:
|
if 'class="empty-box"' in resp.text:
|
||||||
break
|
break
|
||||||
soup = BeautifulSoup(resp.text, 'html.parser')
|
soup = BeautifulSoup(resp.text, 'html.parser')
|
||||||
subdomains = self.match(self.domain, soup.text)
|
subdomains = self.match_subdomains(self.domain, soup.text)
|
||||||
if not subdomains:
|
if not subdomains:
|
||||||
break
|
break
|
||||||
if not full_search:
|
if not full_search:
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
import requests
|
import requests
|
||||||
from config import api
|
from config import api
|
||||||
from common.utils import match_subdomain
|
|
||||||
from common.search import Search
|
from common.search import Search
|
||||||
from config.log import logger
|
from config.log import logger
|
||||||
|
|
||||||
@@ -61,7 +60,7 @@ class GithubAPI(Search):
|
|||||||
if resp.status_code != 200:
|
if resp.status_code != 200:
|
||||||
logger.log('ERROR', f'{self.source}模块搜索出错')
|
logger.log('ERROR', f'{self.source}模块搜索出错')
|
||||||
break
|
break
|
||||||
subdomains = match_subdomain(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
if not subdomains:
|
if not subdomains:
|
||||||
break
|
break
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ class Google(Search):
|
|||||||
resp = self.get(url=self.addr, params=payload)
|
resp = self.get(url=self.addr, params=payload)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(domain, resp.text)
|
subdomains = self.match_subdomains(domain, resp.text)
|
||||||
if not subdomains:
|
if not subdomains:
|
||||||
break
|
break
|
||||||
if not full_search:
|
if not full_search:
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ class GoogleAPI(Search):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(domain, str(resp.json()))
|
subdomains = self.match_subdomains(domain, str(resp.json()))
|
||||||
if not subdomains:
|
if not subdomains:
|
||||||
break
|
break
|
||||||
if not full_search:
|
if not full_search:
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ class ShodanAPI(Search):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||||
break
|
break
|
||||||
if subdomains:
|
if subdomains:
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ class So(Search):
|
|||||||
resp = self.get(url=self.addr, params=payload)
|
resp = self.get(url=self.addr, params=payload)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(domain, resp.text)
|
subdomains = self.match_subdomains(domain, resp.text)
|
||||||
if not subdomains:
|
if not subdomains:
|
||||||
break
|
break
|
||||||
if not full_search:
|
if not full_search:
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ class Sogou(Search):
|
|||||||
resp = self.get(self.addr, payload)
|
resp = self.get(self.addr, payload)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(domain, resp.text)
|
subdomains = self.match_subdomains(domain, resp.text)
|
||||||
if not subdomains:
|
if not subdomains:
|
||||||
break
|
break
|
||||||
if not full_search:
|
if not full_search:
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ class Yahoo(Search):
|
|||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
text = resp.text.replace('<b>', '').replace('</b>', '')
|
text = resp.text.replace('<b>', '').replace('</b>', '')
|
||||||
subdomains = self.match(domain, text)
|
subdomains = self.match_subdomains(domain, text)
|
||||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||||
break
|
break
|
||||||
if not full_search:
|
if not full_search:
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ class Yandex(Search):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(domain, resp.text)
|
subdomains = self.match_subdomains(domain, resp.text)
|
||||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||||
break
|
break
|
||||||
if not full_search:
|
if not full_search:
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ class ZoomEyeAPI(Search):
|
|||||||
resp = self.get(self.addr, params)
|
resp = self.get(self.addr, params)
|
||||||
if not resp:
|
if not resp:
|
||||||
return
|
return
|
||||||
subdomains = self.match(self.domain, resp.text)
|
subdomains = self.match_subdomains(self.domain, resp.text)
|
||||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||||
break
|
break
|
||||||
self.subdomains = self.subdomains.union(subdomains)
|
self.subdomains = self.subdomains.union(subdomains)
|
||||||
|
|||||||
Reference in New Issue
Block a user