From 74d5354ead893ec9528f18b08a207402db96bf44 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Mon, 11 May 2020 21:17:25 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BB=9F=E4=B8=80=E8=B0=83=E6=95=B4=E5=AD=90?= =?UTF-8?q?=E5=9F=9F=E5=8C=B9=E9=85=8D=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/lookup.py | 2 +- common/module.py | 18 ++++++-------- common/search.py | 33 +++++++++++++++++++++++--- common/utils.py | 21 ---------------- modules/certificates/censys_api.py | 4 ++-- modules/certificates/certspotter.py | 2 +- modules/certificates/crtsh.py | 2 +- modules/certificates/google.py | 2 +- modules/check/axfr.py | 2 +- modules/check/cdx.py | 2 +- modules/check/cert.py | 2 +- modules/check/csp.py | 2 +- modules/check/robots.py | 2 +- modules/check/sitemap.py | 2 +- modules/crawl/archivecrawl.py | 4 ++-- modules/crawl/commoncrawl.py | 2 +- modules/datasets/binaryedge_api.py | 2 +- modules/datasets/bufferover.py | 2 +- modules/datasets/cebaidu.py | 2 +- modules/datasets/chinaz.py | 2 +- modules/datasets/chinaz_api.py | 2 +- modules/datasets/circl_api.py | 2 +- modules/datasets/dnsdb_api.py | 2 +- modules/datasets/dnsdumpster.py | 2 +- modules/datasets/hackertarget.py | 2 +- modules/datasets/ip138.py | 2 +- modules/datasets/ipv4info_api.py | 2 +- modules/datasets/netcraft.py | 2 +- modules/datasets/passivedns_api.py | 2 +- modules/datasets/qianxun.py | 2 +- modules/datasets/rapiddns.py | 2 +- modules/datasets/riddler.py | 2 +- modules/datasets/robtex.py | 2 +- modules/datasets/sitedossier.py | 2 +- modules/datasets/spyse_api.py | 2 +- modules/datasets/threatcrowd.py | 2 +- modules/datasets/wzpc.py | 2 +- modules/datasets/ximcx.py | 2 +- modules/dnsquery/srv.py | 2 +- modules/intelligence/alienvault.py | 4 ++-- modules/intelligence/threatbook_api.py | 2 +- modules/intelligence/threatminer.py | 2 +- modules/search/ask.py | 2 +- modules/search/baidu.py | 2 +- modules/search/bing.py | 2 +- modules/search/bing_api.py | 2 +- modules/search/fofa_api.py | 2 +- modules/search/gitee.py | 2 +- modules/search/github_api.py | 3 +-- modules/search/google.py | 2 +- modules/search/google_api.py | 2 +- modules/search/shodan_api.py | 2 +- modules/search/so.py | 2 +- modules/search/sogou.py | 2 +- modules/search/yahoo.py | 2 +- modules/search/yandex.py | 2 +- modules/search/zoomeye_api.py | 2 +- 57 files changed, 94 insertions(+), 93 deletions(-) diff --git a/common/lookup.py b/common/lookup.py index cf72d81..37bb281 100644 --- a/common/lookup.py +++ b/common/lookup.py @@ -20,7 +20,7 @@ class Lookup(Module): return None for item in answer: record = item.to_text() - subdomains = utils.match_subdomain(self.domain, record) + subdomains = self.match_subdomains(self.domain, record) self.subdomains = self.subdomains.union(subdomains) self.gen_record(subdomains, record) return self.subdomains diff --git a/common/module.py b/common/module.py index 98a9ceb..374b12a 100644 --- a/common/module.py +++ b/common/module.py @@ -128,7 +128,7 @@ class Module(object): Custom post request :param str url: request url - :param dict params: request parameters + :param dict data: request parameters :param bool check: check response :param kwargs: other params :return: requests's response object @@ -184,24 +184,22 @@ class Module(object): return self.proxy @staticmethod - def match(domain, html, distinct=True): + def match_subdomains(domain, text, distinct=True): """ Use regexp to match subdomains :param str domain: domain - :param str html: response html text + :param str text: text :param bool distinct: deduplicate results or not (default True) :return set/list: result set or list """ logger.log('TRACE', f'Use regexp to match subdomains in the response body') - regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \ - r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + domain.replace('.', r'\.') - result = re.findall(regexp, html, re.I) + result = re.findall(regexp, text, re.I) if not result: return set() - regexp = r'(?:http://|https://)' - deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result) + deal = map(lambda s: s.lower(), result) if distinct: return set(deal) else: @@ -340,14 +338,12 @@ class Module(object): 'elapse': self.elapse, 'find': find, 'brute': brute, - 'valid': valid, - } + 'valid': valid} self.results.append(result) def save_db(self): """ Save module results into the database - """ logger.log('DEBUG', f'Saving results to database') lock.acquire() diff --git a/common/search.py b/common/search.py index 16c12a3..8dadfad 100644 --- a/common/search.py +++ b/common/search.py @@ -1,6 +1,9 @@ +import re + from config import setting -from .module import Module -from . import utils +from config.log import logger +from common.module import Module +from common import utils class Search(Module): @@ -51,4 +54,28 @@ class Search(Module): location = resp.headers.get('location') if not location: return set() - return set(utils.match_subdomain(domain, location)) + return set(self.match_subdomains(domain, location)) + + @staticmethod + def match_subdomains(domain, html, distinct=True): + """ + Use regexp to match subdomains + + :param str domain: domain + :param str html: response html text + :param bool distinct: deduplicate results or not (default True) + :return set/list: result set or list + """ + logger.log('TRACE', f'Use regexp to match subdomains in the response body') + regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \ + r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + + domain.replace('.', r'\.') + result = re.findall(regexp, html, re.I) + if not result: + return set() + regexp = r'(?:http://|https://)' + deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result) + if distinct: + return set(deal) + else: + return list(deal) diff --git a/common/utils.py b/common/utils.py index 4f9cb32..67d9de9 100644 --- a/common/utils.py +++ b/common/utils.py @@ -32,27 +32,6 @@ user_agents = [ 'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0'] -def match_subdomain(domain, text, distinct=True): - """ - Use regexp to match subdomains in text - - :param str domain: domain - :param str text: response text - :param bool distinct: deduplicate results - :return set/list: match result - """ - regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ - + domain.replace('.', r'\.') - result = re.findall(regexp, text, re.I) - if not result: - return set() - deal = map(lambda s: s.lower(), result) - if distinct: - return set(deal) - else: - return list(deal) - - def gen_random_ip(): """ Generate random decimal IP string diff --git a/modules/certificates/censys_api.py b/modules/certificates/censys_api.py index be98141..bd692a8 100644 --- a/modules/certificates/censys_api.py +++ b/modules/certificates/censys_api.py @@ -33,7 +33,7 @@ class CensysAPI(Query): if status != 'ok': logger.log('ALERT', status) return - subdomains = self.match(self.domain, str(json)) + subdomains = self.match_subdomains(self.domain, str(json)) self.subdomains = self.subdomains.union(subdomains) pages = json.get('metadata').get('pages') for page in range(2, pages + 1): @@ -41,7 +41,7 @@ class CensysAPI(Query): resp = self.post(self.addr, json=data, auth=(self.id, self.secret)) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/certificates/certspotter.py b/modules/certificates/certspotter.py index 153debb..b6b32cf 100644 --- a/modules/certificates/certspotter.py +++ b/modules/certificates/certspotter.py @@ -22,7 +22,7 @@ class CertSpotter(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = utils.match_subdomain(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/certificates/crtsh.py b/modules/certificates/crtsh.py index 830e7af..90b61e4 100644 --- a/modules/certificates/crtsh.py +++ b/modules/certificates/crtsh.py @@ -20,7 +20,7 @@ class Crtsh(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = utils.match_subdomain(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/certificates/google.py b/modules/certificates/google.py index c82c88f..a7ad98f 100644 --- a/modules/certificates/google.py +++ b/modules/certificates/google.py @@ -23,7 +23,7 @@ class Google(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = utils.match_subdomain(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/check/axfr.py b/modules/check/axfr.py index e92b8a1..666a308 100644 --- a/modules/check/axfr.py +++ b/modules/check/axfr.py @@ -44,7 +44,7 @@ class CheckAXFR(Module): names = zone.nodes.keys() for name in names: full_domain = str(name) + '.' + self.domain - subdomain = utils.match_subdomain(self.domain, full_domain) + subdomain = self.match_subdomains(self.domain, full_domain) self.subdomains = self.subdomains.union(subdomain) record = zone[name].to_text(name) self.results.append(record) diff --git a/modules/check/cdx.py b/modules/check/cdx.py index ec2f49f..bdc3156 100644 --- a/modules/check/cdx.py +++ b/modules/check/cdx.py @@ -31,7 +31,7 @@ class CheckCDX(Module): if not response: return if response and len(response.content): - self.subdomains = utils.match_subdomain(self.domain, + self.subdomains = self.match_subdomains(self.domain, response.text) def run(self): diff --git a/modules/check/cert.py b/modules/check/cert.py index e122ef5..659fe43 100644 --- a/modules/check/cert.py +++ b/modules/check/cert.py @@ -32,7 +32,7 @@ class CheckCert(Module): except Exception as e: logger.log('DEBUG', e.args) return - subdomains = utils.match_subdomain(self.domain, str(cert_dict)) + subdomains = self.match_subdomains(self.domain, str(cert_dict)) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/check/csp.py b/modules/check/csp.py index bce94fb..11850a1 100644 --- a/modules/check/csp.py +++ b/modules/check/csp.py @@ -52,7 +52,7 @@ class CheckCSP(Module): if not csp: logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段') return - self.subdomains = utils.match_subdomain(self.domain, csp) + self.subdomains = self.match_subdomains(self.domain, csp) def run(self): """ diff --git a/modules/check/robots.py b/modules/check/robots.py index 1c87719..8116c7a 100644 --- a/modules/check/robots.py +++ b/modules/check/robots.py @@ -30,7 +30,7 @@ class CheckRobots(Module): if not response: return if response and len(response.content): - self.subdomains = utils.match_subdomain(self.domain, + self.subdomains = self.match_subdomains(self.domain, response.text) def run(self): diff --git a/modules/check/sitemap.py b/modules/check/sitemap.py index 54db09f..c03122a 100644 --- a/modules/check/sitemap.py +++ b/modules/check/sitemap.py @@ -43,7 +43,7 @@ class CheckRobots(Module): if not response: return if response and len(response.content): - self.subdomains = utils.match_subdomain(self.domain, + self.subdomains = self.match_subdomains(self.domain, response.text) diff --git a/modules/crawl/archivecrawl.py b/modules/crawl/archivecrawl.py index 5ef1181..70ac9a1 100644 --- a/modules/crawl/archivecrawl.py +++ b/modules/crawl/archivecrawl.py @@ -26,8 +26,8 @@ class ArchiveCrawl(Crawl): for resp in cdx.iter(url, limit=limit): if resp.data.get('status') not in ['301', '302']: url = resp.data.get('url') - subdomains = self.match(self.register(domain), - url + resp.text) + subdomains = self.match_subdomains(self.register(domain), + url + resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/crawl/commoncrawl.py b/modules/crawl/commoncrawl.py index 52779c4..6504910 100644 --- a/modules/crawl/commoncrawl.py +++ b/modules/crawl/commoncrawl.py @@ -26,7 +26,7 @@ class CommonCrawl(Crawl): for resp in tqdm(cdx.iter(url, limit=limit), total=limit): if resp.data.get('status') not in ['301', '302']: - subdomains = self.match(self.register(domain), resp.text) + subdomains = self.match_subdomains(self.register(domain), resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/binaryedge_api.py b/modules/datasets/binaryedge_api.py index 95d48ad..6a63e1f 100644 --- a/modules/datasets/binaryedge_api.py +++ b/modules/datasets/binaryedge_api.py @@ -22,7 +22,7 @@ class BinaryEdgeAPI(Query): resp = self.get(url) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/datasets/bufferover.py b/modules/datasets/bufferover.py index cdea847..b5eedb6 100644 --- a/modules/datasets/bufferover.py +++ b/modules/datasets/bufferover.py @@ -26,7 +26,7 @@ class BufferOver(Query): return if resp.status_code != 200: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/cebaidu.py b/modules/datasets/cebaidu.py index 44ed6d4..5311246 100644 --- a/modules/datasets/cebaidu.py +++ b/modules/datasets/cebaidu.py @@ -19,7 +19,7 @@ class CeBaidu(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/chinaz.py b/modules/datasets/chinaz.py index c852c51..82a0128 100644 --- a/modules/datasets/chinaz.py +++ b/modules/datasets/chinaz.py @@ -19,7 +19,7 @@ class Chinaz(Query): resp = self.get(self.addr) if not resp: return - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/chinaz_api.py b/modules/datasets/chinaz_api.py index 50227ee..69ba907 100644 --- a/modules/datasets/chinaz_api.py +++ b/modules/datasets/chinaz_api.py @@ -21,7 +21,7 @@ class ChinazAPI(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/circl_api.py b/modules/datasets/circl_api.py index 2147479..b2548f2 100644 --- a/modules/datasets/circl_api.py +++ b/modules/datasets/circl_api.py @@ -21,7 +21,7 @@ class CirclAPI(Query): resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd)) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/dnsdb_api.py b/modules/datasets/dnsdb_api.py index f957261..6d4f839 100644 --- a/modules/datasets/dnsdb_api.py +++ b/modules/datasets/dnsdb_api.py @@ -23,7 +23,7 @@ class DNSdbAPI(Query): resp = self.get(url) if not resp: return - subdomains = utils.match_subdomain(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/dnsdumpster.py b/modules/datasets/dnsdumpster.py index 3226200..1b0e415 100644 --- a/modules/datasets/dnsdumpster.py +++ b/modules/datasets/dnsdumpster.py @@ -26,7 +26,7 @@ class DNSdumpster(Query): resp = self.post(self.addr, data) if not resp: return - subdomains = utils.match_subdomain(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) if subdomains: # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/hackertarget.py b/modules/datasets/hackertarget.py index 84ee387..967aad5 100644 --- a/modules/datasets/hackertarget.py +++ b/modules/datasets/hackertarget.py @@ -21,7 +21,7 @@ class HackerTarget(Query): if not resp: return if resp.status_code == 200: - subdomains = utils.match_subdomain(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) if subdomains: # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/ip138.py b/modules/datasets/ip138.py index 46a19d7..54072fc 100644 --- a/modules/datasets/ip138.py +++ b/modules/datasets/ip138.py @@ -19,7 +19,7 @@ class IP138(Query): resp = self.get(self.addr) if not resp: return - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/ipv4info_api.py b/modules/datasets/ipv4info_api.py index 236b6e7..681a50e 100644 --- a/modules/datasets/ipv4info_api.py +++ b/modules/datasets/ipv4info_api.py @@ -32,7 +32,7 @@ class IPv4InfoAPI(Query): except Exception as e: logger.log('DEBUG', e.args) break - subdomains = self.match(self.domain, str(json)) + subdomains = self.match_subdomains(self.domain, str(json)) if not subdomains: break # 合并搜索子域名搜索结果 diff --git a/modules/datasets/netcraft.py b/modules/datasets/netcraft.py index 5c6089e..60999e9 100644 --- a/modules/datasets/netcraft.py +++ b/modules/datasets/netcraft.py @@ -49,7 +49,7 @@ class NetCraft(Query): resp = self.get(self.addr + last, params) if not resp: return - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break # 合并搜索子域名搜索结果 diff --git a/modules/datasets/passivedns_api.py b/modules/datasets/passivedns_api.py index 2755a91..30a9dfa 100644 --- a/modules/datasets/passivedns_api.py +++ b/modules/datasets/passivedns_api.py @@ -22,7 +22,7 @@ class PassiveDnsAPI(Query): resp = self.get(url) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/qianxun.py b/modules/datasets/qianxun.py index c82144f..f261689 100644 --- a/modules/datasets/qianxun.py +++ b/modules/datasets/qianxun.py @@ -27,7 +27,7 @@ class QianXun(Query): resp = self.post(url, data) if not resp: break - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) self.subdomains = self.subdomains.union(subdomains) if '
' not in resp.text: break diff --git a/modules/datasets/rapiddns.py b/modules/datasets/rapiddns.py index 4ac9f27..058599f 100644 --- a/modules/datasets/rapiddns.py +++ b/modules/datasets/rapiddns.py @@ -20,7 +20,7 @@ class RapidDNS(Query): resp = self.get(url, params) if not resp: return - subdomains = utils.match_subdomain(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/riddler.py b/modules/datasets/riddler.py index 5e9e096..109ebb4 100644 --- a/modules/datasets/riddler.py +++ b/modules/datasets/riddler.py @@ -19,7 +19,7 @@ class Riddler(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/robtex.py b/modules/datasets/robtex.py index 3fd0455..3a9f22e 100644 --- a/modules/datasets/robtex.py +++ b/modules/datasets/robtex.py @@ -32,7 +32,7 @@ class Robtex(Query): resp = self.get(url) if not resp: return - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) if subdomains: # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/sitedossier.py b/modules/datasets/sitedossier.py index 6ac7a55..81cb9e9 100644 --- a/modules/datasets/sitedossier.py +++ b/modules/datasets/sitedossier.py @@ -24,7 +24,7 @@ class SiteDossier(Query): resp = self.get(url) if not resp: return - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break # 合并搜索子域名搜索结果 diff --git a/modules/datasets/spyse_api.py b/modules/datasets/spyse_api.py index ce707b5..ba99178 100644 --- a/modules/datasets/spyse_api.py +++ b/modules/datasets/spyse_api.py @@ -27,7 +27,7 @@ class SpyseAPI(Query): if not resp: return json = resp.json() - subdomains = utils.match_subdomain(self.domain, str(json)) + subdomains = self.match_subdomains(self.domain, str(json)) if not subdomains: # 搜索没有发现子域名则停止搜索 break # 合并搜索子域名搜索结果 diff --git a/modules/datasets/threatcrowd.py b/modules/datasets/threatcrowd.py index d064cb4..03b118a 100644 --- a/modules/datasets/threatcrowd.py +++ b/modules/datasets/threatcrowd.py @@ -25,7 +25,7 @@ class ThreatCrowd(Query): return if resp.status_code != 200: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/wzpc.py b/modules/datasets/wzpc.py index b775b50..ec60237 100644 --- a/modules/datasets/wzpc.py +++ b/modules/datasets/wzpc.py @@ -31,7 +31,7 @@ class WZPCQuery(Query): break if not resp: break - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) self.subdomains = self.subdomains.union(subdomains) if not subdomains: break diff --git a/modules/datasets/ximcx.py b/modules/datasets/ximcx.py index 943b6ca..57a1a09 100644 --- a/modules/datasets/ximcx.py +++ b/modules/datasets/ximcx.py @@ -20,7 +20,7 @@ class Ximcx(Query): if not resp: return json = resp.json() - subdomains = self.match(self.domain, str(json)) + subdomains = self.match_subdomains(self.domain, str(json)) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/dnsquery/srv.py b/modules/dnsquery/srv.py index e1bad11..1f00ec7 100644 --- a/modules/dnsquery/srv.py +++ b/modules/dnsquery/srv.py @@ -50,7 +50,7 @@ class BruteSRV(Module): continue for item in answer: record = str(item) - subdomains = utils.match_subdomain(self.domain, record) + subdomains = self.match_subdomains(self.domain, record) self.subdomains = self.subdomains.union(subdomains) self.gen_record(subdomains, record) diff --git a/modules/intelligence/alienvault.py b/modules/intelligence/alienvault.py index a7eeeee..925beec 100644 --- a/modules/intelligence/alienvault.py +++ b/modules/intelligence/alienvault.py @@ -21,7 +21,7 @@ class AlienVault(Query): if not resp: return json = resp.json() - subdomains = self.match(self.domain, str(json)) + subdomains = self.match_subdomains(self.domain, str(json)) self.subdomains = self.subdomains.union(subdomains) url = f'{base}/{self.domain}/url_list' @@ -29,7 +29,7 @@ class AlienVault(Query): if not resp: return json = resp.json() - subdomains = self.match(self.domain, str(json)) + subdomains = self.match_subdomains(self.domain, str(json)) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/intelligence/threatbook_api.py b/modules/intelligence/threatbook_api.py index 99d8393..8e50d42 100644 --- a/modules/intelligence/threatbook_api.py +++ b/modules/intelligence/threatbook_api.py @@ -23,7 +23,7 @@ class ThreatBookAPI(Query): resp = self.post(self.addr, params) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/intelligence/threatminer.py b/modules/intelligence/threatminer.py index a4ca1c9..8832d78 100644 --- a/modules/intelligence/threatminer.py +++ b/modules/intelligence/threatminer.py @@ -20,7 +20,7 @@ class ThreatMiner(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/search/ask.py b/modules/search/ask.py index 784caa5..a8fd235 100644 --- a/modules/search/ask.py +++ b/modules/search/ask.py @@ -30,7 +30,7 @@ class Ask(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(domain, resp.text) + subdomains = self.match_subdomains(domain, resp.text) if not subdomains: break if not full_search: diff --git a/modules/search/baidu.py b/modules/search/baidu.py index 6e79028..a58a31a 100644 --- a/modules/search/baidu.py +++ b/modules/search/baidu.py @@ -54,7 +54,7 @@ class Baidu(Search): # 获取百度跳转URL响应头的Location字段获取直链 subdomains = self.redirect_match(domain, resp.text) else: - subdomains = self.match(domain, resp.text) + subdomains = self.match_subdomains(domain, resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break if not full_search: diff --git a/modules/search/bing.py b/modules/search/bing.py index 2693a5e..461c8b8 100644 --- a/modules/search/bing.py +++ b/modules/search/bing.py @@ -36,7 +36,7 @@ class Bing(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(domain, resp.text) + subdomains = self.match_subdomains(domain, resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break if not full_search: diff --git a/modules/search/bing_api.py b/modules/search/bing_api.py index eaa4670..2864811 100644 --- a/modules/search/bing_api.py +++ b/modules/search/bing_api.py @@ -37,7 +37,7 @@ class BingAPI(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(domain, str(resp.json())) + subdomains = self.match_subdomains(domain, str(resp.json())) if not subdomains: # 搜索没有发现子域名则停止搜索 break if not full_search: diff --git a/modules/search/fofa_api.py b/modules/search/fofa_api.py index 725d609..06ff17d 100644 --- a/modules/search/fofa_api.py +++ b/modules/search/fofa_api.py @@ -36,7 +36,7 @@ class FoFa(Search): if not resp: return resp_json = resp.json() - subdomains = self.match(self.domain, str(resp_json)) + subdomains = self.match_subdomains(self.domain, str(resp_json)) if not subdomains: # 搜索没有发现子域名则停止搜索 break self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/search/gitee.py b/modules/search/gitee.py index 979c5a1..970c003 100644 --- a/modules/search/gitee.py +++ b/modules/search/gitee.py @@ -34,7 +34,7 @@ class Gitee(Search): if 'class="empty-box"' in resp.text: break soup = BeautifulSoup(resp.text, 'html.parser') - subdomains = self.match(self.domain, soup.text) + subdomains = self.match_subdomains(self.domain, soup.text) if not subdomains: break if not full_search: diff --git a/modules/search/github_api.py b/modules/search/github_api.py index 8f7a7cf..1c93351 100644 --- a/modules/search/github_api.py +++ b/modules/search/github_api.py @@ -1,6 +1,5 @@ import requests from config import api -from common.utils import match_subdomain from common.search import Search from config.log import logger @@ -61,7 +60,7 @@ class GithubAPI(Search): if resp.status_code != 200: logger.log('ERROR', f'{self.source}模块搜索出错') break - subdomains = match_subdomain(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) if not subdomains: break self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/search/google.py b/modules/search/google.py index 640a760..457cb34 100644 --- a/modules/search/google.py +++ b/modules/search/google.py @@ -40,7 +40,7 @@ class Google(Search): resp = self.get(url=self.addr, params=payload) if not resp: return - subdomains = self.match(domain, resp.text) + subdomains = self.match_subdomains(domain, resp.text) if not subdomains: break if not full_search: diff --git a/modules/search/google_api.py b/modules/search/google_api.py index 3a35c1b..ae9613c 100644 --- a/modules/search/google_api.py +++ b/modules/search/google_api.py @@ -35,7 +35,7 @@ class GoogleAPI(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(domain, str(resp.json())) + subdomains = self.match_subdomains(domain, str(resp.json())) if not subdomains: break if not full_search: diff --git a/modules/search/shodan_api.py b/modules/search/shodan_api.py index 0cd758f..36628cf 100644 --- a/modules/search/shodan_api.py +++ b/modules/search/shodan_api.py @@ -25,7 +25,7 @@ class ShodanAPI(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break if subdomains: diff --git a/modules/search/so.py b/modules/search/so.py index 7d0f0eb..b2d88ab 100644 --- a/modules/search/so.py +++ b/modules/search/so.py @@ -31,7 +31,7 @@ class So(Search): resp = self.get(url=self.addr, params=payload) if not resp: return - subdomains = self.match(domain, resp.text) + subdomains = self.match_subdomains(domain, resp.text) if not subdomains: break if not full_search: diff --git a/modules/search/sogou.py b/modules/search/sogou.py index 49c120d..481498b 100644 --- a/modules/search/sogou.py +++ b/modules/search/sogou.py @@ -28,7 +28,7 @@ class Sogou(Search): resp = self.get(self.addr, payload) if not resp: return - subdomains = self.match(domain, resp.text) + subdomains = self.match_subdomains(domain, resp.text) if not subdomains: break if not full_search: diff --git a/modules/search/yahoo.py b/modules/search/yahoo.py index 29cab13..cb790fd 100644 --- a/modules/search/yahoo.py +++ b/modules/search/yahoo.py @@ -36,7 +36,7 @@ class Yahoo(Search): if not resp: return text = resp.text.replace('', '').replace('', '') - subdomains = self.match(domain, text) + subdomains = self.match_subdomains(domain, text) if not subdomains: # 搜索没有发现子域名则停止搜索 break if not full_search: diff --git a/modules/search/yandex.py b/modules/search/yandex.py index fe69410..b7af641 100644 --- a/modules/search/yandex.py +++ b/modules/search/yandex.py @@ -36,7 +36,7 @@ class Yandex(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(domain, resp.text) + subdomains = self.match_subdomains(domain, resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break if not full_search: diff --git a/modules/search/zoomeye_api.py b/modules/search/zoomeye_api.py index af58f5d..819caec 100644 --- a/modules/search/zoomeye_api.py +++ b/modules/search/zoomeye_api.py @@ -48,7 +48,7 @@ class ZoomEyeAPI(Search): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break self.subdomains = self.subdomains.union(subdomains)