diff --git a/common/lookup.py b/common/lookup.py index cf72d81..37bb281 100644 --- a/common/lookup.py +++ b/common/lookup.py @@ -20,7 +20,7 @@ class Lookup(Module): return None for item in answer: record = item.to_text() - subdomains = utils.match_subdomain(self.domain, record) + subdomains = self.match_subdomains(self.domain, record) self.subdomains = self.subdomains.union(subdomains) self.gen_record(subdomains, record) return self.subdomains diff --git a/common/module.py b/common/module.py index 98a9ceb..374b12a 100644 --- a/common/module.py +++ b/common/module.py @@ -128,7 +128,7 @@ class Module(object): Custom post request :param str url: request url - :param dict params: request parameters + :param dict data: request parameters :param bool check: check response :param kwargs: other params :return: requests's response object @@ -184,24 +184,22 @@ class Module(object): return self.proxy @staticmethod - def match(domain, html, distinct=True): + def match_subdomains(domain, text, distinct=True): """ Use regexp to match subdomains :param str domain: domain - :param str html: response html text + :param str text: text :param bool distinct: deduplicate results or not (default True) :return set/list: result set or list """ logger.log('TRACE', f'Use regexp to match subdomains in the response body') - regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \ - r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + domain.replace('.', r'\.') - result = re.findall(regexp, html, re.I) + result = re.findall(regexp, text, re.I) if not result: return set() - regexp = r'(?:http://|https://)' - deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result) + deal = map(lambda s: s.lower(), result) if distinct: return set(deal) else: @@ -340,14 +338,12 @@ class Module(object): 'elapse': self.elapse, 'find': find, 'brute': brute, - 'valid': valid, - } + 'valid': valid} self.results.append(result) def save_db(self): """ Save module results into the database - """ logger.log('DEBUG', f'Saving results to database') lock.acquire() diff --git a/common/search.py b/common/search.py index 16c12a3..8dadfad 100644 --- a/common/search.py +++ b/common/search.py @@ -1,6 +1,9 @@ +import re + from config import setting -from .module import Module -from . import utils +from config.log import logger +from common.module import Module +from common import utils class Search(Module): @@ -51,4 +54,28 @@ class Search(Module): location = resp.headers.get('location') if not location: return set() - return set(utils.match_subdomain(domain, location)) + return set(self.match_subdomains(domain, location)) + + @staticmethod + def match_subdomains(domain, html, distinct=True): + """ + Use regexp to match subdomains + + :param str domain: domain + :param str html: response html text + :param bool distinct: deduplicate results or not (default True) + :return set/list: result set or list + """ + logger.log('TRACE', f'Use regexp to match subdomains in the response body') + regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \ + r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + + domain.replace('.', r'\.') + result = re.findall(regexp, html, re.I) + if not result: + return set() + regexp = r'(?:http://|https://)' + deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result) + if distinct: + return set(deal) + else: + return list(deal) diff --git a/common/utils.py b/common/utils.py index 4f9cb32..67d9de9 100644 --- a/common/utils.py +++ b/common/utils.py @@ -32,27 +32,6 @@ user_agents = [ 'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0'] -def match_subdomain(domain, text, distinct=True): - """ - Use regexp to match subdomains in text - - :param str domain: domain - :param str text: response text - :param bool distinct: deduplicate results - :return set/list: match result - """ - regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ - + domain.replace('.', r'\.') - result = re.findall(regexp, text, re.I) - if not result: - return set() - deal = map(lambda s: s.lower(), result) - if distinct: - return set(deal) - else: - return list(deal) - - def gen_random_ip(): """ Generate random decimal IP string diff --git a/modules/certificates/censys_api.py b/modules/certificates/censys_api.py index be98141..bd692a8 100644 --- a/modules/certificates/censys_api.py +++ b/modules/certificates/censys_api.py @@ -33,7 +33,7 @@ class CensysAPI(Query): if status != 'ok': logger.log('ALERT', status) return - subdomains = self.match(self.domain, str(json)) + subdomains = self.match_subdomains(self.domain, str(json)) self.subdomains = self.subdomains.union(subdomains) pages = json.get('metadata').get('pages') for page in range(2, pages + 1): @@ -41,7 +41,7 @@ class CensysAPI(Query): resp = self.post(self.addr, json=data, auth=(self.id, self.secret)) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/certificates/certspotter.py b/modules/certificates/certspotter.py index 153debb..b6b32cf 100644 --- a/modules/certificates/certspotter.py +++ b/modules/certificates/certspotter.py @@ -22,7 +22,7 @@ class CertSpotter(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = utils.match_subdomain(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/certificates/crtsh.py b/modules/certificates/crtsh.py index 830e7af..90b61e4 100644 --- a/modules/certificates/crtsh.py +++ b/modules/certificates/crtsh.py @@ -20,7 +20,7 @@ class Crtsh(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = utils.match_subdomain(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/certificates/google.py b/modules/certificates/google.py index c82c88f..a7ad98f 100644 --- a/modules/certificates/google.py +++ b/modules/certificates/google.py @@ -23,7 +23,7 @@ class Google(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = utils.match_subdomain(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/check/axfr.py b/modules/check/axfr.py index e92b8a1..666a308 100644 --- a/modules/check/axfr.py +++ b/modules/check/axfr.py @@ -44,7 +44,7 @@ class CheckAXFR(Module): names = zone.nodes.keys() for name in names: full_domain = str(name) + '.' + self.domain - subdomain = utils.match_subdomain(self.domain, full_domain) + subdomain = self.match_subdomains(self.domain, full_domain) self.subdomains = self.subdomains.union(subdomain) record = zone[name].to_text(name) self.results.append(record) diff --git a/modules/check/cdx.py b/modules/check/cdx.py index ec2f49f..bdc3156 100644 --- a/modules/check/cdx.py +++ b/modules/check/cdx.py @@ -31,7 +31,7 @@ class CheckCDX(Module): if not response: return if response and len(response.content): - self.subdomains = utils.match_subdomain(self.domain, + self.subdomains = self.match_subdomains(self.domain, response.text) def run(self): diff --git a/modules/check/cert.py b/modules/check/cert.py index e122ef5..659fe43 100644 --- a/modules/check/cert.py +++ b/modules/check/cert.py @@ -32,7 +32,7 @@ class CheckCert(Module): except Exception as e: logger.log('DEBUG', e.args) return - subdomains = utils.match_subdomain(self.domain, str(cert_dict)) + subdomains = self.match_subdomains(self.domain, str(cert_dict)) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/check/csp.py b/modules/check/csp.py index bce94fb..11850a1 100644 --- a/modules/check/csp.py +++ b/modules/check/csp.py @@ -52,7 +52,7 @@ class CheckCSP(Module): if not csp: logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段') return - self.subdomains = utils.match_subdomain(self.domain, csp) + self.subdomains = self.match_subdomains(self.domain, csp) def run(self): """ diff --git a/modules/check/robots.py b/modules/check/robots.py index 1c87719..8116c7a 100644 --- a/modules/check/robots.py +++ b/modules/check/robots.py @@ -30,7 +30,7 @@ class CheckRobots(Module): if not response: return if response and len(response.content): - self.subdomains = utils.match_subdomain(self.domain, + self.subdomains = self.match_subdomains(self.domain, response.text) def run(self): diff --git a/modules/check/sitemap.py b/modules/check/sitemap.py index 54db09f..c03122a 100644 --- a/modules/check/sitemap.py +++ b/modules/check/sitemap.py @@ -43,7 +43,7 @@ class CheckRobots(Module): if not response: return if response and len(response.content): - self.subdomains = utils.match_subdomain(self.domain, + self.subdomains = self.match_subdomains(self.domain, response.text) diff --git a/modules/crawl/archivecrawl.py b/modules/crawl/archivecrawl.py index 5ef1181..70ac9a1 100644 --- a/modules/crawl/archivecrawl.py +++ b/modules/crawl/archivecrawl.py @@ -26,8 +26,8 @@ class ArchiveCrawl(Crawl): for resp in cdx.iter(url, limit=limit): if resp.data.get('status') not in ['301', '302']: url = resp.data.get('url') - subdomains = self.match(self.register(domain), - url + resp.text) + subdomains = self.match_subdomains(self.register(domain), + url + resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/crawl/commoncrawl.py b/modules/crawl/commoncrawl.py index 52779c4..6504910 100644 --- a/modules/crawl/commoncrawl.py +++ b/modules/crawl/commoncrawl.py @@ -26,7 +26,7 @@ class CommonCrawl(Crawl): for resp in tqdm(cdx.iter(url, limit=limit), total=limit): if resp.data.get('status') not in ['301', '302']: - subdomains = self.match(self.register(domain), resp.text) + subdomains = self.match_subdomains(self.register(domain), resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/binaryedge_api.py b/modules/datasets/binaryedge_api.py index 95d48ad..6a63e1f 100644 --- a/modules/datasets/binaryedge_api.py +++ b/modules/datasets/binaryedge_api.py @@ -22,7 +22,7 @@ class BinaryEdgeAPI(Query): resp = self.get(url) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) self.subdomains = self.subdomains.union(subdomains) def run(self): diff --git a/modules/datasets/bufferover.py b/modules/datasets/bufferover.py index cdea847..b5eedb6 100644 --- a/modules/datasets/bufferover.py +++ b/modules/datasets/bufferover.py @@ -26,7 +26,7 @@ class BufferOver(Query): return if resp.status_code != 200: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/cebaidu.py b/modules/datasets/cebaidu.py index 44ed6d4..5311246 100644 --- a/modules/datasets/cebaidu.py +++ b/modules/datasets/cebaidu.py @@ -19,7 +19,7 @@ class CeBaidu(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/chinaz.py b/modules/datasets/chinaz.py index c852c51..82a0128 100644 --- a/modules/datasets/chinaz.py +++ b/modules/datasets/chinaz.py @@ -19,7 +19,7 @@ class Chinaz(Query): resp = self.get(self.addr) if not resp: return - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/chinaz_api.py b/modules/datasets/chinaz_api.py index 50227ee..69ba907 100644 --- a/modules/datasets/chinaz_api.py +++ b/modules/datasets/chinaz_api.py @@ -21,7 +21,7 @@ class ChinazAPI(Query): resp = self.get(self.addr, params) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/circl_api.py b/modules/datasets/circl_api.py index 2147479..b2548f2 100644 --- a/modules/datasets/circl_api.py +++ b/modules/datasets/circl_api.py @@ -21,7 +21,7 @@ class CirclAPI(Query): resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd)) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/dnsdb_api.py b/modules/datasets/dnsdb_api.py index f957261..6d4f839 100644 --- a/modules/datasets/dnsdb_api.py +++ b/modules/datasets/dnsdb_api.py @@ -23,7 +23,7 @@ class DNSdbAPI(Query): resp = self.get(url) if not resp: return - subdomains = utils.match_subdomain(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/dnsdumpster.py b/modules/datasets/dnsdumpster.py index 3226200..1b0e415 100644 --- a/modules/datasets/dnsdumpster.py +++ b/modules/datasets/dnsdumpster.py @@ -26,7 +26,7 @@ class DNSdumpster(Query): resp = self.post(self.addr, data) if not resp: return - subdomains = utils.match_subdomain(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) if subdomains: # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/hackertarget.py b/modules/datasets/hackertarget.py index 84ee387..967aad5 100644 --- a/modules/datasets/hackertarget.py +++ b/modules/datasets/hackertarget.py @@ -21,7 +21,7 @@ class HackerTarget(Query): if not resp: return if resp.status_code == 200: - subdomains = utils.match_subdomain(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) if subdomains: # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/ip138.py b/modules/datasets/ip138.py index 46a19d7..54072fc 100644 --- a/modules/datasets/ip138.py +++ b/modules/datasets/ip138.py @@ -19,7 +19,7 @@ class IP138(Query): resp = self.get(self.addr) if not resp: return - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/ipv4info_api.py b/modules/datasets/ipv4info_api.py index 236b6e7..681a50e 100644 --- a/modules/datasets/ipv4info_api.py +++ b/modules/datasets/ipv4info_api.py @@ -32,7 +32,7 @@ class IPv4InfoAPI(Query): except Exception as e: logger.log('DEBUG', e.args) break - subdomains = self.match(self.domain, str(json)) + subdomains = self.match_subdomains(self.domain, str(json)) if not subdomains: break # 合并搜索子域名搜索结果 diff --git a/modules/datasets/netcraft.py b/modules/datasets/netcraft.py index 5c6089e..60999e9 100644 --- a/modules/datasets/netcraft.py +++ b/modules/datasets/netcraft.py @@ -49,7 +49,7 @@ class NetCraft(Query): resp = self.get(self.addr + last, params) if not resp: return - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) if not subdomains: # 搜索没有发现子域名则停止搜索 break # 合并搜索子域名搜索结果 diff --git a/modules/datasets/passivedns_api.py b/modules/datasets/passivedns_api.py index 2755a91..30a9dfa 100644 --- a/modules/datasets/passivedns_api.py +++ b/modules/datasets/passivedns_api.py @@ -22,7 +22,7 @@ class PassiveDnsAPI(Query): resp = self.get(url) if not resp: return - subdomains = self.match(self.domain, str(resp.json())) + subdomains = self.match_subdomains(self.domain, str(resp.json())) # 合并搜索子域名搜索结果 self.subdomains = self.subdomains.union(subdomains) diff --git a/modules/datasets/qianxun.py b/modules/datasets/qianxun.py index c82144f..f261689 100644 --- a/modules/datasets/qianxun.py +++ b/modules/datasets/qianxun.py @@ -27,7 +27,7 @@ class QianXun(Query): resp = self.post(url, data) if not resp: break - subdomains = self.match(self.domain, resp.text) + subdomains = self.match_subdomains(self.domain, resp.text) self.subdomains = self.subdomains.union(subdomains) if '