From f8f3ad0352fd53b4e8d6983a33893c92fa543da6 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Thu, 20 Aug 2020 13:49:28 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=86=85=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- brute.py | 27 ++++++++++++--------------- common/utils.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/brute.py b/brute.py index 289e0d5..8c93f5e 100644 --- a/brute.py +++ b/brute.py @@ -90,9 +90,9 @@ def gen_subdomains(expression, path): :param str expression: generate subdomains's expression :param str path: path of wordlist - :return list subdomains: list of subdomains + :return set subdomains: list of subdomains """ - subdomains = list() + subdomains = set() with open(path, encoding='utf-8', errors='ignore') as fd: for line in fd: word = line.strip().lower() @@ -105,14 +105,13 @@ def gen_subdomains(expression, path): if word.endswith('.'): word = word[:-1] subdomain = expression.replace('*', word) - subdomains.append(subdomain) + subdomains.add(subdomain) size = len(subdomains) logger.log('DEBUG', f'The size of the dictionary generated by {path} is {size}') if size == 0: logger.log('ALERT', 'Please check the dictionary content!') else: - random_domain = random.choice(subdomains) - logger.log('ALERT', f'Please check whether {random_domain} is correct or not') + utils.check_random_subdomain(subdomains) return subdomains @@ -123,12 +122,12 @@ def gen_fuzz_subdomains(expression, rule, fuzzlist): :param str expression: generate subdomains's expression :param str rule: regexp rule :param str fuzzlist: fuzz dictionary - :return list subdomains: list of subdomains + :return set subdomains: list of subdomains """ - subdomains = list() + subdomains = set() if fuzzlist: fuzz_domain = gen_subdomains(expression, fuzzlist) - subdomains = subdomains.extend(fuzz_domain) + subdomains.update(fuzz_domain) if rule: fuzz_count = exrex.count(rule) if fuzz_count > 10000000: @@ -139,9 +138,8 @@ def gen_fuzz_subdomains(expression, rule, fuzzlist): if not fuzz_string.isalnum(): continue fuzz_domain = expression.replace('*', fuzz_string) - subdomains.append(fuzz_domain) - random_domain = random.choice(subdomains) - logger.log('ALERT', f'Please check whether {random_domain} is correct or not') + subdomains.add(fuzz_domain) + utils.check_random_subdomain(subdomains) logger.log('DEBUG', f'Dictionary size based on fuzz mode: {len(subdomains)}') return subdomains @@ -152,7 +150,7 @@ def gen_word_subdomains(expression, path): :param str expression: generate subdomains's expression :param str path: path of wordlist - :return list subdomains: list of subdomains + :return set subdomains: list of subdomains """ subdomains = gen_subdomains(expression, path) logger.log('DEBUG', f'Dictionary based on word mode size: {len(subdomains)}') @@ -543,11 +541,10 @@ class Brute(Module): wordlist = self.recursive_nextlist if self.word: word_subdomains = gen_word_subdomains(self.place, wordlist) - # set可以合并list - dict_set = dict_set.union(word_subdomains) + dict_set.update(word_subdomains) if self.fuzz: fuzz_subdomains = gen_fuzz_subdomains(self.place, self.rule, self.fuzzlist) - dict_set = dict_set.union(fuzz_subdomains) + dict_set.update(fuzz_subdomains) count = len(dict_set) logger.log('INFOR', f'Dictionary size: {count}') if count > 10000000: diff --git a/common/utils.py b/common/utils.py index 12cd446..fc987c7 100644 --- a/common/utils.py +++ b/common/utils.py @@ -728,3 +728,13 @@ def match_subdomains(domain, html, distinct=True, fuzzy=True): return set(deal) else: return list(deal) + + +def check_random_subdomain(subdomains): + if not subdomains: + logger.log('ALERT', f'The generated dictionary is empty') + return False + for subdomain in subdomains: + if subdomain: + logger.log('ALERT', f'Please check whether {subdomain} is correct or not') + return True From e056834f1d3bf5989b96cc7450f8e0f18063ef73 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Thu, 20 Aug 2020 14:01:17 +0800 Subject: [PATCH 2/6] =?UTF-8?q?set=E4=BD=BF=E7=94=A8update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- brute.py | 2 +- common/lookup.py | 2 +- common/module.py | 3 ++- modules/certificates/censys_api.py | 2 +- modules/certificates/crtsh.py | 2 +- modules/check/axfr.py | 2 +- modules/check/cert.py | 2 +- modules/check/nsec.py | 2 +- modules/crawl/archivecrawl.py | 2 +- modules/crawl/commoncrawl.py | 2 +- modules/datasets/cloudflare_api.py | 4 ++-- modules/datasets/ipv4info_api.py | 2 +- modules/datasets/netcraft.py | 2 +- modules/datasets/qianxun.py | 2 +- modules/datasets/securitytrails_api.py | 2 +- modules/datasets/sitedossier.py | 2 +- modules/datasets/spyse_api.py | 2 +- modules/datasets/threatcrowd.py | 2 +- modules/datasets/wzpc.py | 2 +- modules/dnsquery/srv.py | 2 +- modules/finder.py | 4 ++-- modules/intelligence/virustotal.py | 2 +- modules/intelligence/virustotal_api.py | 2 +- modules/search/ask.py | 2 +- modules/search/baidu.py | 4 ++-- modules/search/bing.py | 2 +- modules/search/bing_api.py | 2 +- modules/search/fofa_api.py | 2 +- modules/search/gitee.py | 2 +- modules/search/github_api.py | 2 +- modules/search/google.py | 2 +- modules/search/google_api.py | 2 +- modules/search/shodan_api.py | 2 +- modules/search/so.py | 2 +- modules/search/sogou.py | 2 +- modules/search/yahoo.py | 2 +- modules/search/yandex.py | 2 +- modules/search/zoomeye_api.py | 2 +- 38 files changed, 42 insertions(+), 41 deletions(-) diff --git a/brute.py b/brute.py index 8c93f5e..76712d8 100644 --- a/brute.py +++ b/brute.py @@ -247,7 +247,7 @@ def collect_wildcard_record(domain, authoritative_ns): continue if ip is None: continue - ips = ips.union(ip) + ips.update(ip) # 统计每个泛解析IP出现次数 for addr in ip: count = ips_stat.setdefault(addr, 0) diff --git a/common/lookup.py b/common/lookup.py index 66945b0..0978130 100644 --- a/common/lookup.py +++ b/common/lookup.py @@ -23,6 +23,6 @@ class Lookup(Module): for item in answer: record = item.to_text() subdomains = self.match_subdomains(record) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) logger.log('DEBUG', record) return self.subdomains diff --git a/common/module.py b/common/module.py index d7481fb..cb51ae6 100644 --- a/common/module.py +++ b/common/module.py @@ -220,7 +220,8 @@ class Module(object): def collect_subdomains(self, resp): subdomains = self.match_subdomains(resp) - return self.subdomains.union(subdomains) + self.subdomains.update(subdomains) + return self.subdomains def save_json(self): """ diff --git a/modules/certificates/censys_api.py b/modules/certificates/censys_api.py index 247167a..33d6682 100644 --- a/modules/certificates/censys_api.py +++ b/modules/certificates/censys_api.py @@ -34,7 +34,7 @@ class CensysAPI(Query): logger.log('ALERT', f'{self.source} module {status}') return subdomains = self.match_subdomains(resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) pages = json.get('metadata').get('pages') for page in range(2, pages + 1): data['page'] = page diff --git a/modules/certificates/crtsh.py b/modules/certificates/crtsh.py index 6c55dbd..ee6918e 100644 --- a/modules/certificates/crtsh.py +++ b/modules/certificates/crtsh.py @@ -21,7 +21,7 @@ class Crtsh(Query): return text = resp.text.replace(r'\n', ' ') subdomains = self.match_subdomains(text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) def run(self): """ diff --git a/modules/check/axfr.py b/modules/check/axfr.py index 188fd95..7e60fc6 100644 --- a/modules/check/axfr.py +++ b/modules/check/axfr.py @@ -46,7 +46,7 @@ class CheckAXFR(Module): for name in names: full_domain = str(name) + '.' + self.domain subdomain = self.match_subdomains(full_domain) - self.subdomains = self.subdomains.union(subdomain) + self.subdomains.update(subdomain) record = zone[name].to_text(name) self.results.append(record) if self.results: diff --git a/modules/check/cert.py b/modules/check/cert.py index 1eae2d9..e33c72b 100644 --- a/modules/check/cert.py +++ b/modules/check/cert.py @@ -32,7 +32,7 @@ class CheckCert(Module): logger.log('DEBUG', e.args) return subdomains = self.match_subdomains(str(cert_dict)) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) def run(self): """ diff --git a/modules/check/nsec.py b/modules/check/nsec.py index 893797c..0a6c190 100644 --- a/modules/check/nsec.py +++ b/modules/check/nsec.py @@ -23,7 +23,7 @@ class CheckNSEC(Module): record = item.to_text() subdomains = self.match_subdomains(record) subdomain = ''.join(subdomains) # 其实这里的subdomains的长度为1 也就是说只会有一个子域 - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) self.gen_record(subdomains, record) if subdomain == self.domain: # 当查出子域为主域 说明完成了一个循环 不再继续查询 break diff --git a/modules/crawl/archivecrawl.py b/modules/crawl/archivecrawl.py index 1b6445a..fb49c0c 100644 --- a/modules/crawl/archivecrawl.py +++ b/modules/crawl/archivecrawl.py @@ -28,7 +28,7 @@ class ArchiveCrawl(Crawl): url = resp.data.get('url') subdomains = self.match_subdomains(self.get_maindomain(domain), url + resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) def run(self): """ diff --git a/modules/crawl/commoncrawl.py b/modules/crawl/commoncrawl.py index d79e3c0..707033f 100644 --- a/modules/crawl/commoncrawl.py +++ b/modules/crawl/commoncrawl.py @@ -27,7 +27,7 @@ class CommonCrawl(Crawl): for resp in tqdm(cdx.iter(url, limit=limit), total=limit): if resp.data.get('status') not in ['301', '302']: subdomains = self.match_subdomains(self.get_maindomain(domain), resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) def run(self): """ diff --git a/modules/datasets/cloudflare_api.py b/modules/datasets/cloudflare_api.py index 988df60..d183474 100644 --- a/modules/datasets/cloudflare_api.py +++ b/modules/datasets/cloudflare_api.py @@ -81,7 +81,7 @@ class CloudFlareAPI(Query): f'{list_dns_resp.status_code} {list_dns_resp.text}') return subdomains = self.match_subdomains(list_dns_resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) if not self.subdomains: # waiting for cloudflare enumerate subdomains sleep(5) @@ -96,7 +96,7 @@ class CloudFlareAPI(Query): return total_pages = list_dns_resp.json()['result_info']['total_pages'] subdomains = (self.match_subdomains(list_dns_resp.text)) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) page += 1 if page > total_pages: break diff --git a/modules/datasets/ipv4info_api.py b/modules/datasets/ipv4info_api.py index 18cc097..c41d7f3 100644 --- a/modules/datasets/ipv4info_api.py +++ b/modules/datasets/ipv4info_api.py @@ -35,7 +35,7 @@ class IPv4InfoAPI(Query): subdomains = self.match_subdomains(str(json)) if not subdomains: break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名 subdomains = json.get('Subdomains') if subdomains and len(subdomains) < 300: diff --git a/modules/datasets/netcraft.py b/modules/datasets/netcraft.py index 6cc37b8..0970d97 100644 --- a/modules/datasets/netcraft.py +++ b/modules/datasets/netcraft.py @@ -50,7 +50,7 @@ class NetCraft(Query): subdomains = self.match_subdomains(resp) if not subdomains: # 搜索没有发现子域名则停止搜索 break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) if 'Next Page' not in resp.text: # 搜索页面没有出现下一页时停止搜索 break last = re.search(r'&last=.*' + self.domain, resp.text).group(0) diff --git a/modules/datasets/qianxun.py b/modules/datasets/qianxun.py index c1aa096..360443f 100644 --- a/modules/datasets/qianxun.py +++ b/modules/datasets/qianxun.py @@ -28,7 +28,7 @@ class QianXun(Query): subdomains = self.match_subdomains(resp) if not subdomains: # 没有发现子域名则停止查询 break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) if '
' not in resp.text: break if '
  • »
    • ' in resp.text: diff --git a/modules/datasets/securitytrails_api.py b/modules/datasets/securitytrails_api.py index 82c381d..7a8e34d 100644 --- a/modules/datasets/securitytrails_api.py +++ b/modules/datasets/securitytrails_api.py @@ -26,7 +26,7 @@ class SecurityTrailsAPI(Query): prefixs = resp.json()['subdomains'] subdomains = [f'{prefix}.{self.domain}' for prefix in prefixs] if subdomains: - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) def run(self): """ diff --git a/modules/datasets/sitedossier.py b/modules/datasets/sitedossier.py index 728815b..7a616d0 100644 --- a/modules/datasets/sitedossier.py +++ b/modules/datasets/sitedossier.py @@ -25,7 +25,7 @@ class SiteDossier(Query): subdomains = self.match_subdomains(resp) if not subdomains: # 没有发现子域名则停止查询 break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) # 搜索页面没有出现下一页时停止搜索 if 'Show next 100 items' not in resp.text: break diff --git a/modules/datasets/spyse_api.py b/modules/datasets/spyse_api.py index 369d363..59651f3 100644 --- a/modules/datasets/spyse_api.py +++ b/modules/datasets/spyse_api.py @@ -29,7 +29,7 @@ class SpyseAPI(Query): subdomains = self.match_subdomains(str(json)) if not subdomains: # 没有发现子域名则停止查询 break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) offset += limit if len(json.get('data').get('items')) < limit: break diff --git a/modules/datasets/threatcrowd.py b/modules/datasets/threatcrowd.py index 5ea17c9..f01f02f 100644 --- a/modules/datasets/threatcrowd.py +++ b/modules/datasets/threatcrowd.py @@ -26,7 +26,7 @@ class ThreatCrowd(Query): if resp.status_code != 200: return subdomains = self.match_subdomains(resp.text) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) def run(self): """ diff --git a/modules/datasets/wzpc.py b/modules/datasets/wzpc.py index 9db0dcf..62a38c7 100644 --- a/modules/datasets/wzpc.py +++ b/modules/datasets/wzpc.py @@ -34,7 +34,7 @@ class WZPCQuery(Query): subdomains = self.match_subdomains(resp.text) if not subdomains: # 没有发现子域名则停止查询 break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) if not subdomains: break if page_num > 10: diff --git a/modules/dnsquery/srv.py b/modules/dnsquery/srv.py index 88ba9dc..c28d8e0 100644 --- a/modules/dnsquery/srv.py +++ b/modules/dnsquery/srv.py @@ -52,7 +52,7 @@ class BruteSRV(Module): for item in answer: record = str(item) subdomains = self.match_subdomains(record) - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) self.gen_record(subdomains, record) def run(self): diff --git a/modules/finder.py b/modules/finder.py index 418115b..57f5db9 100644 --- a/modules/finder.py +++ b/modules/finder.py @@ -163,7 +163,7 @@ def find_subdomains(domain, data): if not rsp_html: continue logger.log('DEBUG', f'matching subdomains from response of {req_url}') - subdomains = subdomains.union(match_subdomains(domain, rsp_html)) + subdomains.update(match_subdomains(domain, rsp_html)) urls = find_url(rsp_html) if not urls: continue @@ -175,5 +175,5 @@ def find_subdomains(domain, data): for resp, text in resp_data: if text: logger.log('DEBUG', f'matching subdomains from response of {resp.url}') - subdomains = subdomains.union(match_subdomains(domain, text)) + subdomains.update(match_subdomains(domain, text)) return subdomains diff --git a/modules/intelligence/virustotal.py b/modules/intelligence/virustotal.py index 542d87b..67f6a75 100644 --- a/modules/intelligence/virustotal.py +++ b/modules/intelligence/virustotal.py @@ -30,7 +30,7 @@ class VirusTotal(Query): subdomains = self.match_subdomains(resp) if not subdomains: break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) data = resp.json() next_cursor = data.get('meta').get('cursor') diff --git a/modules/intelligence/virustotal_api.py b/modules/intelligence/virustotal_api.py index b4058a6..19ae9e5 100644 --- a/modules/intelligence/virustotal_api.py +++ b/modules/intelligence/virustotal_api.py @@ -25,7 +25,7 @@ class VirusTotalAPI(Query): subdomains = self.match_subdomains(resp) if not subdomains: break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) data = resp.json() next_cursor = data.get('meta').get('cursor') diff --git a/modules/search/ask.py b/modules/search/ask.py index dca0e23..1532dff 100644 --- a/modules/search/ask.py +++ b/modules/search/ask.py @@ -30,7 +30,7 @@ class Ask(Search): subdomains = self.match_subdomains(resp, fuzzy=False) if not self.check_subdomains(subdomains): break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) self.page_num += 1 if '>Next<' not in resp.text: break diff --git a/modules/search/baidu.py b/modules/search/baidu.py index b062345..8552e49 100644 --- a/modules/search/baidu.py +++ b/modules/search/baidu.py @@ -25,7 +25,7 @@ class Baidu(Search): for find_res in bs.find_all('a', {'class': 'c-showurl'}): url = find_res.get('href') subdomains = self.match_location(url) - subdomains_all = subdomains_all.union(subdomains) + subdomains_all.update(subdomains) return subdomains_all def search(self, domain, filtered_subdomain=''): @@ -54,7 +54,7 @@ class Baidu(Search): subdomains = self.match_subdomains(resp, fuzzy=False) if not self.check_subdomains(subdomains): break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) self.page_num += self.per_page_num # 搜索页面没有出现下一页时停止搜索 if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text: diff --git a/modules/search/bing.py b/modules/search/bing.py index fea3d0e..52dc448 100644 --- a/modules/search/bing.py +++ b/modules/search/bing.py @@ -36,7 +36,7 @@ class Bing(Search): subdomains = self.match_subdomains(resp, fuzzy=False) if not self.check_subdomains(subdomains): break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) # 搜索页面没有出现下一页时停止搜索 if '
    ' not in resp.text: break diff --git a/modules/search/bing_api.py b/modules/search/bing_api.py index f59cd8d..9c6f787 100644 --- a/modules/search/bing_api.py +++ b/modules/search/bing_api.py @@ -36,7 +36,7 @@ class BingAPI(Search): subdomains = self.match_subdomains(resp) if not self.check_subdomains(subdomains): break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) self.page_num += self.per_page_num if self.page_num >= self.limit_num: # 搜索条数限制 break diff --git a/modules/search/fofa_api.py b/modules/search/fofa_api.py index 3297328..ef2b32e 100644 --- a/modules/search/fofa_api.py +++ b/modules/search/fofa_api.py @@ -40,7 +40,7 @@ class FoFa(Search): subdomains = self.match_subdomains(resp) if not subdomains: # 搜索没有发现子域名则停止搜索 break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) size = resp_json.get('size') if size < 10000: break diff --git a/modules/search/gitee.py b/modules/search/gitee.py index 4b5faa9..fc994d1 100644 --- a/modules/search/gitee.py +++ b/modules/search/gitee.py @@ -37,7 +37,7 @@ class Gitee(Search): subdomains = self.match_subdomains(soup, fuzzy=False) if not self.check_subdomains(subdomains): break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) if '
  • ' in resp.text: break page_num += 1 diff --git a/modules/search/github_api.py b/modules/search/github_api.py index eb72ca7..b862f2a 100644 --- a/modules/search/github_api.py +++ b/modules/search/github_api.py @@ -63,7 +63,7 @@ class GithubAPI(Search): subdomains = self.match_subdomains(resp) if not subdomains: break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) page += 1 try: resp_json = resp.json() diff --git a/modules/search/google.py b/modules/search/google.py index 7f66230..cfe1cf8 100644 --- a/modules/search/google.py +++ b/modules/search/google.py @@ -40,7 +40,7 @@ class Google(Search): subdomains = self.match_subdomains(resp, fuzzy=False) if not self.check_subdomains(subdomains): break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) page_num += per_page_num if 'start=' + str(page_num) not in resp.text: break diff --git a/modules/search/google_api.py b/modules/search/google_api.py index 9a3b5e0..2027df7 100644 --- a/modules/search/google_api.py +++ b/modules/search/google_api.py @@ -35,7 +35,7 @@ class GoogleAPI(Search): subdomains = self.match_subdomains(resp) if not self.check_subdomains(subdomains): break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) self.page_num += self.per_page_num if self.page_num > 100: # 免费的API只能查询前100条结果 break diff --git a/modules/search/shodan_api.py b/modules/search/shodan_api.py index 5de5128..e0ae022 100644 --- a/modules/search/shodan_api.py +++ b/modules/search/shodan_api.py @@ -26,7 +26,7 @@ class ShodanAPI(Search): subdomains = self.match_subdomains(resp) if not subdomains: # 搜索没有发现子域名则停止搜索 break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) page += 1 def run(self): diff --git a/modules/search/so.py b/modules/search/so.py index 596fd11..5081c1d 100644 --- a/modules/search/so.py +++ b/modules/search/so.py @@ -31,7 +31,7 @@ class So(Search): subdomains = self.match_subdomains(resp, fuzzy=False) if not self.check_subdomains(subdomains): break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) page_num += 1 # 搜索页面没有出现下一页时停止搜索 if 'Next' not in resp.text: # 搜索页面没有出现下一页时停止搜索 break self.page_num += self.per_page_num diff --git a/modules/search/yandex.py b/modules/search/yandex.py index 1a38041..6e5f1ea 100644 --- a/modules/search/yandex.py +++ b/modules/search/yandex.py @@ -36,7 +36,7 @@ class Yandex(Search): subdomains = self.match_subdomains(resp, fuzzy=False) if not self.check_subdomains(subdomains): break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) if '>next' not in resp.text: # 搜索页面没有出现下一页时停止搜索 break self.page_num += 1 diff --git a/modules/search/zoomeye_api.py b/modules/search/zoomeye_api.py index 30fa46d..b95e10b 100644 --- a/modules/search/zoomeye_api.py +++ b/modules/search/zoomeye_api.py @@ -50,7 +50,7 @@ class ZoomEyeAPI(Search): subdomains = self.match_subdomains(resp) if not subdomains: # 搜索没有发现子域名则停止搜索 break - self.subdomains = self.subdomains.union(subdomains) + self.subdomains.update(subdomains) page_num += 1 if page_num > 500: break From 0a770916cf5b13e69455ee7759a49de59a0a2611 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Thu, 20 Aug 2020 14:07:31 +0800 Subject: [PATCH 3/6] typos --- modules/autotake/github.py | 9 ++++++--- modules/banner.py | 4 ++-- modules/check/axfr.py | 9 ++++++--- modules/check/nsec.py | 1 - modules/crawl/archivecrawl.py | 3 +-- modules/crawl/commoncrawl.py | 2 +- modules/dnsquery/srv.py | 1 - modules/finder.py | 4 ++-- 8 files changed, 18 insertions(+), 15 deletions(-) diff --git a/modules/autotake/github.py b/modules/autotake/github.py index 32f7e92..b708d51 100644 --- a/modules/autotake/github.py +++ b/modules/autotake/github.py @@ -13,7 +13,9 @@ from config import settings HEADERS = { "Accept": "application/json, text/javascript, */*; q=0.01", "Accept-Language": "zh-CN,zh;q=0.9", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/63.0.3239.84 Safari/537.36", } @@ -73,8 +75,9 @@ def github_takeover(url): }, "content": cname_url64 } - html_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/index.html' - url_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/CNAME' + base_url = 'https://api.github.com/repos/' + html_url = base_url + user + '/' + repo_name + '/contents/index.html' + url_url = base_url + user + '/' + repo_name + '/contents/CNAME' html_r = requests.put(url=html_url, data=json.dumps(html_dict), headers=headers) # 上传index.html cname_r = requests.put(url=url_url, data=json.dumps(url_dict), diff --git a/modules/banner.py b/modules/banner.py index 772c2fc..25f51cd 100644 --- a/modules/banner.py +++ b/modules/banner.py @@ -388,8 +388,8 @@ class Condition(object): else: name = [] - while self.index < len( - self.condstr) and self.condstr[self.index] in self.allow_character: + while self.index < len(self.condstr) and self.condstr[self.index] \ + in self.allow_character: name.append(self.condstr[self.index]) self.index += 1 diff --git a/modules/check/axfr.py b/modules/check/axfr.py index 7e60fc6..55fb43d 100644 --- a/modules/check/axfr.py +++ b/modules/check/axfr.py @@ -33,14 +33,16 @@ class CheckAXFR(Module): :param server: domain server """ - logger.log('DEBUG', f'Trying to perform domain transfer in {server} of {self.domain}') + logger.log('DEBUG', f'Trying to perform domain transfer in {server} ' + f'of {self.domain}') try: xfr = dns.query.xfr(where=server, zone=self.domain, timeout=5.0, lifetime=10.0) zone = dns.zone.from_xfr(xfr) except Exception as e: logger.log('DEBUG', e.args) - logger.log('DEBUG', f'Domain transfer to server {server} of {self.domain} failed') + logger.log('DEBUG', f'Domain transfer to server {server} of ' + f'{self.domain} failed') return names = zone.nodes.keys() for name in names: @@ -50,7 +52,8 @@ class CheckAXFR(Module): record = zone[name].to_text(name) self.results.append(record) if self.results: - logger.log('DEBUG', f'Found the domain transfer record of {self.domain} on {server}') + logger.log('DEBUG', f'Found the domain transfer record of ' + f'{self.domain} on {server}') logger.log('DEBUG', '\n'.join(self.results)) self.results = [] diff --git a/modules/check/nsec.py b/modules/check/nsec.py index 0a6c190..952b648 100644 --- a/modules/check/nsec.py +++ b/modules/check/nsec.py @@ -24,7 +24,6 @@ class CheckNSEC(Module): subdomains = self.match_subdomains(record) subdomain = ''.join(subdomains) # 其实这里的subdomains的长度为1 也就是说只会有一个子域 self.subdomains.update(subdomains) - self.gen_record(subdomains, record) if subdomain == self.domain: # 当查出子域为主域 说明完成了一个循环 不再继续查询 break domain = subdomain diff --git a/modules/crawl/archivecrawl.py b/modules/crawl/archivecrawl.py index fb49c0c..eb4c7bf 100644 --- a/modules/crawl/archivecrawl.py +++ b/modules/crawl/archivecrawl.py @@ -26,8 +26,7 @@ class ArchiveCrawl(Crawl): for resp in cdx.iter(url, limit=limit): if resp.data.get('status') not in ['301', '302']: url = resp.data.get('url') - subdomains = self.match_subdomains(self.get_maindomain(domain), - url + resp.text) + subdomains = self.match_subdomains(domain, url + resp.text) self.subdomains.update(subdomains) def run(self): diff --git a/modules/crawl/commoncrawl.py b/modules/crawl/commoncrawl.py index 707033f..7979f41 100644 --- a/modules/crawl/commoncrawl.py +++ b/modules/crawl/commoncrawl.py @@ -26,7 +26,7 @@ class CommonCrawl(Crawl): for resp in tqdm(cdx.iter(url, limit=limit), total=limit): if resp.data.get('status') not in ['301', '302']: - subdomains = self.match_subdomains(self.get_maindomain(domain), resp.text) + subdomains = self.match_subdomains(domain, resp.text) self.subdomains.update(subdomains) def run(self): diff --git a/modules/dnsquery/srv.py b/modules/dnsquery/srv.py index c28d8e0..f21bf14 100644 --- a/modules/dnsquery/srv.py +++ b/modules/dnsquery/srv.py @@ -53,7 +53,6 @@ class BruteSRV(Module): record = str(item) subdomains = self.match_subdomains(record) self.subdomains.update(subdomains) - self.gen_record(subdomains, record) def run(self): """ diff --git a/modules/finder.py b/modules/finder.py index 57f5db9..668aff3 100644 --- a/modules/finder.py +++ b/modules/finder.py @@ -102,8 +102,8 @@ def filter_name(path, black_name): if path.endswith(name): return True black_ext = ['io.js', 'ui.js', 'fp.js', 'en.js', 'en-us,js', 'zh.js', 'zh-cn.js', - 'zh_cn.js', 'dev.js', 'min.js', 'umd.js', 'esm.js', 'all.js', 'cjs.js', 'prod.js', - 'slim.js', 'core.js', 'global.js', 'bundle.js', 'browser.js', + 'zh_cn.js', 'dev.js', 'min.js', 'umd.js', 'esm.js', 'all.js', 'cjs.js', + 'prod.js', 'slim.js', 'core.js', 'global.js', 'bundle.js', 'browser.js', 'brands.js', 'simple.js', 'common.js', 'development.js', 'banner.js', 'production.js'] for ext in black_ext: From 9f72a3af3244b06664ff7b87a5f0f0338d345208 Mon Sep 17 00:00:00 2001 From: Tinker <41416976+tinker-li@users.noreply.github.com> Date: Thu, 20 Aug 2020 16:45:39 +0800 Subject: [PATCH 4/6] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 代码里改里这里没改 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 86d70e4..9f6ac04 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ DESCRIPTION Example: python3 oneforall.py version python3 oneforall.py --target example.com run - python3 oneforall.py --target ./domains.txt run + python3 oneforall.py --targets ./domains.txt run python3 oneforall.py --target example.com --valid None run python3 oneforall.py --target example.com --brute True run python3 oneforall.py --target example.com --port small run From 8834a87f013013dc9048b472a6614677ccf5f39e Mon Sep 17 00:00:00 2001 From: Tinker <41416976+tinker-li@users.noreply.github.com> Date: Thu, 20 Aug 2020 17:04:39 +0800 Subject: [PATCH 5/6] Update README.md fix --- docs/en-us/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en-us/README.md b/docs/en-us/README.md index 63a00fc..0f84ce0 100644 --- a/docs/en-us/README.md +++ b/docs/en-us/README.md @@ -163,7 +163,7 @@ DESCRIPTION Example: python3 oneforall.py version python3 oneforall.py --target example.com run - python3 oneforall.py --target ./domains.txt run + python3 oneforall.py --targets ./domains.txt run python3 oneforall.py --target example.com --alive False run python3 oneforall.py --target example.com --brute True run python3 oneforall.py --target example.com --port medium run From 42b1bc441d636446256ed5e5ac54f867bdf29f76 Mon Sep 17 00:00:00 2001 From: JrD <421273918@qq.com> Date: Thu, 20 Aug 2020 17:12:30 +0800 Subject: [PATCH 6/6] typo --- README.md | 5 ++++- docs/en-us/README.md | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 86d70e4..f809152 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ docker run -it --rm -v ~/results:/OneForAll/results -v ~/.config:/OneForAll/conf 1. 如果你是通过pip3安装的依赖则使用以下命令运行示例： ```bash python3 oneforall.py --target example.com run +python3 oneforall.py --targets ./example.txt run ``` ![Example](./docs/usage_example.svg) @@ -191,7 +192,9 @@ DESCRIPTION ARGUMENTS TARGET - 单个域名或者每行一个域名的文件路径(必需参数) + 单个域名(二选一必需参数) + TARGETS + 每行一个域名的文件路径(二选一必需参数) FLAGS --brute=BRUTE diff --git a/docs/en-us/README.md b/docs/en-us/README.md index 63a00fc..8e96ec4 100644 --- a/docs/en-us/README.md +++ b/docs/en-us/README.md @@ -91,6 +91,7 @@ Result will be saved in `~/results`. 1. If you are use pip3, run the following command: ```bash python3 oneforall.py --target example.com run +python3 oneforall.py --targets ./example.txt run ``` ![Example](../usage_example.svg) @@ -163,7 +164,7 @@ DESCRIPTION Example: python3 oneforall.py version python3 oneforall.py --target example.com run - python3 oneforall.py --target ./domains.txt run + python3 oneforall.py --targets ./domains.txt run python3 oneforall.py --target example.com --alive False run python3 oneforall.py --target example.com --brute True run python3 oneforall.py --target example.com --port medium run @@ -181,7 +182,9 @@ DESCRIPTION ARGUMENTS TARGET - One domain or File path of one domain per line (required) + One domain (required) + TARGETS + File path of one domain per line (required) FLAGS --brute=BRUTE