Merge remote-tracking branch 'origin/master'

# Conflicts:
#	modules/dnsquery/srv.py
#	modules/finder.py

README.md

@@ -99,6 +99,7 @@ docker run -it --rm -v ~/results:/OneForAll/results -v ~/.config:/OneForAll/conf
 1. 如果你是通过pip3安装的依赖则使用以下命令运行示例：   
 ```bash
 python3 oneforall.py --target example.com run
+python3 oneforall.py --targets ./example.txt run
 ```
 
 ![Example](./docs/usage_example.svg)
@@ -172,7 +173,7 @@ DESCRIPTION
     Example:
         python3 oneforall.py version
         python3 oneforall.py --target example.com run
-        python3 oneforall.py --target ./domains.txt run
+        python3 oneforall.py --targets ./domains.txt run
         python3 oneforall.py --target example.com --valid None run
         python3 oneforall.py --target example.com --brute True run
         python3 oneforall.py --target example.com --port small run
@@ -191,7 +192,9 @@ DESCRIPTION
 
 ARGUMENTS
     TARGET
-        单个域名或者每行一个域名的文件路径(必需参数)
+        单个域名(二选一必需参数)
+    TARGETS
+        每行一个域名的文件路径(二选一必需参数)
 
 FLAGS
     --brute=BRUTE

brute.py

@@ -90,9 +90,9 @@ def gen_subdomains(expression, path):
 
     :param  str  expression: generate subdomains's expression
     :param  str  path: path of wordlist
-    :return list subdomains: list of subdomains
+    :return set  subdomains: list of subdomains
     """
-    subdomains = list()
+    subdomains = set()
     with open(path, encoding='utf-8', errors='ignore') as fd:
         for line in fd:
             word = line.strip().lower()
@@ -105,14 +105,13 @@ def gen_subdomains(expression, path):
             if word.endswith('.'):
                 word = word[:-1]
             subdomain = expression.replace('*', word)
-            subdomains.append(subdomain)
+            subdomains.add(subdomain)
     size = len(subdomains)
     logger.log('DEBUG', f'The size of the dictionary generated by {path} is {size}')
     if size == 0:
         logger.log('ALERT', 'Please check the dictionary content!')
     else:
-        random_domain = random.choice(subdomains)
+        utils.check_random_subdomain(subdomains)
-        logger.log('ALERT', f'Please check whether {random_domain} is correct or not')
     return subdomains
 
 
@@ -123,12 +122,12 @@ def gen_fuzz_subdomains(expression, rule, fuzzlist):
     :param  str  expression: generate subdomains's expression
     :param  str  rule: regexp rule
     :param  str  fuzzlist: fuzz dictionary
-    :return list subdomains: list of subdomains
+    :return set  subdomains: list of subdomains
     """
-    subdomains = list()
+    subdomains = set()
     if fuzzlist:
         fuzz_domain = gen_subdomains(expression, fuzzlist)
-        subdomains = subdomains.extend(fuzz_domain)
+        subdomains.update(fuzz_domain)
     if rule:
         fuzz_count = exrex.count(rule)
         if fuzz_count > 10000000:
@@ -139,9 +138,8 @@ def gen_fuzz_subdomains(expression, rule, fuzzlist):
             if not fuzz_string.isalnum():
                 continue
             fuzz_domain = expression.replace('*', fuzz_string)
-            subdomains.append(fuzz_domain)
+            subdomains.add(fuzz_domain)
-        random_domain = random.choice(subdomains)
+        utils.check_random_subdomain(subdomains)
-        logger.log('ALERT', f'Please check whether {random_domain} is correct or not')
     logger.log('DEBUG', f'Dictionary size based on fuzz mode: {len(subdomains)}')
     return subdomains
 
@@ -152,7 +150,7 @@ def gen_word_subdomains(expression, path):
 
     :param  str  expression: generate subdomains's expression
     :param  str  path: path of wordlist
-    :return list subdomains: list of subdomains
+    :return set  subdomains: list of subdomains
     """
     subdomains = gen_subdomains(expression, path)
     logger.log('DEBUG', f'Dictionary based on word mode size: {len(subdomains)}')
@@ -249,7 +247,7 @@ def collect_wildcard_record(domain, authoritative_ns):
             continue
         if ip is None:
             continue
-        ips = ips.union(ip)
+        ips.update(ip)
         # 统计每个泛解析IP出现次数
         for addr in ip:
             count = ips_stat.setdefault(addr, 0)
@@ -543,11 +541,10 @@ class Brute(Module):
             wordlist = self.recursive_nextlist
         if self.word:
             word_subdomains = gen_word_subdomains(self.place, wordlist)
-            # set可以合并list
+            dict_set.update(word_subdomains)
-            dict_set = dict_set.union(word_subdomains)
         if self.fuzz:
             fuzz_subdomains = gen_fuzz_subdomains(self.place, self.rule, self.fuzzlist)
-            dict_set = dict_set.union(fuzz_subdomains)
+            dict_set.update(fuzz_subdomains)
         count = len(dict_set)
         logger.log('INFOR', f'Dictionary size: {count}')
         if count > 10000000:

common/lookup.py

@@ -23,6 +23,6 @@ class Lookup(Module):
         for item in answer:
             record = item.to_text()
             subdomains = self.match_subdomains(record)
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             logger.log('DEBUG', record)
         return self.subdomains

common/module.py

@@ -220,7 +220,8 @@ class Module(object):
 
     def collect_subdomains(self, resp):
         subdomains = self.match_subdomains(resp)
-        return self.subdomains.union(subdomains)
+        self.subdomains.update(subdomains)
+        return self.subdomains
 
     def save_json(self):
         """

common/utils.py

@@ -728,3 +728,13 @@ def match_subdomains(domain, html, distinct=True, fuzzy=True):
         return set(deal)
     else:
         return list(deal)
+
+
+def check_random_subdomain(subdomains):
+    if not subdomains:
+        logger.log('ALERT', f'The generated dictionary is empty')
+        return False
+    for subdomain in subdomains:
+        if subdomain:
+            logger.log('ALERT', f'Please check whether {subdomain} is correct or not')
+            return True

docs/en-us/README.md

@@ -91,6 +91,7 @@ Result will be saved in `~/results`.
 1. If you are use pip3, run the following command: 
 ```bash
 python3 oneforall.py --target example.com run
+python3 oneforall.py --targets ./example.txt run
 ```
 
 ![Example](../usage_example.svg)
@@ -163,7 +164,7 @@ DESCRIPTION
     Example:
         python3 oneforall.py version
         python3 oneforall.py --target example.com run
-        python3 oneforall.py --target ./domains.txt run
+        python3 oneforall.py --targets ./domains.txt run
         python3 oneforall.py --target example.com --alive False run
         python3 oneforall.py --target example.com --brute True run
         python3 oneforall.py --target example.com --port medium run
@@ -181,7 +182,9 @@ DESCRIPTION
 
 ARGUMENTS
     TARGET
-        One domain or File path of one domain per line (required)
+        One domain (required)
+    TARGETS
+        File path of one domain per line (required)
 
 FLAGS
     --brute=BRUTE

modules/autotake/github.py

@@ -13,7 +13,9 @@ from config import settings
 HEADERS = {
     "Accept": "application/json, text/javascript, */*; q=0.01",
     "Accept-Language": "zh-CN,zh;q=0.9",
-    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) "
+                  "AppleWebKit/537.36 (KHTML, like Gecko) "
+                  "Chrome/63.0.3239.84 Safari/537.36",
 }
 
 
@@ -73,8 +75,9 @@ def github_takeover(url):
                     },
                     "content": cname_url64
                 }
-                html_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/index.html'
+                base_url = 'https://api.github.com/repos/'
-                url_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/CNAME'
+                html_url = base_url + user + '/' + repo_name + '/contents/index.html'
+                url_url = base_url + user + '/' + repo_name + '/contents/CNAME'
                 html_r = requests.put(url=html_url, data=json.dumps(html_dict),
                                       headers=headers)  # 上传index.html
                 cname_r = requests.put(url=url_url, data=json.dumps(url_dict),

modules/banner.py

@@ -388,8 +388,8 @@ class Condition(object):
 
             else:
                 name = []
-                while self.index < len(
+                while self.index < len(self.condstr) and self.condstr[self.index] \
-                        self.condstr) and self.condstr[self.index] in self.allow_character:
+                        in self.allow_character:
                     name.append(self.condstr[self.index])
                     self.index += 1
 

modules/certificates/censys_api.py

@@ -34,7 +34,7 @@ class CensysAPI(Query):
             logger.log('ALERT', f'{self.source} module {status}')
             return
         subdomains = self.match_subdomains(resp.text)
-        self.subdomains = self.subdomains.union(subdomains)
+        self.subdomains.update(subdomains)
         pages = json.get('metadata').get('pages')
         for page in range(2, pages + 1):
             data['page'] = page

modules/certificates/crtsh.py

@@ -21,7 +21,7 @@ class Crtsh(Query):
             return
         text = resp.text.replace(r'\n', ' ')
         subdomains = self.match_subdomains(text)
-        self.subdomains = self.subdomains.union(subdomains)
+        self.subdomains.update(subdomains)
 
     def run(self):
         """

modules/check/axfr.py

@@ -33,24 +33,27 @@ class CheckAXFR(Module):
 
         :param server: domain server
         """
-        logger.log('DEBUG', f'Trying to perform domain transfer in {server} of {self.domain}')
+        logger.log('DEBUG', f'Trying to perform domain transfer in {server} '
+                            f'of {self.domain}')
         try:
             xfr = dns.query.xfr(where=server, zone=self.domain,
                                 timeout=5.0, lifetime=10.0)
             zone = dns.zone.from_xfr(xfr)
         except Exception as e:
             logger.log('DEBUG', e.args)
-            logger.log('DEBUG', f'Domain transfer to server {server} of {self.domain} failed')
+            logger.log('DEBUG', f'Domain transfer to server {server} of '
+                                f'{self.domain} failed')
             return
         names = zone.nodes.keys()
         for name in names:
             full_domain = str(name) + '.' + self.domain
             subdomain = self.match_subdomains(full_domain)
-            self.subdomains = self.subdomains.union(subdomain)
+            self.subdomains.update(subdomain)
             record = zone[name].to_text(name)
             self.results.append(record)
         if self.results:
-            logger.log('DEBUG', f'Found the domain transfer record of {self.domain} on {server}')
+            logger.log('DEBUG', f'Found the domain transfer record of '
+                                f'{self.domain} on {server}')
             logger.log('DEBUG', '\n'.join(self.results))
             self.results = []
 

modules/check/cert.py

@@ -32,7 +32,7 @@ class CheckCert(Module):
             logger.log('DEBUG', e.args)
             return
         subdomains = self.match_subdomains(str(cert_dict))
-        self.subdomains = self.subdomains.union(subdomains)
+        self.subdomains.update(subdomains)
 
     def run(self):
         """

modules/check/nsec.py

@@ -23,8 +23,7 @@ class CheckNSEC(Module):
                 record = item.to_text()
                 subdomains = self.match_subdomains(record)
                 subdomain = ''.join(subdomains)  # 其实这里的subdomains的长度为1 也就是说只会有一个子域
-                self.subdomains = self.subdomains.union(subdomains)
+                self.subdomains.update(subdomains)
-                self.gen_record(subdomains, record)
             if subdomain == self.domain:  # 当查出子域为主域 说明完成了一个循环 不再继续查询
                 break
             domain = subdomain

modules/crawl/archivecrawl.py

@@ -26,9 +26,8 @@ class ArchiveCrawl(Crawl):
         for resp in cdx.iter(url, limit=limit):
             if resp.data.get('status') not in ['301', '302']:
                 url = resp.data.get('url')
-                subdomains = self.match_subdomains(self.get_maindomain(domain),
+                subdomains = self.match_subdomains(domain, url + resp.text)
-                                                   url + resp.text)
+                self.subdomains.update(subdomains)
-                self.subdomains = self.subdomains.union(subdomains)
 
     def run(self):
         """

modules/crawl/commoncrawl.py

@@ -26,8 +26,8 @@ class CommonCrawl(Crawl):
 
         for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
             if resp.data.get('status') not in ['301', '302']:
-                subdomains = self.match_subdomains(self.get_maindomain(domain), resp.text)
+                subdomains = self.match_subdomains(domain, resp.text)
-                self.subdomains = self.subdomains.union(subdomains)
+                self.subdomains.update(subdomains)
 
     def run(self):
         """

modules/datasets/cloudflare_api.py

@@ -81,7 +81,7 @@ class CloudFlareAPI(Query):
                        f'{list_dns_resp.status_code} {list_dns_resp.text}')
             return
         subdomains = self.match_subdomains(list_dns_resp.text)
-        self.subdomains = self.subdomains.union(subdomains)
+        self.subdomains.update(subdomains)
         if not self.subdomains:
             # waiting for cloudflare enumerate subdomains
             sleep(5)
@@ -96,7 +96,7 @@ class CloudFlareAPI(Query):
                     return
                 total_pages = list_dns_resp.json()['result_info']['total_pages']
                 subdomains = (self.match_subdomains(list_dns_resp.text))
-                self.subdomains = self.subdomains.union(subdomains)
+                self.subdomains.update(subdomains)
                 page += 1
                 if page > total_pages:
                     break

modules/datasets/ipv4info_api.py

@@ -35,7 +35,7 @@ class IPv4InfoAPI(Query):
             subdomains = self.match_subdomains(str(json))
             if not subdomains:
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
             subdomains = json.get('Subdomains')
             if subdomains and len(subdomains) < 300:

modules/datasets/netcraft.py

@@ -50,7 +50,7 @@ class NetCraft(Query):
             subdomains = self.match_subdomains(resp)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             if 'Next Page' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                 break
             last = re.search(r'&last=.*' + self.domain, resp.text).group(0)

modules/datasets/qianxun.py

@@ -28,7 +28,7 @@ class QianXun(Query):
             subdomains = self.match_subdomains(resp)
             if not subdomains:  # 没有发现子域名则停止查询
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             if '<div id="page" class="pagelist">' not in resp.text:
                 break
             if '<li class="disabled"><span>&raquo;</span></li>' in resp.text:

modules/datasets/securitytrails_api.py

@@ -26,7 +26,7 @@ class SecurityTrailsAPI(Query):
         prefixs = resp.json()['subdomains']
         subdomains = [f'{prefix}.{self.domain}' for prefix in prefixs]
         if subdomains:
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
 
     def run(self):
         """

modules/datasets/sitedossier.py

@@ -25,7 +25,7 @@ class SiteDossier(Query):
             subdomains = self.match_subdomains(resp)
             if not subdomains:  # 没有发现子域名则停止查询
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             # 搜索页面没有出现下一页时停止搜索
             if 'Show next 100 items' not in resp.text:
                 break

modules/datasets/spyse_api.py

@@ -29,7 +29,7 @@ class SpyseAPI(Query):
             subdomains = self.match_subdomains(str(json))
             if not subdomains:  # 没有发现子域名则停止查询
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             offset += limit
             if len(json.get('data').get('items')) < limit:
                 break

modules/datasets/threatcrowd.py

@@ -26,7 +26,7 @@ class ThreatCrowd(Query):
         if resp.status_code != 200:
             return
         subdomains = self.match_subdomains(resp.text)
-        self.subdomains = self.subdomains.union(subdomains)
+        self.subdomains.update(subdomains)
 
     def run(self):
         """

modules/datasets/wzpc.py

@@ -34,7 +34,7 @@ class WZPCQuery(Query):
             subdomains = self.match_subdomains(resp.text)
             if not subdomains:  # 没有发现子域名则停止查询
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             if not subdomains:
                 break
             if page_num > 10:

modules/dnsquery/srv.py

@@ -52,7 +52,7 @@ class BruteSRV(Module):
             for item in answer:
                 record = str(item)
                 subdomains = self.match_subdomains(record)
-                self.subdomains = self.subdomains.union(subdomains)
+                self.subdomains.update(subdomains)
 
     def run(self):
         """

modules/intelligence/virustotal.py

@@ -30,7 +30,7 @@ class VirusTotal(Query):
             subdomains = self.match_subdomains(resp)
             if not subdomains:
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             data = resp.json()
             next_cursor = data.get('meta').get('cursor')
 

modules/intelligence/virustotal_api.py

@@ -25,7 +25,7 @@ class VirusTotalAPI(Query):
             subdomains = self.match_subdomains(resp)
             if not subdomains:
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             data = resp.json()
             next_cursor = data.get('meta').get('cursor')
 

modules/search/ask.py

@@ -30,7 +30,7 @@ class Ask(Search):
             subdomains = self.match_subdomains(resp, fuzzy=False)
             if not self.check_subdomains(subdomains):
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             self.page_num += 1
             if '>Next<' not in resp.text:
                 break

modules/search/baidu.py

@@ -25,7 +25,7 @@ class Baidu(Search):
         for find_res in bs.find_all('a', {'class': 'c-showurl'}):
             url = find_res.get('href')
             subdomains = self.match_location(url)
-            subdomains_all = subdomains_all.union(subdomains)
+            subdomains_all.update(subdomains)
         return subdomains_all
 
     def search(self, domain, filtered_subdomain=''):
@@ -54,7 +54,7 @@ class Baidu(Search):
                 subdomains = self.match_subdomains(resp, fuzzy=False)
             if not self.check_subdomains(subdomains):
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             self.page_num += self.per_page_num
             # 搜索页面没有出现下一页时停止搜索
             if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:

modules/search/bing.py

@@ -36,7 +36,7 @@ class Bing(Search):
             subdomains = self.match_subdomains(resp, fuzzy=False)
             if not self.check_subdomains(subdomains):
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             # 搜索页面没有出现下一页时停止搜索
             if '<div class="sw_next">' not in resp.text:
                 break

modules/search/bing_api.py

@@ -36,7 +36,7 @@ class BingAPI(Search):
             subdomains = self.match_subdomains(resp)
             if not self.check_subdomains(subdomains):
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             self.page_num += self.per_page_num
             if self.page_num >= self.limit_num:  # 搜索条数限制
                 break

modules/search/fofa_api.py

@@ -40,7 +40,7 @@ class FoFa(Search):
             subdomains = self.match_subdomains(resp)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             size = resp_json.get('size')
             if size < 10000:
                 break

modules/search/gitee.py

@@ -37,7 +37,7 @@ class Gitee(Search):
             subdomains = self.match_subdomains(soup, fuzzy=False)
             if not self.check_subdomains(subdomains):
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             if '<li class="disabled"><a href="###">' in resp.text:
                 break
             page_num += 1

modules/search/github_api.py

@@ -63,7 +63,7 @@ class GithubAPI(Search):
             subdomains = self.match_subdomains(resp)
             if not subdomains:
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             page += 1
             try:
                 resp_json = resp.json()

modules/search/google.py

@@ -40,7 +40,7 @@ class Google(Search):
             subdomains = self.match_subdomains(resp, fuzzy=False)
             if not self.check_subdomains(subdomains):
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             page_num += per_page_num
             if 'start=' + str(page_num) not in resp.text:
                 break

modules/search/google_api.py

@@ -35,7 +35,7 @@ class GoogleAPI(Search):
             subdomains = self.match_subdomains(resp)
             if not self.check_subdomains(subdomains):
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             self.page_num += self.per_page_num
             if self.page_num > 100:  # 免费的API只能查询前100条结果
                 break

modules/search/shodan_api.py

@@ -26,7 +26,7 @@ class ShodanAPI(Search):
             subdomains = self.match_subdomains(resp)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             page += 1
 
     def run(self):

modules/search/so.py

@@ -31,7 +31,7 @@ class So(Search):
             subdomains = self.match_subdomains(resp, fuzzy=False)
             if not self.check_subdomains(subdomains):
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             page_num += 1
             # 搜索页面没有出现下一页时停止搜索
             if '<a id="snext"' not in resp.text:

modules/search/sogou.py

@@ -28,7 +28,7 @@ class Sogou(Search):
             subdomains = self.match_subdomains(resp, fuzzy=False)
             if not self.check_subdomains(subdomains):
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             self.page_num += 1
             # 搜索页面没有出现下一页时停止搜索
             if '<a id="sogou_next"' not in resp.text:

modules/search/yahoo.py

@@ -38,7 +38,7 @@ class Yahoo(Search):
             subdomains = self.match_subdomains(text, fuzzy=False)
             if not self.check_subdomains(subdomains):
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             if '>Next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                 break
             self.page_num += self.per_page_num

modules/search/yandex.py

@@ -36,7 +36,7 @@ class Yandex(Search):
             subdomains = self.match_subdomains(resp, fuzzy=False)
             if not self.check_subdomains(subdomains):
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             if '>next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                 break
             self.page_num += 1

modules/search/zoomeye_api.py

@@ -50,7 +50,7 @@ class ZoomEyeAPI(Search):
             subdomains = self.match_subdomains(resp)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
-            self.subdomains = self.subdomains.union(subdomains)
+            self.subdomains.update(subdomains)
             page_num += 1
             if page_num > 500:
                 break