typo

README.md

@@ -5,7 +5,7 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
 [![python](https://img.shields.io/badge/python-3.6|3.7|3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.4.0-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/release-v0.4.3-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
 
 👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md)
 
@@ -233,6 +233,8 @@ FLAGS
   7. 利用搜索引擎发现子域（目前有18个模块：`ask`, `baidu`, `bing`, `bing_api`, `duckduckgo`, `exalead`, `fofa_api`, `gitee`, `github`, `github_api`, `google`, `google_api`, `shodan_api`, `so`, `sogou`, `yahoo`, `yandex`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
 * **支持子域爆破**，该模块有常规的字典爆破，也有自定义的fuzz模式，支持批量爆破和递归爆破，自动判断泛解析并处理。
 * **支持子域验证**，默认开启子域验证，自动解析子域DNS，自动请求子域获取title和banner，并综合判断子域存活情况。
+* **支持子域爬取**，根据已有的子域，请求子域响应体以及响应体里的JS，从中再次发现新的子域。
+* **支持子域置换**，根据已有的子域，使用子域替换技术再次发现新的子域。
 * **支持子域接管**，默认开启子域接管风险检查，支持子域自动接管（目前只有Github，有待完善），支持批量检查。
 * **处理功能强大**，发现的子域结果支持自动去除，自动DNS解析，HTTP请求探测，自动筛选出有效子域，拓展子域的Banner信息，最终支持的导出格式有`txt`, `csv`, `json`。
 * **速度极快**，[收集模块](https://github.com/shmilylty/OneForAll/tree/master/collect.py)使用多线程调用，[爆破模块](https://github.com/shmilylty/OneForAll/tree/master/brute.py)使用[massdns](https://github.com/blechschmidt/massdns)，DNS解析速度每秒可解析350000以上个域名，子域验证中DNS解析和HTTP请求使用异步多协程，多线程检查[子域接管](https://github.com/shmilylty/OneForAll/tree/master/takeover.py)风险。
@@ -267,7 +269,6 @@ FLAGS
 ## ⌛后续计划
 
 - [ ] 各模块持续优化和完善
-- [x] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
 - [ ] 操作强大交互人性的前端界面实现
 
 更多信息请参阅[后续开发计划](https://github.com/shmilylty/OneForAll/tree/master/docs/todo.md)。

brute.py

@@ -296,7 +296,7 @@ class Brute(Module):
         brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
 
     Note:
-        --fmt csv/json (result fmt)
+        --fmt csv/json (result format)
         --path   Result path (default None, automatically generated)
 
 

common/database.py

@@ -221,7 +221,7 @@ class Database(object):
         """
         table_name = table_name.replace('.', '_')
         logger.log('TRACE', f'Get all the data from {table_name} table')
-        return self.query(f'select * from {table_name}')
+        return self.query(f'select * from "{table_name}"')
 
     def export_data(self, table_name, alive, limit):
         """
@@ -234,7 +234,7 @@ class Database(object):
         table_name = table_name.replace('.', '_')
         sql = f'select id, alive, request, resolve, url, subdomain, level,' \
               f'cname, ip, public, cdn, port, status, reason, title, banner,' \
-              f'cidr, asn, org, addr, isp, source from {table_name} order by subdomain'
+              f'cidr, asn, org, addr, isp, source from "{table_name}" '
         if alive and limit:
             if limit in ['resolve', 'request']:
                 where = f' where {limit} = 1'
@@ -242,31 +242,32 @@ class Database(object):
         elif alive:
             where = f' where alive = 1'
             sql += where
+        sql += ' order by subdomain'
         logger.log('TRACE', f'Get the data from {table_name} table')
         return self.query(sql)
 
     def count_alive(self, table_name):
         table_name = table_name.replace('.', '_')
-        sql = f'select count() from {table_name} where alive = 1'
+        sql = f'select count() from "{table_name}" where alive = 1'
         return self.query(sql)
 
     def get_resp_by_url(self, table_name, url):
         table_name = table_name.replace('.', '_')
-        sql = f'select response from {table_name} where url = "{url}"'
+        sql = f'select response from "{table_name}" where url = "{url}"'
         logger.log('TRACE', f'Get response data from {url}')
         return self.query(sql).scalar()
 
     def get_data_by_fields(self, table_name, fields):
         table_name = table_name.replace('.', '_')
         field_str = ', '.join(fields)
-        sql = f"select {field_str} from {table_name}"
+        sql = f'select {field_str} from "{table_name}"'
         logger.log('TRACE', f'Get specified field data {fields} from {table_name} table')
         return self.query(sql)
 
     def update_data_by_url(self, table_name, info, url):
         table_name = table_name.replace('.', '_')
         field_str = ', '.join(map(lambda kv: f'{kv[0]} = "{kv[1]}"', info.items()))
-        sql = f'update {table_name} set {field_str} where url = "{url}"'
+        sql = f'update "{table_name}" set {field_str} where url = "{url}"'
         return self.query(sql)
 
     def close(self):

common/utils.py

@@ -1,4 +1,3 @@
-import os
 import re
 import sys
 import time
@@ -10,10 +9,10 @@ import platform
 import subprocess
 from urllib.parse import scheme_chars
 from ipaddress import IPv4Address, ip_address
+from distutils.version import LooseVersion
 from pathlib import Path
 from stat import S_IXUSR
 
-import dns
 import requests
 import tenacity
 from dns.resolver import Resolver
@@ -526,7 +525,7 @@ def check_dep():
     if implementation != 'CPython':
         logger.log('FATAL', f'OneForAll only passed the test under CPython')
         exit(1)
-    if version < '3.6':
+    if LooseVersion(version) < LooseVersion('3.6'):
         logger.log('FATAL', 'OneForAll requires Python 3.6 or higher')
         exit(1)
 
@@ -771,3 +770,10 @@ def get_ns_path(in_china=None, enable_wildcard=None, ns_ip_list=None):
     ns_data = '\n'.join(ns_ip_list)
     save_to_file(path, ns_data)
     return path
+
+
+def init_table(domain):
+    db = Database()
+    db.drop_table(domain)
+    db.create_table(domain)
+    db.close()

config/default.py

@@ -23,7 +23,7 @@ enable_brute_module = True  # 使用爆破模块(默认True)
 enable_dns_resolve = True  # 使用DNS解析子域(默认True)
 enable_http_request = True  # 使用HTTP请求子域(默认True)
 enable_finder_module = True  # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True)
-enable_altdns_module = False  # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
+enable_altdns_module = True  # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
 enable_enrich_module = True  # 开启enrich模块，开启会富化出信息，如ip的cdn，cidr，asn，org，addr和isp等信息
 enable_banner_identify = True  # 开启WEB指纹识别模块(默认True)
 enable_takeover_check = False  # 开启子域接管风险检查(默认False)
@@ -75,7 +75,12 @@ brute_ip_blacklist = {'0.0.0.0', '0.0.0.1'}  # IP黑名单 子域解析到IP黑
 ip_appear_maximum = 100  # 多个子域解析到同一IP次数超过100次则标记为非法(泛解析)子域
 
 # altdns模块设置
-enable_fast_alt = True  # 是否开启快速置换(默认True，只使用部分置换规则)
+altdns_increase_num = True
+altdns_decrease_num = True
+altdns_replace_word = False
+altdns_insert_word = False
+altdns_add_word = False
+
 
 # banner识别模块设置
 banner_process_number = 4  # 识别进程数量(默认4)

config/setting.py

@@ -15,6 +15,7 @@ enable_brute_module = True  # 使用爆破模块(默认True)
 enable_dns_resolve = True  # 使用DNS解析子域(默认True)
 enable_http_request = True  # 使用HTTP请求子域(默认True)
 enable_finder_module = True  # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True)
+enable_altdns_module = True  # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
 enable_cdn_check = True  # 开启cdn检查模块(默认True)
 enable_banner_identify = True  # 开启WEB指纹识别模块(默认True)
 enable_takeover_check = False  # 开启子域接管风险检查(默认False)

docs/changes.md

@@ -8,6 +8,16 @@ OneForAll遵守[语义化版本格式](https://semver.org/)。
 # Unreleased
 
 # Released
+## [0.4.3](https://github.com/shmilylty/oneforall/releases/tag/v0.4.3) - 2020-11-29
+- 修复了已知问题
+- 更新了文档
+
+## [0.4.2](https://github.com/shmilylty/oneforall/releases/tag/v0.4.2) - 2020-11-23
+- 添加了数据表初始化处理流程，修复了#163中出现的问题。
+
+## [0.4.1](https://github.com/shmilylty/oneforall/releases/tag/v0.4.1) - 2020-11-18
+- 修复了数字开头主域（如58.com）出现数据库报错的问题
+
 ## [0.4.0](https://github.com/shmilylty/oneforall/releases/tag/v0.4.0) - 2020-11-18
 - 重构了子域请求模块，解决了内存占用过大问题
 - 新增了子域置换模块，能从现有的子域发现更多新子域

docs/en-us/README.md

@@ -5,7 +5,7 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
 [![python](https://img.shields.io/badge/python-3.6|3.7|3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.4.0-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/release-v0.4.3-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
 
 👊**OneForAll is a powerful subdomain integration tool**  📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
 
@@ -234,7 +234,9 @@ At present, OneForAll is under development, there must be a lot of problems and
 6. Use 6 threat intelligence modules: `alienvault`, `riskiq_ api`, `threatbook_ api`, `threatkeeper `, `virustotal`, `virustotal_ api`, which need to be added and improved.
 7. Use 16 search engines modules: `ask`, `baidu`, `bing`, `bing_api`, `fofa_api`, `gitee`, `github_api`, `google`, `google_api`, `shodan_api`, `so`, `sogou`, `yahoo`, `yandex`, `zoomeye_api`, except for special search engines. General search engines support automatic exclusion of search, full search and recursive search. 
 * **Support subdomain brute force**, can use dictionary mode or custom fuzz mode. Supports bulk brute and recursive brute, and automatically determine wildcard or not and processing.
-* **Support subdmain verification**, default enable, automatically resolve DNS, request subdomain to obtain response, and determine subdomain alive or not.
+* **Support subdomain verification**, default enable, automatically resolve DNS, request subdomain to obtain response, and determine subdomain alive or not.
+* **Support subdomain crawling**, according to the existing subdomains, the response body of the request subdomain and the JS in the response body can be found again from the new subdomain.
+* **Support subdomain replacement**, according to the existing subdomain, use subdomain replacement technology to discover new subdomains again.
 * **Support subdomain takeover**, default enable, supports bulk inspection, and automatic takeover subdomain (only Github, remains to be improved at present).
 * **Powerful processing feature**, support automatic deduplicate, DNS resolve, HTTP request, filter valid subdomains and information for subdomains. Supported export formats: `txt`, `csv`, `json`.
 * **Very fast**, [collection module](https://github.com/shmilylty/OneForAll/tree/master/collect.py) uses multi-threading, [brute module](https://github.com/shmilylty/OneForAll/tree/master/brute.py) uses [MassDNS](https://github.com/blechschmidt/massdns), MassDNS is capable of resolving over 350,000 names per second using publicly available resolvers. DNS resolve and HTTP requests use async-coroutine. [subdomain takeover](https://github.com/shmilylty/OneForAll/tree/master/takeover.py) uses multi-threading.
@@ -268,7 +270,6 @@ The project uses [SemVer](https://semver.org/) for version management, and you c
 ## ⌛Follow-up plan
 
 - [ ] Continuous optimize and improve of each module
-- [x] Subdomain collection crawler (collect subdomains from static files such as JS)
 - [ ] Implementation of front-end interface for powerful interaction
 
 For more details, read [todo.md](https://github.com/shmilylty/OneForAll/tree/master/docs/todo.md).

export.py

@@ -46,7 +46,7 @@ def export_data(target, db=None, alive=False, limit=None, path=None, fmt='csv',
             rows = database.export_data(table_name, alive, limit)
             if rows is None:
                 continue
-            data, _, _ = do_export(fmt, path, rows, show, table_name, target)
+            data, _, _ = do_export(fmt, path, rows, show, domain, target)
             datas.extend(data)
     database.close()
     if len(domains) > 1:
@@ -54,14 +54,14 @@ def export_data(target, db=None, alive=False, limit=None, path=None, fmt='csv',
     return datas
 
 
-def do_export(fmt, path, rows, show, table_name, target):
+def do_export(fmt, path, rows, show, domain, target):
     fmt = utils.check_format(fmt)
     path = utils.check_path(path, target, fmt)
     if show:
         print(rows.dataset)
     data = rows.export(fmt)
     utils.save_to_file(path, data)
-    logger.log('ALERT', f'The subdomain result for {table_name}: {path}')
+    logger.log('ALERT', f'The subdomain result for {domain}: {path}')
     data = rows.as_dict()
     return data, fmt, path
 

modules/altdns.py

@@ -7,6 +7,7 @@ import itertools
 
 from config import settings
 
+from modules import wildcard
 from common import utils
 from common import resolve
 from common import request
@@ -75,14 +76,16 @@ class Altdns(Module):
         # test1.example.com -> test2.example.com, test3.example.com, ...
         # test01.example.com -> test02.example.com, test03.example.com, ...
 
+        count = 0
         digits = re.findall(r'\d{1,3}', subname)
-
         for d in digits:
             for m in range(self.num_count):
                 replacement = str(int(d) + 1 + m).zfill(len(d))
                 tmp_domain = subname.replace(d, replacement)
                 new_domain = f'{tmp_domain}.{self.domain}'
                 self.new_subdomains.add(new_domain)
+                count += 1
+        logger.log('DEBUG', f'The increase_num generated {count} subdomains')
 
     def decrease_num(self, subname):
         """
@@ -94,8 +97,8 @@ class Altdns(Module):
         # test4.example.com -> test3.example.com, test2.example.com, ...
         # test04.example.com -> test03.example.com, test02.example.com, ...
 
+        count = 0
         digits = re.findall(r'\d{1,3}', subname)
-
         for d in digits:
             for m in range(self.num_count):
                 new_digit = (int(d) - 1 - m)
@@ -106,6 +109,8 @@ class Altdns(Module):
                 tmp_domain = subname.replace(d, replacement)
                 new_domain = f'{tmp_domain}.{self.domain}'
                 self.new_subdomains.add(new_domain)
+                count += 1
+        logger.log('DEBUG', f'The decrease_num generated {count} subdomains')
 
     def insert_word(self, parts):
         """
@@ -118,12 +123,15 @@ class Altdns(Module):
         #                           test.1.foo.WORD.example.com,
         #                           ...
 
+        count = 0
         for word in self.words:
             for index in range(len(parts)):
                 tmp_parts = parts.copy()
                 tmp_parts.insert(index, word)
                 new_domain = '.'.join(tmp_parts)
                 self.new_subdomains.add(new_domain)
+                count += 1
+        logger.log('DEBUG', f'The insert_word generated {count} subdomains')
 
     def add_word(self, subnames):
         """
@@ -131,6 +139,7 @@ class Altdns(Module):
         append existing content with `-WORD`
         """
 
+        count = 0
         for word in self.words:
             for index, name in enumerate(subnames):
                 # Prepend with `-`
@@ -146,6 +155,8 @@ class Altdns(Module):
                 tmp_subnames[index] = f'{name}-{word}'
                 new_subname = '.'.join(tmp_subnames + [self.domain])
                 self.new_subdomains.add(new_subname)
+                count += 1
+        logger.log('DEBUG', f'The add_word generated {count} subdomains')
 
     def replace_word(self, subname):
         """
@@ -158,6 +169,7 @@ class Altdns(Module):
         #                            WORD4.1.foo.example.com,
         #                            ..
 
+        count = 0
         for word in self.words:
             if word not in subname:
                 continue
@@ -167,16 +179,22 @@ class Altdns(Module):
                 new_subname = subname.replace(word, word_alt)
                 new_subdomain = f'{new_subname}.{self.domain}'
                 self.new_subdomains.add(new_subdomain)
+                count += 1
+        logger.log('DEBUG', f'The replace_word generated {count} subdomains')
 
     def gen_new_subdomains(self):
         for subdomain in self.now_subdomains:
             subname, parts = split_domain(subdomain)
             subnames = subname.split('.')
+            if settings.altdns_increase_num:
                 self.increase_num(subname)
+            if settings.altdns_decrease_num:
                 self.decrease_num(subname)
+            if settings.altdns_replace_word:
                 self.replace_word(subname)
-            if not settings.enable_fast_alt:
+            if settings.altdns_insert_word:
                 self.insert_word(parts)
+            if settings.altdns_add_word:
                 self.add_word(subnames)
         count = len(self.new_subdomains)
         logger.log('DEBUG', f'The altdns module generated {count} subdomains')
@@ -194,6 +212,5 @@ class Altdns(Module):
         self.elapse = round(self.end - self.start, 1)
         self.gen_result()
         resolved_data = resolve.run_resolve(self.domain, self.results)
-        request.run_request(self.domain, resolved_data, port)
-        logger.log('INFOR', f'Saving altdns results')
-        utils.save_to_db(self.domain, data, 'altdns')
+        valid_data = wildcard.deal_wildcard(resolved_data)  # 强制开启泛解析处理
+        request.run_request(self.domain, valid_data, port)

modules/search/fofa_api.py

@@ -21,7 +21,7 @@ class FoFa(Search):
         发送搜索请求并做子域匹配
         """
         self.page_num = 1
-        subdomain_encode = f'domain={self.domain} || cert={self.domain}'.encode('utf-8')
+        subdomain_encode = f'domain="{self.domain}"'.encode('utf-8')
         query_data = base64.b64encode(subdomain_encode)
         while True:
             time.sleep(self.delay)
@@ -32,7 +32,7 @@ class FoFa(Search):
                      'qbase64': query_data,
                      'page': self.page_num,
                      'full': 'true',
-                     'size': 5000}
+                     'size': 1000}
             resp = self.get(self.addr, query)
             if not resp:
                 return
@@ -42,7 +42,7 @@ class FoFa(Search):
                 break
             self.subdomains.update(subdomains)
             size = resp_json.get('size')
-            if size < 5000:
+            if size < 1000:
                 break
             self.page_num += 1
 

modules/search/github_api.py

@@ -1,3 +1,5 @@
+import time
+
 import requests
 from config import settings
 from common.search import Search
@@ -50,6 +52,7 @@ class GithubAPI(Search):
             return
         page = 1
         while True:
+            time.sleep(self.delay)
             params = {'q': self.domain, 'per_page': 100,
                       'page': page, 'sort': 'indexed'}
             try:

oneforall.py

@@ -32,7 +32,7 @@ blue = '\033[01;34m'
 red = '\033[1;31m'
 end = '\033[0m'
 
-version = 'v0.4.0'
+version = 'v0.4.3'
 message = white + '{' + red + version + ' #dev' + white + '}'
 
 oneforall_banner = f"""
@@ -147,6 +147,8 @@ class OneForAll(object):
         :return: subdomain results
         :rtype: list
         """
+        utils.init_table(self.domain)
+
         if not self.access_internet:
             logger.log('ALERT', 'Because it cannot access the Internet, '
                                 'OneForAll will not execute the subdomain collection module!')