v0.4.2

README.md

@@ -5,7 +5,7 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
 [![python](https://img.shields.io/badge/python-3.6|3.7|3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.4.0-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/release-v0.4.2-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
 
 👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md)
 

common/database.py

@@ -221,7 +221,7 @@ class Database(object):
         """
         table_name = table_name.replace('.', '_')
         logger.log('TRACE', f'Get all the data from {table_name} table')
-        return self.query(f'select * from {table_name}')
+        return self.query(f'select * from "{table_name}"')
 
     def export_data(self, table_name, alive, limit):
         """
@@ -234,7 +234,7 @@ class Database(object):
         table_name = table_name.replace('.', '_')
         sql = f'select id, alive, request, resolve, url, subdomain, level,' \
               f'cname, ip, public, cdn, port, status, reason, title, banner,' \
-              f'cidr, asn, org, addr, isp, source from {table_name} order by subdomain'
+              f'cidr, asn, org, addr, isp, source from "{table_name}" order by subdomain'
         if alive and limit:
             if limit in ['resolve', 'request']:
                 where = f' where {limit} = 1'
@@ -247,26 +247,26 @@ class Database(object):
 
     def count_alive(self, table_name):
         table_name = table_name.replace('.', '_')
-        sql = f'select count() from {table_name} where alive = 1'
+        sql = f'select count() from "{table_name}" where alive = 1'
         return self.query(sql)
 
     def get_resp_by_url(self, table_name, url):
         table_name = table_name.replace('.', '_')
-        sql = f'select response from {table_name} where url = "{url}"'
+        sql = f'select response from "{table_name}" where url = "{url}"'
         logger.log('TRACE', f'Get response data from {url}')
         return self.query(sql).scalar()
 
     def get_data_by_fields(self, table_name, fields):
         table_name = table_name.replace('.', '_')
         field_str = ', '.join(fields)
-        sql = f"select {field_str} from {table_name}"
+        sql = f'select {field_str} from "{table_name}"'
         logger.log('TRACE', f'Get specified field data {fields} from {table_name} table')
         return self.query(sql)
 
     def update_data_by_url(self, table_name, info, url):
         table_name = table_name.replace('.', '_')
         field_str = ', '.join(map(lambda kv: f'{kv[0]} = "{kv[1]}"', info.items()))
-        sql = f'update {table_name} set {field_str} where url = "{url}"'
+        sql = f'update "{table_name}" set {field_str} where url = "{url}"'
         return self.query(sql)
 
     def close(self):

common/utils.py

@@ -771,3 +771,10 @@ def get_ns_path(in_china=None, enable_wildcard=None, ns_ip_list=None):
     ns_data = '\n'.join(ns_ip_list)
     save_to_file(path, ns_data)
     return path
+
+
+def init_table(domain):
+    db = Database()
+    db.drop_table(domain)
+    db.create_table(domain)
+    db.close()

config/default.py

@@ -23,7 +23,7 @@ enable_brute_module = True  # 使用爆破模块(默认True)
 enable_dns_resolve = True  # 使用DNS解析子域(默认True)
 enable_http_request = True  # 使用HTTP请求子域(默认True)
 enable_finder_module = True  # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True)
-enable_altdns_module = False  # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
+enable_altdns_module = True  # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
 enable_enrich_module = True  # 开启enrich模块，开启会富化出信息，如ip的cdn，cidr，asn，org，addr和isp等信息
 enable_banner_identify = True  # 开启WEB指纹识别模块(默认True)
 enable_takeover_check = False  # 开启子域接管风险检查(默认False)

config/setting.py

@@ -15,6 +15,7 @@ enable_brute_module = True  # 使用爆破模块(默认True)
 enable_dns_resolve = True  # 使用DNS解析子域(默认True)
 enable_http_request = True  # 使用HTTP请求子域(默认True)
 enable_finder_module = True  # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True)
+enable_altdns_module = True  # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
 enable_cdn_check = True  # 开启cdn检查模块(默认True)
 enable_banner_identify = True  # 开启WEB指纹识别模块(默认True)
 enable_takeover_check = False  # 开启子域接管风险检查(默认False)

docs/changes.md

@@ -8,6 +8,12 @@ OneForAll遵守[语义化版本格式](https://semver.org/)。
 # Unreleased
 
 # Released
+## [0.4.2](https://github.com/shmilylty/oneforall/releases/tag/v0.4.2) - 2020-11-23
+- 添加了数据表初始化处理流程，修复了#163中出现的问题。
+
+## [0.4.1](https://github.com/shmilylty/oneforall/releases/tag/v0.4.1) - 2020-11-18
+- 修复了数字开头主域（如58.com）出现数据库报错的问题
+
 ## [0.4.0](https://github.com/shmilylty/oneforall/releases/tag/v0.4.0) - 2020-11-18
 - 重构了子域请求模块，解决了内存占用过大问题
 - 新增了子域置换模块，能从现有的子域发现更多新子域

docs/en-us/README.md

@@ -5,7 +5,7 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
 [![python](https://img.shields.io/badge/python-3.6|3.7|3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.4.0-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/release-v0.4.2-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
 
 👊**OneForAll is a powerful subdomain integration tool**  📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
 

export.py

@@ -46,7 +46,7 @@ def export_data(target, db=None, alive=False, limit=None, path=None, fmt='csv',
             rows = database.export_data(table_name, alive, limit)
             if rows is None:
                 continue
-            data, _, _ = do_export(fmt, path, rows, show, table_name, target)
+            data, _, _ = do_export(fmt, path, rows, show, domain, target)
             datas.extend(data)
     database.close()
     if len(domains) > 1:
@@ -54,14 +54,14 @@ def export_data(target, db=None, alive=False, limit=None, path=None, fmt='csv',
     return datas
 
 
-def do_export(fmt, path, rows, show, table_name, target):
+def do_export(fmt, path, rows, show, domain, target):
     fmt = utils.check_format(fmt)
     path = utils.check_path(path, target, fmt)
     if show:
         print(rows.dataset)
     data = rows.export(fmt)
     utils.save_to_file(path, data)
-    logger.log('ALERT', f'The subdomain result for {table_name}: {path}')
+    logger.log('ALERT', f'The subdomain result for {domain}: {path}')
     data = rows.as_dict()
     return data, fmt, path
 

modules/altdns.py

@@ -7,6 +7,7 @@ import itertools
 
 from config import settings
 
+from modules import wildcard
 from common import utils
 from common import resolve
 from common import request
@@ -75,14 +76,16 @@ class Altdns(Module):
         # test1.example.com -> test2.example.com, test3.example.com, ...
         # test01.example.com -> test02.example.com, test03.example.com, ...
 
+        count = 0
         digits = re.findall(r'\d{1,3}', subname)
-
         for d in digits:
             for m in range(self.num_count):
                 replacement = str(int(d) + 1 + m).zfill(len(d))
                 tmp_domain = subname.replace(d, replacement)
                 new_domain = f'{tmp_domain}.{self.domain}'
                 self.new_subdomains.add(new_domain)
+                count += 1
+        logger.log('DEBUG', f'The increase_num generated {count} subdomains')
 
     def decrease_num(self, subname):
         """
@@ -94,8 +97,8 @@ class Altdns(Module):
         # test4.example.com -> test3.example.com, test2.example.com, ...
         # test04.example.com -> test03.example.com, test02.example.com, ...
 
+        count = 0
         digits = re.findall(r'\d{1,3}', subname)
-
         for d in digits:
             for m in range(self.num_count):
                 new_digit = (int(d) - 1 - m)
@@ -106,6 +109,8 @@ class Altdns(Module):
                 tmp_domain = subname.replace(d, replacement)
                 new_domain = f'{tmp_domain}.{self.domain}'
                 self.new_subdomains.add(new_domain)
+                count += 1
+        logger.log('DEBUG', f'The decrease_num generated {count} subdomains')
 
     def insert_word(self, parts):
         """
@@ -118,12 +123,15 @@ class Altdns(Module):
         #                           test.1.foo.WORD.example.com,
         #                           ...
 
+        count = 0
         for word in self.words:
             for index in range(len(parts)):
                 tmp_parts = parts.copy()
                 tmp_parts.insert(index, word)
                 new_domain = '.'.join(tmp_parts)
                 self.new_subdomains.add(new_domain)
+                count += 1
+        logger.log('DEBUG', f'The insert_word generated {count} subdomains')
 
     def add_word(self, subnames):
         """
@@ -131,6 +139,7 @@ class Altdns(Module):
         append existing content with `-WORD`
         """
 
+        count = 0
         for word in self.words:
             for index, name in enumerate(subnames):
                 # Prepend with `-`
@@ -146,6 +155,8 @@ class Altdns(Module):
                 tmp_subnames[index] = f'{name}-{word}'
                 new_subname = '.'.join(tmp_subnames + [self.domain])
                 self.new_subdomains.add(new_subname)
+                count += 1
+        logger.log('DEBUG', f'The add_word generated {count} subdomains')
 
     def replace_word(self, subname):
         """
@@ -158,6 +169,7 @@ class Altdns(Module):
         #                            WORD4.1.foo.example.com,
         #                            ..
 
+        count = 0
         for word in self.words:
             if word not in subname:
                 continue
@@ -167,6 +179,8 @@ class Altdns(Module):
                 new_subname = subname.replace(word, word_alt)
                 new_subdomain = f'{new_subname}.{self.domain}'
                 self.new_subdomains.add(new_subdomain)
+                count += 1
+        logger.log('DEBUG', f'The replace_word generated {count} subdomains')
 
     def gen_new_subdomains(self):
         for subdomain in self.now_subdomains:
@@ -194,6 +208,5 @@ class Altdns(Module):
         self.elapse = round(self.end - self.start, 1)
         self.gen_result()
         resolved_data = resolve.run_resolve(self.domain, self.results)
-        request.run_request(self.domain, resolved_data, port)
+        valid_data = wildcard.deal_wildcard(resolved_data)  # 强制开启泛解析处理
-        logger.log('INFOR', f'Saving altdns results')
+        request.run_request(self.domain, valid_data, port)
-        utils.save_to_db(self.domain, data, 'altdns')

modules/search/fofa_api.py

@@ -21,7 +21,7 @@ class FoFa(Search):
         发送搜索请求并做子域匹配
         """
         self.page_num = 1
-        subdomain_encode = f'domain={self.domain} || cert={self.domain}'.encode('utf-8')
+        subdomain_encode = f'domain="{self.domain}"'.encode('utf-8')
         query_data = base64.b64encode(subdomain_encode)
         while True:
             time.sleep(self.delay)
@@ -32,7 +32,7 @@ class FoFa(Search):
                      'qbase64': query_data,
                      'page': self.page_num,
                      'full': 'true',
-                     'size': 5000}
+                     'size': 1000}
             resp = self.get(self.addr, query)
             if not resp:
                 return
@@ -42,7 +42,7 @@ class FoFa(Search):
                 break
             self.subdomains.update(subdomains)
             size = resp_json.get('size')
-            if size < 5000:
+            if size < 1000:
                 break
             self.page_num += 1
 

modules/search/github_api.py

@@ -1,3 +1,5 @@
+import time
+
 import requests
 from config import settings
 from common.search import Search
@@ -50,6 +52,7 @@ class GithubAPI(Search):
             return
         page = 1
         while True:
+            time.sleep(self.delay)
             params = {'q': self.domain, 'per_page': 100,
                       'page': page, 'sort': 'indexed'}
             try:

oneforall.py

@@ -32,7 +32,7 @@ blue = '\033[01;34m'
 red = '\033[1;31m'
 end = '\033[0m'
 
-version = 'v0.4.0'
+version = 'v0.4.1'
 message = white + '{' + red + version + ' #dev' + white + '}'
 
 oneforall_banner = f"""
@@ -147,6 +147,8 @@ class OneForAll(object):
         :return: subdomain results
         :rtype: list
         """
+        utils.init_table(self.domain)
+
         if not self.access_internet:
             logger.log('ALERT', 'Because it cannot access the Internet, '
                                 'OneForAll will not execute the subdomain collection module!')