Merge remote-tracking branch 'origin/master'

.github/workflows/test.yml

@@ -0,0 +1,34 @@
+name: OneForAll test
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Lint with flake8
+      run: |
+        pip install flake8
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+#     - name: Test with pytest
+#       run: |
+#       pip install pytest
+#       pytest
+    - name: Test with example
+      run: |
+        pip install coverage
+        coverage run oneforall/example.py

README.en.md

@@ -206,7 +206,7 @@ The OneForAll command line interface is based on [Fire](https://github.com/googl
        --format=FORMAT
            Export format (default xls)
        --takeover=TAKEOVER
-           Check subdomain takeover (default True)
+           Check subdomain takeover (default False)
        --show=SHOW
            Terminal display exported data (default False)
    ```

README.md

@@ -199,7 +199,7 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
        --format=FORMAT
            导出格式(默认xls)
        --takeover=TAKEOVER
-           检查子域接管(默认True)
+           检查子域接管(默认False)
        --show=SHOW
            终端显示导出数据(默认False)
    ```

oneforall/common/search.py

@@ -47,7 +47,7 @@ class Search(Module):
         :return: 匹配的子域
         :rtype set
         """
-        resp = self.head(url, allow_redirects=False)
+        resp = self.head(url, check=False, allow_redirects=False)
         if not resp:
             return set()
         location = resp.headers.get('location')

oneforall/modules/certificates/censys_api.py

@@ -1,4 +1,3 @@
-import time
 import config
 from common.query import Query
 from config import logger
@@ -38,7 +37,6 @@ class CensysAPI(Query):
         self.subdomains = self.subdomains.union(subdomains)
         pages = json.get('metadata').get('pages')
         for page in range(2, pages + 1):
-            time.sleep(self.delay)
             data['page'] = page
             resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
             if not resp:

oneforall/modules/certificates/certspotter.py

@@ -1,4 +1,3 @@
-import time
 from common import utils
 from common.query import Query
 
@@ -15,7 +14,6 @@ class CertSpotter(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         params = {'domain': self.domain,

oneforall/modules/certificates/crtsh.py

@@ -1,4 +1,3 @@
-import time
 from common import utils
 from common.query import Query
 
@@ -15,7 +14,6 @@ class Crtsh(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         params = {'q': f'%.{self.domain}', 'output': 'json'}

oneforall/modules/certificates/entrust.py

@@ -1,4 +1,3 @@
-import time
 from common import utils
 from common.query import Query
 
@@ -15,7 +14,6 @@ class Entrust(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         params = {'fields': 'subjectDN',

oneforall/modules/certificates/google.py

@@ -1,4 +1,3 @@
-import time
 from common import utils
 from common.query import Query
 
@@ -16,7 +15,6 @@ class Google(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         params = {'include_expired': 'true',

oneforall/modules/certificates/spyse_api.py

@@ -1,4 +1,3 @@
-import time
 import config
 from common import utils
 from common.query import Query
@@ -19,7 +18,6 @@ class SpyseAPI(Query):
         """
         page_num = 1
         while True:
-            time.sleep(self.delay)
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             params = {'domain': self.domain,

oneforall/modules/check/cdx.py

@@ -24,16 +24,15 @@ class CheckCDX(Module):
                 f'https://{self.domain}/crossdomain.xml',
                 f'http://www.{self.domain}/crossdomain.xml',
                 f'https://www.{self.domain}/crossdomain.xml']
-        response = None
         for url in urls:
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
-            response = self.get(url)
+            response = self.get(url, check=False)
-            if response:
-                break
             if not response:
                 return
-        self.subdomains = utils.match_subdomain(self.domain, response.text)
+            if response and len(response.content):
+                self.subdomains = utils.match_subdomain(self.domain,
+                                                        response.text)
 
     def run(self):
         """

oneforall/modules/check/csp.py

@@ -12,38 +12,46 @@ class CheckCSP(Module):
     """
     检查内容安全策略收集子域名
     """
-
     def __init__(self, domain, header):
         Module.__init__(self)
         self.domain = self.register(domain)
         self.module = 'Check'
         self.source = 'ContentSecurityPolicy'
-        self.header = header
+        self.csp_header = header
+
+    def grab_header(self):
+        """
+        抓取请求头
+
+        :return: 请求头
+        """
+        csp_header = dict()
+        urls = [f'http://{self.domain}',
+                f'https://{self.domain}',
+                f'http://www.{self.domain}',
+                f'https://www.{self.domain}']
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            response = self.get(url, check=False)
+            if response:
+                csp_header = response.headers
+                break
+        return csp_header
 
     def check(self):
         """
         正则匹配响应头中的内容安全策略字段以发现子域名
         """
-        if not self.header:
+        if not self.csp_header:
-            urls = [f'http://{self.domain}',
+            self.csp_header = self.grab_header()
-                    f'https://{self.domain}',
-                    f'http://www.{self.domain}',
-                    f'https://www.{self.domain}']
-            response = None
-            for url in urls:
-                self.header = self.get_header()
-                self.proxy = self.get_proxy(self.source)
-                response = self.get(url)
-                if response:
-                    break
-            if not response:
-                return
-            self.header = response.headers
         csp = self.header.get('Content-Security-Policy')
+        if not self.csp_header:
+            logger.log('DEBUG', f'获取{self.domain}域的请求头失败')
+            return
         if not csp:
             logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段')
             return
-        logger.log('DEBUG', f'{self.domain}域的响应头存在内容安全策略字段')
         self.subdomains = utils.match_subdomain(self.domain, csp)
 
     def run(self):

oneforall/modules/check/robots.py

@@ -1,18 +1,14 @@
 """
 检查内容安全策略收集子域名收集子域名
 """
-import requests
-
 from common.module import Module
 from common import utils
-from config import logger
 
 
 class CheckRobots(Module):
     """
     检查robots.txt收集子域名
     """
-
     def __init__(self, domain):
         Module.__init__(self)
         self.domain = self.register(domain)
@@ -27,16 +23,15 @@ class CheckRobots(Module):
                 f'https://{self.domain}/robots.txt',
                 f'http://www.{self.domain}/robots.txt',
                 f'https://www.{self.domain}/robots.txt']
-        response = None
         for url in urls:
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
-            response = self.get(url, allow_redirects=False)
+            response = self.get(url, check=False, allow_redirects=False)
-            if response:
-                break
             if not response:
                 return
-        self.subdomains = utils.match_subdomain(self.domain, response.text)
+            if response and len(response.content):
+                self.subdomains = utils.match_subdomain(self.domain,
+                                                        response.text)
 
     def run(self):
         """

oneforall/modules/check/sitemap.py

@@ -1,18 +1,14 @@
 """
 检查内容安全策略收集子域名收集子域名
 """
-import requests
-
 from common.module import Module
 from common import utils
-from config import logger
 
 
 class CheckRobots(Module):
     """
     检查sitemap收集子域名
     """
-
     def __init__(self, domain):
         Module.__init__(self)
         self.domain = self.register(domain)
@@ -39,17 +35,17 @@ class CheckRobots(Module):
                 f'https://{self.domain}/sitemap_index.xml',
                 f'http://www.{self.domain}/sitemap_index.xml',
                 f'https://www.{self.domain}/sitemap_index.xml']
-        response = None
         for url in urls:
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             self.timeout = 10
-            response = self.get(url, allow_redirects=False)
+            response = self.get(url, check=False, allow_redirects=False)
-            if response:
-                break
             if not response:
                 return
-        self.subdomains = utils.match_subdomain(self.domain, response.text)
+            if response and len(response.content):
+                self.subdomains = utils.match_subdomain(self.domain,
+                                                        response.text)
+
 
     def run(self):
         """

oneforall/modules/datasets/binaryedge_api.py

@@ -1,4 +1,3 @@
-import time
 import config
 from common.query import Query
 
@@ -16,7 +15,6 @@ class BinaryEdgeAPI(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.header.update({'X-Key': self.api})
         self.proxy = self.get_proxy(self.source)

oneforall/modules/datasets/bufferover.py

@@ -1,4 +1,3 @@
-import time
 import cloudscraper
 from common.query import Query
 from config import logger
@@ -16,7 +15,6 @@ class BufferOver(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         # 绕过cloudFlare验证
         scraper = cloudscraper.create_scraper()
         scraper.interpreter = 'js2py'

oneforall/modules/datasets/cebaidu.py

@@ -1,4 +1,3 @@
-import time
 from common.query import Query
 
 
@@ -14,7 +13,6 @@ class CeBaidu(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         params = {'site_address': self.domain}

oneforall/modules/datasets/chinaz.py

@@ -1,4 +1,3 @@
-import time
 from common.query import Query
 
 
@@ -14,7 +13,6 @@ class Chinaz(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         self.addr = self.addr + self.domain

oneforall/modules/datasets/chinaz_api.py

@@ -1,4 +1,3 @@
-import time
 import config
 from common.query import Query
 
@@ -16,7 +15,6 @@ class ChinazAPI(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         params = {'key': self.api, 'domainName': self.domain}

oneforall/modules/datasets/circl_api.py

@@ -1,4 +1,3 @@
-import time
 import config
 from common.query import Query
 
@@ -17,7 +16,6 @@ class CirclAPI(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))

oneforall/modules/datasets/dnsdb_api.py

@@ -1,4 +1,3 @@
-import time
 import config
 from common import utils
 from common.query import Query
@@ -17,7 +16,6 @@ class DNSdbAPI(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.header.update({'X-API-Key': self.api})
         self.proxy = self.get_proxy(self.source)

oneforall/modules/datasets/dnsdumpster.py

@@ -1,4 +1,3 @@
-import time
 from common import utils
 from common.query import Query
 
@@ -15,7 +14,6 @@ class DNSdumpster(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.header.update({'Referer': 'https://dnsdumpster.com'})
         self.proxy = self.get_proxy(self.source)

oneforall/modules/datasets/ip138.py

@@ -1,4 +1,3 @@
-import time
 from common.query import Query
 
 
@@ -14,7 +13,6 @@ class IP138(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         data = {'domain': self.domain}

oneforall/modules/datasets/passivedns_api.py

@@ -1,4 +1,3 @@
-import time
 import config
 from common.query import Query
 
@@ -16,7 +15,6 @@ class PassiveDnsAPI(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.header.update({'X-AuthToken': self.token})
         self.proxy = self.get_proxy(self.source)

oneforall/modules/datasets/riddler.py

@@ -1,4 +1,3 @@
-import time
 from common.query import Query
 
 
@@ -14,7 +13,6 @@ class Riddler(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         params = {'q': 'pld:' + self.domain}

oneforall/modules/datasets/securitytrails_api.py

@@ -1,4 +1,3 @@
-import time
 import config
 from common.query import Query
 
@@ -17,7 +16,6 @@ class SecurityTrailsAPI(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         params = {'apikey': self.api}

oneforall/modules/datasets/sitedossier.py

@@ -1,4 +1,3 @@
-import time
 
 from common.query import Query
 
@@ -19,7 +18,6 @@ class SiteDossier(Query):
         向接口查询子域并做子域匹配
         """
         while True:
-            time.sleep(self.delay)
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             url = f'{self.addr}{self.domain}/{self.page_num}'

oneforall/modules/datasets/ximcx.py

@@ -1,4 +1,3 @@
-import time
 from common.query import Query
 
 
@@ -14,7 +13,6 @@ class Ximcx(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         data = {'domain': self.domain}

oneforall/modules/intelligence/alienvault.py

@@ -1,5 +1,3 @@
-import time
-
 from common.query import Query
 
 
@@ -14,7 +12,6 @@ class AlienVault(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
 

oneforall/modules/intelligence/threatminer.py

@@ -1,5 +1,3 @@
-import time
-
 from common.query import Query
 
 
@@ -15,7 +13,6 @@ class ThreatMiner(Query):
         """
         向接口查询子域并做子域匹配
         """
-        time.sleep(self.delay)
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
         params = {'e': 'subdomains_container',

oneforall/modules/intelligence/virustotal.py

@@ -1,5 +1,3 @@
-import time
-
 from common.query import Query
 
 '''
@@ -21,7 +19,6 @@ class VirusTotal(Query):
         """
         next_cursor = ''
         while True:
-            time.sleep(self.delay)
             self.header = self.get_header()
             self.header.update({'Referer': 'https://www.virustotal.com/',
                                 'TE': 'Trailers'})

oneforall/modules/search/baidu.py

@@ -15,18 +15,19 @@ class Baidu(Search):
 
     def redirect_match(self, domain, html):
         """
+        获取跳转地址并传递地址进行跳转head请求
 
-        :param domain:
+        :param domain: 域名
-        :param html:
+        :param html: 响应体
-        :return:
+        :return: 子域
         """
         bs = BeautifulSoup(html, features='lxml')
         subdomains_all = set()
         # 获取搜索结果中所有的跳转URL地址
         for find_res in bs.find_all('a', {'class': 'c-showurl'}):
             url = find_res.get('href')
-            subdomain = self.match_location(domain, url)
+            subdomains = self.match_location(domain, url)
-            subdomains_all = subdomains_all.union(subdomain)
+            subdomains_all = subdomains_all.union(subdomains)
         return subdomains_all
 
     def search(self, domain, filtered_subdomain='', full_search=False):
@@ -43,7 +44,8 @@ class Baidu(Search):
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             query = 'site:' + domain + filtered_subdomain
-            params = {'wd': query, 'pn': self.page_num,
+            params = {'wd': query,
+                      'pn': self.page_num,
                       'rn': self.per_page_num}
             resp = self.get(self.addr, params)
             if not resp:
@@ -73,9 +75,7 @@ class Baidu(Search):
         类执行入口
         """
         self.begin()
-
         self.search(self.domain, full_search=True)
-
         # 排除同一子域搜索结果过多的子域以发现新的子域
         for statement in self.filter(self.domain, self.subdomains):
             self.search(self.domain, filtered_subdomain=statement)
@@ -106,4 +106,4 @@ def do(domain):  # 统一入口名字 方便多线程调用
 
 
 if __name__ == '__main__':
-    do('example.com')
+    do('huayunshuzi.com')

oneforall/oneforall.py

@@ -70,11 +70,11 @@ class OneForAll(object):
     :param str port:       请求验证子域的端口范围(默认只探测80端口)
     :param int valid:      导出子域的有效性(默认1)
     :param str format:     导出格式(默认csv)
-    :param bool takeover:  检查子域接管(默认True)
+    :param bool takeover:  检查子域接管(默认False)
     :param bool show:      终端显示导出数据(默认False)
     """
     def __init__(self, target, brute=None, dns=None, req=None, port='default',
-                 valid=1, format='csv', takeover=True, show=False):
+                 valid=1, format='csv', takeover=False, show=False):
         self.target = target
         self.port = port
         self.domains = set()