优化常规检查模块

common/check.py

@@ -0,0 +1,26 @@
+from common.module import Module
+
+
+class Check(Module):
+    """
+    Check base class
+    """
+
+    def __init__(self):
+        Module.__init__(self)
+        self.request_status = 1
+
+    def to_check(self, filenames):
+        urls = set()
+        for filename in filenames:
+            urls.update((f'http://{self.domain}/{filename}',
+                         f'https://{self.domain}/{filename}',
+                         f'http://www.{self.domain}/{filename}',
+                         f'https://www.{self.domain}/{filename}'))
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            resp = self.get(url, check=False, ignore=True)
+            self.subdomains = self.collect_subdomains(resp)
+            if self.subdomains:
+                break

common/module.py

@@ -73,7 +73,7 @@ class Module(object):
         :param dict params: request parameters
         :param bool check: check response
         :param kwargs: other params
-        :return: requests's response object
+        :return: response object
         """
         try:
             resp = requests.head(url,
@@ -93,16 +93,20 @@ class Module(object):
             return resp
         return None
 
-    def get(self, url, params=None, check=True, **kwargs):
+    def get(self, url, params=None, check=True, ignore=False, **kwargs):
         """
         Custom get request
 
         :param str  url: request url
         :param dict params: request parameters
         :param bool check: check response
+        :param bool ignore: ignore error
         :param kwargs: other params
-        :return: requests's response object
+        :return: response object
         """
+        level = 'ERROR'
+        if ignore:
+            level = 'DEBUG'
         try:
             resp = requests.get(url,
                                 params=params,
@@ -113,7 +117,7 @@ class Module(object):
                                 verify=self.verify,
                                 **kwargs)
         except Exception as e:
-            logger.log('ERROR', e.args)
+            logger.log(level, e.args)
             return None
         if not check:
             return resp
@@ -129,7 +133,7 @@ class Module(object):
         :param dict data: request data
         :param bool check: check response
         :param kwargs: other params
-        :return: requests's response object
+        :return: response object
         """
         try:
             resp = requests.post(url,

modules/check/axfr.py

@@ -19,7 +19,6 @@ class CheckAXFR(Module):
     """
     DNS zone transfer vulnerability base class
     """
-
     def __init__(self, domain: str):
         Module.__init__(self)
         self.domain = domain

modules/check/cdx.py

@@ -1,16 +1,15 @@
 """
 检查crossdomain.xml文件收集子域名
 """
-
+from common.check import Check
-from common.module import Module
 
 
-class CheckCDX(Module):
+class CheckCDX(Check):
     """
     检查crossdomain.xml文件收集子域名
     """
     def __init__(self, domain: str):
-        Module.__init__(self)
+        Check.__init__(self)
         self.domain = domain
         self.module = 'Check'
         self.source = "CrossDomainXml"
@@ -19,15 +18,8 @@ class CheckCDX(Module):
         """
         检查crossdomain.xml收集子域名
         """
-        urls = [f'http://{self.domain}/crossdomain.xml',
+        filenames = {'crossdomain.xml'}
-                f'https://{self.domain}/crossdomain.xml',
+        self.to_check(filenames)
-                f'http://www.{self.domain}/crossdomain.xml',
-                f'https://www.{self.domain}/crossdomain.xml']
-        for url in urls:
-            self.header = self.get_header()
-            self.proxy = self.get_proxy(self.source)
-            resp = self.get(url, check=False)
-            self.subdomains = self.collect_subdomains(resp)
 
     def run(self):
         """

modules/check/cert.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 """
 检查域名证书收集子域名
 """

modules/check/csp.py

@@ -3,16 +3,16 @@ Collect subdomains from ContentSecurityPolicy
 """
 import requests
 
-from common.module import Module
+from common.check import Check
 from config.log import logger
 
 
-class CheckCSP(Module):
+class CheckCSP(Check):
     """
     Collect subdomains from ContentSecurityPolicy
     """
     def __init__(self, domain, header):
-        Module.__init__(self)
+        Check.__init__(self)
         self.domain = domain
         self.module = 'Check'
         self.source = 'ContentSecurityPolicy'
@@ -34,8 +34,7 @@ class CheckCSP(Module):
             self.proxy = self.get_proxy(self.source)
             response = self.get(url, check=False)
             if response:
-                csp_header = response.headers
+                return response.headers
-                break
         return csp_header
 
     def check(self):

modules/check/robots.py

@@ -1,15 +1,15 @@
 """
 检查内容安全策略收集子域名收集子域名
 """
-from common.module import Module
+from common.check import Check
 
 
-class CheckRobots(Module):
+class CheckRobots(Check):
     """
     检查robots.txt收集子域名
     """
     def __init__(self, domain):
-        Module.__init__(self)
+        Check.__init__(self)
         self.domain = domain
         self.module = 'Check'
         self.source = 'Robots'
@@ -18,15 +18,8 @@ class CheckRobots(Module):
         """
         正则匹配域名的robots.txt文件中的子域
         """
-        urls = [f'http://{self.domain}/robots.txt',
+        filenames = {'robots.txt'}
-                f'https://{self.domain}/robots.txt',
+        self.to_check(filenames)
-                f'http://www.{self.domain}/robots.txt',
-                f'https://www.{self.domain}/robots.txt']
-        for url in urls:
-            self.header = self.get_header()
-            self.proxy = self.get_proxy(self.source)
-            resp = self.get(url, check=False, allow_redirects=False)
-            self.subdomains = self.collect_subdomains(resp)
 
     def run(self):
         """

modules/check/sitemap.py

@@ -1,15 +1,15 @@
 """
 检查内容安全策略收集子域名收集子域名
 """
-from common.module import Module
+from common.check import Check
 
 
-class CheckRobots(Module):
+class CheckRobots(Check):
     """
     检查sitemap收集子域名
     """
     def __init__(self, domain):
-        Module.__init__(self)
+        Check.__init__(self)
         self.domain = domain
         self.module = 'Check'
         self.source = 'Sitemap'
@@ -18,20 +18,8 @@ class CheckRobots(Module):
         """
         正则匹配域名的sitemap文件中的子域
         """
-        urls = [f'http://{self.domain}/sitemap.xml',
+        filenames = {'sitemap.xml', 'sitemap.txt', 'sitemap.html'}
-                f'http://www.{self.domain}/sitemap.xml',
+        self.to_check(filenames)
-                f'http://{self.domain}/sitemap.txt',
-                f'http://www.{self.domain}/sitemap.txt',
-                f'http://{self.domain}/sitemap.html',
-                f'http://www.{self.domain}/sitemap.html',
-                f'http://{self.domain}/sitemap_index.xml',
-                f'http://www.{self.domain}/sitemap_index.xml']
-        for url in urls:
-            self.header = self.get_header()
-            self.proxy = self.get_proxy(self.source)
-            self.timeout = 10
-            resp = self.get(url, check=False)
-            self.subdomains = self.collect_subdomains(resp)
 
     def run(self):
         """