统一调整子域匹配函数

common/lookup.py

@@ -20,7 +20,7 @@ class Lookup(Module):
             return None
         for item in answer:
             record = item.to_text()
-            subdomains = utils.match_subdomain(self.domain, record)
+            subdomains = self.match_subdomains(self.domain, record)
             self.subdomains = self.subdomains.union(subdomains)
             self.gen_record(subdomains, record)
         return self.subdomains

common/module.py

@@ -128,7 +128,7 @@ class Module(object):
         Custom post request
 
         :param str  url: request url
-        :param dict params: request parameters
+        :param dict data: request parameters
         :param bool check: check response
         :param kwargs: other params
         :return: requests's response object
@@ -184,24 +184,22 @@ class Module(object):
             return self.proxy
 
     @staticmethod
-    def match(domain, html, distinct=True):
+    def match_subdomains(domain, text, distinct=True):
         """
         Use regexp to match subdomains
 
         :param  str domain: domain
-        :param  str html: response html text
+        :param  str text: text
         :param  bool distinct: deduplicate results or not (default True)
         :return set/list: result set or list
         """
         logger.log('TRACE', f'Use regexp to match subdomains in the response body')
-        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
-                 r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+        regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
                  + domain.replace('.', r'\.')
-        result = re.findall(regexp, html, re.I)
+        result = re.findall(regexp, text, re.I)
         if not result:
             return set()
-        regexp = r'(?:http://|https://)'
-        deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
+        deal = map(lambda s: s.lower(), result)
         if distinct:
             return set(deal)
         else:
@@ -340,14 +338,12 @@ class Module(object):
                           'elapse': self.elapse,
                           'find': find,
                           'brute': brute,
-                          'valid': valid,
-                          }
+                          'valid': valid}
                 self.results.append(result)
 
     def save_db(self):
         """
         Save module results into the database
-
         """
         logger.log('DEBUG', f'Saving results to database')
         lock.acquire()

common/search.py

@@ -1,6 +1,9 @@
+import re
+
 from config import setting
-from .module import Module
-from . import utils
+from config.log import logger
+from common.module import Module
+from common import utils
 
 
 class Search(Module):
@@ -51,4 +54,28 @@ class Search(Module):
         location = resp.headers.get('location')
         if not location:
             return set()
-        return set(utils.match_subdomain(domain, location))
+        return set(self.match_subdomains(domain, location))
+
+    @staticmethod
+    def match_subdomains(domain, html, distinct=True):
+        """
+        Use regexp to match subdomains
+
+        :param  str domain: domain
+        :param  str html: response html text
+        :param  bool distinct: deduplicate results or not (default True)
+        :return set/list: result set or list
+        """
+        logger.log('TRACE', f'Use regexp to match subdomains in the response body')
+        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
+                 r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+                 + domain.replace('.', r'\.')
+        result = re.findall(regexp, html, re.I)
+        if not result:
+            return set()
+        regexp = r'(?:http://|https://)'
+        deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
+        if distinct:
+            return set(deal)
+        else:
+            return list(deal)

common/utils.py

@@ -32,27 +32,6 @@ user_agents = [
     'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0']
 
 
-def match_subdomain(domain, text, distinct=True):
-    """
-    Use regexp to match subdomains in text
-
-    :param  str domain: domain
-    :param  str text: response text
-    :param  bool distinct: deduplicate results
-    :return set/list: match result
-    """
-    regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
-             + domain.replace('.', r'\.')
-    result = re.findall(regexp, text, re.I)
-    if not result:
-        return set()
-    deal = map(lambda s: s.lower(), result)
-    if distinct:
-        return set(deal)
-    else:
-        return list(deal)
-
-
 def gen_random_ip():
     """
     Generate random decimal IP string