实现finder模块（从响应体和JS文件中再次发现新子域）

modules/collect.py

@@ -31,7 +31,7 @@ class Collect(object):
             # The crawl module has some problems
             modules = ['certificates', 'check', 'datasets',
                        'dnsquery', 'intelligence', 'search']
-            # modules = ['datasets']
+            # modules = ['certificates']
             for module in modules:
                 module_path = setting.module_dir.joinpath(module)
                 for path in module_path.rglob('*.py'):

modules/finder.py

@@ -1,47 +1,39 @@
 import re
+import json
+import time
 from urllib import parse
 
 from common import utils
+from common import resolve
 from common import request
 from common.module import Module
 from config import setting
+from config.log import logger
 
 
 class Finder(Module):
-    def __init__(self, domain):
+    def __init__(self):
         Module.__init__(self)
         self.module = 'Finder'
         self.source = ''
-        self.domain = domain
-        self.ts = utils.get_timestring()
-        self.name = f'found_subdomains_{self.domain}_{self.ts}'
-        self.path = setting.temp_save_dir.joinpath(self.name)
+        self.start = time.time()  # 模块开始执行时间
 
-    def save_subdomain(self, data, path):
-        if not path:
-            path = self.path
-        utils.save_data(path, data)
-
-    def remove_subdomain(self, path):
-        if not path:
-            path = self.path
-        utils.remove_data(path)
-
-    def find_url(self):
-        pass
-
-    def batch_find_url(self):
-        pass
-
-    def find_subdomain(self, html):
-        subdomains = self.match_subdomains(html, fuzzy=False)
-        self.subdomains.add(subdomains)
-
-    def find_js(self):
-        pass
-
-    def filter_js(self):
-        pass
+    def run(self, domain, data, port):
+        logger.log('INFOR', f'Start finder module')
+        existing_subdomains = set(map(lambda x: x.get('subdomain'), data))  # 已有的子域
+        found_subdomains = find_subdomains(domain, data)
+        new_subdomains = found_subdomains - existing_subdomains
+        if not len(new_subdomains):  # 未发现新的子域就直接返回
+            return data
+        self.subdomains = new_subdomains
+        self.gen_result()
+        temp_data = resolve.run_resolve(domain, self.results)
+        fina_data = request.run_request(domain, temp_data, port)
+        data = data + fina_data
+        self.finish()
+        logger.log('INFOR', f'Saving finder results')
+        utils.save_db(domain, data, 'finder')
+        return data
 
 
 # Regular expression comes from https://github.com/GerbenJavado/LinkFinder
@@ -104,29 +96,78 @@ def process_url(req_url, rel_url):
     return result
 
 
-def filter_url(urs):
-    pass
+def filter_name(path, black_name):
+    for name in black_name:
+        if path.endswith(name):
+            return True
+    black_ext = ['io.js', 'ui.js', 'fp.js', 'en.js', 'dev.js', 'min.js', 'umd.js',
+                 'esm.js', 'all.js', 'cjs.js', 'prod.js', 'slim.js', 'core.js',
+                 'global.js', 'bundle.js', 'browser.js', 'brands.js', 'simple.js',
+                 'common.js', 'development.js', 'production.js']
+    for ext in black_ext:
+        if path.endswith(ext):
+            return True
+    r = re.compile(r'\d+.\d+.\d+')
+    if r.search(path):
+        return True
+    return False
 
 
-def run_finder(domain, data):
-    existing_subdomains = set(map(lambda x: x.get('subdomain'), data))  # 已有的子域
-    found_subdomains = set()
-    found_urls = set()
-    finder = Finder(domain)
+def filter_url(domain, url, black_name):
+    raw_url = parse.urlparse(url)
+    scheme = raw_url.scheme.lower()
+    if not scheme:
+        return True
+    if scheme not in ['http', 'https']:
+        return True
+    netloc = raw_url.netloc.lower()
+    if not netloc:
+        return True
+    if not netloc.endswith(domain):
+        return True
+    path = raw_url.path.lower()
+    if not path:
+        return True
+    if not path.endswith('.js'):
+        return True
+    if path.endswith('min.js'):
+        return True
+    return filter_name(path, black_name)
+
+
+def get_black_name():
+    path = setting.data_storage_dir.joinpath('common_js_library.json')
+    with open(path) as fp:
+        return json.load(fp)
+
+
+def match_subdomains(domain, text):
+    subdomains = utils.match_subdomains(domain, text)
+    logger.log('DEBUG', f'matched subdomains: {subdomains}')
+    return subdomains
+
+
+def find_subdomains(domain, data):
+    subdomains = set()
+    js_urls = set()
+    black_name = get_black_name()
     for item in data:
         req_url = item.get('url')
         rsp_html = item.get('response')
-        found_subdomains.add(finder.match_subdomains(rsp_html))
+        if not rsp_html:
+            continue
+        logger.log('DEBUG', f'matching subdomains from response of {req_url}')
+        subdomains = subdomains.union(match_subdomains(domain, rsp_html))
         urls = find_url(rsp_html)
         if not urls:
             continue
         for rel_url in urls:
-            found_url = process_url(req_url, rel_url)
-            found_urls.add(found_url)
-
-    resp_data = request.urls_request(found_urls)
-    new_subdomains = found_subdomains - existing_subdomains
-
-
-def save_db(domain, data):
-    pass
+            url = process_url(req_url, rel_url)
+            if not filter_url(domain, url, black_name):
+                js_urls.add(url)
+    resp_data = request.urls_request(js_urls)
+    for resp, text in resp_data:
+        if text:
+            logger.log('DEBUG', f'matching subdomains from response of {resp.url}')
+            subdomains = subdomains.union(match_subdomains(domain, text))
+    return subdomains