实现finder模块（从响应体和JS文件中再次发现新子域）

README.md

@@ -276,8 +276,8 @@ FLAGS
 
 - [ ] 各模块持续优化和完善
 - [x] 子域监控（标记每次新发现的子域）
-- [ ] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
+- [x] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
-- [ ] 操作强大交互人性的前端界面实现（暂定：前端：Element + 后端：Flask）
+- [ ] 操作强大交互人性的前端界面实现
 
 更多详细信息请阅读[todo.md](https://github.com/shmilylty/OneForAll/tree/master/docs/todo.md)。
 

common/database.py

@@ -95,7 +95,7 @@ class Database(object):
 
         :param str table_name: table name
         :param list results: results list
-        :param str module_name: mo
+        :param str module_name: module
         """
         logger.log('TRACE', f'Saving the subdomain results of {table_name} '
                             f'found by module {module_name} into database')

common/module.py

@@ -212,39 +212,7 @@ class Module(object):
             return self.proxy
 
     def match_subdomains(self, html, distinct=True, fuzzy=True):
-        """
+        return utils.match_subdomains(self.domain, html, distinct, fuzzy)
-        Use regexp to match subdomains
-
-        :param  str html: response html text
-        :param  bool distinct: deduplicate results or not (default True)
-        :param  bool fuzzy: fuzzy match subdomain or not (default True)
-        :return set/list: result set or list
-        """
-        logger.log('TRACE', f'Use regexp to match subdomains in the response body')
-        if fuzzy:
-            regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
-                     + self.domain.replace('.', r'\.')
-            result = re.findall(regexp, html, re.I)
-            if not result:
-                return set()
-            deal = map(lambda s: s.lower(), result)
-            if distinct:
-                return set(deal)
-            else:
-                return list(deal)
-        else:
-            regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
-                     r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
-                     + self.domain.replace('.', r'\.')
-            result = re.findall(regexp, html, re.I)
-        if not result:
-            return set()
-        regexp = r'(?:http://|https://)'
-        deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
-        if distinct:
-            return set(deal)
-        else:
-            return list(deal)
 
     @staticmethod
     def get_maindomain(domain):
@@ -305,11 +273,11 @@ class Module(object):
                       'new': None,
                       'url': None,
                       'subdomain': None,
+                      'port': None,
                       'level': None,
                       'cname': None,
                       'content': None,
                       'public': None,
-                      'port': None,
                       'status': None,
                       'reason': None,
                       'title': None,
@@ -369,11 +337,11 @@ class Module(object):
                           'new': None,
                           'url': url,
                           'subdomain': subdomain,
+                          'port': 80,
                           'level': level,
                           'cname': cname,
                           'content': content,
                           'public': public,
-                          'port': 80,
                           'status': None,
                           'reason': reason,
                           'title': None,

common/request.py

@@ -9,7 +9,6 @@ from bs4 import BeautifulSoup
 from common import utils
 from config.log import logger
 from config import setting
-from common.database import Database
 
 
 def get_limit_conn():
@@ -35,7 +34,7 @@ def get_ports(port):
     if not ports:  # 意外情况
         logger.log('ERROR', 'The specified request port range is incorrect')
         ports = {80}
-    logger.log('INFOR', 'Port range:{ports}')
+    logger.log('INFOR', f'Port range:{ports}')
     return set(ports)
 
 
@@ -110,7 +109,7 @@ async def fetch(session, method, url):
                         text = await resp.text(encoding=None, errors='ignore')
         return resp, text
     except Exception as e:
-        return e
+        return e, None
 
 
 def get_title(markup):
@@ -154,15 +153,15 @@ def get_title(markup):
 
 
 def request_callback(future, index, datas):
-    result = future.result()
+    resp, text = future.result()
-    if isinstance(result, BaseException):
+    if isinstance(resp, BaseException):
-        logger.log('TRACE', result.args)
+        exception = resp
-        name = utils.get_classname(result)
+        logger.log('TRACE', exception.args)
-        datas[index]['reason'] = name + ' ' + str(result)
+        name = utils.get_classname(exception)
+        datas[index]['reason'] = name + ' ' + str(exception)
         datas[index]['request'] = 0
         datas[index]['alive'] = 0
-    elif isinstance(result, tuple):
+    else:
-        resp, text = result
         datas[index]['reason'] = resp.reason
         datas[index]['status'] = resp.status
         datas[index]['request'] = 1
@@ -257,17 +256,17 @@ def run_request(domain, data, port):
 
     :param  str domain: domain to be requested
     :param  list data: subdomains data to be requested
-    :param  str port: range of ports to be requested
+    :param  any port: range of ports to be requested
     :return list: result
     """
-    logger.log('INFOR', 'Start subdomain request module')
+    logger.log('INFOR', f'Start requesting subdomains of {domain}')
     loop = set_loop()
     data = utils.set_id_none(data)
     request_coroutine = bulk_request(data, port)
     data = loop.run_until_complete(request_coroutine)
     loop.run_until_complete(asyncio.sleep(0.25))
     count = utils.count_alive(data)
-    logger.log('INFOR', f'Request module found {domain} have {count} alive subdomains')
+    logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
     return data
 
 
@@ -277,7 +276,6 @@ def urls_request(urls):
     request_coroutine = async_request(urls)
     data = loop.run_until_complete(request_coroutine)
     loop.run_until_complete(asyncio.sleep(0.25))
-    logger.log('INFOR', 'End urls request module')
     return data
 
 
@@ -288,8 +286,5 @@ def save_db(name, data):
     :param str  name: table name
     :param list data: data to be saved
     """
-    db = Database()
+    logger.log('INFOR', f'Saving requested results')
-    db.drop_table(name)
+    utils.save_db(name, data, 'request')
-    db.create_table(name)
-    db.save_db(name, data, 'request')
-    db.close()

common/resolve.py

@@ -50,17 +50,13 @@ def update_data(data, records):
 
 def save_db(name, data):
     """
-    保存解析结果到数据库
+    Save resolved results to database
 
-    :param str name: 保存表名
+    :param str  name: table name
-    :param list data: 待保存的数据
+    :param list data: data to be saved
     """
     logger.log('INFOR', f'Saving resolved results')
-    db = Database()
+    utils.save_db(name, data, 'resolve')
-    db.drop_table(name)
-    db.create_table(name)
-    db.save_db(name, data, 'resolve')
-    db.close()
 
 
 def save_subdomains(save_path, subdomain_list):
@@ -161,7 +157,7 @@ def run_resolve(domain, data):
     :return: 解析得到的结果列表
     :rtype: list
     """
-    logger.log('INFOR', f'Start resolve subdomains of {domain}')
+    logger.log('INFOR', f'Start resolving subdomains of {domain}')
     subdomains = filter_subdomain(data)
     if not subdomains:
         return data

common/utils.py

@@ -17,6 +17,7 @@ from records import Record, RecordCollection
 from dns.resolver import Resolver
 
 from common.domain import Domain
+from common.database import Database
 from config import setting
 from config.log import logger
 
@@ -214,6 +215,21 @@ def check_format(format, count):
         return 'csv'
 
 
+def save_db(name, data, module):
+    """
+    Save request results to database
+
+    :param str  name: table name
+    :param list data: data to be saved
+    :param str module: module name
+    """
+    db = Database()
+    db.drop_table(name)
+    db.create_table(name)
+    db.save_db(name, data, module)
+    db.close()
+
+
 def save_data(path, data):
     """
     保存数据到文件
@@ -663,3 +679,40 @@ def ip_to_int(ip):
         logger.log('ERROR', e.args)
         return None
     return int(ipv4)
+
+
+def match_subdomains(domain, html, distinct=True, fuzzy=True):
+    """
+    Use regexp to match subdomains
+
+    :param  str domain: main domain
+    :param  str html: response html text
+    :param  bool distinct: deduplicate results or not (default True)
+    :param  bool fuzzy: fuzzy match subdomain or not (default True)
+    :return set/list: result set or list
+    """
+    logger.log('TRACE', f'Use regexp to match subdomains in the response body')
+    if fuzzy:
+        regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+                 + domain.replace('.', r'\.')
+        result = re.findall(regexp, html, re.I)
+        if not result:
+            return set()
+        deal = map(lambda s: s.lower(), result)
+        if distinct:
+            return set(deal)
+        else:
+            return list(deal)
+    else:
+        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
+                 r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+                 + domain.replace('.', r'\.')
+        result = re.findall(regexp, html, re.I)
+    if not result:
+        return set()
+    regexp = r'(?:http://|https://)'
+    deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
+    if distinct:
+        return set(deal)
+    else:
+        return list(deal)

config/setting.py

@@ -18,6 +18,7 @@ temp_save_dir = result_save_dir.joinpath('temp')
 enable_check_version = True  # 开启最新版本检查
 enable_dns_resolve = True  # 使用DNS解析子域(默认True)
 enable_http_request = True  # 使用HTTP请求子域(默认True)
+enable_finder_module = True  # 开启finder模块(默认True)
 enable_takeover_check = False  # 开启子域接管风险检查(默认False)
 # 参数port可选值有'default', 'small', 'large'
 http_request_port = 'default'  # HTTP请求子域(默认'default'，探测80端口)

data/common_js_library.json

@@ -0,0 +1,621 @@
+[
+    "npm.js",
+    "bower.js",
+    "component.js",
+    "spm.js",
+    "jam.js",
+    "jspm.js",
+    "ender.js",
+    "volo.js",
+    "duo.js",
+    "yarn.js",
+    "requirejs.js",
+    "browserify.js",
+    "seajs.js",
+    "headjs.js",
+    "curl.js",
+    "lazyload.js",
+    "systemjs.js",
+    "lodjs.js",
+    "esl.js",
+    "modulejs.js",
+    "webpack.js",
+    "rollup.js",
+    "brunch.js",
+    "parcel.js",
+    "microbundle.js",
+    "typescript.js",
+    "hegel.js",
+    "typl.js",
+    "mocha.js",
+    "jasmine.js",
+    "qunit.js",
+    "jest.js",
+    "prova.js",
+    "dalekjs.js",
+    "protractor.js",
+    "tape.js",
+    "testcafe.js",
+    "ava.js",
+    "cypress.js",
+    "chai.js",
+    "enzyme.js",
+    "proxyquire.js",
+    "istanbul.js",
+    "blanket.js",
+    "jscover.js",
+    "phantomjs.js",
+    "slimerjs.js",
+    "casperjs.js",
+    "zombie.js",
+    "totoro.js",
+    "karma.js",
+    "nightwatch.js",
+    "intern.js",
+    "yolpo.js",
+    "puppeteer.js",
+    "webdriverio.js",
+    "prettier.js",
+    "jshint.js",
+    "jscs.js",
+    "jsfmt.js",
+    "jsinspect.js",
+    "eslint.js",
+    "jslint.js",
+    "aurelia.js",
+    "backbone.js",
+    "meteor.js",
+    "ractive.js",
+    "vue.js",
+    "svelte.js",
+    "knockout.js",
+    "spine.js",
+    "canjs.js",
+    "react.js",
+    "hyperapp.js",
+    "preact.js",
+    "nativescript.js",
+    "riot.js",
+    "thorax.js",
+    "chaplin.js",
+    "marionette.js",
+    "ripple.js",
+    "rivets.js",
+    "derby.js",
+    "jsblocks.js",
+    "liquidlava.js",
+    "feathers.js",
+    "keo.js",
+    "atvjs.js",
+    "makefun.js",
+    "keystonejs.js",
+    "ghost.js",
+    "apostrophe.js",
+    "taracotjs.js",
+    "nodizecms.js",
+    "cody.js",
+    "pencilblue.js",
+    "strapi.js",
+    "factor.js",
+    "nunjucks.js",
+    "dot.js",
+    "dustjs.js",
+    "eco.js",
+    "pug.js",
+    "ejs.js",
+    "xtemplate.js",
+    "marko.js",
+    "swig.js",
+    "ehtml.js",
+    "d3.js",
+    "peity.js",
+    "raphael.js",
+    "echarts.js",
+    "vis.js",
+    "arbor.js",
+    "cubism.js",
+    "vega.js",
+    "envisionjs.js",
+    "rickshaw.js",
+    "flot.js",
+    "nvd3.js",
+    "trianglify.js",
+    "d4.js",
+    "epoch.js",
+    "c3.js",
+    "babylonjs.js",
+    "recharts.js",
+    "graphicsjs.js",
+    "mxgraph.js",
+    "amchart.js",
+    "anychart.js",
+    "plotly.js",
+    "highchart.js",
+    "handsontable.js",
+    "ace.js",
+    "codemirror.js",
+    "esprima.js",
+    "quill.js",
+    "pen.js",
+    "editor.js",
+    "epiceditor.js",
+    "jsoneditor.js",
+    "squire.js",
+    "tinymce.js",
+    "trix.js",
+    "trumbowyg.js",
+    "wysihtml5.js",
+    "popline.js",
+    "summernote.js",
+    "devdocs.js",
+    "dexy.js",
+    "docco.js",
+    "styledocco.js",
+    "ronn.js",
+    "dox.js",
+    "jsdox.js",
+    "esdoc.js",
+    "yuidoc.js",
+    "coddoc.js",
+    "sphinx.js",
+    "jsduck.js",
+    "codecrumbs.js",
+    "jbinary.js",
+    "diff2html.js",
+    "jspdf.js",
+    "underscore.js",
+    "lodash.js",
+    "sugar.js",
+    "ramda.js",
+    "mout.js",
+    "mesh.js",
+    "preludejs.js",
+    "rxjs.js",
+    "bacon.js",
+    "kefir.js",
+    "highland.js",
+    "mobx.js",
+    "mori.js",
+    "buckets.js",
+    "hashmap.js",
+    "moment.js",
+    "date.js",
+    "fecha.js",
+    "dayjs.js",
+    "voca.js",
+    "selecting.js",
+    "he.js",
+    "multiline.js",
+    "jsurl.js",
+    "plexis.js",
+    "odometer.js",
+    "localforage.js",
+    "jstorage.js",
+    "cookies.js",
+    "crumbsjs.js",
+    "randomcolor.js",
+    "color.js",
+    "colors.js",
+    "pleasejs.js",
+    "tinycolor.js",
+    "i18next.js",
+    "polyglot.js",
+    "babelfish.js",
+    "ttag.js",
+    "async.js",
+    "q.js",
+    "step.js",
+    "contra.js",
+    "bluebird.js",
+    "when.js",
+    "objecteventtarget.js",
+    "sporadic.js",
+    "director.js",
+    "pathjs.js",
+    "crossroads.js",
+    "navaid.js",
+    "dompurify.js",
+    "log.js",
+    "conzole.js",
+    "loglevel.js",
+    "minilog.js",
+    "storyboard.js",
+    "regex101.js",
+    "regexr.js",
+    "regexpbuilder.js",
+    "annyang.js",
+    "axios.js",
+    "bottleneck.js",
+    "amygdala.js",
+    "wretch.js",
+    "farfetch.js",
+    "tailor.js",
+    "convnetjs.js",
+    "dn2a.js",
+    "synapses.js",
+    "bowser.js",
+    "matcha.js",
+    "prismjs.js",
+    "nprogress.js",
+    "pace.js",
+    "topbar.js",
+    "nanobar.js",
+    "pageloadingeffects.js",
+    "spinkit.js",
+    "ladda.js",
+    "ajaxload.js",
+    "preloaders.js",
+    "cssload.js",
+    "validatr.js",
+    "formvalidation.js",
+    "fieldval.js",
+    "funval.js",
+    "mousetrap.js",
+    "keymaster.js",
+    "keypress.js",
+    "keyboardjs.js",
+    "jwerty.js",
+    "shepherd.js",
+    "tourist.js",
+    "pageguide.js",
+    "hopscotch.js",
+    "joyride.js",
+    "focusable.js",
+    "izitoast.js",
+    "messenger.js",
+    "noty.js",
+    "pnotify.js",
+    "toastr.js",
+    "notie.js",
+    "swiper.js",
+    "slick.js",
+    "slidesjs.js",
+    "flexslider.js",
+    "unslider.js",
+    "sly.js",
+    "vegas.js",
+    "sequence.js",
+    "strut.js",
+    "photoswipe.js",
+    "jcslider.js",
+    "slidr.js",
+    "flickity.js",
+    "jqrangeslider.js",
+    "nouislider.js",
+    "fancyinput.js",
+    "awesomplete.js",
+    "pikaday.js",
+    "fullcalendar.js",
+    "rome.js",
+    "datedropper.js",
+    "select2.js",
+    "chosen.js",
+    "dropzone.js",
+    "fileapi.js",
+    "plupload.js",
+    "form.js",
+    "countable.js",
+    "card.js",
+    "stretchy.js",
+    "analytics.js",
+    "tipsy.js",
+    "opentip.js",
+    "qtip2.js",
+    "tooltipster.js",
+    "simptip.js",
+    "toolbar.js",
+    "vex.js",
+    "sweetalert.js",
+    "colorbox.js",
+    "fancybox.js",
+    "swipebox.js",
+    "jbox.js",
+    "scrollmonitor.js",
+    "headroom.js",
+    "iscroll.js",
+    "skrollr.js",
+    "parallax.js",
+    "plax.js",
+    "jparallax.js",
+    "fullpage.js",
+    "scrollmenu.js",
+    "simpleparallax.js",
+    "slideout.js",
+    "jtable.js",
+    "datatables.js",
+    "tabulator.js",
+    "floatthead.js",
+    "masonry.js",
+    "packery.js",
+    "isotope.js",
+    "flexboxgrid.js",
+    "w2ui.js",
+    "fluidity.js",
+    "ink.js",
+    "dataformsjs.js",
+    "webplate.js",
+    "cerberus.js",
+    "touchemulator.js",
+    "dragula.js",
+    "leaflet.js",
+    "cesium.js",
+    "gmaps.js",
+    "polymaps.js",
+    "jqvmap.js",
+    "openlayers3.js",
+    "html5media.js",
+    "polyplayer.js",
+    "flowplayer.js",
+    "mediaelement.js",
+    "soundjs.js",
+    "clappr.js",
+    "exifr.js",
+    "bigtext.js",
+    "circletype.js",
+    "slabtext.js",
+    "velocity.js",
+    "transitionend.js",
+    "textillate.js",
+    "animatable.js",
+    "tsparticles.js",
+    "pica.js",
+    "cropper.js",
+    "es6features.js",
+    "gridsome.js",
+    "docusaurus.js",
+    "echo.js",
+    "picturefill.js",
+    "json3.js",
+    "mixitup.js",
+    "grid.js",
+    "ky.js",
+    "fcal.js",
+    "iooxa.js",
+    "idyll.js",
+    "primer.js",
+    "glue.js",
+    "postcss.js",
+    "mui.js",
+    "img2css.js",
+    "weui.js",
+    "csscss.js",
+    "simditor.js",
+    "htmlhint.js",
+    "csslint.js",
+    "grunt.js",
+    "yeoman.js",
+    "gulp.js",
+    "zrender.js",
+    "highcharts.js",
+    "tweenjs.js",
+    "swipe.js",
+    "superslides.js",
+    "slider.js",
+    "polymer.js",
+    "ionic.js",
+    "timelinejs.js",
+    "togetherjs.js",
+    "foundation.js",
+    "todomvc.js",
+    "vuejs.js",
+    "webuploader.js",
+    "fastclick.js",
+    "wangeditor.js",
+    "tooling.js",
+    "judge.js",
+    "amdoc.js",
+    "amazeui.js",
+    "zepto.js",
+    "nodeclub.js",
+    "nodeppt.js",
+    "hexo.js",
+    "koa.js",
+    "connect.js",
+    "nvm.js",
+    "flux.js",
+    "browserquest.js",
+    "html5shiv.js",
+    "ulkit.js",
+    "arttemplate.js",
+    "jade.js",
+    "modernizr.js",
+    "css3please.js",
+    "babel.js",
+    "f2etest.js",
+    "brackets.js",
+    "ueditor.js",
+    "electron.js",
+    "base.js",
+    "basscss.js",
+    "bootflat.js",
+    "bootswatch.js",
+    "bulma.js",
+    "cardinal.js",
+    "caramel.js",
+    "corpus.js",
+    "kube.js",
+    "materialize.js",
+    "milligram.js",
+    "papercss.js",
+    "papier.js",
+    "pavilion.js",
+    "picnicss.js",
+    "pure.js",
+    "skeleton.js",
+    "tachyons.js",
+    "tacit.js",
+    "uikit.js",
+    "wing.js",
+    "angular.js",
+    "choo.js",
+    "deku.js",
+    "displayjs.js",
+    "inferno.js",
+    "mercury.js",
+    "mithril.js",
+    "moon.js",
+    "skatejs.js",
+    "solid.js",
+    "bliss.js",
+    "cash.js",
+    "jquery.js",
+    "nanojs.js",
+    "selector.js",
+    "umbrella.js",
+    "zeptojs.js",
+    "chartist.js",
+    "charts.js",
+    "chartjs.js",
+    "dc.js",
+    "dimple.js",
+    "d3xter.js",
+    "f2.js",
+    "frappe.js",
+    "ggraph.js",
+    "jsplumb.js",
+    "metricsgraphics.js",
+    "morrisjs.js",
+    "muze.js",
+    "sparkline.js",
+    "sparky.js",
+    "taucharts.js",
+    "uvcharts.js",
+    "vivagraph.js",
+    "z3d.js",
+    "kartograph.js",
+    "mapsicon.js",
+    "osmbuildings.js",
+    "planetary.js",
+    "smallworld.js",
+    "tangram.js",
+    "topojson.js",
+    "turf.js",
+    "dynatables.js",
+    "listjs.js",
+    "sortable.js",
+    "tablesaw.js",
+    "flipside.js",
+    "nudge.js",
+    "glide.js",
+    "lory.js",
+    "siema.js",
+    "blotter.js",
+    "fitty.js",
+    "flowtype.js",
+    "lettering.js",
+    "shave.js",
+    "typeplate.js",
+    "fitvid.js",
+    "medialementjs.js",
+    "plyr.js",
+    "talkie.js",
+    "videojs.js",
+    "abcjs.js",
+    "audio5js.js",
+    "bap.js",
+    "blip.js",
+    "howler.js",
+    "soundcite.js",
+    "tonal.js",
+    "vexflow.js",
+    "easeljs.js",
+    "konva.js",
+    "panzoom.js",
+    "p5js.js",
+    "vizflow.js",
+    "zdog.js",
+    "scenejs.js",
+    "whitestormjs.js",
+    "camanjs.js",
+    "grafijs.js",
+    "smartcrop.js",
+    "basicscroll.js",
+    "moveto.js",
+    "scrollmagic.js",
+    "scrollreveal.js",
+    "verge.js",
+    "alloyfinger.js",
+    "anime.js",
+    "choreographer.js",
+    "gsap.js",
+    "impulse.js",
+    "mojs.js",
+    "popmotion.js",
+    "rebound.js",
+    "repaintless.js",
+    "shifty.js",
+    "snabbt.js",
+    "snapsvg.js",
+    "vivus.js",
+    "dotjs.js",
+    "handlebars.js",
+    "hogan.js",
+    "mustache.js",
+    "vdo.js",
+    "aja.js",
+    "fetch.js",
+    "qwest.js",
+    "reqwest.js",
+    "superagent.js",
+    "bean.js",
+    "eventemitter2.js",
+    "mitt.js",
+    "elegant.js",
+    "feather.js",
+    "flaticon.js",
+    "fontawesome.js",
+    "fontello.js",
+    "icomoon.js",
+    "ikonate.js",
+    "ionicons.js",
+    "octicons.js",
+    "weloveiconfonts.js",
+    "chromajs.js",
+    "coolors.js",
+    "colorbrewer2.js",
+    "colorhexa.js",
+    "colourco.js",
+    "colormind.js",
+    "kewler.js",
+    "khroma.js",
+    "polychrome.js",
+    "uigradients.js",
+    "forerunnerdb.js",
+    "lokijs.js",
+    "lovefield.js",
+    "pouchdb.js",
+    "rxdb.js",
+    "taffydb.js",
+    "zangodb.js",
+    "parsley.js",
+    "dateformat.js",
+    "flatpickr.js",
+    "instadate.js",
+    "luxon.js",
+    "tinytime.js",
+    "l10ns.js",
+    "globalize.js",
+    "datakit.js",
+    "datalib.js",
+    "gauss.js",
+    "jstat.js",
+    "statkit.js",
+    "stdlib.js",
+    "theoremjs.js",
+    "stealjs.js",
+    "aload.js",
+    "lazysizes.js",
+    "loadxt.js",
+    "unveil.js",
+    "brain.js",
+    "mind.js",
+    "neurojs.js",
+    "rrssb.js",
+    "sharingbuttons.js",
+    "socialcount.js",
+    "moutjs.js",
+    "ramdajs.js",
+    "formstone.js",
+    "tether.js",
+    "upup.js"
+]

docs/en-us/README.md

@@ -274,8 +274,8 @@ Very warmly welcome all people to make OneForAll better together!
 
 - [ ] Continuous optimize and improve of each module
 - [x] Subdomain monitoring (mark newly discovered subdomain)
-- [ ] Subdomain collection crawler (collect subdomains from static files such as JS)
+- [x] Subdomain collection crawler (collect subdomains from static files such as JS)
-- [ ] Implementation of front-end interface for powerful interaction (tentative: front-end: Element + back-end: Flask)
+- [ ] Implementation of front-end interface for powerful interaction
 
 For more details, read [todo.md](https://github.com/shmilylty/OneForAll/tree/master/docs/todo.md).
 

modules/collect.py

@@ -31,7 +31,7 @@ class Collect(object):
             # The crawl module has some problems
             modules = ['certificates', 'check', 'datasets',
                        'dnsquery', 'intelligence', 'search']
-            # modules = ['datasets']
+            # modules = ['certificates']
             for module in modules:
                 module_path = setting.module_dir.joinpath(module)
                 for path in module_path.rglob('*.py'):

modules/finder.py

@@ -1,47 +1,39 @@
 import re
+import json
+import time
 from urllib import parse
 
 from common import utils
+from common import resolve
 from common import request
 from common.module import Module
 from config import setting
+from config.log import logger
 
 
 class Finder(Module):
-    def __init__(self, domain):
+    def __init__(self):
         Module.__init__(self)
         self.module = 'Finder'
         self.source = ''
-        self.domain = domain
+        self.start = time.time()  # 模块开始执行时间
-        self.ts = utils.get_timestring()
-        self.name = f'found_subdomains_{self.domain}_{self.ts}'
-        self.path = setting.temp_save_dir.joinpath(self.name)
 
-    def save_subdomain(self, data, path):
+    def run(self, domain, data, port):
-        if not path:
+        logger.log('INFOR', f'Start finder module')
-            path = self.path
+        existing_subdomains = set(map(lambda x: x.get('subdomain'), data))  # 已有的子域
-        utils.save_data(path, data)
+        found_subdomains = find_subdomains(domain, data)
-
+        new_subdomains = found_subdomains - existing_subdomains
-    def remove_subdomain(self, path):
+        if not len(new_subdomains):  # 未发现新的子域就直接返回
-        if not path:
+            return data
-            path = self.path
+        self.subdomains = new_subdomains
-        utils.remove_data(path)
+        self.gen_result()
-
+        temp_data = resolve.run_resolve(domain, self.results)
-    def find_url(self):
+        fina_data = request.run_request(domain, temp_data, port)
-        pass
+        data = data + fina_data
-
+        self.finish()
-    def batch_find_url(self):
+        logger.log('INFOR', f'Saving finder results')
-        pass
+        utils.save_db(domain, data, 'finder')
-
+        return data
-    def find_subdomain(self, html):
-        subdomains = self.match_subdomains(html, fuzzy=False)
-        self.subdomains.add(subdomains)
-
-    def find_js(self):
-        pass
-
-    def filter_js(self):
-        pass
 
 
 # Regular expression comes from https://github.com/GerbenJavado/LinkFinder
@@ -104,29 +96,78 @@ def process_url(req_url, rel_url):
     return result
 
 
-def filter_url(urs):
+def filter_name(path, black_name):
-    pass
+    for name in black_name:
+        if path.endswith(name):
+            return True
+    black_ext = ['io.js', 'ui.js', 'fp.js', 'en.js', 'dev.js', 'min.js', 'umd.js',
+                 'esm.js', 'all.js', 'cjs.js', 'prod.js', 'slim.js', 'core.js',
+                 'global.js', 'bundle.js', 'browser.js', 'brands.js', 'simple.js',
+                 'common.js', 'development.js', 'production.js']
+    for ext in black_ext:
+        if path.endswith(ext):
+            return True
+    r = re.compile(r'\d+.\d+.\d+')
+    if r.search(path):
+        return True
+    return False
 
 
-def run_finder(domain, data):
+def filter_url(domain, url, black_name):
-    existing_subdomains = set(map(lambda x: x.get('subdomain'), data))  # 已有的子域
+    raw_url = parse.urlparse(url)
-    found_subdomains = set()
+    scheme = raw_url.scheme.lower()
-    found_urls = set()
+    if not scheme:
-    finder = Finder(domain)
+        return True
+    if scheme not in ['http', 'https']:
+        return True
+    netloc = raw_url.netloc.lower()
+    if not netloc:
+        return True
+    if not netloc.endswith(domain):
+        return True
+    path = raw_url.path.lower()
+    if not path:
+        return True
+    if not path.endswith('.js'):
+        return True
+    if path.endswith('min.js'):
+        return True
+    return filter_name(path, black_name)
+
+
+def get_black_name():
+    path = setting.data_storage_dir.joinpath('common_js_library.json')
+    with open(path) as fp:
+        return json.load(fp)
+
+
+def match_subdomains(domain, text):
+    subdomains = utils.match_subdomains(domain, text)
+    logger.log('DEBUG', f'matched subdomains: {subdomains}')
+    return subdomains
+
+
+def find_subdomains(domain, data):
+    subdomains = set()
+    js_urls = set()
+    black_name = get_black_name()
     for item in data:
         req_url = item.get('url')
         rsp_html = item.get('response')
-        found_subdomains.add(finder.match_subdomains(rsp_html))
+        if not rsp_html:
+            continue
+        logger.log('DEBUG', f'matching subdomains from response of {req_url}')
+        subdomains = subdomains.union(match_subdomains(domain, rsp_html))
         urls = find_url(rsp_html)
         if not urls:
             continue
         for rel_url in urls:
-            found_url = process_url(req_url, rel_url)
+            url = process_url(req_url, rel_url)
-            found_urls.add(found_url)
+            if not filter_url(domain, url, black_name):
-
+                js_urls.add(url)
-    resp_data = request.urls_request(found_urls)
+    resp_data = request.urls_request(js_urls)
-    new_subdomains = found_subdomains - existing_subdomains
+    for resp, text in resp_data:
-
+        if text:
-
+            logger.log('DEBUG', f'matching subdomains from response of {resp.url}')
-def save_db(domain, data):
+            subdomains = subdomains.union(match_subdomains(domain, text))
-    pass
+    return subdomains

oneforall.py

@@ -17,6 +17,7 @@ from brute import Brute
 from common import utils, resolve, request
 from common.database import Database
 from modules.collect import Collect
+from modules.finder import Finder
 from config import setting
 from config.log import logger
 from takeover import Takeover
@@ -209,6 +210,11 @@ class OneForAll(object):
         # Save HTTP request result
         request.save_db(self.domain, self.data)
 
+        # Finder module
+        if setting.enable_finder_module:
+            finder = Finder()
+            self.data = finder.run(self.domain, self.data, self.port)
+
         # Add the final result list to the total data list
         self.datas.extend(self.data)