From 62d8b76c2172394ea641d1f0d551ab3986e2a547 Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Fri, 24 Jul 2020 01:43:59 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9E=E7=8E=B0finder=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=EF=BC=88=E4=BB=8E=E5=93=8D=E5=BA=94=E4=BD=93=E5=92=8CJS?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E4=B8=AD=E5=86=8D=E6=AC=A1=E5=8F=91=E7=8E=B0?= =?UTF-8?q?=E6=96=B0=E5=AD=90=E5=9F=9F=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +- common/database.py | 2 +- common/module.py | 38 +-- common/request.py | 33 +- common/resolve.py | 14 +- common/utils.py | 53 +++ config/setting.py | 1 + data/common_js_library.json | 621 ++++++++++++++++++++++++++++++++++++ docs/en-us/README.md | 4 +- modules/collect.py | 2 +- modules/finder.py | 135 +++++--- oneforall.py | 6 + 12 files changed, 797 insertions(+), 116 deletions(-) create mode 100644 data/common_js_library.json diff --git a/README.md b/README.md index fe367d0..cd3acba 100644 --- a/README.md +++ b/README.md @@ -276,8 +276,8 @@ FLAGS - [ ] 各模块持续优化和完善 - [x] 子域监控（标记每次新发现的子域） -- [ ] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域） -- [ ] 操作强大交互人性的前端界面实现（暂定：前端：Element + 后端：Flask） +- [x] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域） +- [ ] 操作强大交互人性的前端界面实现 更多详细信息请阅读[todo.md](https://github.com/shmilylty/OneForAll/tree/master/docs/todo.md)。 diff --git a/common/database.py b/common/database.py index b4e7de9..92839dc 100644 --- a/common/database.py +++ b/common/database.py @@ -95,7 +95,7 @@ class Database(object): :param str table_name: table name :param list results: results list - :param str module_name: mo + :param str module_name: module """ logger.log('TRACE', f'Saving the subdomain results of {table_name} ' f'found by module {module_name} into database') diff --git a/common/module.py b/common/module.py index 4bd7060..81248ef 100644 --- a/common/module.py +++ b/common/module.py @@ -212,39 +212,7 @@ class Module(object): return self.proxy def match_subdomains(self, html, distinct=True, fuzzy=True): - """ - Use regexp to match subdomains - - :param str html: response html text - :param bool distinct: deduplicate results or not (default True) - :param bool fuzzy: fuzzy match subdomain or not (default True) - :return set/list: result set or list - """ - logger.log('TRACE', f'Use regexp to match subdomains in the response body') - if fuzzy: - regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ - + self.domain.replace('.', r'\.') - result = re.findall(regexp, html, re.I) - if not result: - return set() - deal = map(lambda s: s.lower(), result) - if distinct: - return set(deal) - else: - return list(deal) - else: - regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \ - r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ - + self.domain.replace('.', r'\.') - result = re.findall(regexp, html, re.I) - if not result: - return set() - regexp = r'(?:http://|https://)' - deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result) - if distinct: - return set(deal) - else: - return list(deal) + return utils.match_subdomains(self.domain, html, distinct, fuzzy) @staticmethod def get_maindomain(domain): @@ -305,11 +273,11 @@ class Module(object): 'new': None, 'url': None, 'subdomain': None, + 'port': None, 'level': None, 'cname': None, 'content': None, 'public': None, - 'port': None, 'status': None, 'reason': None, 'title': None, @@ -369,11 +337,11 @@ class Module(object): 'new': None, 'url': url, 'subdomain': subdomain, + 'port': 80, 'level': level, 'cname': cname, 'content': content, 'public': public, - 'port': 80, 'status': None, 'reason': reason, 'title': None, diff --git a/common/request.py b/common/request.py index 1849ca5..5bc22e9 100644 --- a/common/request.py +++ b/common/request.py @@ -9,7 +9,6 @@ from bs4 import BeautifulSoup from common import utils from config.log import logger from config import setting -from common.database import Database def get_limit_conn(): @@ -35,7 +34,7 @@ def get_ports(port): if not ports: # 意外情况 logger.log('ERROR', 'The specified request port range is incorrect') ports = {80} - logger.log('INFOR', 'Port range:{ports}') + logger.log('INFOR', f'Port range:{ports}') return set(ports) @@ -110,7 +109,7 @@ async def fetch(session, method, url): text = await resp.text(encoding=None, errors='ignore') return resp, text except Exception as e: - return e + return e, None def get_title(markup): @@ -154,15 +153,15 @@ def get_title(markup): def request_callback(future, index, datas): - result = future.result() - if isinstance(result, BaseException): - logger.log('TRACE', result.args) - name = utils.get_classname(result) - datas[index]['reason'] = name + ' ' + str(result) + resp, text = future.result() + if isinstance(resp, BaseException): + exception = resp + logger.log('TRACE', exception.args) + name = utils.get_classname(exception) + datas[index]['reason'] = name + ' ' + str(exception) datas[index]['request'] = 0 datas[index]['alive'] = 0 - elif isinstance(result, tuple): - resp, text = result + else: datas[index]['reason'] = resp.reason datas[index]['status'] = resp.status datas[index]['request'] = 1 @@ -257,17 +256,17 @@ def run_request(domain, data, port): :param str domain: domain to be requested :param list data: subdomains data to be requested - :param str port: range of ports to be requested + :param any port: range of ports to be requested :return list: result """ - logger.log('INFOR', 'Start subdomain request module') + logger.log('INFOR', f'Start requesting subdomains of {domain}') loop = set_loop() data = utils.set_id_none(data) request_coroutine = bulk_request(data, port) data = loop.run_until_complete(request_coroutine) loop.run_until_complete(asyncio.sleep(0.25)) count = utils.count_alive(data) - logger.log('INFOR', f'Request module found {domain} have {count} alive subdomains') + logger.log('INFOR', f'Found that {domain} has {count} alive subdomains') return data @@ -277,7 +276,6 @@ def urls_request(urls): request_coroutine = async_request(urls) data = loop.run_until_complete(request_coroutine) loop.run_until_complete(asyncio.sleep(0.25)) - logger.log('INFOR', 'End urls request module') return data @@ -288,8 +286,5 @@ def save_db(name, data): :param str name: table name :param list data: data to be saved """ - db = Database() - db.drop_table(name) - db.create_table(name) - db.save_db(name, data, 'request') - db.close() + logger.log('INFOR', f'Saving requested results') + utils.save_db(name, data, 'request') diff --git a/common/resolve.py b/common/resolve.py index 73e68a6..dd4ea27 100644 --- a/common/resolve.py +++ b/common/resolve.py @@ -50,17 +50,13 @@ def update_data(data, records): def save_db(name, data): """ - 保存解析结果到数据库 + Save resolved results to database - :param str name: 保存表名 - :param list data: 待保存的数据 + :param str name: table name + :param list data: data to be saved """ logger.log('INFOR', f'Saving resolved results') - db = Database() - db.drop_table(name) - db.create_table(name) - db.save_db(name, data, 'resolve') - db.close() + utils.save_db(name, data, 'resolve') def save_subdomains(save_path, subdomain_list): @@ -161,7 +157,7 @@ def run_resolve(domain, data): :return: 解析得到的结果列表 :rtype: list """ - logger.log('INFOR', f'Start resolve subdomains of {domain}') + logger.log('INFOR', f'Start resolving subdomains of {domain}') subdomains = filter_subdomain(data) if not subdomains: return data diff --git a/common/utils.py b/common/utils.py index 79ef3ac..2d35374 100644 --- a/common/utils.py +++ b/common/utils.py @@ -17,6 +17,7 @@ from records import Record, RecordCollection from dns.resolver import Resolver from common.domain import Domain +from common.database import Database from config import setting from config.log import logger @@ -214,6 +215,21 @@ def check_format(format, count): return 'csv' +def save_db(name, data, module): + """ + Save request results to database + + :param str name: table name + :param list data: data to be saved + :param str module: module name + """ + db = Database() + db.drop_table(name) + db.create_table(name) + db.save_db(name, data, module) + db.close() + + def save_data(path, data): """ 保存数据到文件 @@ -663,3 +679,40 @@ def ip_to_int(ip): logger.log('ERROR', e.args) return None return int(ipv4) + + +def match_subdomains(domain, html, distinct=True, fuzzy=True): + """ + Use regexp to match subdomains + + :param str domain: main domain + :param str html: response html text + :param bool distinct: deduplicate results or not (default True) + :param bool fuzzy: fuzzy match subdomain or not (default True) + :return set/list: result set or list + """ + logger.log('TRACE', f'Use regexp to match subdomains in the response body') + if fuzzy: + regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + + domain.replace('.', r'\.') + result = re.findall(regexp, html, re.I) + if not result: + return set() + deal = map(lambda s: s.lower(), result) + if distinct: + return set(deal) + else: + return list(deal) + else: + regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \ + r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \ + + domain.replace('.', r'\.') + result = re.findall(regexp, html, re.I) + if not result: + return set() + regexp = r'(?:http://|https://)' + deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result) + if distinct: + return set(deal) + else: + return list(deal) diff --git a/config/setting.py b/config/setting.py index ac01035..6bceb6e 100644 --- a/config/setting.py +++ b/config/setting.py @@ -18,6 +18,7 @@ temp_save_dir = result_save_dir.joinpath('temp') enable_check_version = True # 开启最新版本检查 enable_dns_resolve = True # 使用DNS解析子域(默认True) enable_http_request = True # 使用HTTP请求子域(默认True) +enable_finder_module = True # 开启finder模块(默认True) enable_takeover_check = False # 开启子域接管风险检查(默认False) # 参数port可选值有'default', 'small', 'large' http_request_port = 'default' # HTTP请求子域(默认'default'，探测80端口) diff --git a/data/common_js_library.json b/data/common_js_library.json new file mode 100644 index 0000000..424fe50 --- /dev/null +++ b/data/common_js_library.json @@ -0,0 +1,621 @@ +[ + "npm.js", + "bower.js", + "component.js", + "spm.js", + "jam.js", + "jspm.js", + "ender.js", + "volo.js", + "duo.js", + "yarn.js", + "requirejs.js", + "browserify.js", + "seajs.js", + "headjs.js", + "curl.js", + "lazyload.js", + "systemjs.js", + "lodjs.js", + "esl.js", + "modulejs.js", + "webpack.js", + "rollup.js", + "brunch.js", + "parcel.js", + "microbundle.js", + "typescript.js", + "hegel.js", + "typl.js", + "mocha.js", + "jasmine.js", + "qunit.js", + "jest.js", + "prova.js", + "dalekjs.js", + "protractor.js", + "tape.js", + "testcafe.js", + "ava.js", + "cypress.js", + "chai.js", + "enzyme.js", + "proxyquire.js", + "istanbul.js", + "blanket.js", + "jscover.js", + "phantomjs.js", + "slimerjs.js", + "casperjs.js", + "zombie.js", + "totoro.js", + "karma.js", + "nightwatch.js", + "intern.js", + "yolpo.js", + "puppeteer.js", + "webdriverio.js", + "prettier.js", + "jshint.js", + "jscs.js", + "jsfmt.js", + "jsinspect.js", + "eslint.js", + "jslint.js", + "aurelia.js", + "backbone.js", + "meteor.js", + "ractive.js", + "vue.js", + "svelte.js", + "knockout.js", + "spine.js", + "canjs.js", + "react.js", + "hyperapp.js", + "preact.js", + "nativescript.js", + "riot.js", + "thorax.js", + "chaplin.js", + "marionette.js", + "ripple.js", + "rivets.js", + "derby.js", + "jsblocks.js", + "liquidlava.js", + "feathers.js", + "keo.js", + "atvjs.js", + "makefun.js", + "keystonejs.js", + "ghost.js", + "apostrophe.js", + "taracotjs.js", + "nodizecms.js", + "cody.js", + "pencilblue.js", + "strapi.js", + "factor.js", + "nunjucks.js", + "dot.js", + "dustjs.js", + "eco.js", + "pug.js", + "ejs.js", + "xtemplate.js", + "marko.js", + "swig.js", + "ehtml.js", + "d3.js", + "peity.js", + "raphael.js", + "echarts.js", + "vis.js", + "arbor.js", + "cubism.js", + "vega.js", + "envisionjs.js", + "rickshaw.js", + "flot.js", + "nvd3.js", + "trianglify.js", + "d4.js", + "epoch.js", + "c3.js", + "babylonjs.js", + "recharts.js", + "graphicsjs.js", + "mxgraph.js", + "amchart.js", + "anychart.js", + "plotly.js", + "highchart.js", + "handsontable.js", + "ace.js", + "codemirror.js", + "esprima.js", + "quill.js", + "pen.js", + "editor.js", + "epiceditor.js", + "jsoneditor.js", + "squire.js", + "tinymce.js", + "trix.js", + "trumbowyg.js", + "wysihtml5.js", + "popline.js", + "summernote.js", + "devdocs.js", + "dexy.js", + "docco.js", + "styledocco.js", + "ronn.js", + "dox.js", + "jsdox.js", + "esdoc.js", + "yuidoc.js", + "coddoc.js", + "sphinx.js", + "jsduck.js", + "codecrumbs.js", + "jbinary.js", + "diff2html.js", + "jspdf.js", + "underscore.js", + "lodash.js", + "sugar.js", + "ramda.js", + "mout.js", + "mesh.js", + "preludejs.js", + "rxjs.js", + "bacon.js", + "kefir.js", + "highland.js", + "mobx.js", + "mori.js", + "buckets.js", + "hashmap.js", + "moment.js", + "date.js", + "fecha.js", + "dayjs.js", + "voca.js", + "selecting.js", + "he.js", + "multiline.js", + "jsurl.js", + "plexis.js", + "odometer.js", + "localforage.js", + "jstorage.js", + "cookies.js", + "crumbsjs.js", + "randomcolor.js", + "color.js", + "colors.js", + "pleasejs.js", + "tinycolor.js", + "i18next.js", + "polyglot.js", + "babelfish.js", + "ttag.js", + "async.js", + "q.js", + "step.js", + "contra.js", + "bluebird.js", + "when.js", + "objecteventtarget.js", + "sporadic.js", + "director.js", + "pathjs.js", + "crossroads.js", + "navaid.js", + "dompurify.js", + "log.js", + "conzole.js", + "loglevel.js", + "minilog.js", + "storyboard.js", + "regex101.js", + "regexr.js", + "regexpbuilder.js", + "annyang.js", + "axios.js", + "bottleneck.js", + "amygdala.js", + "wretch.js", + "farfetch.js", + "tailor.js", + "convnetjs.js", + "dn2a.js", + "synapses.js", + "bowser.js", + "matcha.js", + "prismjs.js", + "nprogress.js", + "pace.js", + "topbar.js", + "nanobar.js", + "pageloadingeffects.js", + "spinkit.js", + "ladda.js", + "ajaxload.js", + "preloaders.js", + "cssload.js", + "validatr.js", + "formvalidation.js", + "fieldval.js", + "funval.js", + "mousetrap.js", + "keymaster.js", + "keypress.js", + "keyboardjs.js", + "jwerty.js", + "shepherd.js", + "tourist.js", + "pageguide.js", + "hopscotch.js", + "joyride.js", + "focusable.js", + "izitoast.js", + "messenger.js", + "noty.js", + "pnotify.js", + "toastr.js", + "notie.js", + "swiper.js", + "slick.js", + "slidesjs.js", + "flexslider.js", + "unslider.js", + "sly.js", + "vegas.js", + "sequence.js", + "strut.js", + "photoswipe.js", + "jcslider.js", + "slidr.js", + "flickity.js", + "jqrangeslider.js", + "nouislider.js", + "fancyinput.js", + "awesomplete.js", + "pikaday.js", + "fullcalendar.js", + "rome.js", + "datedropper.js", + "select2.js", + "chosen.js", + "dropzone.js", + "fileapi.js", + "plupload.js", + "form.js", + "countable.js", + "card.js", + "stretchy.js", + "analytics.js", + "tipsy.js", + "opentip.js", + "qtip2.js", + "tooltipster.js", + "simptip.js", + "toolbar.js", + "vex.js", + "sweetalert.js", + "colorbox.js", + "fancybox.js", + "swipebox.js", + "jbox.js", + "scrollmonitor.js", + "headroom.js", + "iscroll.js", + "skrollr.js", + "parallax.js", + "plax.js", + "jparallax.js", + "fullpage.js", + "scrollmenu.js", + "simpleparallax.js", + "slideout.js", + "jtable.js", + "datatables.js", + "tabulator.js", + "floatthead.js", + "masonry.js", + "packery.js", + "isotope.js", + "flexboxgrid.js", + "w2ui.js", + "fluidity.js", + "ink.js", + "dataformsjs.js", + "webplate.js", + "cerberus.js", + "touchemulator.js", + "dragula.js", + "leaflet.js", + "cesium.js", + "gmaps.js", + "polymaps.js", + "jqvmap.js", + "openlayers3.js", + "html5media.js", + "polyplayer.js", + "flowplayer.js", + "mediaelement.js", + "soundjs.js", + "clappr.js", + "exifr.js", + "bigtext.js", + "circletype.js", + "slabtext.js", + "velocity.js", + "transitionend.js", + "textillate.js", + "animatable.js", + "tsparticles.js", + "pica.js", + "cropper.js", + "es6features.js", + "gridsome.js", + "docusaurus.js", + "echo.js", + "picturefill.js", + "json3.js", + "mixitup.js", + "grid.js", + "ky.js", + "fcal.js", + "iooxa.js", + "idyll.js", + "primer.js", + "glue.js", + "postcss.js", + "mui.js", + "img2css.js", + "weui.js", + "csscss.js", + "simditor.js", + "htmlhint.js", + "csslint.js", + "grunt.js", + "yeoman.js", + "gulp.js", + "zrender.js", + "highcharts.js", + "tweenjs.js", + "swipe.js", + "superslides.js", + "slider.js", + "polymer.js", + "ionic.js", + "timelinejs.js", + "togetherjs.js", + "foundation.js", + "todomvc.js", + "vuejs.js", + "webuploader.js", + "fastclick.js", + "wangeditor.js", + "tooling.js", + "judge.js", + "amdoc.js", + "amazeui.js", + "zepto.js", + "nodeclub.js", + "nodeppt.js", + "hexo.js", + "koa.js", + "connect.js", + "nvm.js", + "flux.js", + "browserquest.js", + "html5shiv.js", + "ulkit.js", + "arttemplate.js", + "jade.js", + "modernizr.js", + "css3please.js", + "babel.js", + "f2etest.js", + "brackets.js", + "ueditor.js", + "electron.js", + "base.js", + "basscss.js", + "bootflat.js", + "bootswatch.js", + "bulma.js", + "cardinal.js", + "caramel.js", + "corpus.js", + "kube.js", + "materialize.js", + "milligram.js", + "papercss.js", + "papier.js", + "pavilion.js", + "picnicss.js", + "pure.js", + "skeleton.js", + "tachyons.js", + "tacit.js", + "uikit.js", + "wing.js", + "angular.js", + "choo.js", + "deku.js", + "displayjs.js", + "inferno.js", + "mercury.js", + "mithril.js", + "moon.js", + "skatejs.js", + "solid.js", + "bliss.js", + "cash.js", + "jquery.js", + "nanojs.js", + "selector.js", + "umbrella.js", + "zeptojs.js", + "chartist.js", + "charts.js", + "chartjs.js", + "dc.js", + "dimple.js", + "d3xter.js", + "f2.js", + "frappe.js", + "ggraph.js", + "jsplumb.js", + "metricsgraphics.js", + "morrisjs.js", + "muze.js", + "sparkline.js", + "sparky.js", + "taucharts.js", + "uvcharts.js", + "vivagraph.js", + "z3d.js", + "kartograph.js", + "mapsicon.js", + "osmbuildings.js", + "planetary.js", + "smallworld.js", + "tangram.js", + "topojson.js", + "turf.js", + "dynatables.js", + "listjs.js", + "sortable.js", + "tablesaw.js", + "flipside.js", + "nudge.js", + "glide.js", + "lory.js", + "siema.js", + "blotter.js", + "fitty.js", + "flowtype.js", + "lettering.js", + "shave.js", + "typeplate.js", + "fitvid.js", + "medialementjs.js", + "plyr.js", + "talkie.js", + "videojs.js", + "abcjs.js", + "audio5js.js", + "bap.js", + "blip.js", + "howler.js", + "soundcite.js", + "tonal.js", + "vexflow.js", + "easeljs.js", + "konva.js", + "panzoom.js", + "p5js.js", + "vizflow.js", + "zdog.js", + "scenejs.js", + "whitestormjs.js", + "camanjs.js", + "grafijs.js", + "smartcrop.js", + "basicscroll.js", + "moveto.js", + "scrollmagic.js", + "scrollreveal.js", + "verge.js", + "alloyfinger.js", + "anime.js", + "choreographer.js", + "gsap.js", + "impulse.js", + "mojs.js", + "popmotion.js", + "rebound.js", + "repaintless.js", + "shifty.js", + "snabbt.js", + "snapsvg.js", + "vivus.js", + "dotjs.js", + "handlebars.js", + "hogan.js", + "mustache.js", + "vdo.js", + "aja.js", + "fetch.js", + "qwest.js", + "reqwest.js", + "superagent.js", + "bean.js", + "eventemitter2.js", + "mitt.js", + "elegant.js", + "feather.js", + "flaticon.js", + "fontawesome.js", + "fontello.js", + "icomoon.js", + "ikonate.js", + "ionicons.js", + "octicons.js", + "weloveiconfonts.js", + "chromajs.js", + "coolors.js", + "colorbrewer2.js", + "colorhexa.js", + "colourco.js", + "colormind.js", + "kewler.js", + "khroma.js", + "polychrome.js", + "uigradients.js", + "forerunnerdb.js", + "lokijs.js", + "lovefield.js", + "pouchdb.js", + "rxdb.js", + "taffydb.js", + "zangodb.js", + "parsley.js", + "dateformat.js", + "flatpickr.js", + "instadate.js", + "luxon.js", + "tinytime.js", + "l10ns.js", + "globalize.js", + "datakit.js", + "datalib.js", + "gauss.js", + "jstat.js", + "statkit.js", + "stdlib.js", + "theoremjs.js", + "stealjs.js", + "aload.js", + "lazysizes.js", + "loadxt.js", + "unveil.js", + "brain.js", + "mind.js", + "neurojs.js", + "rrssb.js", + "sharingbuttons.js", + "socialcount.js", + "moutjs.js", + "ramdajs.js", + "formstone.js", + "tether.js", + "upup.js" +] \ No newline at end of file diff --git a/docs/en-us/README.md b/docs/en-us/README.md index 2ec08e2..1831819 100644 --- a/docs/en-us/README.md +++ b/docs/en-us/README.md @@ -274,8 +274,8 @@ Very warmly welcome all people to make OneForAll better together! - [ ] Continuous optimize and improve of each module - [x] Subdomain monitoring (mark newly discovered subdomain) -- [ ] Subdomain collection crawler (collect subdomains from static files such as JS) -- [ ] Implementation of front-end interface for powerful interaction (tentative: front-end: Element + back-end: Flask) +- [x] Subdomain collection crawler (collect subdomains from static files such as JS) +- [ ] Implementation of front-end interface for powerful interaction For more details, read [todo.md](https://github.com/shmilylty/OneForAll/tree/master/docs/todo.md). diff --git a/modules/collect.py b/modules/collect.py index c8ae637..70ae21a 100644 --- a/modules/collect.py +++ b/modules/collect.py @@ -31,7 +31,7 @@ class Collect(object): # The crawl module has some problems modules = ['certificates', 'check', 'datasets', 'dnsquery', 'intelligence', 'search'] - # modules = ['datasets'] + # modules = ['certificates'] for module in modules: module_path = setting.module_dir.joinpath(module) for path in module_path.rglob('*.py'): diff --git a/modules/finder.py b/modules/finder.py index aec6e98..7818560 100644 --- a/modules/finder.py +++ b/modules/finder.py @@ -1,47 +1,39 @@ import re +import json +import time from urllib import parse from common import utils +from common import resolve from common import request from common.module import Module from config import setting +from config.log import logger class Finder(Module): - def __init__(self, domain): + def __init__(self): Module.__init__(self) self.module = 'Finder' self.source = '' - self.domain = domain - self.ts = utils.get_timestring() - self.name = f'found_subdomains_{self.domain}_{self.ts}' - self.path = setting.temp_save_dir.joinpath(self.name) + self.start = time.time() # 模块开始执行时间 - def save_subdomain(self, data, path): - if not path: - path = self.path - utils.save_data(path, data) - - def remove_subdomain(self, path): - if not path: - path = self.path - utils.remove_data(path) - - def find_url(self): - pass - - def batch_find_url(self): - pass - - def find_subdomain(self, html): - subdomains = self.match_subdomains(html, fuzzy=False) - self.subdomains.add(subdomains) - - def find_js(self): - pass - - def filter_js(self): - pass + def run(self, domain, data, port): + logger.log('INFOR', f'Start finder module') + existing_subdomains = set(map(lambda x: x.get('subdomain'), data)) # 已有的子域 + found_subdomains = find_subdomains(domain, data) + new_subdomains = found_subdomains - existing_subdomains + if not len(new_subdomains): # 未发现新的子域就直接返回 + return data + self.subdomains = new_subdomains + self.gen_result() + temp_data = resolve.run_resolve(domain, self.results) + fina_data = request.run_request(domain, temp_data, port) + data = data + fina_data + self.finish() + logger.log('INFOR', f'Saving finder results') + utils.save_db(domain, data, 'finder') + return data # Regular expression comes from https://github.com/GerbenJavado/LinkFinder @@ -104,29 +96,78 @@ def process_url(req_url, rel_url): return result -def filter_url(urs): - pass +def filter_name(path, black_name): + for name in black_name: + if path.endswith(name): + return True + black_ext = ['io.js', 'ui.js', 'fp.js', 'en.js', 'dev.js', 'min.js', 'umd.js', + 'esm.js', 'all.js', 'cjs.js', 'prod.js', 'slim.js', 'core.js', + 'global.js', 'bundle.js', 'browser.js', 'brands.js', 'simple.js', + 'common.js', 'development.js', 'production.js'] + for ext in black_ext: + if path.endswith(ext): + return True + r = re.compile(r'\d+.\d+.\d+') + if r.search(path): + return True + return False -def run_finder(domain, data): - existing_subdomains = set(map(lambda x: x.get('subdomain'), data)) # 已有的子域 - found_subdomains = set() - found_urls = set() - finder = Finder(domain) +def filter_url(domain, url, black_name): + raw_url = parse.urlparse(url) + scheme = raw_url.scheme.lower() + if not scheme: + return True + if scheme not in ['http', 'https']: + return True + netloc = raw_url.netloc.lower() + if not netloc: + return True + if not netloc.endswith(domain): + return True + path = raw_url.path.lower() + if not path: + return True + if not path.endswith('.js'): + return True + if path.endswith('min.js'): + return True + return filter_name(path, black_name) + + +def get_black_name(): + path = setting.data_storage_dir.joinpath('common_js_library.json') + with open(path) as fp: + return json.load(fp) + + +def match_subdomains(domain, text): + subdomains = utils.match_subdomains(domain, text) + logger.log('DEBUG', f'matched subdomains: {subdomains}') + return subdomains + + +def find_subdomains(domain, data): + subdomains = set() + js_urls = set() + black_name = get_black_name() for item in data: req_url = item.get('url') rsp_html = item.get('response') - found_subdomains.add(finder.match_subdomains(rsp_html)) + if not rsp_html: + continue + logger.log('DEBUG', f'matching subdomains from response of {req_url}') + subdomains = subdomains.union(match_subdomains(domain, rsp_html)) urls = find_url(rsp_html) if not urls: continue for rel_url in urls: - found_url = process_url(req_url, rel_url) - found_urls.add(found_url) - - resp_data = request.urls_request(found_urls) - new_subdomains = found_subdomains - existing_subdomains - - -def save_db(domain, data): - pass + url = process_url(req_url, rel_url) + if not filter_url(domain, url, black_name): + js_urls.add(url) + resp_data = request.urls_request(js_urls) + for resp, text in resp_data: + if text: + logger.log('DEBUG', f'matching subdomains from response of {resp.url}') + subdomains = subdomains.union(match_subdomains(domain, text)) + return subdomains diff --git a/oneforall.py b/oneforall.py index 65285cf..c0a54f9 100644 --- a/oneforall.py +++ b/oneforall.py @@ -17,6 +17,7 @@ from brute import Brute from common import utils, resolve, request from common.database import Database from modules.collect import Collect +from modules.finder import Finder from config import setting from config.log import logger from takeover import Takeover @@ -209,6 +210,11 @@ class OneForAll(object): # Save HTTP request result request.save_db(self.domain, self.data) + # Finder module + if setting.enable_finder_module: + finder = Finder() + self.data = finder.run(self.domain, self.data, self.port) + # Add the final result list to the total data list self.datas.extend(self.data)