diff --git a/README.md b/README.md index dbb392f..2832b04 100644 --- a/README.md +++ b/README.md @@ -227,7 +227,7 @@ FLAGS 1. 利用证书透明度收集子域（目前有6个模块：`censys_api`，`certspotter`，`crtsh`，`entrust`，`google`，`spyse_api`） 2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，检查robots文件`robots`，检查sitemap文件`sitemap`，利用NSEC记录遍历DNS域`dnssec`，后续会添加NSEC3记录等模块） 3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善） - 4. 利用DNS数据集收集子域（目前有24个模块：`binaryedge_api`, `bufferover`, `cebaidu`, `chinaz`, `chinaz_api`, `circl_api`, `cloudflare`, `dnsdb_api`, `dnsdumpster`, `hackertarget`, `ip138`, `ipv4info_api`, `netcraft`, `passivedns_api`, `ptrarchive`, `qianxun`, `rapiddns`, `riddler`, `robtex`, `securitytrails_api`, `sitedossier`, `threatcrowd`, `wzpc`, `ximcx`） + 4. 利用DNS数据集收集子域（目前有24个模块：`bevigil_api`, `binaryedge_api`, `bufferover`, `cebaidu`, `chinaz`, `chinaz_api`, `circl_api`, `cloudflare`, `dnsdb_api`, `dnsdumpster`, `hackertarget`, `ip138`, `ipv4info_api`, `netcraft`, `passivedns_api`, `ptrarchive`, `qianxun`, `rapiddns`, `riddler`, `robtex`, `securitytrails_api`, `sitedossier`, `threatcrowd`, `wzpc`, `ximcx`） 5. 利用DNS查询收集子域（目前有5个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，以及通过查询域名的DNS记录中的MX,NS,SOA,TXT记录来收集子域） 6. 利用威胁情报平台数据收集子域（目前有6个模块：`alienvault`, `riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善） 7. 利用搜索引擎发现子域（目前有18个模块：`ask`, `baidu`, `bing`, `bing_api`, `duckduckgo`, `exalead`, `fofa_api`, `gitee`, `github`, `github_api`, `google`, `google_api`, `shodan_api`, `so`, `sogou`, `yahoo`, `yandex`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。 diff --git a/config/api.py b/config/api.py index d8b0c47..00b0019 100644 --- a/config/api.py +++ b/config/api.py @@ -7,6 +7,9 @@ censys_api_secret = '' # 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。 binaryedge_api = '' +# BeVigil API: https://bevigil.com/osint-api +bevigil_api = '' + # Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa chinaz_api = '' diff --git a/config/default.py b/config/default.py index 58ae0a3..a0e8e5f 100644 --- a/config/default.py +++ b/config/default.py @@ -173,6 +173,9 @@ censys_api_secret = '' # 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。 binaryedge_api = '' +# BeVigil API: https://bevigil.com/osint-api +bevigil_api = '' + # Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa chinaz_api = '' diff --git a/docs/en-us/README.md b/docs/en-us/README.md index 8e92629..f765fdd 100644 --- a/docs/en-us/README.md +++ b/docs/en-us/README.md @@ -229,7 +229,7 @@ At present, OneForAll is under development, there must be a lot of problems and 1. Use 6 certificate modules: `censys_api`, `certspotter`, `crtsh`, `entrust`, `google`, `spyse_api`. 2. Use 6 baseline testing modules: scan domain transfer vulnerability `axfr`, cross-domain policy file `cdx`, HTTPS certificate `cert`, content security policy `csp`, robots file `robots`, and sitemap file `sitemap`, NSEC record `nsec`. NSEC3 record and other modules will be added later. 3. Use 2 web crawler modules: `archirawl`, `commoncrawl`, which is still being debugged and needs to be added and improved). -4. Use 24 DNS datasets modules: `binaryedge_api`, `bufferover`, `cebaidu`, `chinaz`, `chinaz_api`, `circl_api`, `cloudflare`, `dnsdb_api`, `dnsdumpster`, `hackertarget`, `ip138`, `ipv4info_api`, `netcraft`, `passivedns_api`, `ptrarchive`, `qianxun`, `rapiddns`, `riddler`, `robtex`, `securitytrails_api`, `sitedossier`, `threatcrowd`, `wzpc`, `ximcx`. +4. Use 24 DNS datasets modules: `bevigil`, `binaryedge_api`, `bufferover`, `cebaidu`, `chinaz`, `chinaz_api`, `circl_api`, `cloudflare`, `dnsdb_api`, `dnsdumpster`, `hackertarget`, `ip138`, `ipv4info_api`, `netcraft`, `passivedns_api`, `ptrarchive`, `qianxun`, `rapiddns`, `riddler`, `robtex`, `securitytrails_api`, `sitedossier`, `threatcrowd`, `wzpc`, `ximcx`. 5. Use 6 DNS queries modules: enumerating SRV records `srv` and collect from `MX`, `NS`, `SOA`, `TXT`, `SPF`. 6. Use 6 threat intelligence modules: `alienvault`, `riskiq_ api`, `threatbook_ api`, `threatkeeper `, `virustotal`, `virustotal_ api`, which need to be added and improved. 7. Use 16 search engines modules: `ask`, `baidu`, `bing`, `bing_api`, `fofa_api`, `gitee`, `github_api`, `google`, `google_api`, `shodan_api`, `so`, `sogou`, `yahoo`, `yandex`, `zoomeye_api`, except for special search engines. General search engines support automatic exclusion of search, full search and recursive search. diff --git a/modules/datasets/bevigil.py b/modules/datasets/bevigil.py new file mode 100644 index 0000000..30b0bbb --- /dev/null +++ b/modules/datasets/bevigil.py @@ -0,0 +1,50 @@ +from config import settings +from common.query import Query + + +class BeVigilAPI(Query): + def __init__(self, domain): + Query.__init__(self) + self.domain = domain + self.module = 'Dataset' + self.source = 'BeVigilOsintApi' + self.addr = 'http://osint.bevigil.com/api/{}/subdomains/' + self.api = settings.bevigil_api + + def query(self): + """ + 向接口查询子域并做子域匹配 + """ + + self.header = self.get_header() + self.header.update({"X-Access-Token": self.api}) + self.proxy = self.get_proxy(self.source) + url = self.addr.format(self.domain) + resp = self.get(url) + self.subdomains = self.collect_subdomains(resp) + + def run(self): + """ + 类执行入口 + """ + if not self.have_api(self.api): + return + self.begin() + self.query() + self.finish() + self.save_json() + self.gen_result() + self.save_db() + + +def run(domain): + """ + 类统一调用入口 + + :param str domain: 域名 + """ + query = BeVigilAPI(domain) + query.run() + +if __name__ == '__main__': + run('example.com')