river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-26 04:47:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' of https://github.com/shmilylty/OneForAll into dev

Browse Source
This commit is contained in:
JrD
2020-11-10 16:31:34 +08:00
parent 112d06f748 8f988a441c
commit d0a5f3d08e
30 changed files with 651 additions and 726 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Pipfile.lock
Generated
+60 -49
View File
@@ -18,11 +18,11 @@
"default": { "default": {
"beautifulsoup4": { "beautifulsoup4": {
"hashes": [ "hashes": [
"sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7", "sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35",
"sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8", "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25",
"sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c" "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"
], ],
"version": "==4.9.1" "version": "==4.9.3"
}, },
"bs4": { "bs4": {
"hashes": [ "hashes": [
@@ -47,11 +47,11 @@
}, },
"colorama": { "colorama": {
"hashes": [ "hashes": [
"sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b",
"sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1" "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"
], ],
"markers": "sys_platform == 'win32'", "markers": "sys_platform == 'win32'",
"version": "==0.4.3" "version": "==0.4.4"
}, },
"dnspython": { "dnspython": {
"hashes": [ "hashes": [
@@ -79,6 +79,7 @@
"hashes": [ "hashes": [
"sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
], ],
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==0.18.2" "version": "==0.18.2"
}, },
"idna": { "idna": {
@@ -86,6 +87,7 @@
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==2.10" "version": "==2.10"
}, },
"loguru": { "loguru": {
@@ -118,6 +120,7 @@
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
"sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==1.15.0" "version": "==1.15.0"
}, },
"soupsieve": { "soupsieve": {
@@ -125,45 +128,52 @@
"sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
"sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
], ],
"markers": "python_version >= '3.0'",
"version": "==2.0.1" "version": "==2.0.1"
}, },
"sqlalchemy": { "sqlalchemy": {
"hashes": [ "hashes": [
"sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb", "sha256:009e8388d4d551a2107632921320886650b46332f61dc935e70c8bcf37d8e0d6",
"sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804", "sha256:0157c269701d88f5faf1fa0e4560e4d814f210c01a5b55df3cab95e9346a8bcc",
"sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6", "sha256:0a92745bb1ebbcb3985ed7bda379b94627f0edbc6c82e9e4bac4fb5647ae609a",
"sha256:26c5ca9d09f0e21b8671a32f7d83caad5be1f6ff45eef5ec2f6fd0db85fc5dc0", "sha256:0cca1844ba870e81c03633a99aa3dc62256fb96323431a5dec7d4e503c26372d",
"sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe", "sha256:166917a729b9226decff29416f212c516227c2eb8a9c9f920d69ced24e30109f",
"sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de", "sha256:1f5f369202912be72fdf9a8f25067a5ece31a2b38507bb869306f173336348da",
"sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36", "sha256:2909dffe5c9a615b7e6c92d1ac2d31e3026dc436440a4f750f4749d114d88ceb",
"sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e", "sha256:2b5dafed97f778e9901b79cc01b88d39c605e0545b4541f2551a2fd785adc15b",
"sha256:465c999ef30b1c7525f81330184121521418a67189053bcf585824d833c05b66", "sha256:2e9bd5b23bba8ae8ce4219c9333974ff5e103c857d9ff0e4b73dc4cb244c7d86",
"sha256:51064ee7938526bab92acd049d41a1dc797422256086b39c08bafeffb9d304c6", "sha256:3aa6d45e149a16aa1f0c46816397e12313d5e37f22205c26e06975e150ffcf2a",
"sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc", "sha256:4bdbdb8ca577c6c366d15791747c1de6ab14529115a2eb52774240c412a7b403",
"sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d", "sha256:53fd857c6c8ffc0aa6a5a3a2619f6a74247e42ec9e46b836a8ffa4abe7aab327",
"sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce", "sha256:5cdfe54c1e37279dc70d92815464b77cd8ee30725adc9350f06074f91dbfeed2",
"sha256:6557af9e0d23f46b8cd56f8af08eaac72d2e3c632ac8d5cf4e20215a8dca7cea", "sha256:5d92c18458a4aa27497a986038d5d797b5279268a2de303cd00910658e8d149c",
"sha256:73a40d4fcd35fdedce07b5885905753d5d4edf413fbe53544dd871f27d48bd4f", "sha256:632b32183c0cb0053194a4085c304bc2320e5299f77e3024556fa2aa395c2a8b",
"sha256:8280f9dae4adb5889ce0bb3ec6a541bf05434db5f9ab7673078c00713d148365", "sha256:7c735c7a6db8ee9554a3935e741cf288f7dcbe8706320251eb38c412e6a4281d",
"sha256:83469ad15262402b0e0974e612546bc0b05f379b5aa9072ebf66d0f8fef16bea", "sha256:7cd40cb4bc50d9e87b3540b23df6e6b24821ba7e1f305c1492b0806c33dbdbec",
"sha256:860d0fe234922fd5552b7f807fbb039e3e7ca58c18c8d38aa0d0a95ddf4f6c23", "sha256:84f0ac4a09971536b38cc5d515d6add7926a7e13baa25135a1dbb6afa351a376",
"sha256:883c9fb62cebd1e7126dd683222b3b919657590c3e2db33bdc50ebbad53e0338", "sha256:8dcbf377529a9af167cbfc5b8acec0fadd7c2357fc282a1494c222d3abfc9629",
"sha256:8afcb6f4064d234a43fea108859942d9795c4060ed0fbd9082b0f280181a15c1", "sha256:950f0e17ffba7a7ceb0dd056567bc5ade22a11a75920b0e8298865dc28c0eff6",
"sha256:96f51489ac187f4bab588cf51f9ff2d40b6d170ac9a4270ffaed535c8404256b", "sha256:9e379674728f43a0cd95c423ac0e95262500f9bfd81d33b999daa8ea1756d162",
"sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e", "sha256:b15002b9788ffe84e42baffc334739d3b68008a973d65fad0a410ca5d0531980",
"sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba", "sha256:b6f036ecc017ec2e2cc2a40615b41850dc7aaaea6a932628c0afc73ab98ba3fb",
"sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02", "sha256:bad73f9888d30f9e1d57ac8829f8a12091bdee4949b91db279569774a866a18e",
"sha256:b6ff91356354b7ff3bd208adcf875056d3d886ed7cef90c571aef2ab8a554b12", "sha256:bbc58fca72ce45a64bb02b87f73df58e29848b693869e58bd890b2ddbb42d83b",
"sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86", "sha256:bca4d367a725694dae3dfdc86cf1d1622b9f414e70bd19651f5ac4fb3aa96d61",
"sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d", "sha256:be41d5de7a8e241864189b7530ca4aaf56a5204332caa70555c2d96379e18079",
"sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7", "sha256:bf53d8dddfc3e53a5bda65f7f4aa40fae306843641e3e8e701c18a5609471edf",
"sha256:e49947d583fe4d29af528677e4f0aa21f5e535ca2ae69c48270ebebd0d8843c0", "sha256:c092fe282de83d48e64d306b4bce03114859cdbfe19bf8a978a78a0d44ddadb1",
"sha256:eb1d71643e4154398b02e88a42fc8b29db8c44ce4134cf0f4474bfc5cb5d4dac", "sha256:c3ab23ee9674336654bf9cac30eb75ac6acb9150dc4b1391bec533a7a4126471",
"sha256:f2e8a9c0c8813a468aa659a01af6592f71cd30237ec27c4cc0683f089f90dcfc", "sha256:ce64a44c867d128ab8e675f587aae7f61bd2db836a3c4ba522d884cd7c298a77",
"sha256:fe7fe11019fc3e6600819775a7d55abc5446dda07e9795f5954fdbf8a49e1c37" "sha256:d05cef4a164b44ffda58200efcb22355350979e000828479971ebca49b82ddb1",
"sha256:d2f25c7f410338d31666d7ddedfa67570900e248b940d186b48461bd4e5569a1",
"sha256:d3b709d64b5cf064972b3763b47139e4a0dc4ae28a36437757f7663f67b99710",
"sha256:e32e3455db14602b6117f0f422f46bc297a3853ae2c322ecd1e2c4c04daf6ed5",
"sha256:ed53209b5f0f383acb49a927179fa51a6e2259878e164273ebc6815f3a752465",
"sha256:f605f348f4e6a2ba00acb3399c71d213b92f27f2383fc4abebf7a37368c12142",
"sha256:fcdb3755a7c355bc29df1b5e6fb8226d5c8b90551d202d69d0076a8a5649d68b"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.3.19" "version": "==1.3.20"
}, },
"tenacity": { "tenacity": {
"hashes": [ "hashes": [
@@ -181,11 +191,11 @@
}, },
"tqdm": { "tqdm": {
"hashes": [ "hashes": [
"sha256:8f3c5815e3b5e20bc40463fa6b42a352178859692a68ffaa469706e6d38342a5", "sha256:9ad44aaf0fc3697c06f6e05c7cf025dd66bc7bcb7613c66d85f4464c47ac8fad",
"sha256:faf9c671bd3fad5ebaeee366949d969dca2b2be32c872a7092a1e1a9048d105b" "sha256:ef54779f1c09f346b2b5a8e5c61f96fbcb639929e640e59f8cf810794f406432"
], ],
"index": "pypi", "index": "pypi",
"version": "==4.49.0" "version": "==4.51.0"
}, },
"treelib": { "treelib": {
"hashes": [ "hashes": [
@@ -196,18 +206,19 @@
}, },
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2",
"sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e"
], ],
"version": "==1.25.10" "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
"version": "==1.25.11"
}, },
"win32-setctime": { "win32-setctime": {
"hashes": [ "hashes": [
"sha256:02b4c5959ca0b195f45c98115826c6e8a630b7cf648e724feaab1a5aa6250640", "sha256:4e88556c32fdf47f64165a2180ba4552f8bb32c1103a2fafd05723a0bd42bd4b",
"sha256:47aa7c43548c1fc0a4f026d1944b748b37036df116c7c4cf908e82638d854313" "sha256:dc925662de0a6eb987f0b01f599c01a8236cb8c62831c22d9cada09ad958243e"
], ],
"markers": "sys_platform == 'win32'", "markers": "sys_platform == 'win32'",
"version": "==1.0.2" "version": "==1.0.3"
} }
}, },
"develop": {} "develop": {}
README.md
+3 -6
View File
@@ -97,8 +97,6 @@ docker run -it --rm -v ~/results:/OneForAll/results -v ~/.config:/OneForAll/conf
<details> <details>
<summary><b>✨使用演示</b></summary> <summary><b>✨使用演示</b></summary>
如果你的主机不在中国，请把 [setting](https://github.com/shmilylty/OneForAll/blob/master/config/setting.py#L46) 中`brute_nameservers_path`选项的`cn_nameservers.txt`修改为`nameservers.txt`。
如果你是通过pip3安装的依赖则使用以下命令运行示例： 如果你是通过pip3安装的依赖则使用以下命令运行示例：
```bash ```bash
python3 oneforall.py --target example.com run python3 oneforall.py --target example.com run
@@ -170,7 +168,7 @@ DESCRIPTION
python3 oneforall.py --target example.com --valid None run python3 oneforall.py --target example.com --valid None run
python3 oneforall.py --target example.com --brute True run python3 oneforall.py --target example.com --brute True run
python3 oneforall.py --target example.com --port small run python3 oneforall.py --target example.com --port small run
python3 oneforall.py --target example.com --format csv run python3 oneforall.py --target example.com --fmt csv run
python3 oneforall.py --target example.com --dns False run python3 oneforall.py --target example.com --dns False run
python3 oneforall.py --target example.com --req False run python3 oneforall.py --target example.com --req False run
python3 oneforall.py --target example.com --takeover False run python3 oneforall.py --target example.com --takeover False run
@@ -179,7 +177,7 @@ DESCRIPTION
Note: Note:
参数alive可选值True，False分别表示导出存活，全部子域结果 参数alive可选值True，False分别表示导出存活，全部子域结果
参数port可选值有'default', 'small', 'large', 详见config.py配置 参数port可选值有'default', 'small', 'large', 详见config.py配置
参数format可选格式有 'csv','json' 参数fmt可选格式有 'csv','json'
参数path默认None使用OneForAll结果目录生成路径 参数path默认None使用OneForAll结果目录生成路径
ARGUMENTS ARGUMENTS
@@ -199,7 +197,7 @@ FLAGS
请求验证子域的端口范围(默认只探测80端口) 请求验证子域的端口范围(默认只探测80端口)
--valid=VALID --valid=VALID
只导出存活的子域结果(默认False) 只导出存活的子域结果(默认False)
--format=FORMAT --fmt=FMT
结果保存格式(默认csv) 结果保存格式(默认csv)
--path=PATH --path=PATH
结果保存路径(默认None) 结果保存路径(默认None)
@@ -269,7 +267,6 @@ FLAGS
## ⌛后续计划 ## ⌛后续计划
- [ ] 各模块持续优化和完善 - [ ] 各模块持续优化和完善
- [x] 子域监控（标记每次新发现的子域）
- [x] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域） - [x] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
- [ ] 操作强大交互人性的前端界面实现 - [ ] 操作强大交互人性的前端界面实现
brute.py
+18 -25
View File
@@ -349,18 +349,6 @@ def collect_wildcard_record(domain, authoritative_ns):
return ips, ttl return ips, ttl
def get_nameservers_path(enable_wildcard, ns_ip_list):
path = settings.brute_nameservers_path
if not enable_wildcard:
return path
if not ns_ip_list:
return path
path = settings.authoritative_dns_path
ns_data = '\n'.join(ns_ip_list)
utils.save_data(path, ns_data)
return path
def check_dict(): def check_dict():
if not settings.enable_check_dict: if not settings.enable_check_dict:
return return
@@ -384,7 +372,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
info = dict() info = dict()
cnames = list() cnames = list()
ips = list() ips = list()
public = list()
times = list() times = list()
ttls = list() ttls = list()
is_valid_flags = list() is_valid_flags = list()
@@ -401,7 +388,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
cnames.append(cname) # 去除最右边的`.`点号 cnames.append(cname) # 去除最右边的`.`点号
ip = answer.get('data') ip = answer.get('data')
ips.append(ip) ips.append(ip)
public.append(utils.ip_is_public(ip))
num = ip_times.get(ip) num = ip_times.get(ip)
times.append(num) times.append(num)
isvalid, reason = is_valid_subdomain(ip, ttl, num, wc_ips, wc_ttl, cname) isvalid, reason = is_valid_subdomain(ip, ttl, num, wc_ips, wc_ttl, cname)
@@ -416,7 +402,6 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
info['ttl'] = ttls info['ttl'] = ttls
info['cname'] = cnames info['cname'] = cnames
info['ip'] = ips info['ip'] = ips
info['public'] = public
info['times'] = times info['times'] = times
info['resolver'] = resolver info['resolver'] = resolver
infos[qname] = info infos[qname] = info
@@ -559,7 +544,7 @@ class Brute(Module):
brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
Note: Note:
--format csv/json (result format) --fmt csv/json (result fmt)
--path Result path (default None, automatically generated) --path Result path (default None, automatically generated)
@@ -578,13 +563,13 @@ class Brute(Module):
:param str rule: Specify the regexp rules used in fuzz mode (required if use fuzz mode) :param str rule: Specify the regexp rules used in fuzz mode (required if use fuzz mode)
:param str fuzzlist: Dictionary path used in fuzz mode (default use ./config/default.py) :param str fuzzlist: Dictionary path used in fuzz mode (default use ./config/default.py)
:param bool export: Export the results (default True) :param bool export: Export the results (default True)
:param str format: Result format (default csv) :param str fmt: Result format (default csv)
:param str path: Result directory (default None) :param str path: Result directory (default None)
""" """
def __init__(self, target=None, targets=None, process=None, concurrent=None, def __init__(self, target=None, targets=None, process=None, concurrent=None,
word=False, wordlist=None, recursive=False, depth=None, nextlist=None, word=False, wordlist=None, recursive=False, depth=None, nextlist=None,
fuzz=False, place=None, rule=None, fuzzlist=None, export=True, fuzz=False, place=None, rule=None, fuzzlist=None, export=True,
alive=True, format='csv', path=None): alive=True, fmt='csv', path=None):
Module.__init__(self) Module.__init__(self)
self.module = 'Brute' self.module = 'Brute'
self.source = 'Brute' self.source = 'Brute'
@@ -603,15 +588,15 @@ class Brute(Module):
self.fuzzlist = fuzzlist or settings.fuzz_list self.fuzzlist = fuzzlist or settings.fuzz_list
self.export = export self.export = export
self.alive = alive self.alive = alive
self.format = format self.fmt = fmt
self.path = path self.path = path
self.bulk = False # 是否是批量爆破场景 self.bulk = False # 是否是批量爆破场景
self.domains = list() # 待爆破的所有域名集合 self.domains = list() # 待爆破的所有域名集合
self.domain = str() # 当前正在进行爆破的域名 self.domain = str() # 当前正在进行爆破的域名
self.ips_times = dict() # IP集合出现次数 self.ips_times = dict() # IP集合出现次数
self.enable_wildcard = False # 当前域名是否使用泛解析 self.enable_wildcard = False # 当前域名是否使用泛解析
self.check_env = True
self.quite = False self.quite = False
self.in_china = None
def gen_brute_dict(self, domain): def gen_brute_dict(self, domain):
logger.log('INFOR', f'Generating dictionary for {domain}') logger.log('INFOR', f'Generating dictionary for {domain}')
@@ -668,6 +653,13 @@ class Brute(Module):
logger.log('FATAL', f'Incorrect domain for fuzz') logger.log('FATAL', f'Incorrect domain for fuzz')
exit(1) exit(1)
def init_dict_path(self):
data_dir = settings.data_storage_dir
if self.wordlist is None:
self.wordlist = settings.brute_wordlist_path or data_dir.joinpath('subnames.txt')
if self.recursive_nextlist is None:
self.recursive_nextlist = settings.recursive_nextlist_path or data_dir.joinpath('subnames_next.txt')
def main(self, domain): def main(self, domain):
start = time.time() start = time.time()
logger.log('INFOR', f'Blasting {domain} ') logger.log('INFOR', f'Blasting {domain} ')
@@ -687,7 +679,7 @@ class Brute(Module):
if self.enable_wildcard: if self.enable_wildcard:
wildcard_ips, wildcard_ttl = collect_wildcard_record(domain, wildcard_ips, wildcard_ttl = collect_wildcard_record(domain,
ns_ip_list) ns_ip_list)
ns_path = get_nameservers_path(self.enable_wildcard, ns_ip_list) ns_path = utils.get_ns_path(self.in_china, self.enable_wildcard, ns_ip_list)
dict_set = self.gen_brute_dict(domain) dict_set = self.gen_brute_dict(domain)
@@ -730,12 +722,13 @@ class Brute(Module):
def run(self): def run(self):
logger.log('INFOR', f'Start running {self.source} module') logger.log('INFOR', f'Start running {self.source} module')
if self.check_env: if self.in_china is None:
utils.check_env() _, self.in_china = utils.get_net_env()
self.domains = utils.get_domains(self.target, self.targets) self.domains = utils.get_domains(self.target, self.targets)
for self.domain in self.domains: for self.domain in self.domains:
self.results = list() # 置空 self.results = list() # 置空
all_subdomains = list() all_subdomains = list()
self.init_dict_path()
self.check_brute_params() self.check_brute_params()
if self.recursive_brute: if self.recursive_brute:
logger.log('INFOR', f'Start recursively brute the 1 layer subdomain' logger.log('INFOR', f'Start recursively brute the 1 layer subdomain'
@@ -761,7 +754,7 @@ class Brute(Module):
logger.log('INFOR', f'Finished {self.source} module to brute {self.domain}') logger.log('INFOR', f'Finished {self.source} module to brute {self.domain}')
if not self.path: if not self.path:
name = f'{self.domain}_brute_result.{self.format}' name = f'{self.domain}_brute_result.{self.fmt}'
self.path = settings.result_save_dir.joinpath(name) self.path = settings.result_save_dir.joinpath(name)
# 数据库导出 # 数据库导出
if self.export: if self.export:
@@ -770,7 +763,7 @@ class Brute(Module):
alive=self.alive, alive=self.alive,
limit='resolve', limit='resolve',
path=self.path, path=self.path,
format=self.format) fmt=self.fmt)
if __name__ == '__main__': if __name__ == '__main__':
common/database.py
+48 -22
View File
@@ -57,7 +57,6 @@ class Database(object):
f'alive int,' f'alive int,'
f'request int,' f'request int,'
f'resolve int,' f'resolve int,'
f'new int,'
f'url text,' f'url text,'
f'subdomain text,' f'subdomain text,'
f'port int,' f'port int,'
@@ -86,6 +85,20 @@ class Database(object):
f'elapse float,' f'elapse float,'
f'find int)') f'find int)')
def insert_table(self, table_name, result):
table_name = table_name.replace('.', '_')
self.conn.query(
f'insert into "{table_name}" '
f'(id, alive, resolve, request, url, subdomain, port, level,'
f'cname, ip, public, cdn, status, reason, title, banner, header,'
f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,'
f'module, source, elapse, find) '
f'values (:id, :alive, :resolve, :request, :url,'
f':subdomain, :port, :level, :cname, :ip, :public, :cdn,'
f':status, :reason, :title, :banner, :header, :history, :response,'
f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,'
f':source, :elapse, :find)', **result)
def save_db(self, table_name, results, module_name=None): def save_db(self, table_name, results, module_name=None):
""" """
Save the results of each module in the database Save the results of each module in the database
@@ -101,11 +114,11 @@ class Database(object):
try: try:
self.conn.bulk_query( self.conn.bulk_query(
f'insert into "{table_name}" ' f'insert into "{table_name}" '
f'(id, alive, resolve, request, new, url, subdomain, port, level,' f'(id, alive, resolve, request, url, subdomain, port, level,'
f'cname, ip, public, cdn, status, reason, title, banner, header,' f'cname, ip, public, cdn, status, reason, title, banner, header,'
f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,' f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,'
f'module, source, elapse, find) ' f'module, source, elapse, find) '
f'values (:id, :alive, :resolve, :request, :new, :url,' f'values (:id, :alive, :resolve, :request, :url,'
f':subdomain, :port, :level, :cname, :ip, :public, :cdn,' f':subdomain, :port, :level, :cname, :ip, :public, :cdn,'
f':status, :reason, :title, :banner, :header, :history, :response,' f':status, :reason, :title, :banner, :header, :history, :response,'
f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,' f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,'
@@ -178,7 +191,7 @@ class Database(object):
def deduplicate_subdomain(self, table_name): def deduplicate_subdomain(self, table_name):
""" """
Deduplicates of subdomains in the table Deduplicate subdomains in the table
:param str table_name: table name :param str table_name: table name
""" """
@@ -199,17 +212,6 @@ class Database(object):
self.query(f'delete from "{table_name}" where ' self.query(f'delete from "{table_name}" where '
f'subdomain is null or resolve == 0') f'subdomain is null or resolve == 0')
def deal_table(self, deal_table_name, backup_table_name):
"""
Process the table when the collection task is complete
:param str deal_table_name: Pending table name
:param str backup_table_name: Table name for backup
"""
self.copy_table(deal_table_name, backup_table_name)
self.remove_invalid(deal_table_name)
self.deduplicate_subdomain(deal_table_name)
def get_data(self, table_name): def get_data(self, table_name):
""" """
Get all the data in the table Get all the data in the table
@@ -218,7 +220,7 @@ class Database(object):
""" """
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
logger.log('TRACE', f'Get all the data from {table_name} table') logger.log('TRACE', f'Get all the data from {table_name} table')
return self.query(f'select * from "{table_name}"') return self.query(f'select * from {table_name}')
def export_data(self, table_name, alive, limit): def export_data(self, table_name, alive, limit):
""" """
@@ -229,18 +231,42 @@ class Database(object):
:param str limit: limit value :param str limit: limit value
""" """
table_name = table_name.replace('.', '_') table_name = table_name.replace('.', '_')
query = f'select id, new, alive, request, resolve, url, subdomain, level,' \ sql = f'select id, alive, request, resolve, url, subdomain, level,' \
f'cname, ip, public, cdn, port, status, reason, title, banner,' \ f'cname, ip, public, cdn, port, status, reason, title, banner,' \
f'cidr, asn, org, addr, isp, source from "{table_name}"' f'cidr, asn, org, addr, isp, source from {table_name} order by subdomain'
if alive and limit: if alive and limit:
if limit in ['resolve', 'request']: if limit in ['resolve', 'request']:
where = f' where {limit} = 1' where = f' where {limit} = 1'
query += where sql += where
elif alive: elif alive:
where = f' where alive = 1' where = f' where alive = 1'
query += where sql += where
logger.log('TRACE', f'Get the data from {table_name} table') logger.log('TRACE', f'Get the data from {table_name} table')
return self.query(query) return self.query(sql)
def count_alive(self, table_name):
table_name = table_name.replace('.', '_')
sql = f'select count() from {table_name} where alive = 1'
return self.query(sql)
def get_resp_by_url(self, table_name, url):
table_name = table_name.replace('.', '_')
sql = f'select response from {table_name} where url = "{url}"'
logger.log('TRACE', f'Get response data from {url}')
return self.query(sql).scalar()
def get_data_by_fields(self, table_name, fields):
table_name = table_name.replace('.', '_')
field_str = ', '.join(fields)
sql = f"select {field_str} from {table_name}"
logger.log('TRACE', f'Get specified field data {fields} from {table_name} table')
return self.query(sql)
def update_data_by_url(self, table_name, info, url):
table_name = table_name.replace('.', '_')
field_str = ', '.join(map(lambda kv: f'{kv[0]} = "{kv[1]}"', info.items()))
sql = f'update {table_name} set {field_str} where url = "{url}"'
return self.query(sql)
def close(self): def close(self):
""" """
common/module.py
+1 -5
View File
@@ -275,7 +275,6 @@ class Module(object):
'alive': None, 'alive': None,
'request': None, 'request': None,
'resolve': None, 'resolve': None,
'new': None,
'url': None, 'url': None,
'subdomain': None, 'subdomain': None,
'port': None, 'port': None,
@@ -315,25 +314,22 @@ class Module(object):
ip = info.get('ip') ip = info.get('ip')
times = info.get('times') times = info.get('times')
ttl = info.get('ttl') ttl = info.get('ttl')
public = info.get('public')
if isinstance(cname, list): if isinstance(cname, list):
cname = ','.join(cname) cname = ','.join(cname)
ip = ','.join(ip) ip = ','.join(ip)
times = ','.join([str(num) for num in times]) times = ','.join([str(num) for num in times])
ttl = ','.join([str(num) for num in ttl]) ttl = ','.join([str(num) for num in ttl])
public = ','.join([str(num) for num in public])
result = {'id': None, result = {'id': None,
'alive': info.get('alive'), 'alive': info.get('alive'),
'request': info.get('request'), 'request': info.get('request'),
'resolve': info.get('resolve'), 'resolve': info.get('resolve'),
'new': None,
'url': url, 'url': url,
'subdomain': subdomain, 'subdomain': subdomain,
'port': 80, 'port': 80,
'level': level, 'level': level,
'cname': cname, 'cname': cname,
'ip': ip, 'ip': ip,
'public': public, 'public': info.get('public'),
'cdn': info.get('cdn'), 'cdn': info.get('cdn'),
'status': None, 'status': None,
'reason': info.get('reason'), 'reason': info.get('reason'),
common/request.py
+57 -54
View File
@@ -1,7 +1,6 @@
import json import json
from threading import Thread from threading import Thread
from queue import Queue from queue import Queue
from operator import attrgetter
import tqdm import tqdm
import requests import requests
@@ -9,6 +8,7 @@ from bs4 import BeautifulSoup
from common import utils from common import utils
from config.log import logger from config.log import logger
from common.database import Database
from config import settings from config import settings
@@ -127,7 +127,7 @@ def get_progress_bar(total):
return bar return bar
def get(url, resp_list, session): def get_resp(url, session):
timeout = settings.request_timeout_second timeout = settings.request_timeout_second
redirect = settings.request_allow_redirect redirect = settings.request_allow_redirect
proxy = utils.get_proxy() proxy = utils.get_proxy()
@@ -136,13 +136,14 @@ def get(url, resp_list, session):
except Exception as e: except Exception as e:
logger.log('DEBUG', e.args) logger.log('DEBUG', e.args)
resp = e resp = e
resp_list.append((url, resp)) return resp
def request(urls_queue, resp_list, session): def request(urls_queue, resp_queue, session):
while not urls_queue.empty(): while not urls_queue.empty():
url = urls_queue.get() index, url = urls_queue.get()
get(url, resp_list, session) resp = get_resp(url, session)
resp_queue.put((index, resp))
urls_queue.task_done() urls_queue.task_done()
@@ -168,31 +169,6 @@ def get_session():
return session return session
def bulk_request(urls):
logger.log('INFOR', 'Requesting urls in bulk')
resp_list = list()
urls_queue = Queue()
for url in urls:
urls_queue.put(url)
total = len(urls)
session = get_session()
thread_count = req_thread_count()
bar = get_progress_bar(total)
progress_thread = Thread(target=progress, name='ProgressThread',
args=(bar, total, urls_queue), daemon=True)
progress_thread.start()
for i in range(thread_count):
request_thread = Thread(target=request, name=f'RequestThread-{i}',
args=(urls_queue, resp_list, session), daemon=True)
request_thread.start()
urls_queue.join()
return resp_list
def gen_new_info(info, resp): def gen_new_info(info, resp):
if isinstance(resp, Exception): if isinstance(resp, Exception):
info['reason'] = str(resp.args) info['reason'] = str(resp.args)
@@ -220,13 +196,54 @@ def gen_new_info(info, resp):
return info return info
def gen_new_data(data, resp_list): def save(name, total, req_data, resp_queue):
new_data = list() db = Database()
for url, resp in resp_list: db.create_table(name)
for info in data: i = 0
if info.get('url') == url: while True:
new_data.append(gen_new_info(info, resp)) if not resp_queue.empty():
return new_data i += 1
index, resp = resp_queue.get()
old_info = req_data[index]
new_info = gen_new_info(old_info, resp)
db.insert_table(name, new_info)
resp_queue.task_done()
if i >= total:
break
db.close()
def bulk_request(domain, req_data, ret=False):
logger.log('INFOR', 'Requesting urls in bulk')
resp_queue = Queue()
urls_queue = Queue()
total = len(req_data)
for index, info in enumerate(req_data):
try:
url = info.get('url')
except Exception:
pass
urls_queue.put((index, url))
session = get_session()
thread_count = req_thread_count()
bar = get_progress_bar(total)
progress_thread = Thread(target=progress, name='ProgressThread',
args=(bar, total, urls_queue), daemon=True)
progress_thread.start()
for i in range(thread_count):
request_thread = Thread(target=request, name=f'RequestThread-{i}',
args=(urls_queue, resp_queue, session), daemon=True)
request_thread.start()
if ret:
urls_queue.join()
return resp_queue
save_thread = Thread(target=save, name=f'SaveThread',
args=(domain, total, req_data, resp_queue), daemon=True)
save_thread.start()
urls_queue.join()
save_thread.join()
def run_request(domain, data, port): def run_request(domain, data, port):
@@ -242,20 +259,6 @@ def run_request(domain, data, port):
data = utils.set_id_none(data) data = utils.set_id_none(data)
ports = get_port_seq(port) ports = get_port_seq(port)
req_data, req_urls = gen_req_data(data, ports) req_data, req_urls = gen_req_data(data, ports)
resp_list = bulk_request(req_urls) bulk_request(domain, req_data)
new_data = gen_new_data(req_data, resp_list) count = utils.count_alive(domain)
count = utils.count_alive(new_data)
logger.log('INFOR', f'Found that {domain} has {count} alive subdomains') logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
sorted_data = utils.sort_by_subdomain(new_data)
return sorted_data
def save_db(name, data):
"""
Save request results to database
:param str name: table name
:param list data: data to be saved
"""
logger.log('INFOR', f'Saving requested results')
utils.save_db(name, data, 'request')
common/resolve.py
+1 -28
View File
@@ -4,12 +4,6 @@ import json
from config.log import logger from config.log import logger
from config import settings from config import settings
from common import utils from common import utils
from common.ipasn import IPAsnInfo
from common.ipreg import IpRegData
ip_asn = IPAsnInfo()
ip_reg = IpRegData()
def filter_subdomain(data): def filter_subdomain(data):
@@ -79,13 +73,7 @@ def gen_infos(data, qname, info, infos):
flag = False flag = False
cname = list() cname = list()
ips = list() ips = list()
public = list()
ttl = list() ttl = list()
cidr = list()
asn = list()
org = list()
addr = list()
isp = list()
answers = data.get('answers') answers = data.get('answers')
for answer in answers: for answer in answers:
if answer.get('type') == 'A': if answer.get('type') == 'A':
@@ -94,25 +82,11 @@ def gen_infos(data, qname, info, infos):
ip = answer.get('data') ip = answer.get('data')
ips.append(ip) ips.append(ip)
ttl.append(str(answer.get('ttl'))) ttl.append(str(answer.get('ttl')))
public.append(str(utils.ip_is_public(ip)))
asn_info = ip_asn.find(ip)
cidr.append(asn_info.get('cidr'))
asn.append(asn_info.get('asn'))
org.append(asn_info.get('org'))
ip_info = ip_reg.query(ip)
addr.append(ip_info.get('addr'))
isp.append(ip_info.get('isp'))
info['resolve'] = 1 info['resolve'] = 1
info['reason'] = 'OK' info['reason'] = 'OK'
info['cname'] = ','.join(cname) info['cname'] = ','.join(cname)
info['ip'] = ','.join(ips) info['ip'] = ','.join(ips)
info['public'] = ','.join(public)
info['ttl'] = ','.join(ttl) info['ttl'] = ','.join(ttl)
info['cidr'] = ','.join(cidr)
info['asn'] = ','.join(asn)
info['org'] = ','.join(org)
info['addr'] = ','.join(addr)
info['isp'] = ','.join(isp)
infos[qname] = info infos[qname] = info
if not flag: if not flag:
info['alive'] = 0 info['alive'] = 0
@@ -181,8 +155,7 @@ def run_resolve(domain, data):
output_name = f'resolved_result_{domain}_{timestring}.json' output_name = f'resolved_result_{domain}_{timestring}.json'
output_path = temp_dir.joinpath(output_name) output_path = temp_dir.joinpath(output_name)
log_path = result_dir.joinpath('massdns.log') log_path = result_dir.joinpath('massdns.log')
ns_path = utils.get_ns_path()
ns_path = settings.brute_nameservers_path
logger.log('INFOR', f'Running massdns to resolve subdomains') logger.log('INFOR', f'Running massdns to resolve subdomains')
utils.call_massdns(massdns_path, save_path, ns_path, utils.call_massdns(massdns_path, save_path, ns_path,
common/utils.py
+80 -115
View File
@@ -176,16 +176,16 @@ def check_dir(dir_path):
dir_path.mkdir(parents=True, exist_ok=True) dir_path.mkdir(parents=True, exist_ok=True)
def check_path(path, name, format): def check_path(path, name, fmt):
""" """
检查结果输出目录路径 检查结果输出目录路径
:param path: 保存路径 :param path: 保存路径
:param name: 导出名字 :param name: 导出名字
:param format: 保存格式 :param fmt: 保存格式
:return: 保存路径 :return: 保存路径
""" """
filename = f'{name}.{format}' filename = f'{name}.{fmt}'
default_path = settings.result_save_dir.joinpath(filename) default_path = settings.result_save_dir.joinpath(filename)
if isinstance(path, str): if isinstance(path, str):
path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠 path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠
@@ -204,19 +204,18 @@ def check_path(path, name, format):
return path return path
def check_format(format, count): def check_format(fmt):
""" """
检查导出格式 检查导出格式
:param format: 传入的导出格式 :param fmt: 传入的导出格式
:param count: 数量
:return: 导出格式 :return: 导出格式
""" """
formats = ['csv', 'json', ] formats = ['csv', 'json', ]
if format in formats: if fmt in formats:
return format return fmt
else: else:
logger.log('ALERT', f'Does not support {format} format') logger.log('ALERT', f'Does not support {fmt} format')
logger.log('ALERT', 'So use csv format by default') logger.log('ALERT', 'So use csv format by default')
return 'csv' return 'csv'
@@ -333,8 +332,8 @@ def remove_invalid_string(string):
return re.sub(r'[\000-\010]|[\013-\014]|[\016-\037]', r'', string) return re.sub(r'[\000-\010]|[\013-\014]|[\016-\037]', r'', string)
def export_all_results(path, name, format, datas): def export_all_results(path, name, fmt, datas):
path = check_path(path, name, format) path = check_path(path, name, fmt)
logger.log('ALERT', f'The subdomain result for all main domains: {path}') logger.log('ALERT', f'The subdomain result for all main domains: {path}')
row_list = list() row_list = list()
for row in datas: for row in datas:
@@ -346,7 +345,7 @@ def export_all_results(path, name, format, datas):
values = row.values() values = row.values()
row_list.append(Record(keys, values)) row_list.append(Record(keys, values))
rows = RecordCollection(iter(row_list)) rows = RecordCollection(iter(row_list))
content = rows.export(format) content = rows.export(fmt)
save_data(path, content) save_data(path, content)
@@ -366,19 +365,19 @@ def export_all_subdomains(alive, path, name, datas):
save_data(path, data) save_data(path, data)
def export_all(alive, format, path, datas): def export_all(alive, fmt, path, datas):
""" """
将所有结果数据导出 将所有结果数据导出
:param bool alive: 只导出存活子域结果 :param bool alive: 只导出存活子域结果
:param str format: 导出文件格式 :param str fmt: 导出文件格式
:param str path: 导出文件路径 :param str path: 导出文件路径
:param list datas: 待导出的结果数据 :param list datas: 待导出的结果数据
""" """
format = check_format(format, len(datas)) fmt = check_format(fmt)
timestamp = get_timestring() timestamp = get_timestring()
name = f'all_subdomain_result_{timestamp}' name = f'all_subdomain_result_{timestamp}'
export_all_results(path, name, format, datas) export_all_results(path, name, fmt, datas)
export_all_subdomains(alive, path, name, datas) export_all_subdomains(alive, path, name, datas)
@@ -430,10 +429,18 @@ def python_version():
return sys.version return sys.version
def count_alive(data): def calc_alive(data):
return len(list(filter(lambda item: item.get('alive') == 1, data))) return len(list(filter(lambda item: item.get('alive') == 1, data)))
def count_alive(name):
db = Database()
result = db.count_alive(name)
count = result.scalar()
db.close()
return count
def get_subdomains(data): def get_subdomains(data):
return set(map(lambda item: item.get('subdomain'), data)) return set(map(lambda item: item.get('subdomain'), data))
@@ -511,15 +518,16 @@ def delete_file(*paths):
@tenacity.retry(stop=tenacity.stop_after_attempt(3)) @tenacity.retry(stop=tenacity.stop_after_attempt(3))
def check_net(): def check_net():
logger.log('INFOR', 'Checking Internet environment') urls = ['http://ipinfo.io/json', 'http://ipconfig.io/json']
urls = ['http://www.baidu.com', 'http://www.bing.com',
'http://www.apple.com', 'http://www.microsoft.com']
url = random.choice(urls) url = random.choice(urls)
logger.log('INFOR', f'Trying to access {url}') header = {'User_Agent': 'curl'}
timeout = settings.request_timeout_second
verify = settings.request_ssl_verify
logger.log('DEBUG', f'Trying to access {url}')
session = requests.Session() session = requests.Session()
session.trust_env = False session.trust_env = False
try: try:
rsp = session.get(url, proxies=get_proxy()) rsp = session.get(url, headers=header, timeout=timeout, verify=verify)
except Exception as e: except Exception as e:
logger.log('ERROR', e.args) logger.log('ERROR', e.args)
logger.log('ALERT', 'Can not access Internet, retrying') logger.log('ALERT', 'Can not access Internet, retrying')
@@ -529,10 +537,16 @@ def check_net():
f'{rsp.status_code} {rsp.reason}') f'{rsp.status_code} {rsp.reason}')
logger.log('ALERT', 'Can not access Internet normally, retrying') logger.log('ALERT', 'Can not access Internet normally, retrying')
raise tenacity.TryAgain raise tenacity.TryAgain
logger.log('INFOR', 'Access to Internet OK') logger.log('DEBUG', 'Access to Internet OK')
country = rsp.json().get('country').lower()
if country in ['cn', 'china']:
logger.log('DEBUG', f'The host in china')
return True, True
else:
return True, False
def check_pre(): def check_dep():
logger.log('INFOR', 'Checking dependent environment') logger.log('INFOR', 'Checking dependent environment')
implementation = platform.python_implementation() implementation = platform.python_implementation()
version = platform.python_version() version = platform.python_version()
@@ -544,15 +558,15 @@ def check_pre():
exit(1) exit(1)
def check_env(): def get_net_env():
logger.log('INFOR', 'Checking the environment') logger.log('INFOR', 'Checking network environment')
try: try:
check_net() result = check_net()
except Exception as e: except Exception as e:
logger.log('DEBUG', e.args) logger.log('DEBUG', e.args)
logger.log('FATAL', 'Can not access Internet') logger.log('ALERT', 'Can not access Internet')
exit(1) return False, None
check_pre() return result
def check_version(local): def check_version(local):
@@ -570,7 +584,7 @@ def check_version(local):
resp_json = resp.json() resp_json = resp.json()
latest = resp_json['tag_name'] latest = resp_json['tag_name']
except Exception as e: except Exception as e:
logger.log('ERROR', 'An error occurred while checking the latest version') logger.log('ALERT', 'An error occurred while checking the latest version')
logger.log('DEBUG', e.args) logger.log('DEBUG', e.args)
return return
if latest > local: if latest > local:
@@ -736,90 +750,6 @@ def sort_by_subdomain(data):
return sorted(data, key=lambda item: item.get('subdomain')) return sorted(data, key=lambda item: item.get('subdomain'))
def ping(host, path):
param = '-n' if platform.system().lower() == 'windows' else '-c'
command = ['ping', param, '5', host]
with open(path, "w") as f:
return subprocess.call(command, stdout=f, stderr=f)
def ping_avg_time(nameserver):
check_dir(settings.temp_save_dir)
temp_path = settings.temp_save_dir.joinpath('ping')
ping(nameserver, path=temp_path)
with open(temp_path, 'r') as f:
text = f.read()
if '100.0% packet loss' in text or '100% packet loss' in text or '100% 丢失' in text:
logger.log('ALERT', f'100.0% packet loss, ping {nameserver} failed.')
return None
elif platform.system() in ('Darwin', 'Linux'):
try:
avg_time = re.findall(r'(?:min/avg/max/.+ )(?:\d+\.\d+)/(\d+\.\d+)/', text)[0]
logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.')
except IndexError:
return None
return avg_time
elif platform.system() == 'Windows':
try:
avg_time = re.findall(r'(?:Average|平均).+(\d.?)ms', text)[0]
logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.')
except IndexError:
return None
return avg_time
else:
logger.log('ALERT', f'{text}')
return None
def auto_select_nameserver():
logger.log('INFOR', f'Ping test start, to select nameservers.')
avg_time1 = ping_avg_time('114.114.114.114')
avg_time2 = ping_avg_time('8.8.8.8')
if avg_time1 and avg_time2:
if avg_time1 < avg_time2:
change_nameservers_file('cn')
logger.log('INFOR', f'Ping test finished, use cn nameservers.')
else:
change_nameservers_file('common')
logger.log('INFOR', f'Ping test finished, use common nameservers.')
elif avg_time1 and not avg_time2:
change_nameservers_file('cn')
logger.log('INFOR', f'Ping test finished, use cn nameservers.')
elif not avg_time1 and avg_time1:
change_nameservers_file('common')
logger.log('INFOR', f'Ping test finished, use common nameservers.')
elif not avg_time1 and not avg_time1:
change_nameservers_file('default')
logger.log('INFOR', f'Ping test finished, use default nameservers.')
return
def change_nameservers_file(option):
text = ''
if option == 'cn':
with open(settings.data_storage_dir.joinpath('cn_nameservers.txt'), 'r') as f:
text = f.read()
elif option == 'common':
with open(settings.data_storage_dir.joinpath('common_nameservers.txt'), 'r') as f:
text = f.read()
elif option == 'default':
for n in default_nameserver():
text = '\n'.join(n)
with open(settings.data_storage_dir.joinpath('nameservers.txt'), 'w') as f:
f.write(text)
return
def default_nameserver():
try:
resolver = dns.resolver.Resolver()
return resolver.nameservers
except dns.resolver.NoResolverConfiguration:
logger.log('ERROR', 'Resolver configuration could not be read '
'or specified no nameservers.')
exit(1)
def looks_like_ip(maybe_ip): def looks_like_ip(maybe_ip):
"""Does the given str look like an IP address?""" """Does the given str look like an IP address?"""
if not maybe_ip[0].isdigit(): if not maybe_ip[0].isdigit():
@@ -833,3 +763,38 @@ def looks_like_ip(maybe_ip):
return True return True
except socket.error: except socket.error:
return False return False
def deal_data(domain):
db = Database()
db.remove_invalid(domain)
db.deduplicate_subdomain(domain)
db.close()
def get_data(domain):
db = Database()
data = db.get_data(domain).as_dict()
db.close()
return data
def clear_data(domain):
db = Database()
db.drop_table(domain)
db.close()
def get_ns_path(in_china=None, enable_wildcard=None, ns_ip_list=None):
data_dir = settings.data_storage_dir
path = data_dir.joinpath('nameservers.txt')
if in_china:
path = data_dir.joinpath('nameservers_cn.txt')
if not enable_wildcard:
return path
if not ns_ip_list:
return path
path = settings.authoritative_dns_path
ns_data = '\n'.join(ns_ip_list)
save_data(path, ns_data)
return path
config/default.py
+5 -8
View File
@@ -24,7 +24,7 @@ enable_dns_resolve = True # 使用DNS解析子域(默认True)
enable_http_request = True # 使用HTTP请求子域(默认True) enable_http_request = True # 使用HTTP请求子域(默认True)
enable_finder_module = True # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True) enable_finder_module = True # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True)
enable_altdns_module = False # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True) enable_altdns_module = False # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
enable_cdn_check = True # 开启cdn检查模块(默认True) enable_enrich_module = True # 开启enrich模块，开启会富化出信息，如ip的cdn，cidr，asn，org，addr和isp等信息
enable_banner_identify = True # 开启WEB指纹识别模块(默认True) enable_banner_identify = True # 开启WEB指纹识别模块(默认True)
enable_takeover_check = False # 开启子域接管风险检查(默认False) enable_takeover_check = False # 开启子域接管风险检查(默认False)
# 参数可选值有'small', 'medium', 'large' # 参数可选值有'small', 'medium', 'large'
@@ -56,17 +56,14 @@ brute_process_num = 1 # 默认1
brute_concurrent_num = 2000 # 并发查询数量(默认2000，最大推荐10000) brute_concurrent_num = 2000 # 并发查询数量(默认2000，最大推荐10000)
brute_socket_num = 1 # 爆破时每个进程下的socket数量 brute_socket_num = 1 # 爆破时每个进程下的socket数量
brute_resolve_num = 15 # 解析失败时尝试换名称服务器重查次数 brute_resolve_num = 15 # 解析失败时尝试换名称服务器重查次数
# 爆破所使用的字典路径 默认data/subdomains.txt # 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径)
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt') brute_wordlist_path = None
# 爆破所使用的字典路径 默认data/cn_nameservers.txt
# 如果你不在中国请改为nameservers.txt
brute_nameservers_path = data_storage_dir.joinpath('cn_nameservers.txt')
# 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询 # 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询
authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt') authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt')
enable_recursive_brute = False # 是否使用递归爆破(默认False) enable_recursive_brute = False # 是否使用递归爆破(默认False)
brute_recursive_depth = 2 # 递归爆破深度(默认2层) brute_recursive_depth = 2 # 递归爆破深度(默认2层)
# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt # 爆破下一层子域所使用的字典路径(默认None则使用data/subnames_next.txt，自定义字典请使用绝对路径)
recursive_nextlist_path = data_storage_dir.joinpath('next_subnames.txt') recursive_nextlist_path = None
enable_check_dict = False # 是否开启字典配置检查提示(默认False) enable_check_dict = False # 是否开启字典配置检查提示(默认False)
delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True) delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True)
delete_massdns_result = True # 是否删除爆破时massdns输出的解析结果 (默认True) delete_massdns_result = True # 是否删除爆破时massdns输出的解析结果 (默认True)
config/setting.py
+6 -12
View File
@@ -18,8 +18,8 @@ enable_finder_module = True # 开启finder模块,开启会从响应体和JS中
enable_cdn_check = True # 开启cdn检查模块(默认True) enable_cdn_check = True # 开启cdn检查模块(默认True)
enable_banner_identify = True # 开启WEB指纹识别模块(默认True) enable_banner_identify = True # 开启WEB指纹识别模块(默认True)
enable_takeover_check = False # 开启子域接管风险检查(默认False) enable_takeover_check = False # 开启子域接管风险检查(默认False)
# 参数可选值有'small', 'medium', 'large' # HTTP请求子域的端口范围 参数可选值有 'small', 'medium', 'large'
http_request_port = 'small' # HTTP请求子域(默认'small'，探测80,443端口) http_request_port = 'small' # 请求端口范围(默认 'small'，表示请求子域的80,443端口)
# 参数可选值True，False分别表示导出存活，全部子域结果 # 参数可选值True，False分别表示导出存活，全部子域结果
result_export_alive = False # 只导出存活的子域结果(默认False) result_export_alive = False # 只导出存活的子域结果(默认False)
result_save_format = 'csv' # 子域结果保存文件格式(默认csv) result_save_format = 'csv' # 子域结果保存文件格式(默认csv)
@@ -36,18 +36,12 @@ enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_mo
# 爆破模块设置 # 爆破模块设置
brute_concurrent_num = 2000 # 爆破时并发查询数量(默认2000，最大推荐10000) brute_concurrent_num = 2000 # 爆破时并发查询数量(默认2000，最大推荐10000)
# 爆破所使用的字典路径 默认data/subdomains.txt # 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径)
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt') brute_wordlist_path = None
# 爆破所使用的DNS服务器路径 默认data/cn_nameservers.txt 如果你不在中国请改为nameservers.txt
# DNS resolve server file path default data/nameservers.txt
# If your computer's location are not in China, change `cn_nameservers.txt` to `nameservers.txt` plz.
brute_nameservers_path = data_storage_dir.joinpath('cn_nameservers.txt')
# 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询
authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt')
enable_recursive_brute = False # 是否使用递归爆破(默认False) enable_recursive_brute = False # 是否使用递归爆破(默认False)
brute_recursive_depth = 2 # 递归爆破深度(默认2层) brute_recursive_depth = 2 # 递归爆破深度(默认2层)
# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt # 爆破下一层子域所使用的字典路径(默认None则使用data/subnames_next.txt，自定义字典请使用绝对路径)
recursive_nextlist_path = data_storage_dir.joinpath('next_subnames.txt') recursive_nextlist_path = None
enable_check_dict = False # 是否开启字典配置检查提示(默认False) enable_check_dict = False # 是否开启字典配置检查提示(默认False)
delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True) delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True)
# 是否删除爆破时massdns输出的解析结果 (默认True) # 是否删除爆破时massdns输出的解析结果 (默认True)
data/cdn_asn_list.json
+132 -132
View File
@@ -1,134 +1,134 @@
[ [
"10576", "AS10576",
"10762", "AS10762",
"11748", "AS11748",
"131099", "AS131099",
"132601", "AS132601",
"133496", "AS133496",
"134409", "AS134409",
"135295", "AS135295",
"136764", "AS136764",
"137187", "AS137187",
"13777", "AS13777",
"13890", "AS13890",
"14103", "AS14103",
"14520", "AS14520",
"17132", "AS17132",
"199251", "AS199251",
"200013", "AS200013",
"200325", "AS200325",
"200856", "AS200856",
"201263", "AS201263",
"202294", "AS202294",
"203075", "AS203075",
"203139", "AS203139",
"204248", "AS204248",
"204286", "AS204286",
"204545", "AS204545",
"206227", "AS206227",
"206734", "AS206734",
"206848", "AS206848",
"206986", "AS206986",
"207158", "AS207158",
"208559", "AS208559",
"209403", "AS209403",
"21030", "AS21030",
"21257", "AS21257",
"23327", "AS23327",
"23393", "AS23393",
"23637", "AS23637",
"23794", "AS23794",
"24997", "AS24997",
"26492", "AS26492",
"268843", "AS268843",
"28709", "AS28709",
"29264", "AS29264",
"30282", "AS30282",
"30637", "AS30637",
"328126", "AS328126",
"36408", "AS36408",
"38107", "AS38107",
"397192", "AS397192",
"40366", "AS40366",
"43303", "AS43303",
"44907", "AS44907",
"46071", "AS46071",
"46177", "AS46177",
"47542", "AS47542",
"49287", "AS49287",
"49689", "AS49689",
"51286", "AS51286",
"55082", "AS55082",
"55254", "AS55254",
"56636", "AS56636",
"57363", "AS57363",
"58127", "AS58127",
"59730", "AS59730",
"59776", "AS59776",
"60068", "AS60068",
"60626", "AS60626",
"60922", "AS60922",
"61107", "AS61107",
"61159", "AS61159",
"62026", "AS62026",
"62229", "AS62229",
"63062", "AS63062",
"64232", "AS64232",
"8868", "AS8868",
"9053", "AS9053",
"55770", "AS55770",
"49846", "AS49846",
"49249", "AS49249",
"48163", "AS48163",
"45700", "AS45700",
"43639", "AS43639",
"39836", "AS39836",
"393560", "AS393560",
"393234", "AS393234",
"36183", "AS36183",
"35994", "AS35994",
"35993", "AS35993",
"35204", "AS35204",
"34850", "AS34850",
"34164", "AS34164",
"33905", "AS33905",
"32787", "AS32787",
"31377", "AS31377",
"31110", "AS31110",
"31109", "AS31109",
"31108", "AS31108",
"31107", "AS31107",
"30675", "AS30675",
"24319", "AS24319",
"23903", "AS23903",
"23455", "AS23455",
"23454", "AS23454",
"22207", "AS22207",
"21399", "AS21399",
"21357", "AS21357",
"21342", "AS21342",
"20940", "AS20940",
"20189", "AS20189",
"18717", "AS18717",
"18680", "AS18680",
"17334", "AS17334",
"16702", "AS16702",
"16625", "AS16625",
"12222", "AS12222",
"209101", "AS209101",
"201585", "AS201585",
"135429", "AS135429",
"395747", "AS395747",
"394536", "AS394536",
"209242", "AS209242",
"203898", "AS203898",
"202623", "AS202623",
"14789", "AS14789",
"133877", "AS133877",
"13335", "AS13335",
"132892", "AS132892",
"21859", "AS21859",
"6185", "AS6185",
"47823", "AS47823",
"4134" "AS4134"
] ]
data/common_nameservers.txt
-30
View File
@@ -1,30 +0,0 @@
8.8.8.8
8.8.4.4
9.9.9.9
9.9.9.10
149.112.112.112
4.2.2.1
4.2.2.2
4.2.2.3
4.2.2.4
4.2.2.5
4.2.2.6
1.1.1.1
1.0.0.1
1.0.0.2
1.0.0.3
1.0.0.19
208.67.222.222
208.67.220.220
8.26.56.26
8.20.247.20
84.200.69.80
84.200.70.40
185.228.168.9
185.228.169.9
64.6.64.6
64.6.65.6
198.101.242.72
23.253.163.53
176.103.130.130
176.103.130.131
data/nameservers.txt
+30
View File
@@ -0,0 +1,30 @@
8.8.8.8
8.8.4.4
9.9.9.9
9.9.9.10
149.112.112.112
4.2.2.1
4.2.2.2
4.2.2.3
4.2.2.4
4.2.2.5
4.2.2.6
1.1.1.1
1.0.0.1
1.0.0.2
1.0.0.3
1.0.0.19
208.67.222.222
208.67.220.220
8.26.56.26
8.20.247.20
84.200.69.80
84.200.70.40
185.228.168.9
185.228.169.9
64.6.64.6
64.6.65.6
198.101.242.72
23.253.163.53
176.103.130.130
176.103.130.131
data/cn_nameservers.txt → data/nameservers_cn.txt
View File
data/big_subnames.7z → data/subnames_big.7z
BIN
View File
Binary file not shown.
data/medium_subnames.txt → data/subnames_medium.txt
View File
data/next_subnames.txt → data/subnames_next.txt
View File
dbexport.py
+12 -12
View File
@@ -15,17 +15,17 @@ from common.database import Database
from config.log import logger from config.log import logger
def export(target, type='target', db=None, alive=False, limit=None, path=None, format='csv', show=False): def export(target, type='target', db=None, alive=False, limit=None, path=None, fmt='csv', show=False):
""" """
OneForAll export from database module OneForAll export from database module
Example: Example:
python3 dbexport.py --target name --format csv --dir= ./result.csv python3 dbexport.py --target name --fmt csv --dir= ./result.csv
python3 dbexport.py --db result.db --target name --show False python3 dbexport.py --db result.db --target name --show False
python3 dbexport.py --target table_name --tb True --show False python3 dbexport.py --target table_name --tb True --show False
Note: Note:
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) --fmt csv/json (result format)
--path Result directory (default directory is ./results) --path Result directory (default directory is ./results)
:param str target: Table to be exported :param str target: Table to be exported
@@ -33,7 +33,7 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f
:param str db: Database path to be exported (default ./results/result.sqlite3) :param str db: Database path to be exported (default ./results/result.sqlite3)
:param bool alive: Only export the results of alive subdomains (default False) :param bool alive: Only export the results of alive subdomains (default False)
:param str limit: Export limit (default None) :param str limit: Export limit (default None)
:param str format: Result format (default csv) :param str fmt: Result format (default csv)
:param str path: Result directory (default None) :param str path: Result directory (default None)
:param bool show: Displays the exported data in terminal (default False) :param bool show: Displays the exported data in terminal (default False)
""" """
@@ -48,29 +48,29 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f
rows = database.export_data(table_name, alive, limit) rows = database.export_data(table_name, alive, limit)
if rows is None: if rows is None:
continue continue
data = export_data(format, path, rows, show, table_name, target) data = export_data(fmt, path, rows, show, table_name, target)
datas.extend(data) datas.extend(data)
database.close() database.close()
if len(domains) > 1: if len(domains) > 1:
utils.export_all(alive, format, path, datas) utils.export_all(alive, fmt, path, datas)
elif type == 'table': elif type == 'table':
database = Database(db) database = Database(db)
rows = database.export_data(target, alive, limit) rows = database.export_data(target, alive, limit)
data = export_data(format, path, rows, show, target, target) data, _, _ = export_data(fmt, path, rows, show, target, target)
database.close() database.close()
return data return data
def export_data(format, path, rows, show, table_name, target): def export_data(fmt, path, rows, show, table_name, target):
format = utils.check_format(format, len(rows)) fmt = utils.check_format(fmt)
path = utils.check_path(path, target, format) path = utils.check_path(path, target, fmt)
if show: if show:
print(rows.dataset) print(rows.dataset)
data = rows.export(format) data = rows.export(fmt)
utils.save_data(path, data) utils.save_data(path, data)
logger.log('ALERT', f'The subdomain result for {table_name}: {path}') logger.log('ALERT', f'The subdomain result for {table_name}: {path}')
data = rows.as_dict() data = rows.as_dict()
return data, format, path return data, fmt, path
def domain_to_table(table): def domain_to_table(table):
docs/directory_structure.md
+3 -3
View File
@@ -47,11 +47,11 @@ D:.
| |
+---data 存放一些所需数据 +---data 存放一些所需数据
| authoritative_dns.txt 临时存放开启了泛解析域名的权威DNS名称服务器IP地址 | authoritative_dns.txt 临时存放开启了泛解析域名的权威DNS名称服务器IP地址
| big_subnames.txt 子域爆破超大字典 | subnames_big.7z 子域爆破超大字典
| cn_nameservers.txt 中国主流名称服务器IP地址 | nameservers_cn.txt 中国主流名称服务器IP地址
| fingerprints.json 检查子域接管风险的指纹 | fingerprints.json 检查子域接管风险的指纹
| nameservers.txt 全球主流名称服务器IP地址 | nameservers.txt 全球主流名称服务器IP地址
| next_subnames.txt 下一层子域字典 | subnames_next.txt 下一层子域字典
| public_suffix_list.dat 顶级域名后缀 | public_suffix_list.dat 顶级域名后缀
| srv_prefixes.json 常见SRV记录前缀名 | srv_prefixes.json 常见SRV记录前缀名
| subnames.txt 子域爆破常见字典 | subnames.txt 子域爆破常见字典
docs/en-us/README.md
+3 -6
View File
@@ -89,8 +89,6 @@ Result will be saved in `~/results`.
<details> <details>
<summary><b>✨OneForAll usage</b></summary> <summary><b>✨OneForAll usage</b></summary>
If your computer are not in China, change [setting](https://github.com/shmilylty/OneForAll/blob/master/config/setting.py#L46) `brute_nameservers_path` param `cn_nameservers.txt` to `nameservers.txt` plz.
If you are use pip3, run the following command: If you are use pip3, run the following command:
```bash ```bash
@@ -164,7 +162,7 @@ DESCRIPTION
python3 oneforall.py --target example.com --alive False run python3 oneforall.py --target example.com --alive False run
python3 oneforall.py --target example.com --brute True run python3 oneforall.py --target example.com --brute True run
python3 oneforall.py --target example.com --port medium run python3 oneforall.py --target example.com --port medium run
python3 oneforall.py --target example.com --format csv run python3 oneforall.py --target example.com --fmt csv run
python3 oneforall.py --target example.com --dns False run python3 oneforall.py --target example.com --dns False run
python3 oneforall.py --target example.com --req False run python3 oneforall.py --target example.com --req False run
python3 oneforall.py --target example.com --takeover False run python3 oneforall.py --target example.com --takeover False run
@@ -173,7 +171,7 @@ DESCRIPTION
Note: Note:
--alive True/False Only export alive subdomains or not (default False) --alive True/False Only export alive subdomains or not (default False)
--port default/small/large See details in ./config/setting.py(default port 80) --port default/small/large See details in ./config/setting.py(default port 80)
--format csv/json (result format) --fmt csv/json (result format)
--path Result directory (default directory is ./results) --path Result directory (default directory is ./results)
ARGUMENTS ARGUMENTS
@@ -193,7 +191,7 @@ FLAGS
The port range request to the subdomains (default port 80) The port range request to the subdomains (default port 80)
--alive=ALIVE --alive=ALIVE
Only export alive subdomains (default False) Only export alive subdomains (default False)
--format=FORMAT --fmt=FMT
Result format (default csv) Result format (default csv)
--path=PATH --path=PATH
Result directory (default None) Result directory (default None)
@@ -270,7 +268,6 @@ The project uses [SemVer](https://semver.org/) for version management, and you c
## ⌛Follow-up plan ## ⌛Follow-up plan
- [ ] Continuous optimize and improve of each module - [ ] Continuous optimize and improve of each module
- [x] Subdomain monitoring (mark newly discovered subdomain)
- [x] Subdomain collection crawler (collect subdomains from static files such as JS) - [x] Subdomain collection crawler (collect subdomains from static files such as JS)
- [ ] Implementation of front-end interface for powerful interaction - [ ] Implementation of front-end interface for powerful interaction
modules/altdns.py
+2 -4
View File
@@ -193,9 +193,7 @@ class Altdns(Module):
self.end = time.time() self.end = time.time()
self.elapse = round(self.end - self.start, 1) self.elapse = round(self.end - self.start, 1)
self.gen_result() self.gen_result()
temp_data = resolve.run_resolve(self.domain, self.results) resolved_data = resolve.run_resolve(self.domain, self.results)
fina_data = request.run_request(self.domain, temp_data, port) request.run_request(self.domain, resolved_data, port)
data = data + fina_data
logger.log('INFOR', f'Saving altdns results') logger.log('INFOR', f'Saving altdns results')
utils.save_db(self.domain, data, 'altdns') utils.save_db(self.domain, data, 'altdns')
return data
modules/datasets/phonebook.py
-61
View File
@@ -1,61 +0,0 @@
from config.log import logger
from common.query import Query
class PhoneBook(Query):
def __init__(self, domain):
Query.__init__(self)
self.domain = domain
self.module = 'Dataset'
self.source = 'PhoneBookQuery'
def query(self):
"""
向接口查询子域并做子域匹配
"""
self.header = self.get_header()
self.proxy = self.get_proxy(self.source)
self.header.update({'Referer': 'https://phonebook.cz/',
'Origin': 'https://phonebook.cz'})
addr = 'https://public.intelx.io/phonebook/search'
key = 'd7d1ed06-f0c5-49d4-a9ca-a167e6d2ffab'
url = f'{addr}?k={key}'
data = {"term": self.domain, "maxresults": 10000,
"media": 0, "target": 1,
"terminate": [], "timeout": 20}
resp = self.post(url, json=data)
if not resp:
return
json = resp.json()
ids = json.get('id')
if not ids:
logger.log('ALERT', f'Get PhoneBook id fail')
return
url = f'{addr}/result?k={key}&id={ids}&limit=10000'
resp = self.get(url)
self.subdomains = self.collect_subdomains(resp)
def run(self):
"""
类执行入口
"""
self.begin()
self.query()
self.finish()
self.save_json()
self.gen_result()
self.save_db()
def run(domain):
"""
类统一调用入口
:param str domain: 域名
"""
query = PhoneBook(domain)
query.run()
if __name__ == '__main__':
run('freebuf.com')
modules/datasets/robtex.py
+1 -1
View File
@@ -17,7 +17,7 @@ class Robtex(Query):
""" """
self.header = self.get_header() self.header = self.get_header()
self.proxy = self.get_proxy(self.source) self.proxy = self.get_proxy(self.source)
base_addr = 'https://freeapi.robtex.com/pdns/' base_addr = 'https://freeapi.robtex.com/pdns'
url = f'{base_addr}/forward/{self.domain}' url = f'{base_addr}/forward/{self.domain}'
resp = self.get(url) resp = self.get(url)
if not resp: if not resp:
modules/enrich.py
+72
View File
@@ -0,0 +1,72 @@
from modules import iscdn
from common import utils
from common.database import Database
from common.ipasn import IPAsnInfo
from common.ipreg import IpRegData
def get_ips(info):
ip = info.get('ip')
if not ip:
return None
ips = ip.split(',')
return ips
def enrich_info(data):
ip_asn = IPAsnInfo()
ip_reg = IpRegData()
for index, info in enumerate(data):
ips = get_ips(info)
if not ips:
continue
public = list()
cidr = list()
asn = list()
org = list()
addr = list()
isp = list()
for ip in ips:
public.append(str(utils.ip_is_public(ip)))
asn_info = ip_asn.find(ip)
cidr.append(asn_info.get('cidr'))
asn.append(asn_info.get('asn'))
org.append(asn_info.get('org'))
ip_info = ip_reg.query(ip)
addr.append(ip_info.get('addr'))
isp.append(ip_info.get('isp'))
data[index]['public'] = ','.join(public)
data[index]['cidr'] = ','.join(cidr)
data[index]['asn'] = ','.join(asn)
data[index]['org'] = ','.join(org)
data[index]['addr'] = ','.join(addr)
data[index]['isp'] = ','.join(isp)
return data
class Enrich(object):
def __init__(self, domain):
self.domain = domain
def get_data(self):
db = Database()
fields = ['url', 'cname', 'ip', 'public', 'cdn', 'header',
'cidr', 'asn', 'org', 'addr', 'isp']
results = db.get_data_by_fields(self.domain, fields)
return results.as_dict()
def save_db(self, data):
db = Database()
for info in data:
url = info.pop('url')
info.pop('cname')
info.pop('ip')
info.pop('header')
db.update_data_by_url(self.domain, info, url)
db.close()
def run(self):
data = self.get_data()
data = enrich_info(data)
data = iscdn.do_check(data)
self.save_db(data)
modules/finder.py
+15 -10
View File
@@ -7,6 +7,7 @@ from common import utils
from common import resolve from common import resolve
from common import request from common import request
from common.module import Module from common.module import Module
from common.database import Database
from config import settings from config import settings
from config.log import logger from config.log import logger
@@ -25,16 +26,11 @@ class Finder(Module):
new_subdomains = found_subdomains - existing_subdomains new_subdomains = found_subdomains - existing_subdomains
if not len(new_subdomains): if not len(new_subdomains):
self.finish() # 未发现新的子域就直接返回 self.finish() # 未发现新的子域就直接返回
return data
self.subdomains = new_subdomains self.subdomains = new_subdomains
self.finish() self.finish()
self.gen_result() self.gen_result()
temp_data = resolve.run_resolve(domain, self.results) resolved_data = resolve.run_resolve(domain, self.results)
fina_data = request.run_request(domain, temp_data, port) request.run_request(domain, resolved_data, port)
data = data + fina_data
logger.log('INFOR', f'Saving finder results')
utils.save_db(domain, data, 'finder')
return data
file_path = settings.data_storage_dir.joinpath('common_js_library.json') file_path = settings.data_storage_dir.joinpath('common_js_library.json')
@@ -177,22 +173,31 @@ def find_js_urls(domain, req_url, rsp_html):
return js_urls return js_urls
def convert_to_dict(url_list):
url_dict = []
for url in url_list:
url_dict.append({'url': url})
return url_dict
def find_subdomains(domain, data): def find_subdomains(domain, data):
subdomains = set() subdomains = set()
js_urls = set() js_urls = set()
db = Database()
for infos in data: for infos in data:
jump_history = infos.get('history') jump_history = infos.get('history')
req_url = infos.get('url') req_url = infos.get('url')
subdomains.update(find_in_history(domain, req_url, jump_history)) subdomains.update(find_in_history(domain, req_url, jump_history))
rsp_html = infos.get('response') rsp_html = db.get_resp_by_url(domain, req_url)
if not rsp_html: if not rsp_html:
logger.log('DEBUG', f'an abnormal response occurred in the request {req_url}') logger.log('DEBUG', f'an abnormal response occurred in the request {req_url}')
continue continue
subdomains.update(find_in_resp(domain, req_url, rsp_html)) subdomains.update(find_in_resp(domain, req_url, rsp_html))
js_urls.update(find_js_urls(domain, req_url, rsp_html)) js_urls.update(find_js_urls(domain, req_url, rsp_html))
resp_data = request.bulk_request(js_urls) req_data = convert_to_dict(js_urls)
for _, resp in resp_data: resp_data = request.bulk_request(domain, req_data, ret=True)
while not resp_data.empty():
_, resp = resp_data.get()
if not isinstance(resp, Response): if not isinstance(resp, Response):
continue continue
text = utils.decode_resp_text(resp) text = utils.decode_resp_text(resp)
modules/iscdn.py
+35 -33
View File
@@ -3,6 +3,7 @@ import ipaddress
from config import settings from config import settings
from common import utils from common import utils
from common.database import Database
from config.log import logger from config.log import logger
data_dir = settings.data_storage_dir data_dir = settings.data_storage_dir
@@ -18,6 +19,8 @@ cdn_header_key = utils.load_json(data_dir.joinpath('cdn_header_keys.json'))
def check_cname_keyword(cname): def check_cname_keyword(cname):
if not cname:
return False
names = cname.lower().split(',') names = cname.lower().split(',')
for name in names: for name in names:
for keyword in cdn_cname_keyword.keys(): for keyword in cdn_cname_keyword.keys():
@@ -26,14 +29,22 @@ def check_cname_keyword(cname):
def check_header_key(header): def check_header_key(header):
header = set(map(lambda x: x.lower(), header.keys())) if isinstance(header, str):
for key in cdn_header_key: header = json.loads(header)
if key in header: if isinstance(header, dict):
return True header = set(map(lambda x: x.lower(), header.keys()))
for key in cdn_header_key:
if key in header:
return True
else:
return False
def check_cdn_cidr(content): def check_cdn_cidr(ips):
ips = set(content.split(',')) if isinstance(ips, str):
ips = set(ips.split(','))
else:
return False
for ip in ips: for ip in ips:
try: try:
ip = ipaddress.ip_address(ip) ip = ipaddress.ip_address(ip)
@@ -46,39 +57,30 @@ def check_cdn_cidr(content):
def check_cdn_asn(asn): def check_cdn_asn(asn):
if str(asn) in cdn_asn_list: if isinstance(asn, str):
return True if asn in cdn_asn_list:
return True
return False
def check_cdn(data): def do_check(data):
logger.log('DEBUG', f'Start cdn check module') logger.log('DEBUG', f'Checking cdn')
for index, item in enumerate(data): for index, item in enumerate(data):
cname = item.get('cname') cname = item.get('cname')
if cname: if check_cname_keyword(cname):
if check_cname_keyword(cname): data[index]['cdn'] = 1
data[index]['cdn'] = 1 continue
continue
header = item.get('header') header = item.get('header')
if header: if check_header_key(header):
header = json.loads(header) data[index]['cdn'] = 1
if check_header_key(header): continue
data[index]['cdn'] = 1
continue
ip = item.get('ip') ip = item.get('ip')
if ip: if check_cdn_cidr(ip):
if check_cdn_cidr(ip): data[index]['cdn'] = 1
data[index]['cdn'] = 1 continue
continue
asn = item.get('asn') asn = item.get('asn')
if asn: if check_cdn_asn(asn):
asn = asn[2:] # 去除AS data[index]['cdn'] = 1
if check_cdn_asn(asn): continue
data[index]['cdn'] = 1
continue
data[index]['cdn'] = 0 data[index]['cdn'] = 0
return data return data
def save_db(name, data):
logger.log('DEBUG', f'Saving cdn check results')
utils.save_db(name, data, 'cdn')
modules/search/baidu.py
+1 -1
View File
@@ -57,7 +57,7 @@ class Baidu(Search):
self.subdomains.update(subdomains) self.subdomains.update(subdomains)
self.page_num += self.per_page_num self.page_num += self.per_page_num
# 搜索页面没有出现下一页时停止搜索 # 搜索页面没有出现下一页时停止搜索
if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text: if f'&pn={self.page_num}&' not in resp.text:
break break
if self.page_num >= self.limit_num: # 搜索条数限制 if self.page_num >= self.limit_num: # 搜索条数限制
break break
oneforall.py
+45 -88
View File
@@ -15,12 +15,11 @@ from datetime import datetime
import dbexport import dbexport
from brute import Brute from brute import Brute
from common import utils, resolve, request from common import utils, resolve, request
from common.database import Database
from modules.collect import Collect from modules.collect import Collect
from modules.srv import BruteSRV from modules.srv import BruteSRV
from modules.finder import Finder from modules.finder import Finder
from modules.altdns import Altdns from modules.altdns import Altdns
from modules import iscdn from modules.enrich import Enrich
from config import settings from config import settings
from config.log import logger from config.log import logger
from takeover import Takeover from takeover import Takeover
@@ -60,7 +59,7 @@ class OneForAll(object):
python3 oneforall.py --target example.com --alive False run python3 oneforall.py --target example.com --alive False run
python3 oneforall.py --target example.com --brute False run python3 oneforall.py --target example.com --brute False run
python3 oneforall.py --target example.com --port medium run python3 oneforall.py --target example.com --port medium run
python3 oneforall.py --target example.com --format csv run python3 oneforall.py --target example.com --fmt csv run
python3 oneforall.py --target example.com --dns False run python3 oneforall.py --target example.com --dns False run
python3 oneforall.py --target example.com --req False run python3 oneforall.py --target example.com --req False run
python3 oneforall.py --target example.com --takeover False run python3 oneforall.py --target example.com --takeover False run
@@ -68,22 +67,22 @@ class OneForAll(object):
Note: Note:
--port small/medium/large See details in ./config/setting.py(default small) --port small/medium/large See details in ./config/setting.py(default small)
--format csv/json (result format) --fmt csv/json (result format)
--path Result path (default None, automatically generated) --path Result path (default None, automatically generated)
:param str target: One domain (target or targets must be provided) :param str target: One domain (target or targets must be provided)
:param str targets: File path of one domain per line :param str targets: File path of one domain per line
:param bool brute: Use brute module (default True) :param bool brute: Use brute module (default True)
:param bool dns: Use DNS resolution (default True) :param bool dns: Use DNS resolution (default True)
:param bool req: HTTP request subdomains (default True) :param bool req: HTTP request subdomains (default True)
:param str port: The port range to request (default small port is 80,443) :param str port: The port range to request (default small port is 80,443)
:param bool alive: Only export alive subdomains (default False) :param bool alive: Only export alive subdomains (default False)
:param str format: Result format (default csv) :param str fmt: Result format (default csv)
:param str path: Result path (default None, automatically generated) :param str path: Result path (default None, automatically generated)
:param bool takeover: Scan subdomain takeover (default False) :param bool takeover: Scan subdomain takeover (default False)
""" """
def __init__(self, target=None, targets=None, brute=None, dns=None, req=None, def __init__(self, target=None, targets=None, brute=None, dns=None, req=None,
port=None, alive=None, format=None, path=None, takeover=None): port=None, alive=None, fmt=None, path=None, takeover=None):
self.target = target self.target = target
self.targets = targets self.targets = targets
self.brute = brute self.brute = brute
@@ -91,17 +90,15 @@ class OneForAll(object):
self.req = req self.req = req
self.port = port self.port = port
self.alive = alive self.alive = alive
self.format = format self.fmt = fmt
self.path = path self.path = path
self.takeover = takeover self.takeover = takeover
self.domain = str() # The domain currently being collected self.domain = str() # The domain currently being collected
self.domains = set() # All domains that are to be collected self.domains = set() # All domains that are to be collected
self.data = list() # The subdomain results of the current domain self.data = list() # The subdomain results of the current domain
self.datas = list() # All subdomain results of the domain self.datas = list() # All subdomain results of the domain
self.old_table = str() # The table name of the last result self.in_china = None
self.new_table = str() # The table name of the current result self.access_internet = False
self.origin_table = str() # The table name of the origin result
self.resolve_table = str() # The table name of the resolute result
def config_param(self): def config_param(self):
""" """
@@ -119,8 +116,8 @@ class OneForAll(object):
self.port = settings.http_request_port self.port = settings.http_request_port
if self.alive is None: if self.alive is None:
self.alive = bool(settings.result_export_alive) self.alive = bool(settings.result_export_alive)
if self.format is None: if self.fmt is None:
self.format = settings.result_save_format self.fmt = settings.result_save_format
if self.path is None: if self.path is None:
self.path = settings.result_save_path self.path = settings.result_save_path
@@ -134,47 +131,13 @@ class OneForAll(object):
def export(self, table): def export(self, table):
""" """
Export data from the database and do some follow-up processing Export data from the database
:param table: table name :param table: table name
:return: export data :return: export data
:rtype: list :rtype: list
""" """
db = Database() return dbexport.export(table, type='table', alive=self.alive, fmt=self.fmt)
data = dbexport.export(table, type='table', alive=self.alive, format=self.format)
db.drop_table(self.new_table)
db.rename_table(self.domain, self.new_table)
db.close()
return data
def deal_db(self):
"""
Process the data when the collection task is completed
"""
db = Database()
db.deal_table(self.domain, self.origin_table)
db.close()
def mark(self):
"""
Mark the new discovered subdomain
:return: marked data
:rtype: list
"""
db = Database()
old_data = list()
now_data = db.get_data(self.domain).as_dict()
# Database pre-processing when it is not the first time to collect this subdomain
if db.exist_table(self.new_table):
# If there is the last collection result table, delete it first
db.drop_table(self.old_table)
# Rename the new table to the old table
db.rename_table(self.new_table, self.old_table)
old_data = db.get_data(self.old_table).as_dict()
db.close()
marked_data = utils.mark_subdomain(old_data, now_data)
return marked_data
def main(self): def main(self):
""" """
@@ -183,13 +146,12 @@ class OneForAll(object):
:return: subdomain results :return: subdomain results
:rtype: list :rtype: list
""" """
self.old_table = self.domain + '_old_result' if not self.access_internet:
self.new_table = self.domain + '_now_result' logger.log('ALERT', 'Because it cannot access the Internet, '
self.origin_table = self.domain + '_origin_result' 'OneForAll will not execute the subdomain collection module!')
self.resolve_table = self.domain + '_resolve_result' if self.access_internet:
collect = Collect(self.domain)
collect = Collect(self.domain) collect.run()
collect.run()
srv = BruteSRV(self.domain) srv = BruteSRV(self.domain)
srv.run() srv.run()
@@ -198,53 +160,48 @@ class OneForAll(object):
# Due to there will be a large number of dns resolution requests, # Due to there will be a large number of dns resolution requests,
# may cause other network tasks to be error # may cause other network tasks to be error
brute = Brute(self.domain, word=True, export=False) brute = Brute(self.domain, word=True, export=False)
brute.check_env = False brute.in_china = self.in_china
brute.quite = True brute.quite = True
brute.run() brute.run()
# Database processing utils.deal_data(self.domain)
self.deal_db()
# Mark the new discovered subdomain
self.data = self.mark()
# Export results without resolve # Export results without resolve
if not self.dns: if not self.dns:
return self.export(self.domain) return self.export(self.domain)
self.data = utils.get_data(self.domain)
# Resolve subdomains # Resolve subdomains
utils.clear_data(self.domain)
self.data = resolve.run_resolve(self.domain, self.data) self.data = resolve.run_resolve(self.domain, self.data)
# Save resolve results # Save resolve results
resolve.save_db(self.resolve_table, self.data) resolve.save_db(self.domain, self.data)
# Export results without HTTP request # Export results without HTTP request
if not self.req: if not self.req:
return self.export(self.resolve_table) return self.export(self.domain)
# HTTP request # HTTP request
self.data = request.run_request(self.domain, self.data, self.port) utils.clear_data(self.domain)
# Save HTTP request result request.run_request(self.domain, self.data, self.port)
request.save_db(self.domain, self.data)
# Finder module # Finder module
if settings.enable_finder_module: if settings.enable_finder_module:
finder = Finder() finder = Finder()
self.data = finder.run(self.domain, self.data, self.port) finder.run(self.domain, self.data, self.port)
# altdns module # altdns module
if settings.enable_altdns_module: if settings.enable_altdns_module:
finder = Altdns(self.domain) altdns = Altdns(self.domain)
self.data = finder.run(self.data, self.port) altdns.run(self.data, self.port)
# check cdn module # Information enrichment module
if settings.enable_cdn_check: if settings.enable_enrich_module:
self.data = iscdn.check_cdn(self.data) enrich = Enrich(self.domain)
iscdn.save_db(self.domain, self.data) enrich.run()
# Add the final result list to the total data list
self.datas.extend(self.data)
# Export # Export
self.export(self.domain) self.datas.extend(self.export(self.domain))
# Scan subdomain takeover # Scan subdomain takeover
if self.takeover: if self.takeover:
@@ -263,12 +220,12 @@ class OneForAll(object):
print(oneforall_banner) print(oneforall_banner)
dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S') dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(f'[*] Starting OneForAll @ {dt}\n') print(f'[*] Starting OneForAll @ {dt}\n')
utils.check_env()
utils.auto_select_nameserver()
if settings.enable_check_version:
utils.check_version(version)
logger.log('DEBUG', 'Python ' + utils.python_version()) logger.log('DEBUG', 'Python ' + utils.python_version())
logger.log('DEBUG', 'OneForAll ' + version) logger.log('DEBUG', 'OneForAll ' + version)
utils.check_dep()
self.access_internet, self.in_china = utils.get_net_env()
if self.access_internet and settings.enable_check_version:
utils.check_version(version)
logger.log('INFOR', 'Start running OneForAll') logger.log('INFOR', 'Start running OneForAll')
self.config_param() self.config_param()
self.check_param() self.check_param()
@@ -277,7 +234,7 @@ class OneForAll(object):
for domain in self.domains: for domain in self.domains:
self.domain = utils.get_main_domain(domain) self.domain = utils.get_main_domain(domain)
self.main() self.main()
utils.export_all(self.alive, self.format, self.path, self.datas) utils.export_all(self.alive, self.fmt, self.path, self.datas)
else: else:
logger.log('FATAL', 'Failed to obtain domain') logger.log('FATAL', 'Failed to obtain domain')
logger.log('INFOR', 'Finished OneForAll') logger.log('INFOR', 'Finished OneForAll')
requirements.txt
+10 -10
View File
@@ -1,23 +1,23 @@
-i https://mirrors.aliyun.com/pypi/simple/ -i https://mirrors.aliyun.com/pypi/simple/
beautifulsoup4==4.9.1 beautifulsoup4==4.9.3
bs4==0.0.1 bs4==0.0.1
certifi==2020.6.20 certifi==2020.6.20
chardet==3.0.4 chardet==3.0.4
colorama==0.4.3 ; sys_platform == 'win32' colorama==0.4.4; sys_platform == 'win32'
dnspython==2.0.0 dnspython==2.0.0
exrex==0.10.5 exrex==0.10.5
fire==0.3.1 fire==0.3.1
future==0.18.2 future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
idna==2.10 idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
loguru==0.5.3 loguru==0.5.3
pysocks==1.7.1 pysocks==1.7.1
requests==2.24.0 requests==2.24.0
six==1.15.0 six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
soupsieve==2.0.1 soupsieve==2.0.1; python_version >= '3.0'
sqlalchemy==1.3.19 sqlalchemy==1.3.20
tenacity==6.2.0 tenacity==6.2.0
termcolor==1.1.0 termcolor==1.1.0
tqdm==4.49.0 tqdm==4.51.0
treelib==1.6.1 treelib==1.6.1
urllib3==1.25.10 urllib3==1.25.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
win32-setctime==1.0.2 ; sys_platform == 'win32' win32-setctime==1.0.3; sys_platform == 'win32'
takeover.py
+9 -9
View File
@@ -45,21 +45,21 @@ class Takeover(Module):
OneForAll subdomain takeover module OneForAll subdomain takeover module
Example: Example:
python3 takeover.py --target www.example.com --format csv run python3 takeover.py --target www.example.com --fmt csv run
python3 takeover.py --targets ./subdomains.txt --thread 10 run python3 takeover.py --targets ./subdomains.txt --thread 10 run
Note: Note:
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) --fmt txt/csv/json (result format)
--path Result directory (default directory is ./results) --path Result directory (default directory is ./results)
:param str target: One domain (target or targets must be provided) :param str target: One domain (target or targets must be provided)
:param str targets: File path of one domain per line :param str targets: File path of one domain per line
:param int thread: threads number (default 20) :param int thread: threads number (default 20)
:param str format: Result format (default csv) :param str fmt: Result format (default csv)
:param str path: Result directory (default None) :param str path: Result directory (default None)
""" """
def __init__(self, target=None, targets=None, thread=20, path=None, format='csv'): def __init__(self, target=None, targets=None, thread=20, path=None, fmt='csv'):
Module.__init__(self) Module.__init__(self)
self.subdomains = set() self.subdomains = set()
self.module = 'Check' self.module = 'Check'
@@ -68,7 +68,7 @@ class Takeover(Module):
self.targets = targets self.targets = targets
self.thread = thread self.thread = thread
self.path = path self.path = path
self.format = format self.fmt = fmt
self.fingerprints = None self.fingerprints = None
self.subdomainq = Queue() self.subdomainq = Queue()
self.cnames = list() self.cnames = list()
@@ -76,10 +76,10 @@ class Takeover(Module):
def save(self): def save(self):
logger.log('DEBUG', 'Saving results') logger.log('DEBUG', 'Saving results')
if self.format == 'txt': if self.fmt == 'txt':
data = str(self.results) data = str(self.results)
else: else:
data = self.results.export(self.format) data = self.results.export(self.fmt)
utils.save_data(self.path, data) utils.save_data(self.path, data)
def compare(self, subdomain, cname, responses): def compare(self, subdomain, cname, responses):
@@ -131,10 +131,10 @@ class Takeover(Module):
self.subdomains = self.targets self.subdomains = self.targets
else: else:
self.subdomains = utils.get_domains(self.target, self.targets) self.subdomains = utils.get_domains(self.target, self.targets)
self.format = utils.check_format(self.format, len(self.subdomains)) self.fmt = utils.check_format(self.fmt)
timestamp = utils.get_timestamp() timestamp = utils.get_timestamp()
name = f'takeover_check_result_{timestamp}' name = f'takeover_check_result_{timestamp}'
self.path = utils.check_path(self.path, name, self.format) self.path = utils.check_path(self.path, name, self.fmt)
if self.subdomains: if self.subdomains:
logger.log('INFOR', f'Checking subdomain takeover') logger.log('INFOR', f'Checking subdomain takeover')
self.fingerprints = get_fingerprint() self.fingerprints = get_fingerprint()