mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-26 04:47:48 +08:00
Compare commits
61 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 shmilylty
|
e662f28407 | ||
|
shmilylty
|
26d32d3cd9 | ||
|
shmilylty
|
55c4cf734a | ||
|
shmilylty
|
c568f73b51 | ||
|
shmilylty
|
9c25b77aa6 | ||
|
shmilylty
|
9fbaf19f35 | ||
|
shmilylty
|
e3c3965020 | ||
|
shmilylty
|
40000472a5 | ||
|
shmilylty
|
cb8ac909d7 | ||
|
shmilylty
|
7627b53650 | ||
|
shmilylty
|
f30f48aea2 | ||
|
shmilylty
|
e66c6d7e21 | ||
|
shmilylty
|
8ede7d9943 | ||
|
shmilylty
|
0d72aefb62 | ||
|
shmilylty
|
893e6628e6 | ||
|
shmilylty
|
baf394fc0b | ||
|
shmilylty
|
035963758d | ||
|
shmilylty
|
d760e02289 | ||
|
shmilylty
|
e6b145d563 | ||
|
shmilylty
|
7afb359918 | ||
|
shmilylty
|
707a5eda93 | ||
|
shmilylty
|
6d3c0ae873 | ||
|
shmilylty
|
476e75869c | ||
|
shmilylty
|
e7110c24c5 | ||
|
shmilylty
|
0e4c7961fe | ||
|
shmilylty
|
6f0cb70e02 | ||
|
shmilylty
|
4dec25d187 | ||
|
shmilylty
|
e5e24804f6 | ||
|
shmilylty
|
f1b24e5674 | ||
|
shmilylty
|
d0edcb9a0b | ||
|
shmilylty
|
2dbb5a1097 | ||
|
shmilylty
|
451992f07c | ||
|
shmilylty
|
5cfbf885ca | ||
|
shmilylty
|
ecc1cb53dc | ||
|
shmilylty
|
39d9adb6bc | ||
|
shmilylty
|
8675dc0202 | ||
|
shmilylty
|
de81a73097 | ||
|
shmilylty
|
9d6f122254 | ||
|
shmilylty
|
be11d5c3a2 | ||
|
shmilylty
|
786be39398 | ||
|
shmilylty
|
c92794263b | ||
|
shmilylty
|
953b7a4e9b | ||
|
shmilylty
|
88b7089d1b | ||
|
shmilylty
|
d4745fe7cf | ||
|
shmilylty
|
94423d2d12 | ||
|
shmilylty
|
f5476ab381 | ||
|
shmilylty
|
1cd52ce2d6 | ||
|
Jing Ling
|
071c115d70 | ||
|
Jing Ling
|
20eea3518a | ||
|
Jing Ling
|
0a049e991d | ||
|
shmilylty
|
6ef7c96a93 | ||
|
shmilylty
|
edcdaa2f05 | ||
|
shmilylty
|
f7c03656bf | ||
|
shmilylty
|
a9ff58061e | ||
|
shmilylty
|
5a269fc601 | ||
|
shmilylty
|
57b581c85b | ||
|
shmilylty
|
dcb1d99b26 | ||
|
shmilylty
|
372a2de406 | ||
|
shmilylty
|
c8fb3a2faf | ||
|
shmilylty
|
c9208b61dd | ||
|
shmilylty
|
e74a738565 |
+2
-2
@@ -19,13 +19,13 @@ matrix:
|
||||
os: osx
|
||||
osx_image: xcode10.2 # Python 3.7 running on macOS 10.14.3
|
||||
language: shell # 'language: python' is an error on Travis CI macOS
|
||||
- name: "Python 3.7 on Windows"
|
||||
- name: "Python 3.8 on Windows"
|
||||
os: windows # Windows 10.0.17134 N/A Build 17134
|
||||
language: shell
|
||||
before_install:
|
||||
- choco install python
|
||||
- python -m pip install --upgrade pip
|
||||
env: PATH=/c/Python37:/c/Python37/Scripts:$PATH
|
||||
env: PATH=/c/Python38:/c/Python38/Scripts:$PATH
|
||||
|
||||
install:
|
||||
- pip3 install -U pip
|
||||
|
||||
+14
@@ -6,6 +6,20 @@ OneForAll的更新日志格式基于[Keep a Changelog](https://keepachangelog.co
|
||||
OneForAll遵守[语义化版本格式](https://semver.org/)。
|
||||
|
||||
## Unreleased
|
||||
## [0.0.8](https://github.com/shmilylty/oneforall/releases/tag/v0.0.8) - 2019-10-30
|
||||
- 添加新子域监控功能
|
||||
- 优化子域爆破字典和默认参数
|
||||
- 修复端口重复问题
|
||||
- 移除aiodns依赖
|
||||
|
||||
## [0.0.7](https://github.com/shmilylty/oneforall/releases/tag/v0.0.7) - 2019-10-18
|
||||
- 修复一些已知问题
|
||||
- 添加百度云观测接口
|
||||
- 添加添加英文Readme文档
|
||||
- 更新有关文档
|
||||
- 优化标题获取
|
||||
- 更新依赖
|
||||
|
||||
## [0.0.6](https://github.com/shmilylty/oneforall/releases/tag/v0.0.6) - 2019-08-27
|
||||
- 修复一些已知问题
|
||||
- 添加PassiveDNS查询和Github子域搜索模块
|
||||
|
||||
+2
-5
@@ -3,13 +3,10 @@
|
||||
* **[Jing Ling](https://github.com/shmilylty)**
|
||||
* 核心开发
|
||||
|
||||
* **[Black Star](https://github.com/blackstar24)**
|
||||
* **[Black Star](https://github.com/blackstar24)**，**[Echocipher](https://github.com/Echocipher)**
|
||||
* 模块贡献
|
||||
|
||||
* **[Echocipher](https://github.com/Echocipher)**
|
||||
* 模块贡献
|
||||
|
||||
* **[iceMatcha](https://github.com/iceMatcha)**
|
||||
* **[iceMatcha](https://github.com/iceMatcha)**，**[mikuKeeper](https://github.com/mikuKeeper)**
|
||||
* 工具测试
|
||||
|
||||
* **Anyone**
|
||||
|
||||
@@ -6,7 +6,6 @@ verify_ssl = true
|
||||
[dev-packages]
|
||||
|
||||
[packages]
|
||||
aiodns = "*"
|
||||
tqdm = "*"
|
||||
aiomultiprocess = "*"
|
||||
loguru = "*"
|
||||
@@ -27,4 +26,4 @@ tablib = "*"
|
||||
brotlipy = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.7"
|
||||
python_version = "3.8"
|
||||
|
||||
Generated
+113
-148
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "9a29beced25222ff7c32a32aaa99c292ee43986700b280c6153e01245a5a71bc"
|
||||
"sha256": "ee0f44c186ca74e3344dfdf99d484042116eb11746a2ffe0245a1cdd9b2ff01c"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3.7"
|
||||
"python_version": "3.8"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
@@ -16,41 +16,23 @@
|
||||
]
|
||||
},
|
||||
"default": {
|
||||
"aiodns": {
|
||||
"hashes": [
|
||||
"sha256:815fdef4607474295d68da46978a54481dd1e7be153c7d60f9e72773cd38d77d",
|
||||
"sha256:aaa5ac584f40fe778013df0aa6544bf157799bd3f608364b451840ed2c8688de"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.0.0"
|
||||
},
|
||||
"aiohttp": {
|
||||
"hashes": [
|
||||
"sha256:00d198585474299c9c3b4f1d5de1a576cc230d562abc5e4a0e81d71a20a6ca55",
|
||||
"sha256:0155af66de8c21b8dba4992aaeeabf55503caefae00067a3b1139f86d0ec50ed",
|
||||
"sha256:09654a9eca62d1bd6d64aa44db2498f60a5c1e0ac4750953fdd79d5c88955e10",
|
||||
"sha256:199f1d106e2b44b6dacdf6f9245493c7d716b01d0b7fbe1959318ba4dc64d1f5",
|
||||
"sha256:296f30dedc9f4b9e7a301e5cc963012264112d78a1d3094cd83ef148fdf33ca1",
|
||||
"sha256:368ed312550bd663ce84dc4b032a962fcb3c7cae099dbbd48663afc305e3b939",
|
||||
"sha256:40d7ea570b88db017c51392349cf99b7aefaaddd19d2c78368aeb0bddde9d390",
|
||||
"sha256:629102a193162e37102c50713e2e31dc9a2fe7ac5e481da83e5bb3c0cee700aa",
|
||||
"sha256:6d5ec9b8948c3d957e75ea14d41e9330e1ac3fed24ec53766c780f82805140dc",
|
||||
"sha256:87331d1d6810214085a50749160196391a712a13336cd02ce1c3ea3d05bcf8d5",
|
||||
"sha256:9a02a04bbe581c8605ac423ba3a74999ec9d8bce7ae37977a3d38680f5780b6d",
|
||||
"sha256:9c4c83f4fa1938377da32bc2d59379025ceeee8e24b89f72fcbccd8ca22dc9bf",
|
||||
"sha256:9cddaff94c0135ee627213ac6ca6d05724bfe6e7a356e5e09ec57bd3249510f6",
|
||||
"sha256:a25237abf327530d9561ef751eef9511ab56fd9431023ca6f4803f1994104d72",
|
||||
"sha256:a5cbd7157b0e383738b8e29d6e556fde8726823dae0e348952a61742b21aeb12",
|
||||
"sha256:a97a516e02b726e089cffcde2eea0d3258450389bbac48cbe89e0f0b6e7b0366",
|
||||
"sha256:acc89b29b5f4e2332d65cd1b7d10c609a75b88ef8925d487a611ca788432dfa4",
|
||||
"sha256:b05bd85cc99b06740aad3629c2585bda7b83bd86e080b44ba47faf905fdf1300",
|
||||
"sha256:c2bec436a2b5dafe5eaeb297c03711074d46b6eb236d002c13c42f25c4a8ce9d",
|
||||
"sha256:cc619d974c8c11fe84527e4b5e1c07238799a8c29ea1c1285149170524ba9303",
|
||||
"sha256:d4392defd4648badaa42b3e101080ae3313e8f4787cb517efd3f5b8157eaefd6",
|
||||
"sha256:e1c3c582ee11af7f63a34a46f0448fca58e59889396ffdae1f482085061a2889"
|
||||
"sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e",
|
||||
"sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326",
|
||||
"sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a",
|
||||
"sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654",
|
||||
"sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a",
|
||||
"sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4",
|
||||
"sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17",
|
||||
"sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec",
|
||||
"sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd",
|
||||
"sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48",
|
||||
"sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59",
|
||||
"sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.5.4"
|
||||
"version": "==3.6.2"
|
||||
},
|
||||
"aiomultiprocess": {
|
||||
"hashes": [
|
||||
@@ -59,13 +41,6 @@
|
||||
"index": "pypi",
|
||||
"version": "==0.6.1"
|
||||
},
|
||||
"asn1crypto": {
|
||||
"hashes": [
|
||||
"sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87",
|
||||
"sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49"
|
||||
],
|
||||
"version": "==0.24.0"
|
||||
},
|
||||
"async-timeout": {
|
||||
"hashes": [
|
||||
"sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
|
||||
@@ -75,25 +50,18 @@
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
|
||||
"sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
|
||||
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
|
||||
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
|
||||
],
|
||||
"version": "==19.1.0"
|
||||
},
|
||||
"backports.csv": {
|
||||
"hashes": [
|
||||
"sha256:1277dfff73130b2e106bf3dd347adb3c5f6c4340882289d88f31240da92cbd6d",
|
||||
"sha256:21f6e09bab589e6c1f877edbc40277b65e626262a86e69a70137db714eaac5ce"
|
||||
],
|
||||
"version": "==1.0.7"
|
||||
"version": "==19.3.0"
|
||||
},
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612",
|
||||
"sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b",
|
||||
"sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469"
|
||||
"sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
|
||||
"sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931",
|
||||
"sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
|
||||
],
|
||||
"version": "==4.8.0"
|
||||
"version": "==4.8.1"
|
||||
},
|
||||
"brotli": {
|
||||
"hashes": [
|
||||
@@ -135,6 +103,7 @@
|
||||
"sha256:09ec3e125d16749b31c74f021aba809541b3564e5359f8c265cbae442810b41a",
|
||||
"sha256:0be698678a114addcf87a4b9496c552c68a2c99bf93cf8e08f5738b392e82057",
|
||||
"sha256:0fa6088a9a87645d43d7e21e32b4a6bf8f7c3939015a50158c10972aa7f425b7",
|
||||
"sha256:1379347337dc3d20b2d61456d44ccce13e0625db2611c368023b4194d5e2477f",
|
||||
"sha256:1ea4e578241504b58f2456a6c69952c88866c794648bdc74baee74839da61d44",
|
||||
"sha256:2699945a0a992c04fc7dc7fa2f1d0575a2c8b4b769f2874a08e8eae46bef36ae",
|
||||
"sha256:2a80319ae13ea8dd60ecdc4f5ccf6da3ae64787765923256b62c598c5bba4121",
|
||||
@@ -142,6 +111,7 @@
|
||||
"sha256:36def0b859beaf21910157b4c33eb3b06d8ce459c942102f16988cca6ea164df",
|
||||
"sha256:3a3e56ced8b15fbbd363380344f70f3b438e0fd1fcf27b7526b6172ea950e867",
|
||||
"sha256:3c1d5e2cf945a46975bdb11a19257fa057b67591eb232f393d260e7246d9e571",
|
||||
"sha256:4e4638b49835d567d447a2cfacec109f9a777f219f071312268b351b6839436d",
|
||||
"sha256:50ca336374131cfad20612f26cc43c637ac0bfd2be3361495e99270883b52962",
|
||||
"sha256:5de6f7d010b7558f72f4b061a07395c5c3fd57f0285c5af7f126a677b976a868",
|
||||
"sha256:637847560d671657f993313ecc6c6c6666a936b7a925779fd044065c7bc035b9",
|
||||
@@ -201,43 +171,45 @@
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939",
|
||||
"sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695"
|
||||
"sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50",
|
||||
"sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"
|
||||
],
|
||||
"version": "==2019.6.16"
|
||||
"version": "==2019.9.11"
|
||||
},
|
||||
"cffi": {
|
||||
"hashes": [
|
||||
"sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774",
|
||||
"sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d",
|
||||
"sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90",
|
||||
"sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b",
|
||||
"sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63",
|
||||
"sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45",
|
||||
"sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25",
|
||||
"sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3",
|
||||
"sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b",
|
||||
"sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647",
|
||||
"sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016",
|
||||
"sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4",
|
||||
"sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb",
|
||||
"sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753",
|
||||
"sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7",
|
||||
"sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9",
|
||||
"sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f",
|
||||
"sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8",
|
||||
"sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f",
|
||||
"sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc",
|
||||
"sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42",
|
||||
"sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3",
|
||||
"sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909",
|
||||
"sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45",
|
||||
"sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d",
|
||||
"sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512",
|
||||
"sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff",
|
||||
"sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201"
|
||||
"sha256:00d890313797d9fe4420506613384b43099ad7d2b905c0752dbcc3a6f14d80fa",
|
||||
"sha256:0cf9e550ac6c5e57b713437e2f4ac2d7fd0cd10336525a27224f5fc1ec2ee59a",
|
||||
"sha256:0ea23c9c0cdd6778146a50d867d6405693ac3b80a68829966c98dd5e1bbae400",
|
||||
"sha256:193697c2918ecdb3865acf6557cddf5076bb39f1f654975e087b67efdff83365",
|
||||
"sha256:1ae14b542bf3b35e5229439c35653d2ef7d8316c1fffb980f9b7647e544baa98",
|
||||
"sha256:1e389e069450609c6ffa37f21f40cce36f9be7643bbe5051ab1de99d5a779526",
|
||||
"sha256:263242b6ace7f9cd4ea401428d2d45066b49a700852334fd55311bde36dcda14",
|
||||
"sha256:33142ae9807665fa6511cfa9857132b2c3ee6ddffb012b3f0933fc11e1e830d5",
|
||||
"sha256:364f8404034ae1b232335d8c7f7b57deac566f148f7222cef78cf8ae28ef764e",
|
||||
"sha256:47368f69fe6529f8f49a5d146ddee713fc9057e31d61e8b6dc86a6a5e38cecc1",
|
||||
"sha256:4895640844f17bec32943995dc8c96989226974dfeb9dd121cc45d36e0d0c434",
|
||||
"sha256:558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b",
|
||||
"sha256:5ba86e1d80d458b338bda676fd9f9d68cb4e7a03819632969cf6d46b01a26730",
|
||||
"sha256:63424daa6955e6b4c70dc2755897f5be1d719eabe71b2625948b222775ed5c43",
|
||||
"sha256:6381a7d8b1ebd0bc27c3bc85bc1bfadbb6e6f756b4d4db0aa1425c3719ba26b4",
|
||||
"sha256:6381ab708158c4e1639da1f2a7679a9bbe3e5a776fc6d1fd808076f0e3145331",
|
||||
"sha256:6fd58366747debfa5e6163ada468a90788411f10c92597d3b0a912d07e580c36",
|
||||
"sha256:728ec653964655d65408949b07f9b2219df78badd601d6c49e28d604efe40599",
|
||||
"sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8",
|
||||
"sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8",
|
||||
"sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa",
|
||||
"sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78",
|
||||
"sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc",
|
||||
"sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e",
|
||||
"sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2",
|
||||
"sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0",
|
||||
"sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71",
|
||||
"sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891",
|
||||
"sha256:ec2fa3ee81707a5232bf2dfbd6623fdb278e070d596effc7e2d788f2ada71a05",
|
||||
"sha256:fd82eb4694be712fcae03c717ca2e0fc720657ac226b80bbb597e971fc6928c2"
|
||||
],
|
||||
"version": "==1.12.3"
|
||||
"version": "==1.13.1"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
@@ -248,11 +220,11 @@
|
||||
},
|
||||
"cloudscraper": {
|
||||
"hashes": [
|
||||
"sha256:7080a4c9ea6fe244422551172eb28a1d6c042da4c9d56d15ff319dbbac2ab6d7",
|
||||
"sha256:79d04cd3c7d782035b404b5093a0cdbacd219cb883134017f4f7ba703f12a93c"
|
||||
"sha256:ea0a52a5c8af2645db4de3eccbf0e0b721a6daa721fb43afacbb51c6a6b8cf48",
|
||||
"sha256:f64dff985d958837864ff24bd245353efbcee200e5e589efb7bbd84d8238b495"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.1.40"
|
||||
"version": "==1.2.5"
|
||||
},
|
||||
"colorama": {
|
||||
"hashes": [
|
||||
@@ -264,24 +236,29 @@
|
||||
},
|
||||
"cryptography": {
|
||||
"hashes": [
|
||||
"sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c",
|
||||
"sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643",
|
||||
"sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216",
|
||||
"sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799",
|
||||
"sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a",
|
||||
"sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9",
|
||||
"sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc",
|
||||
"sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8",
|
||||
"sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53",
|
||||
"sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1",
|
||||
"sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609",
|
||||
"sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292",
|
||||
"sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e",
|
||||
"sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6",
|
||||
"sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed",
|
||||
"sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d"
|
||||
"sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c",
|
||||
"sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595",
|
||||
"sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad",
|
||||
"sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651",
|
||||
"sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2",
|
||||
"sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff",
|
||||
"sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d",
|
||||
"sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42",
|
||||
"sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d",
|
||||
"sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e",
|
||||
"sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912",
|
||||
"sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793",
|
||||
"sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13",
|
||||
"sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7",
|
||||
"sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0",
|
||||
"sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879",
|
||||
"sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f",
|
||||
"sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9",
|
||||
"sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2",
|
||||
"sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf",
|
||||
"sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8"
|
||||
],
|
||||
"version": "==2.7"
|
||||
"version": "==2.8"
|
||||
},
|
||||
"defusedxml": {
|
||||
"hashes": [
|
||||
@@ -382,6 +359,12 @@
|
||||
"index": "pypi",
|
||||
"version": "==4.4.1"
|
||||
},
|
||||
"markuppy": {
|
||||
"hashes": [
|
||||
"sha256:1adee2c0a542af378fe84548ff6f6b0168f3cb7f426b46961038a2bcfaad0d5f"
|
||||
],
|
||||
"version": "==1.14"
|
||||
},
|
||||
"multidict": {
|
||||
"hashes": [
|
||||
"sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f",
|
||||
@@ -428,24 +411,6 @@
|
||||
],
|
||||
"version": "==2.4.11"
|
||||
},
|
||||
"pycares": {
|
||||
"hashes": [
|
||||
"sha256:2ca080db265ea238dc45f997f94effb62b979a617569889e265c26a839ed6305",
|
||||
"sha256:6f79c6afb6ce603009db2042fddc2e348ad093ece9784cbe2daa809499871a23",
|
||||
"sha256:70918d06eb0603016d37092a5f2c0228509eb4e6c5a3faacb4184f6ab7be7650",
|
||||
"sha256:755187d28d24a9ea63aa2b4c0638be31d65fbf7f0ce16d41261b9f8cb55a1b99",
|
||||
"sha256:7baa4b1f2146eb8423ff8303ebde3a20fb444a60db761fba0430d104fe35ddbf",
|
||||
"sha256:90b27d4df86395f465a171386bc341098d6d47b65944df46518814ae298f6cc6",
|
||||
"sha256:9e090dd6b2afa65cb51c133883b2bf2240fd0f717b130b0048714b33fb0f47ce",
|
||||
"sha256:a11b7d63c3718775f6e805d6464cb10943780395ab042c7e5a0a7a9f612735dd",
|
||||
"sha256:b253f5dcaa0ac7076b79388a3ac80dd8f3bd979108f813baade40d3a9b8bf0bd",
|
||||
"sha256:c7f4f65e44ba35e35ad3febc844270665bba21cfb0fb7d749434e705b556e087",
|
||||
"sha256:cdb342e6a254f035bd976d95807a2184038fc088d957a5104dcaab8be602c093",
|
||||
"sha256:cf08e164f8bfb83b9fe633feb56f2754fae6baefcea663593794fa0518f8f98c",
|
||||
"sha256:df9bc694cf03673878ea8ce674082c5acd134991d64d6c306d4bd61c0c1df98f"
|
||||
],
|
||||
"version": "==3.0.0"
|
||||
},
|
||||
"pycparser": {
|
||||
"hashes": [
|
||||
"sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3"
|
||||
@@ -468,19 +433,19 @@
|
||||
},
|
||||
"pysocks": {
|
||||
"hashes": [
|
||||
"sha256:15d38914b60dbcb231d276f64882a20435c049450160e953ca7d313d1405f16f",
|
||||
"sha256:32238918ac0f19e9fd870a8692ac9bd14f5e8752b3c62624cda5851424642210",
|
||||
"sha256:d9031ea45fdfacbe59a99273e9f0448ddb33c1580fe3831c1b09557c5718977c"
|
||||
"sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299",
|
||||
"sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5",
|
||||
"sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.7.0"
|
||||
"version": "==1.7.1"
|
||||
},
|
||||
"pytz": {
|
||||
"hashes": [
|
||||
"sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32",
|
||||
"sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7"
|
||||
"sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
|
||||
"sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
|
||||
],
|
||||
"version": "==2019.2"
|
||||
"version": "==2019.3"
|
||||
},
|
||||
"pyyaml": {
|
||||
"hashes": [
|
||||
@@ -539,25 +504,25 @@
|
||||
},
|
||||
"soupsieve": {
|
||||
"hashes": [
|
||||
"sha256:8662843366b8d8779dec4e2f921bebec9afd856a5ff2e82cd419acc5054a1a92",
|
||||
"sha256:a5a6166b4767725fd52ae55fee8c8b6137d9a51e9f1edea461a062a759160118"
|
||||
"sha256:605f89ad5fdbfefe30cdc293303665eff2d188865d4dbe4eb510bba1edfbfce3",
|
||||
"sha256:b91d676b330a0ebd5b21719cb6e9b57c57d433671f65b9c28dd3461d9a1ed0b6"
|
||||
],
|
||||
"version": "==1.9.3"
|
||||
"version": "==1.9.4"
|
||||
},
|
||||
"sqlalchemy": {
|
||||
"hashes": [
|
||||
"sha256:0459bf0ea6478f3e904de074d65769a11d74cdc34438ab3159250c96d089aef0"
|
||||
"sha256:0f0768b5db594517e1f5e1572c73d14cf295140756431270d89496dc13d5e46c"
|
||||
],
|
||||
"markers": "python_version >= '3.0'",
|
||||
"version": "==1.3.7"
|
||||
"version": "==1.3.10"
|
||||
},
|
||||
"tablib": {
|
||||
"hashes": [
|
||||
"sha256:0f88a9cebdaa1a2cc29ae57387082ee81015d1149ecd34e48a8c8d3b4dd21670",
|
||||
"sha256:5f33c079b07eb10cf9c4b4696add2ecf32c89db7729240546ecdcd5c92f67e13"
|
||||
"sha256:00654241e5beee437ba544e4fa4abef70ccec3668503aa95406c1250bb660770",
|
||||
"sha256:6336e7aa3f0e5894b47270a3dc639cc2e78eb823c7ccb8c4512fb408cdc18d08"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.13.0"
|
||||
"version": "==0.14.0"
|
||||
},
|
||||
"termcolor": {
|
||||
"hashes": [
|
||||
@@ -567,19 +532,19 @@
|
||||
},
|
||||
"tldextract": {
|
||||
"hashes": [
|
||||
"sha256:2c1c5d9d454f79734b4f3da0d603856dd9f820753410a3e9abf0a0c9fde33e97",
|
||||
"sha256:b72bef6013de67c7fa181250bc2c2e089a994d259c09ca95a9771f2f97e29ed1"
|
||||
"sha256:16b2f7e81d89c2a5a914d25bdbddd3932c31a6b510db886c3ce0764a195c0ee7",
|
||||
"sha256:9aa21a1f7827df4209e242ec4fc2293af5940ec730cde46ea80f66ed97bfc808"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.2.1"
|
||||
"version": "==2.2.2"
|
||||
},
|
||||
"tqdm": {
|
||||
"hashes": [
|
||||
"sha256:438d6a735167099d75e5fd9a55175c6727c4dbba345ae406b2886c2728fe3e80",
|
||||
"sha256:ebc205051d79b49989140f5f6c73ec23fce5f590cbc4d9cd6e4c47f168fa0f10"
|
||||
"sha256:abc25d0ce2397d070ef07d8c7e706aede7920da163c64997585d42d3537ece3d",
|
||||
"sha256:dd3fcca8488bb1d416aa7469d2f277902f26260c45aa86b667b074cd44b3b115"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.34.0"
|
||||
"version": "==4.36.1"
|
||||
},
|
||||
"tzlocal": {
|
||||
"hashes": [
|
||||
@@ -590,10 +555,10 @@
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
|
||||
"sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
|
||||
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
|
||||
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
|
||||
],
|
||||
"version": "==1.25.3"
|
||||
"version": "==1.25.6"
|
||||
},
|
||||
"win32-setctime": {
|
||||
"hashes": [
|
||||
|
||||
+378
@@ -0,0 +1,378 @@
|
||||
# OneForAll
|
||||
|
||||
[](https://travis-ci.org/shmilylty/OneForAll)
|
||||
[](https://codecov.io/gh/shmilylty/OneForAll)
|
||||
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/releases)
|
||||
|
||||
👊**OneForAll is a powerful subdomain collection tool** 📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
|
||||
|
||||
|
||||
|
||||
## 🎉Project profile
|
||||
|
||||
Project home：[https://shmilylty.github.io/OneForAll/](https://shmilylty.github.io/OneForAll/)
|
||||
|
||||
Project address ：[https://github.com/shmilylty/OneForAll](https://github.com/shmilylty/OneForAll)
|
||||
|
||||
The importance of information collection in penetration testing is self-evident. Subdomain collection is an essential and very important part of information collection. At present, there are many open source tools for subdomain collection on the Internet, but there are always some of the following problems:
|
||||
|
||||
* **Not powerful enough**，there are not enough interfaces to collect subdomains automatically, and there are no functions such as automatic subdomain resolve, verification, FUZZ and information expansion.
|
||||
* **Not friendly enough**，although the command line module is more convenient, but when there are a lot of optional parameters and the operation to be implemented is complex, using command line mode is a bit unfriendly. If there is a good interaction, With a highly operable front end, the experience will be much better.
|
||||
|
||||
* **Lack of maintenance**，Many tools have not been updated once in years, what issues and PR are, do not exist.
|
||||
|
||||
* **Efficiency issues**，do not take advantage of multi-process, multi-threading and asynchronous cooperation technology, the speed is slow.
|
||||
|
||||
In order to solve the above pain points, the term OneForAll comes from one of my favorite Japanese cartoons,《[My Hero Academia](https://manhua.fzdm.com/131/)》.It's a top personality that has grown from generation to generation, and now [TV series](https://www.bilibili.com/bangumi/media/md7452/)has been updated to season 4. As its name suggests, I want OneForAll to be a collection of 100 strong, powerful and fast subdomains that collect the ultimate artifact 🔨.
|
||||
|
||||
At present, OneForAll is still under development, there must be a lot of problems and areas for improvement. Welcome to submit [Issues](https://github.com/shmilylty/OneForAll/issues) or [PR](https://github.com/shmilylty/OneForAll/pulls)，If you like, give it to a little star ✨，At present, there is a special QQ group for OneForAll communication and feedback: 👨👨👦👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)，You can also 📧[email](mailto://admin@hackfun.org) or [tweet](https://twitter.com/jinglingbiaodi) to me .
|
||||
|
||||
## 👍Features
|
||||
|
||||
* **Powerful collection capability**，For more information, please see [collection module description](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md).
|
||||
1. Collect subdomains using certificate transparency (there are currently 6 modules: `censys_api`，`spyse_api`，`certspotter`，`crtsh`，`entrust`，`google`）
|
||||
|
||||
2. General check collection subdomains (there are currently 4 modules: domain transfer vulnerability exploitation`cdx`, cross-domain policy file `cdx`, HTTPS certificate `cert`, content security policy `csp`, robots file `robots`, and sitemap file `sitemap`. Check NSEC record, NSEC3 record and other modules will be added later).
|
||||
|
||||
3. Collect subdomains using web crawler files (there are currently two modules: `archirawl`, `commoncrawl`, which is still being debugged and needs to be added and improved).
|
||||
|
||||
4. Collect subdomains using DNS datasets (there are currently 19 modules: `CeBaidu`, `binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`, `ptrarchive`, `sitedossier`,`threatcrowd`）
|
||||
|
||||
5. Collect subdomains using DNS queries (there is currently a module to collect subdomains `srv` by enumerating common SRV records and making queries, which needs to be added and improved).
|
||||
|
||||
6. Collect subdomains using threat intelligence platform data (there are currently five modules: `riskiq_ api`, `threatbook_ api`, `threatkeeper `, `virustotal`, `virustotal_ api`, which need to be added and improved).
|
||||
|
||||
7. Use search engines to discover subdomains (there are currently 16 modules: `ask`, `bing_ api`, `fofa_ api`, `shodan_ api`, `yahoo`, `baidu`, `duckduckgo`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_ api`, `sogou`, `zoomeye_ api`), except for special search engines in the search module. General search engines support automatic exclusion of search, full search, recursive search.
|
||||
* **Support subdomain blasting**，This module has both conventional dictionary blasting and custom fuzz mode. It supports batch blasting and recursive blasting, and automatically judges pan-parsing and processing.
|
||||
* **Support subdmain verification**，default to enable subdomain verification, automatically resolve subdomain DNS, automatically request subdomain to obtain title and banner, and comprehensively determine subdomain survival.
|
||||
* **Support subdomain takeover**，By default, subdomain takeover risk checking is enabled. Automatic subdomain takeover is supported (only Github, remains to be improved at present), and batch inspection is supported.
|
||||
* **Powerful processing feature**，The found subdomain results support automatic removal, automatic DNS parsing, HTTP request detection, automatic filtering of valid subdomains, and expansion of Banner information for subdomains. The final supported export formats are `txt`, `rst`, `csv`, `tsv`, `json`, `yaml`, `html`, `xls`, `xlsx`, `dbf`, `latex`, `ods`.
|
||||
* **Very fast**，[collection module](https://github.com/shmilylty/OneForAll/tree/master/oneforall//collect.py) uses multithreaded calls, [blasting module](https://github.com/shmilylty/OneForAll/tree/master/oneforall/aiobrute.py) uses asynchronous multiprocess and multiprogramming, and DNS parsing and HTTP requests use asynchronous multiprogramming in subdomain verification. Multithreaded check [subdomain takeover](https://github.com/shmilylty/OneForAll/tree/master/oneforall/takeover.py) risk.
|
||||
* **Good experience**，Each module has a progress bar, and the results of each module are saved asynchronously.
|
||||
|
||||
## 🚀Start Guide
|
||||
|
||||
📢Currently, the project is **under development**, so it will continue to update iterations. It is best to **clone** the project when downloading and using OneForAll. Please take a moment to read this document to help you quickly get familiar with OneForAll!
|
||||
|
||||
**🐍Installation requirements**
|
||||
|
||||
OneForAll is based on CPython, so you need a Python environment to run. If your system doesn't already have a Python environment, you can refer to the [Python 3 Installation Guide](https://pythonguidecn.readthedocs.io/en/latest/starting/installation.html#python-3), theoretically Python 3.7 and 3.8 can run OneForAll, **but** many tests are done on Python 3.8, so **recommended** you use **Python 3.8** version runs OneForAll. Run the following command to check the Python and pip3 versions:
|
||||
```bash
|
||||
python -V
|
||||
pip3 -V
|
||||
```
|
||||
If you see the following output, there is no problem with the Python environment:
|
||||
```bash
|
||||
Python 3.8.0
|
||||
pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.8)
|
||||
```
|
||||
|
||||
**✔Installation steps**
|
||||
|
||||
1. **Download**
|
||||
|
||||
This project has been mirrored in [Gitee](https://gitee.com/shmilylty/OneForAll.git). If you are in China, it is recommended that you use Gitee for cloning, which is faster:
|
||||
|
||||
```bash
|
||||
git clone https://gitee.com/shmilylty/OneForAll.git
|
||||
```
|
||||
or：
|
||||
```bash
|
||||
git clone https://github.com/shmilylty/OneForAll.git
|
||||
```
|
||||
|
||||
2. **Installation**
|
||||
|
||||
You can install OneForAll dependencies via pip3 (if you are familiar with [pipenv](https://docs.pipenv.org/en/latest/), then it is recommended that you use [pipenv install dependencies](https://github.com/shmilylty/OneForAll/tree/master/docs/Installation_dependency.md), the following is an example of using **pip3** to install dependencies under **Windows system**: (Note: If your Python3 is installed in the system Program Files In the directory, such as: `C:\Program Files\Python37`, then run the command prompt cmd as an administrator to execute the following command!)
|
||||
```bash
|
||||
cd OneForAll/
|
||||
python -m pip install -U pip setuptools wheel
|
||||
pip3 install -r requirements.txt
|
||||
cd oneforall/
|
||||
python oneforall.py --help
|
||||
```
|
||||
For other system platforms, please refer to [dependency installation](https://github.com/shmilylty/OneForAll/tree/master/docs/installation_dependency.md). If you find that compiling a dependent library fails during the installation dependencies, Refer to the solution in the [Q&A](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md) documentation, if not resolved, welcome feedback.
|
||||
|
||||
3. **Update**
|
||||
|
||||
❗Note: If you have cloned the project before, please **back** up your own modified files (such as **config.py**) to the location outside the project before updating, then execute the following command **update** project:
|
||||
|
||||
```bash
|
||||
git fetch --all
|
||||
git reset --hard origin/master
|
||||
git pull
|
||||
```
|
||||
|
||||
**✨Demonstration**
|
||||
|
||||
1. If you are installing dependencies through pip3, run the example using the following command:
|
||||
```bash
|
||||
cd oneforall/
|
||||
python3 oneforall.py --target example.com run
|
||||
```
|
||||
|
||||
|
||||
|
||||
2. If you install dependencies through pipenv, run the example using the following command:
|
||||
```bash
|
||||
cd oneforall/
|
||||
pipenv run python oneforall.py --target example.com run
|
||||
```
|
||||
|
||||
**🤔Help**
|
||||
|
||||
The command line parameters only provide some common parameters. For more detailed parameter configuration, please see [config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py) if you think Some parameters are frequently used in the command interface or missing parameters. Feedback is welcome. For well-known reasons, if you want to use some of the wall's collection interface, please go to [config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py) to configure the proxy, some collection Modules need to provide APIs (most of which are freely available for registered accounts). If you need to use them, please go to [config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py) to configure the API. Information, if not used, please ignore the error message. (For detailed modules, please read [collection module description](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md))
|
||||
|
||||
The OneForAll command line interface is based on [Fire](https://github.com/google/python-fire/). For more advanced usage of Fire, please refer to [using the Fire CLI](https://github.com/google/Python-fire/blob/master/docs/using-cli.md), if you have any doubts during the use, please feel free to give me feedback.
|
||||
|
||||
[oneforall.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/oneforall.py) is the main program entry, and oneforall.py can call [aiobrute.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/aiobrute.py), [takerover.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/takerover.py) and [dbexport.py ](https://github.com/shmilylty/OneForAll/tree/master/oneforall/dbexport.py) and other modules, in order to facilitate the sub-field blasting, aiobrute.py is isolated independently, in order to facilitate the subdomain takeover risk check independently takeover.py, in order to facilitate the database export independently dbexport.py, these modules can be run separately, and the parameters accepted are more abundant.
|
||||
|
||||
❗ Note: When you encounter some problems or doubts during use, please use [Issues](https://github.com/shmilylty/OneForAll/issues) to search for answers. Also see [Q&A](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md).
|
||||
|
||||
1. **oneforall.py help**
|
||||
|
||||
```bash
|
||||
python oneforall.py --help
|
||||
```
|
||||
```bash
|
||||
NAME
|
||||
oneforall.py - OneForAll is a powerful subdomain collection tool
|
||||
|
||||
SYNOPSIS
|
||||
oneforall.py --target=TARGET <flags>
|
||||
|
||||
DESCRIPTION
|
||||
Version: 0.0.6
|
||||
Project: https://git.io/fjHT1
|
||||
|
||||
Example:
|
||||
python3 oneforall.py --target example.com run
|
||||
python3 oneforall.py --target ./domains.txt run
|
||||
python3 oneforall.py --target example.com --brute True run
|
||||
python3 oneforall.py --target example.com --verify False run
|
||||
python3 oneforall.py --target example.com --valid None run
|
||||
python3 oneforall.py --target example.com --port medium run
|
||||
python3 oneforall.py --target example.com --format csv run
|
||||
python3 oneforall.py --target example.com --show True run
|
||||
|
||||
Note:
|
||||
Parameter valid optional value 1, 0, none indicates that the export is
|
||||
valid, invalid, and all subdomains, respectively.
|
||||
Parameter verify for True attempts to resolve and request the subdomain
|
||||
and tag the validity of the subdomain based on the result.
|
||||
Parameter port have optional values 'default' 'small', 'medium', 'large',
|
||||
See config.py configuration for details.
|
||||
Parameter format have optional values 'txt', 'rst', 'csv', 'tsv', 'json',
|
||||
'yaml', 'html', 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'.
|
||||
If the parameter path is None, the appropriate file is generated in the
|
||||
project result directory based on the format parameter and the domain
|
||||
name.
|
||||
|
||||
ARGUMENTS
|
||||
TARGET
|
||||
Single domain name or file path for one domain name per line (required)
|
||||
|
||||
FLAGS
|
||||
--brute=BRUTE
|
||||
Use blasting module (default False)
|
||||
--verify=VERIFY
|
||||
Verify the validity of subdomains (default True)
|
||||
--port=PORT
|
||||
Port range for request authentication (default 80 port)
|
||||
--valid=VALID
|
||||
Export validity of subdomains (default 1)
|
||||
--path=PATH
|
||||
Export path (default None)
|
||||
--format=FORMAT
|
||||
Export format (default xls)
|
||||
--show=SHOW
|
||||
Terminal display exported data (default False)
|
||||
```
|
||||
|
||||
2. **aiobrute.py help**
|
||||
|
||||
With regard to the handling of the universal parsing problem, first of all, OneForAll accesses a random subdomain to determine whether universal parsing is used, and if universal parsing is used, it is handled by the following judgment:
|
||||
- First, it is mainly compared with the pan-parsed IP set and TTL values, see [this article](http://sh3ll.me/archives/201704041222.txt).
|
||||
|
||||
- Second, the number of times to resolve to the same IP collection multiple times (the default is 10, which can be set to size in config.py).
|
||||
|
||||
- Third, considering the blasting efficiency, there is no HTTP response volume similarity comparison and response volume content judgment, this function has not been implemented yet, and will be implemented if necessary.
|
||||
|
||||
After not rigorous testing, in the 16-core CPU, using 16 processes 64 coroutines, 100M network bandwidth environment, set the task to be divided into 50000, run two million dictionaries about 10 minutes to run, about 3333 subdomains per second.
|
||||
|
||||
```bash
|
||||
python aiobrute.py --help
|
||||
```
|
||||
|
||||
```bash
|
||||
NAME
|
||||
aiobrute.py - OneForAll multi-process multi-correlation asynchronous subdomain blasting module
|
||||
|
||||
SYNOPSIS
|
||||
aiobrute.py --target=TARGET <flags>
|
||||
|
||||
DESCRIPTION
|
||||
Example：
|
||||
python3 aiobrute.py --target example.com run
|
||||
python3 aiobrute.py --target ./domains.txt run
|
||||
python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
|
||||
python3 aiobrute.py --target example.com --wordlist subdomains.txt run
|
||||
python3 aiobrute.py --target example.com --recursive True --depth 2 run
|
||||
python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
|
||||
|
||||
Note:
|
||||
The setting of the parameter segment is affected by CPU performance,
|
||||
network bandwidth, and operator restrictions. By default, 500 subdomains
|
||||
are set as task groups. When you feel that your environment is not
|
||||
affected by the above factors, the current blasting speed is slow, so it
|
||||
is strongly recommended to use the dictionary. Size resizing: 100,000
|
||||
dictionary suggestions set to 5000, million dictionary set to 50000.
|
||||
|
||||
Parameter valid optional value 1, 0, none indicates that the export is
|
||||
valid, invalid, and all subdomains, respectively.
|
||||
|
||||
Parameter format have optional values 'txt', 'rst', 'csv', 'tsv', 'json',
|
||||
'yaml', 'html', 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'.
|
||||
If the parameter path is None, the appropriate file is generated in the
|
||||
project result directory based on the format parameter and the domain
|
||||
name.
|
||||
|
||||
ARGUMENTS
|
||||
TARGET
|
||||
Single domain name or file path for one domain name per line (required)
|
||||
|
||||
FLAGS
|
||||
--process=PROCESS
|
||||
Number of processes blasted (default CPU core count)
|
||||
--coroutine=COROUTINE
|
||||
Number of coroutines per blasting process (default 64)
|
||||
--wordlist=WORDLIST
|
||||
Specify the dictionary path used for blasting (config.py is used by default)
|
||||
--segment=SEGMENT
|
||||
Blasting task segmentation (default 500)
|
||||
--recursive=RECURSIVE
|
||||
Whether to use recursive blasting (default False)
|
||||
--depth=DEPTH
|
||||
Depth of recursive blasting (default 2)
|
||||
--namelist=NAMELIST
|
||||
Specifies the dictionary path used by recursive blasting (configured by default using config.py)
|
||||
--fuzz=FUZZ
|
||||
Whether to use the fuzz mode for blasting (default False, you must specify the fuzz regular rule)
|
||||
--rule=RULE
|
||||
Regular rules used by fuzz mode (configured by default using config.py)
|
||||
--export=EXPORT
|
||||
Whether to export the blast result (default True)
|
||||
--valid=VALID
|
||||
Export validity of subdomains (default None)
|
||||
--format=FORMAT
|
||||
Export format (default xls)
|
||||
--path=PATH
|
||||
Export path (default None)
|
||||
--show=SHOW
|
||||
Terminal display exported data (default False)
|
||||
```
|
||||
|
||||
3. For other module usage, please refer to [usage help](https://github.com/shmilylty/OneForAll/tree/master/docs/using_help.md).
|
||||
|
||||
## 👏Framework used
|
||||
|
||||
* [aiodns](https://github.com/saghul/aiodns) - aiodns provides a simple way for doing asynchronous DNS resolutions using [pycares](https://github.com/saghul/pycares).
|
||||
* [aiohttp](https://github.com/aio-libs/aiohttp) - Asynchronous HTTP client/server framework for asyncio and Python
|
||||
* [aiomultiprocess](https://github.com/jreese/aiomultiprocess) - Take a modern Python codebase to the next level of performance. (Multiprocessing and asyncio combine to implement asynchronous multi-process multi-coroutine)
|
||||
* [beautifulsoup4](https://pypi.org/project/beautifulsoup4/) - Beautiful Soup is a library that makes it easy to scrape information from web pages.
|
||||
* [fire](https://github.com/google/python-fire) - Python Fire is a library for automatically generating command line interfaces (CLIs) from absolutely any Python object.
|
||||
* [loguru](https://github.com/Delgan/loguru) - Loguru is a library which aims to bring enjoyable logging in Python.
|
||||
* [records](https://github.com/kennethreitz/records) - Records is a very simple, but powerful, library for making raw SQL queries to most relational databases.
|
||||
* [requests](https://github.com/psf/requests) - A simple, yet elegant HTTP library.
|
||||
* [tqdm](https://github.com/tqdm/tqdm) - A Fast, Extensible Progress Bar for Python and CLI
|
||||
|
||||
Thanks to these great excellent Python libraries!
|
||||
|
||||
## 🌲Directory structure
|
||||
|
||||
```bash
|
||||
D:.
|
||||
|
|
||||
+---.github
|
||||
+---docs
|
||||
| collection_modules.md collection module description
|
||||
+---images
|
||||
\---oneforall
|
||||
| aiobrute.py Asynchronous multi-process multi-correlation subdomain blasting module, can be run separately
|
||||
| collect.py Upper layer call of each collection module
|
||||
| config.py Configuration file
|
||||
| dbexport.py Database export module, can be run separately
|
||||
| domains.txt List of domain names to be blasted
|
||||
| oneforall.py OneForAll main entrance, can be run separately
|
||||
| __init__.py
|
||||
|
|
||||
+---common Common call module
|
||||
+---data Store some of the required data
|
||||
| next_subdomains.txt Next level subdomain dictionary
|
||||
| public_suffix_list.dat Top-level domain name suffix
|
||||
| srv_names.json Common SRV record prefix name
|
||||
| subdomains.txt Common Dictionary of subdomain blasting
|
||||
|
|
||||
\---modules
|
||||
+---certificates Using certificate transparency to collect subdomain
|
||||
+---check Using conventional methods to collect subdomain
|
||||
+---crawl Using web crawler files to collect subdomain
|
||||
+---datasets Using DNS datasets to collect subdomain
|
||||
+---dnsquery Using DNS query to collect subdomain
|
||||
+---intelligence Using threat intelligence platform to collect subdomain
|
||||
\---search Using search engine to collect subdomain
|
||||
|
||||
```
|
||||
|
||||
## 🙏Contribution
|
||||
|
||||
Very warmly welcome all ace to improve the project together!
|
||||
|
||||
## ⌛Follow-up plan
|
||||
|
||||
- [ ] Continuous optimization and improvement of each module
|
||||
- [x] Subdomain monitoring (marking each newly discovered subdomain)
|
||||
- [ ] Subdomain collection crawler implementation (including collection of subdomains from static resource files such as JS)
|
||||
- [ ] Implementation of front-end interface for powerful interaction (tentative: front-end: Element + back-end: Flask)
|
||||
|
||||
For more details, see [TODO.md](https://github.com/shmilylty/OneForAll/tree/master/TODO.md).
|
||||
|
||||
## 🔖Version control
|
||||
|
||||
The project uses [SemVer](https://semver.org/) language version format for version management), and you can view the available version in [Releases](https://github.com/shmilylty/OneForAll/releases).
|
||||
|
||||
## 👨💻Contributors
|
||||
|
||||
* **[Jing Ling](https://github.com/shmilylty)**
|
||||
* Core development
|
||||
|
||||
* **[Black Star](https://github.com/blackstar24)**，**[Echocipher](https://github.com/Echocipher)**
|
||||
* Module contribution
|
||||
|
||||
* **[iceMatcha](https://github.com/iceMatcha)**，**[mikuKeeper](https://github.com/mikuKeeper)**
|
||||
* Tool test
|
||||
|
||||
* **Anyone**
|
||||
* Tool feedback
|
||||
|
||||
You can see all the developers involved in the project in [CONTRIBUTORS.md](https://github.com/shmilylty/OneForAll/tree/master/CONTRIBUTORS.md).
|
||||
|
||||
## 📄License
|
||||
|
||||
The project has signed a GPL-3.0 license, for more information, please see [LICENSE](https://github.com/shmilylty/OneForAll/LICENSE).
|
||||
|
||||
## 😘Acknowledgement
|
||||
|
||||
Thanks to the various subdomain collection projects of online open source!
|
||||
|
||||
Thanks ace of [A-Team](https://github.com/QAX-A-Team) for their enthusiastic and unselfish answers!
|
||||
|
||||
## 📜Disclaimer ##
|
||||
|
||||
This tool is limited to legally authorized enterprise security construction. In the process of using this tool, you should ensure that all your actions comply with local laws and regulations and have obtained sufficient authorization.
|
||||
If you have any illegal behavior in the process of using this tool, you are responsible for all consequences, and all authors and all contributors of this tool do not assume any legal and joint responsibility.
|
||||
Unless you have fully read, fully understood and accepted all the terms of this Agreement, please do not install and use this tool.
|
||||
Your use or any other express or implied representation of you to this Agreement is deemed to have been read and agreed to be bound by this Agreement.
|
||||
@@ -4,8 +4,8 @@
|
||||
[](https://codecov.io/gh/shmilylty/OneForAll)
|
||||
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/releases)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/releases)
|
||||
|
||||
👊**OneForAll是一款功能强大的子域收集工具** 📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/README.en.md)
|
||||
|
||||
@@ -26,20 +26,20 @@
|
||||
|
||||
* **效率问题**，没有利用多进程，多线程以及异步协程技术，速度较慢。
|
||||
|
||||
为了解决以上痛点，此项目应用而生，OneForAll一词是来自我喜欢的一部日漫《[我的英雄学院](https://manhua.fzdm.com/131/)》，它是一种通过一代代的传承不断变强的潜力无穷的顶级个性，目前[番剧](https://www.bilibili.com/bangumi/media/md7452/)也更新到了第三季了，欢迎大佬们入坑😄。正如其名，我希望OneForAll是一款集百家之长，功能强大的全面快速子域收集终极神器🔨。
|
||||
为了解决以上痛点，此项目应用而生，OneForAll一词是来自我喜欢的一部日漫《[我的英雄学院](https://manhua.fzdm.com/131/)》，它是一种通过一代代的传承不断变强的潜力无穷的顶级个性，目前[番剧](https://www.bilibili.com/bangumi/media/md7452/)也更新到了第四季了，欢迎大佬们入坑😄。正如其名，我希望OneForAll是一款集百家之长，功能强大的全面快速子域收集终极神器🔨。
|
||||
|
||||
目前OneForAll还在开发中，肯定有不少问题和需要改进的地方，欢迎大佬们提交[Issues](https://github.com/shmilylty/OneForAll/issues)和[PR](https://github.com/shmilylty/OneForAll/pulls)，用着还行给个小星星✨吧，目前有一个专门用于OneForAll交流和反馈QQ群👨👨👦👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)，也可以给我发邮件📧[admin@hackfun.org]。
|
||||
目前OneForAll还在开发中，肯定有不少问题和需要改进的地方，欢迎大佬们提交[Issues](https://github.com/shmilylty/OneForAll/issues)和[PR](https://github.com/shmilylty/OneForAll/pulls)，用着还行给个小星星✨吧，目前有一个专门用于OneForAll交流和反馈QQ群👨👨👦👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)（加群验证：我的英雄学院），也可以给我发邮件📧[admin@hackfun.org]。
|
||||
|
||||
## 👍功能特性
|
||||
|
||||
* **收集能力强大**，详细模块请阅读[收集模块说明](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md)。
|
||||
1. 利用证书透明度收集子域（目前有6个模块：`censys_api`，`certdb_api`，`certspotter`，`crtsh`，`entrust`，`google`）
|
||||
1. 利用证书透明度收集子域（目前有6个模块：`censys_api`，`spyse_api`，`certspotter`，`crtsh`，`entrust`，`google`）
|
||||
|
||||
2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，检查robots文件`robots`，检查sitemap文件`sitemap`，后续会添加检查NSEC记录，NSEC3记录等模块）
|
||||
|
||||
3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善）
|
||||
|
||||
4. 利用DNS数据集收集子域（目前有18个模块：`binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`, `ptrarchive`, `sitedossier`,`threatcrowd`）
|
||||
4. 利用DNS数据集收集子域（目前有19个模块：`CeBaidu`, `binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`, `ptrarchive`, `sitedossier`,`threatcrowd`）
|
||||
|
||||
5. 利用DNS查询收集子域（目前有1个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，该模块还有待添加和完善）
|
||||
|
||||
@@ -50,8 +50,8 @@
|
||||
* **支持子域验证**，默认开启子域验证，自动解析子域DNS，自动请求子域获取title和banner，并综合判断子域存活情况。
|
||||
* **支持子域接管**，默认开启子域接管风险检查，支持子域自动接管（目前只有Github，有待完善），支持批量检查。
|
||||
* **处理功能强大**，发现的子域结果支持自动去除，自动DNS解析，HTTP请求探测，自动筛选出有效子域，拓展子域的Banner信息，最终支持的导出格式有`txt`, `rst`, `csv`, `tsv`, `json`, `yaml`, `html`, `xls`, `xlsx`, `dbf`, `latex`, `ods`。
|
||||
* **速度极快**，[收集模块](https://github.com/shmilylty/OneForAll/tree/master/oneforall/collect.py)使用多线程调用，[爆破模块](https://github.com/shmilylty/OneForAll/tree/master/oneforall/aiobrute.py)使用异步多进程多协程，子域验证中DNS解析和HTTP请求使用异步多协程，多线程检查[子域接管](https://github.com/shmilylty/OneForAll/tree/master/oneforall/takeover.py)风险。
|
||||
* **体验良好**，日志和终端输出全使用中文，各大模块都有进度条，异步保存各模块结果。
|
||||
* **速度极快**，[收集模块](https://github.com/shmilylty/OneForAll/tree/master/oneforall//collect.py)使用多线程调用，[爆破模块](https://github.com/shmilylty/OneForAll/tree/master/oneforall/aiobrute.py)使用异步多进程多协程，子域验证中DNS解析和HTTP请求使用异步多协程，多线程检查[子域接管](https://github.com/shmilylty/OneForAll/tree/master/oneforall/takeover.py)风险。
|
||||
* **体验良好**，日志和终端输出全使用中文，各模块都有进度条，异步保存各模块结果。
|
||||
|
||||
## 🚀上手指南
|
||||
|
||||
@@ -59,15 +59,15 @@
|
||||
|
||||
**🐍安装要求**
|
||||
|
||||
OneForAll是基于CPython开发的，所以你需要Python环境才能运行，如果你的系统还没有Python环境你可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)，理论上Python 3.6，3.7和3.8都可以正常运行OneForAll，**但是**许多测试都是在Python 3.7上进行的，所以**推荐**你使用**Python 3.7**版本运行OneForAll。运行以下命令检查Python和pip3版本：
|
||||
OneForAll是基于CPython开发的，所以你需要Python环境才能运行，如果你的系统还没有Python环境你可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)，理论上Python 3.7和3.8都可以正常运行OneForAll，**但是**许多测试都是在Python 3.8上进行的，所以**推荐**你使用**Python 3.8**版本运行OneForAll。运行以下命令检查Python和pip3版本：
|
||||
```bash
|
||||
python -V
|
||||
pip3 -V
|
||||
```
|
||||
如果你看到以下类似输出便说明Python环境没有问题：
|
||||
```bash
|
||||
Python 3.7.4
|
||||
pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.7)
|
||||
Python 3.8.0
|
||||
pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python38\site-packages\pip (python 3.8)
|
||||
```
|
||||
|
||||
**✔安装步骤**
|
||||
@@ -84,17 +84,16 @@ pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages
|
||||
```
|
||||
|
||||
2. **安装**
|
||||
首先运行以下命令
|
||||
你可以通过pip3安装OneForAll的依赖（如果你熟悉[pipenv](https://docs.pipenv.org/en/latest/)，那么推荐你使用[pipenv安装依赖]((https://github.com/shmilylty/OneForAll/tree/master/docs/Installation_dependency.md))），以下为**Windows系统**下使用**pip3**安装依赖的示例：（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python37`，那么请以管理员身份运行命令提示符cmd执行以下命令！）
|
||||
|
||||
```bash
|
||||
|
||||
```bash
|
||||
cd OneForAll/
|
||||
python -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
|
||||
python -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
|
||||
cd oneforall/
|
||||
python oneforall.py --help
|
||||
```
|
||||
其他系统平台的请参考[依赖安装](https://github.com/shmilylty/OneForAll/tree/master/docs/installation_dependency.md)，如果在安装依赖过程中发现编译某个依赖库失败时可以参考[Q&A](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md)中解决方法，如果还没有解决欢迎加群反馈。
|
||||
```
|
||||
其他系统平台的请参考[依赖安装](https://github.com/shmilylty/OneForAll/tree/master/docs/installation_dependency.md)，如果在安装依赖过程中发现编译某个依赖库失败时可以参考[Q&A](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md)中解决方法，如果还没有解决欢迎加群反馈。
|
||||
|
||||
3. **更新**
|
||||
❗注意：如果你之前已经克隆了项目运行之前请**备份**自己修改过的文件到项目外的地方（如**config.py**），然后执行以下命令**更新**项目：
|
||||
@@ -129,7 +128,7 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
|
||||
|
||||
[oneforall.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/oneforall.py)是主程序入口，oneforall.py可以调用[aiobrute.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/aiobrute.py)，[takerover.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/takerover.py)及[dbexport.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/dbexport.py)等模块，为了方便进行子域爆破独立出了aiobrute.py，为了方便进行子域接管风险检查独立出了takerover.py，为了方便数据库导出独立出了dbexport.py，这些模块都可以单独运行，并且所接受参数要更丰富一点。
|
||||
|
||||
❗注意：当你在使用过程中遇到一些问题或者疑惑时，请先到[Issues](https://github.com/shmilylty/OneForAll/tree/master/issues)里使用搜索找找答案，还可以参阅[常见问题与回答](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md)。
|
||||
❗注意：当你在使用过程中遇到一些问题或者疑惑时，请先到[Issues](https://github.com/shmilylty/OneForAll/issues)里使用搜索找找答案，还可以参阅[常见问题与回答](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md)。
|
||||
|
||||
1. **oneforall.py使用帮助**
|
||||
|
||||
@@ -160,7 +159,7 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
|
||||
Note:
|
||||
参数valid可选值1，0，None分别表示导出有效，无效，全部子域
|
||||
参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
|
||||
参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
|
||||
参数port可选值有'default', 'small', 'medium', 'large' 详见config.py配置
|
||||
参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
|
||||
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
|
||||
参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
|
||||
@@ -175,13 +174,13 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
|
||||
--verify=VERIFY
|
||||
验证子域有效性(默认True)
|
||||
--port=PORT
|
||||
请求验证的端口范围(默认medium)
|
||||
请求验证的端口范围(默认只探测80端口)
|
||||
--valid=VALID
|
||||
导出子域的有效性(默认1)
|
||||
--path=PATH
|
||||
导出路径(默认None)
|
||||
--format=FORMAT
|
||||
导出格式(默认xlsx)
|
||||
导出格式(默认xls)
|
||||
--show=SHOW
|
||||
终端显示导出数据(默认False)
|
||||
```
|
||||
@@ -251,7 +250,7 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
|
||||
--valid=VALID
|
||||
导出子域的有效性(默认None)
|
||||
--format=FORMAT
|
||||
导出格式(默认xlsx)
|
||||
导出格式(默认xls)
|
||||
--path=PATH
|
||||
导出路径(默认None)
|
||||
--show=SHOW
|
||||
@@ -316,8 +315,8 @@ D:.
|
||||
|
||||
## ⌛后续计划
|
||||
|
||||
- [ ] 各模块支持优化和完善
|
||||
- [ ] 子域监控（标记每次新发现的子域）
|
||||
- [ ] 各模块持续优化和完善
|
||||
- [x] 子域监控（标记每次新发现的子域）
|
||||
- [ ] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
|
||||
- [ ] 操作强大交互人性的前端界面实现（暂定：前端：Element + 后端：Flask）
|
||||
|
||||
@@ -327,28 +326,25 @@ D:.
|
||||
|
||||
该项目使用[SemVer](https://semver.org/)语言化版本格式进行版本管理，你可以在[Releases](https://github.com/shmilylty/OneForAll/releases)查看可用版本。
|
||||
|
||||
## 👨💻作者及贡献者
|
||||
## 👨💻贡献者
|
||||
|
||||
* **[Jing Ling](https://github.com/shmilylty)**
|
||||
* 核心开发
|
||||
|
||||
* **[Black Star](https://github.com/blackstar24)**
|
||||
* **[Black Star](https://github.com/blackstar24)**，**[Echocipher](https://github.com/Echocipher)**
|
||||
* 模块贡献
|
||||
|
||||
* **[Echocipher](https://github.com/Echocipher)**
|
||||
* 模块贡献
|
||||
|
||||
* **[iceMatcha](https://github.com/iceMatcha)**
|
||||
* **[iceMatcha](https://github.com/iceMatcha)**，**[mikuKeeper](https://github.com/mikuKeeper)**
|
||||
* 工具测试
|
||||
|
||||
* **Anyone**
|
||||
* 工具反馈
|
||||
|
||||
*你也可以在[CONTRIBUTORS.md](https://github.com/shmilylty/OneForAll/tree/master/CONTRIBUTORS.md)中参看所有参与该项目的开发者。*
|
||||
你可以在[CONTRIBUTORS.md](https://github.com/shmilylty/OneForAll/tree/master/CONTRIBUTORS.md)中参看所有参与该项目的开发者。
|
||||
|
||||
## 📄版权
|
||||
|
||||
该项目签署了GPL-3.0授权许可，详情请参阅[LICENSE.md](https://github.com/shmilylty/OneForAll/tree/master/LICENSE.md)。
|
||||
该项目签署了GPL-3.0授权许可，详情请参阅[LICENSE](https://github.com/shmilylty/OneForAll/LICENSE)。
|
||||
|
||||
## 😘鸣谢
|
||||
|
||||
@@ -361,4 +357,4 @@ D:.
|
||||
本工具仅限于合法授权的企业安全建设，在使用本工具过程中，您应确保自己所有行为符合当地的法律法规，并且已经取得了足够的授权。
|
||||
如您在使用本工具的过程中存在任何非法行为，您需自行承担所有后果，本工具所有作者和所有贡献者不承担任何法律及连带责任。
|
||||
除非您已充分阅读、完全理解并接受本协议所有条款，否则，请您不要安装并使用本工具。
|
||||
您的使用行为或者您以其他任何明示或者默示方式表示接受本协议的，即视为您已阅读并同意本协议的约束。
|
||||
您的使用行为或者您以其他任何明示或者默示方式表示接受本协议的，即视为您已阅读并同意本协议的约束。
|
||||
|
||||
+13
-4
@@ -10,10 +10,10 @@ A: 可以尝试以下方法:
|
||||
* [https://www.lfd.uci.edu/~gohlke/pythonlibs](https://www.lfd.uci.edu/~gohlke/pythonlibs)
|
||||
* [https://pythonwheels.com/](https://pythonwheels.com/)
|
||||
|
||||
选择好对应版本执行以下命令手动安装。举个例子，当编译pycares时失败时，找到[https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares](https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares)，由于我的系统是Windows 10 64位，使用的Python 3.7便下载`pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl`（一般来说下载最新版本的），然后手动安装：
|
||||
选择好对应版本执行以下命令手动安装。举个例子，当编译 brotlipy 时失败时，找到[https://www.lfd.uci.edu/~gohlke/pythonlibs/#brotlipy](https://www.lfd.uci.edu/~gohlke/pythonlibs/#brotlipy)，由于我的系统是Windows 10 64位，使用的Python 3.8便下载`brotlipy‑0.7.0‑cp38‑cp38‑win_amd64.whl`（一般来说下载最新版本的），然后手动安装：
|
||||
|
||||
```bash
|
||||
pip3 install pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl
|
||||
pip3 install brotlipy‑0.7.0‑cp38‑cp38‑win_amd64.whl
|
||||
```
|
||||
|
||||
2. 到库的项目地址issues和wiki等找找有没有解决方法，如果没有就给他们提issues发邮件😜。
|
||||
@@ -22,10 +22,19 @@ pip3 install pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl
|
||||
|
||||
**Q: 为什么运行OneForAll之后最终结果为空？**
|
||||
|
||||
A: 有几种可能性：第一可能目标域名没有子域。第二由于OneForAll默认会自动验证子域，在导出是只会有效子域，所以存在导出时没有有效子域的情况，你可以在运行OneForAll使用--valid=None指定导出所有发现的子域，你也可以使用--verify=False指定不验证子域的有效性。
|
||||
*A: 有几种可能性：第一可能目标域名没有子域。第二由于OneForAll默认会自动验证子域，在导出是只会有效子域，所以存在导出时没有有效子域的情况，你可以在运行OneForAll使用--valid=None指定导出所有发现的子域，你也可以使用--verify=False指定不验证子域的有效性。*
|
||||
|
||||
**Q: 安装依赖时出现以下类似报错，怎么解决？**
|
||||
Cannot uninstall 'PyYAML'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall.
|
||||
|
||||
A: 安装依赖时尝试加上--ignore-installed参数
|
||||
*A: 安装依赖时尝试加上--ignore-installed参数*
|
||||
|
||||
**Q:使用Excel打开结果的csv中有中文乱码，怎么解决？**
|
||||
|
||||
*A:由于Excel打开文件需要带BOM头识别编码，而默认生成的结果csv文件是UTF-8编码不是UTF-8-BOM，所以会使用Excel打开会出现中文乱码*，有以下解决办法：
|
||||
|
||||
1.在一开始运行OneForAll，设置format参数为xls或者xlsx格式。
|
||||
|
||||
2.使用NotePad++类似工具转化编码格式为UTF-8-BOM。
|
||||
|
||||
3.使用Excel软件导入结果csv文件中的数据。
|
||||
@@ -24,11 +24,11 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch'] # 只代理GoogleQuery和As
|
||||
| 模块名称 | 是否需要代理 | 是否需要API | 其他说明 |
|
||||
| ----------- | ------------ | ----------- | -------------------------------------------------- |
|
||||
| censys_api | 否 | 是 | API使用和申请见[config.py](../oneforall/config.py) |
|
||||
| certdb_api | 否 | 是 | API使用和申请见[config.py](../oneforall/config.py) |
|
||||
| certspotter | 否 | 否 | |
|
||||
| crtsh | 否 | 否 | |
|
||||
| entrust | 否 | 否 | |
|
||||
| google | 是 | 否 | |
|
||||
| spyse_api | 否 | 是 | API使用和申请见[config.py](../oneforall/config.py) |
|
||||
|
||||
|
||||
2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，后续会添加检查NSEC记录，NSEC3记录等模块）
|
||||
@@ -48,12 +48,13 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch'] # 只代理GoogleQuery和As
|
||||
| archivecrawl | 否 | 否 | |
|
||||
| commoncrawl | 否 | 否 | |
|
||||
|
||||
4. 利用DNS数据集收集子域（目前有18个模块：`binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`, `ptrarchive`, `sitedossier`,`threatcrowd`）
|
||||
4. 利用DNS数据集收集子域（目前有19个模块：`cebaidu`, `binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`, `ptrarchive`, `sitedossier`,`threatcrowd`）
|
||||
|
||||
| 模块名称 | 是否需要代理 | 是否需要API | 其他说明 |
|
||||
| ------------------ | ------------ | ----------- | -------------------------------------------------- |
|
||||
| binaryedge_api | 否 | 是 | API使用和申请见[config.py](../oneforall/config.py) |
|
||||
| bufferover | 否 | 否 | |
|
||||
| cebaidu | 否 | 否 | |
|
||||
| chinaz | 否 | 否 | |
|
||||
| chinaz_api | 否 | 是 | API使用和申请见[config.py](../oneforall/config.py) |
|
||||
| circl_api | 否 | 是 | API使用和申请见[config.py](../oneforall/config.py) |
|
||||
@@ -105,4 +106,4 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch'] # 只代理GoogleQuery和As
|
||||
| sogou | 否 | 否 | |
|
||||
| yahoo | 是 | 否 | |
|
||||
| yandex | 是 | 否 | |
|
||||
| zoomeye_api | 否 | 是 | API使用和申请见[config.py](../oneforall/config.py) |
|
||||
| zoomeye_api | 否 | 是 | API使用和申请见[config.py](../oneforall/config.py) |
|
||||
|
||||
@@ -2,15 +2,15 @@
|
||||
|
||||
你可以通过pip3和pipenv两种方法安装OneForAll的依赖（如果你熟悉[pipenv](https://docs.pipenv.org/en/latest/)，那么推荐使用你使用pipenv）：
|
||||
|
||||
* **Windows系统**（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python37`，那么请以管理员身份运行命令提示符cmd执行以下命令！）
|
||||
* **Windows系统**（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python38`，那么请以管理员身份运行命令提示符cmd执行以下命令！）
|
||||
|
||||
1. 使用pipenv
|
||||
|
||||
```bash
|
||||
cd OneForAll/
|
||||
python -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pipenv install --user --python 3.7
|
||||
python -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pip3 install pipenv -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pipenv install --python 3.8
|
||||
cd oneforall
|
||||
pipenv run python oneforall.py --help
|
||||
```
|
||||
@@ -19,8 +19,8 @@
|
||||
|
||||
```bash
|
||||
cd OneForAll/
|
||||
python -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
|
||||
python -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
|
||||
cd oneforall/
|
||||
python oneforall.py --help
|
||||
```
|
||||
@@ -29,17 +29,17 @@
|
||||
1. 使用pipenv
|
||||
```bash
|
||||
cd OneForAll/
|
||||
python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
sudo pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
|
||||
sudo pipenv install --user --python 3.7
|
||||
sudo python3 -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
sudo pip3 install pipenv -i https://mirrors.aliyun.com/pypi/simple/
|
||||
sudo pipenv install --python 3.8
|
||||
cd oneforall
|
||||
pipenv run python3 oneforall.py --help
|
||||
```
|
||||
2. 使用pip3
|
||||
```bash
|
||||
cd OneForAll/
|
||||
python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
|
||||
sudo python3 -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
sudo pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
|
||||
cd oneforall/
|
||||
python3 oneforall.py --help
|
||||
```
|
||||
@@ -48,17 +48,17 @@
|
||||
1. 使用pipenv
|
||||
```bash
|
||||
cd OneForAll/
|
||||
python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pipenv install --user --python 3.7
|
||||
python3 -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pip3 install pipenv -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pipenv install --python 3.8
|
||||
cd oneforall
|
||||
pipenv run python3 oneforall.py --help
|
||||
```
|
||||
2. 使用pip3
|
||||
```bash
|
||||
cd OneForAll/
|
||||
python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
|
||||
python3 -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
|
||||
pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
|
||||
cd oneforall/
|
||||
python3 oneforall.py --help
|
||||
```
|
||||
```
|
||||
|
||||
+34
-25
@@ -8,11 +8,11 @@ OneForAll多进程多协程异步子域爆破模块
|
||||
:license: GNU General Public License v3.0, see LICENSE for more details.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import queue
|
||||
import secrets
|
||||
import signal
|
||||
import time
|
||||
import queue
|
||||
import signal
|
||||
import asyncio
|
||||
import secrets
|
||||
|
||||
import aiomultiprocess as aiomp
|
||||
import exrex
|
||||
@@ -39,19 +39,22 @@ def detect_wildcard(domain):
|
||||
:return: 如果没有使用泛解析返回False 反之返回泛解析的IP集合和ttl整型值
|
||||
"""
|
||||
logger.log('INFOR', f'正在探测{domain}是否使用泛解析')
|
||||
token = secrets.token_hex(16)
|
||||
token = secrets.token_hex(4)
|
||||
random_subdomain = f'{token}.{domain}'
|
||||
try:
|
||||
answers = resolve.dns_query_a(random_subdomain)
|
||||
resolver = resolve.dns_resolver()
|
||||
answers = resolver.query(random_subdomain, 'A')
|
||||
# 如果查询随机域名A记录出错 说明不存在随机子域的A记录 即没有开启泛解析
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e)
|
||||
logger.log('INFOR', f'{domain}没有使用泛解析')
|
||||
return False, None, None
|
||||
ttl = answers.ttl
|
||||
name = answers.name
|
||||
ips = {item.address for item in answers}
|
||||
logger.log('ALERT', f'{domain}使用了泛解析')
|
||||
logger.log('ALERT', f'{random_subdomain} 解析到IP: {ips} TTL: {ttl}')
|
||||
logger.log('ALERT', f'{random_subdomain} 解析到域名: {name} '
|
||||
f'IP: {ips} TTL: {ttl}')
|
||||
return True, ips, ttl
|
||||
|
||||
|
||||
@@ -210,29 +213,35 @@ class AIOBrute(Module):
|
||||
return utils.split_list(domains, self.segment) # 分割任务组
|
||||
|
||||
def deal_results(self, results):
|
||||
for result in results:
|
||||
if result is None:
|
||||
for answer in results:
|
||||
if answer is None:
|
||||
continue
|
||||
if isinstance(result, Exception):
|
||||
if isinstance(answer, Exception):
|
||||
# logger.log('DEBUG', f'爆破{subdomain}时出错 {str(answers)}')
|
||||
continue
|
||||
if isinstance(result, tuple):
|
||||
subdomain, answers = result
|
||||
if not answers:
|
||||
ips = {item.address for item in answer}
|
||||
# 取值 如果是首次出现的IP集合 出现次数先赋值0
|
||||
value = self.ips_times.setdefault(str(ips), 0)
|
||||
self.ips_times[str(ips)] = value + 1
|
||||
ttl = answer.rrset.ttl
|
||||
subdomain = str(answer.rrset.name)
|
||||
# 目前域名开启了泛解析
|
||||
if self.enable_wildcard:
|
||||
# 通过对比查询的子域和响应的子域来判断真实子域
|
||||
# 去掉解析到CDN的情况
|
||||
if not subdomain.endswith(self.domain + '.'):
|
||||
continue
|
||||
ips = {record.host for record in answers}
|
||||
# 取值 如果是首次出现的IP集合 出现次数先赋值0
|
||||
value = self.ips_times.setdefault(str(ips), 0)
|
||||
self.ips_times[str(ips)] = value + 1
|
||||
ttl = answers[0].ttl
|
||||
if self.enable_wildcard:
|
||||
if wildcard_by_compare(ips,
|
||||
ttl,
|
||||
self.wildcard_ips,
|
||||
self.wildcard_ttl):
|
||||
continue
|
||||
# 通过对比解析到的IP集合和TTL确定子域来判断真实子域
|
||||
if wildcard_by_compare(ips,
|
||||
ttl,
|
||||
self.wildcard_ips,
|
||||
self.wildcard_ttl):
|
||||
continue
|
||||
# 通过对比解析到的IP集合的次数来判断真实子域
|
||||
if wildcard_by_times(ips, self.ips_times):
|
||||
continue
|
||||
# 只添加没有出现过的子域
|
||||
if subdomain not in self.subdomains:
|
||||
logger.log('INFOR', f'发现{self.domain}的子域: {subdomain} '
|
||||
f'解析IP: {ips} TTL: {ttl}')
|
||||
self.subdomains.add(subdomain)
|
||||
@@ -252,7 +261,7 @@ class AIOBrute(Module):
|
||||
initializer=init_worker,
|
||||
childconcurrency=self.coroutine) as pool:
|
||||
try:
|
||||
results = await pool.map(resolve.aiodns_query_a, task)
|
||||
results = await pool.map(resolve.dns_query_a, task)
|
||||
except KeyboardInterrupt:
|
||||
logger.log('ALERT', '爆破终止正在退出')
|
||||
pool.terminate() # 关闭pool，结束工作进程，不在处理未完成的任务。
|
||||
|
||||
@@ -37,7 +37,7 @@ class Database(object):
|
||||
|
||||
def create_table(self, table_name):
|
||||
"""
|
||||
初始化数据库
|
||||
创建表结构
|
||||
|
||||
:param str table_name: 要创建的表名
|
||||
"""
|
||||
@@ -53,8 +53,11 @@ class Database(object):
|
||||
f'status int,'
|
||||
f'reason text,'
|
||||
f'valid int,'
|
||||
f'new int,'
|
||||
f'title text,'
|
||||
f'banner text,'
|
||||
f'header text,'
|
||||
f'response text,'
|
||||
f'module text,'
|
||||
f'source text,'
|
||||
f'elapsed float,'
|
||||
@@ -70,21 +73,43 @@ class Database(object):
|
||||
:param list results: 结果列表
|
||||
:param str module_name: 模块名
|
||||
"""
|
||||
logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域结果存入数据库')
|
||||
logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域'
|
||||
'结果存入数据库')
|
||||
table_name = table_name.replace('.', '_')
|
||||
if results:
|
||||
try:
|
||||
self.conn.bulk_query(
|
||||
f'insert into "{table_name}" ('
|
||||
f'id, url, subdomain, port, ips, status, reason, valid,'
|
||||
f'title, banner, module, source, elapsed, count)'
|
||||
f'new, title, banner, header, response, module, source, '
|
||||
f'elapsed, count)'
|
||||
f'values (:id, :url, :subdomain, :port, :ips, :status,'
|
||||
f':reason, :valid, :title, :banner, :module, :source,'
|
||||
f':elapsed, :count)',
|
||||
f':reason, :valid, :new, :title, :banner, :header,'
|
||||
f':response, :module, :source,:elapsed, :count)',
|
||||
results)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
|
||||
def exist_table(self, table_name):
|
||||
"""
|
||||
判断是否存在某表
|
||||
|
||||
:param str table_name: 表名
|
||||
"""
|
||||
table_name = table_name.replace('.', '_')
|
||||
logger.log('DEBUG', f'正在查询是否存在{table_name}表')
|
||||
try:
|
||||
result = self.conn.query(f'select count() from sqlite_master '
|
||||
f'where type = "table" and '
|
||||
f'name = "{table_name}"')
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
else:
|
||||
if len(result) != 0:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def copy_table(self, table_name, bak_table_name):
|
||||
"""
|
||||
复制表创建备份
|
||||
@@ -130,7 +155,7 @@ class Database(object):
|
||||
|
||||
def rename_table(self, table_name, new_table_name):
|
||||
"""
|
||||
复制表创建备份
|
||||
重命名表名
|
||||
|
||||
:param str table_name: 表名
|
||||
:param str new_table_name: 新表名
|
||||
@@ -189,18 +214,22 @@ class Database(object):
|
||||
else:
|
||||
return rows
|
||||
|
||||
def get_subdomain(self, table_name, valid):
|
||||
def export_data(self, table_name, valid):
|
||||
"""
|
||||
获取表中的子域数据
|
||||
获取表中的部分数据
|
||||
|
||||
:param str table_name: 表名
|
||||
:param int valid: 是否有效
|
||||
:param any valid: 有效性
|
||||
"""
|
||||
table_name = table_name.replace('.', '_')
|
||||
query = f'select id, url, subdomain, port, ips, status, reason,' \
|
||||
f'valid, new, title, banner from "{table_name}"'
|
||||
if valid == 0 or valid == 1:
|
||||
where = f' where valid = {valid}'
|
||||
query += where
|
||||
logger.log('DEBUG', f'获取{table_name}表中的所有数据')
|
||||
try:
|
||||
rows = self.conn.query(
|
||||
f'select * from "{table_name}" where valid = {valid}')
|
||||
rows = self.conn.query(query)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
else:
|
||||
|
||||
@@ -220,8 +220,11 @@ class Module(object):
|
||||
'status': None,
|
||||
'reason': None,
|
||||
'valid': None,
|
||||
'new': None,
|
||||
'title': None,
|
||||
'banner': None,
|
||||
'header': None,
|
||||
'response': None,
|
||||
'module': self.module,
|
||||
'source': self.source,
|
||||
'elapsed': self.elapsed,
|
||||
@@ -240,9 +243,12 @@ class Module(object):
|
||||
'status': None,
|
||||
'reason': None,
|
||||
'valid': None,
|
||||
'new': None,
|
||||
'title': None,
|
||||
'banner': None,
|
||||
'module': self.module,
|
||||
'header': None,
|
||||
'response': None,
|
||||
'source': self.source,
|
||||
'elapsed': self.elapsed,
|
||||
'count': len(self.subdomains)}
|
||||
|
||||
+84
-57
@@ -1,70 +1,91 @@
|
||||
# coding=utf-8
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
|
||||
import aiohttp
|
||||
import tqdm
|
||||
from aiohttp import ClientSession
|
||||
from aiohttp.resolver import AsyncResolver
|
||||
from bs4 import BeautifulSoup
|
||||
import config
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
def get_limit_conn():
|
||||
limit_open_conn = config.limit_open_conn
|
||||
if limit_open_conn is None: # 默认情况
|
||||
limit_open_conn = utils.get_semaphore()
|
||||
elif not isinstance(limit_open_conn, int): # 如果传入不是数字的情况
|
||||
limit_open_conn = utils.get_semaphore()
|
||||
return limit_open_conn
|
||||
|
||||
|
||||
def get_ports(port):
|
||||
logger.log('INFOR', f'正在获取请求端口范围')
|
||||
logger.log('DEBUG', f'正在获取请求探测端口范围')
|
||||
ports = set()
|
||||
if isinstance(port, set):
|
||||
ports = port
|
||||
elif isinstance(port, str):
|
||||
if port not in {'small', 'medium', 'large', 'xlarge'}:
|
||||
logger.log('ERROR', f'不存在{port}等端口范围')
|
||||
port = 'medium'
|
||||
elif isinstance(port, list):
|
||||
ports = set(port)
|
||||
elif isinstance(port, tuple):
|
||||
ports = set(port)
|
||||
elif isinstance(port, int):
|
||||
if 0 <= port <= 65535:
|
||||
ports = {port}
|
||||
elif port in {'default', 'small', 'medium', 'large'}:
|
||||
logger.log('INFOR', f'探测{port}等端口范围')
|
||||
ports = config.ports.get(port)
|
||||
logger.log('INFOR', f'使用{port}等端口范围')
|
||||
if not ports: # 意外情况 ports_range为空使用使用中等端口范围
|
||||
logger.log('ALERT', f'使用medium等端口范围')
|
||||
ports = config.ports.get('medium')
|
||||
if not ports: # 意外情况
|
||||
logger.log('ERROR', f'指定探测端口范围有误')
|
||||
ports = {80}
|
||||
if ports == {80}:
|
||||
logger.log('INFOR', f'探测默认端口范围')
|
||||
return ports
|
||||
|
||||
|
||||
def gen_new_datas(datas, ports):
|
||||
logger.log('INFOR', f'正在生成请求地址')
|
||||
new_datas = []
|
||||
protocols = ['http://']
|
||||
protocols = ['http://', 'https://']
|
||||
for data in datas:
|
||||
valid = data.get('valid')
|
||||
if valid is None: # 子域有效性未知的才进行http请求探测
|
||||
subdomain = data.get('subdomain')
|
||||
for port in ports:
|
||||
for protocol in protocols:
|
||||
if port == 443:
|
||||
url = f'https://{subdomain}:{port}'
|
||||
elif port == 8443:
|
||||
url = f'https://{subdomain}:{port}'
|
||||
else:
|
||||
url = f'{protocol}{subdomain}:{port}'
|
||||
if port == 80:
|
||||
url = f'http://{subdomain}'
|
||||
data['id'] = None
|
||||
data['url'] = url
|
||||
data['port'] = port
|
||||
data['port'] = 80
|
||||
new_datas.append(data)
|
||||
data = dict(data) # 需要生成一个新的字典对象
|
||||
elif port == 443:
|
||||
url = f'https://{subdomain}'
|
||||
data['id'] = None
|
||||
data['url'] = url
|
||||
data['port'] = 443
|
||||
new_datas.append(data)
|
||||
data = dict(data) # 需要生成一个新的字典对象
|
||||
else:
|
||||
for protocol in protocols:
|
||||
url = f'{protocol}{subdomain}:{port}'
|
||||
data['id'] = None
|
||||
data['url'] = url
|
||||
data['port'] = port
|
||||
new_datas.append(data)
|
||||
data = dict(data) # 需要生成一个新的字典对象
|
||||
return new_datas
|
||||
|
||||
|
||||
async def fetch(session, url, semaphore):
|
||||
async def fetch(session, url):
|
||||
"""
|
||||
请求
|
||||
|
||||
:param session: session对象
|
||||
:param url: url地址
|
||||
:param semaphore: 并发信号量
|
||||
:return: 响应对象和响应文本
|
||||
"""
|
||||
timeout = aiohttp.ClientTimeout(total=config.get_timeout)
|
||||
async with semaphore:
|
||||
try:
|
||||
async with session.get(url,
|
||||
ssl=config.verify_ssl,
|
||||
allow_redirects=config.get_redirects,
|
||||
@@ -74,8 +95,10 @@ async def fetch(session, url, semaphore):
|
||||
try:
|
||||
text = await resp.text(encoding='gb2312') # 先尝试用fb2312解码
|
||||
except UnicodeDecodeError:
|
||||
text = await resp.text()
|
||||
text = await resp.text(errors='ignore')
|
||||
return resp, text
|
||||
except BaseException as exception:
|
||||
return exception
|
||||
|
||||
|
||||
def get_title(markup):
|
||||
@@ -99,6 +122,10 @@ def get_title(markup):
|
||||
if h2:
|
||||
return h2.text.strip()
|
||||
|
||||
h3 = soup.h3
|
||||
if h2:
|
||||
return h3.text.strip()
|
||||
|
||||
desc = soup.find('meta', attrs={'name': 'description'})
|
||||
if desc:
|
||||
return desc['content'].strip()
|
||||
@@ -110,30 +137,40 @@ def get_title(markup):
|
||||
if len(markup) <= 200:
|
||||
return markup.strip()
|
||||
|
||||
return soup.text.strip()
|
||||
text = soup.text
|
||||
if len(text) <= 200:
|
||||
return text.strip()
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def request_callback(future, index, datas):
|
||||
try:
|
||||
result = future.result()
|
||||
except BaseException as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
logger.log('TRACE', e.args)
|
||||
datas[index]['reason'] = str(e.args)
|
||||
datas[index]['valid'] = 0
|
||||
else:
|
||||
resp, text = result
|
||||
datas[index]['reason'] = resp.reason
|
||||
datas[index]['status'] = resp.status
|
||||
if resp.status >= 500:
|
||||
datas[index]['valid'] = 0
|
||||
if isinstance(result, tuple):
|
||||
resp, text = result
|
||||
datas[index]['reason'] = resp.reason
|
||||
datas[index]['status'] = resp.status
|
||||
if resp.status == 400 or resp.status >= 500:
|
||||
datas[index]['valid'] = 0
|
||||
else:
|
||||
datas[index]['valid'] = 1
|
||||
headers = resp.headers
|
||||
banner = str({'Server': headers.get('Server'),
|
||||
'Via': headers.get('Via'),
|
||||
'X-Powered-By': headers.get('X-Powered-By')})
|
||||
datas[index]['banner'] = banner[1:-1]
|
||||
datas[index]['title'] = get_title(text)
|
||||
datas[index]['header'] = str(dict(headers))[1:-1]
|
||||
datas[index]['response'] = text
|
||||
else:
|
||||
datas[index]['valid'] = 1
|
||||
headers = resp.headers
|
||||
banner = str({'Server': headers.get('Server'),
|
||||
'Via': headers.get('Via'),
|
||||
'X-Powered-By': headers.get('X-Powered-By')})
|
||||
datas[index]['banner'] = banner
|
||||
datas[index]['title'] = get_title(text)
|
||||
datas[index]['reason'] = 'Something error'
|
||||
datas[index]['valid'] = 0
|
||||
|
||||
|
||||
async def bulk_get_request(datas, port):
|
||||
@@ -141,19 +178,11 @@ async def bulk_get_request(datas, port):
|
||||
new_datas = gen_new_datas(datas, ports)
|
||||
logger.log('INFOR', f'正在异步进行子域的GET请求')
|
||||
|
||||
limit_open_conn = config.limit_open_conn
|
||||
if limit_open_conn is None: # 默认情况
|
||||
limit_open_conn = utils.get_semaphore()
|
||||
elif not isinstance(limit_open_conn, int): # 如果传入不是数字的情况
|
||||
limit_open_conn = utils.get_semaphore()
|
||||
# 使用异步域名解析器 自定义域名服务器
|
||||
resolver = AsyncResolver(nameservers=config.resolver_nameservers)
|
||||
conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
|
||||
limit_open_conn = get_limit_conn()
|
||||
conn = aiohttp.TCPConnector(ttl_dns_cache=300,
|
||||
ssl=config.verify_ssl,
|
||||
limit=limit_open_conn,
|
||||
limit_per_host=config.limit_per_host,
|
||||
resolver=resolver)
|
||||
|
||||
semaphore = asyncio.Semaphore(limit_open_conn)
|
||||
limit_per_host=config.limit_per_host)
|
||||
header = None
|
||||
if config.fake_header:
|
||||
header = utils.gen_fake_header()
|
||||
@@ -161,12 +190,13 @@ async def bulk_get_request(datas, port):
|
||||
tasks = []
|
||||
for i, data in enumerate(new_datas):
|
||||
url = data.get('url')
|
||||
task = asyncio.ensure_future(fetch(session, url, semaphore))
|
||||
task = asyncio.ensure_future(fetch(session, url))
|
||||
task.add_done_callback(functools.partial(request_callback,
|
||||
index=i,
|
||||
datas=new_datas))
|
||||
tasks.append(task)
|
||||
if tasks: # 任务列表里有任务不空时才进行解析
|
||||
# 任务列表里有任务不空时才进行解析
|
||||
if tasks:
|
||||
# 等待所有task完成 错误聚合到结果列表里
|
||||
futures = asyncio.as_completed(tasks)
|
||||
for future in tqdm.tqdm(futures,
|
||||
@@ -174,10 +204,7 @@ async def bulk_get_request(datas, port):
|
||||
desc='Progress',
|
||||
smoothing=1.0,
|
||||
ncols=True):
|
||||
try:
|
||||
await future
|
||||
except:
|
||||
pass
|
||||
await future
|
||||
|
||||
logger.log('INFOR', f'完成异步进行子域的GET请求')
|
||||
return new_datas
|
||||
|
||||
+18
-55
@@ -1,13 +1,10 @@
|
||||
# coding=utf-8
|
||||
import asyncio
|
||||
import functools
|
||||
|
||||
import dns.resolver
|
||||
import aiodns
|
||||
import tqdm
|
||||
import dns.resolver
|
||||
|
||||
import config
|
||||
from common import utils
|
||||
from config import logger
|
||||
|
||||
|
||||
@@ -22,7 +19,7 @@ def dns_resolver():
|
||||
return resolver
|
||||
|
||||
|
||||
def dns_query_a(hostname):
|
||||
async def dns_query_a(hostname):
|
||||
"""
|
||||
查询A记录
|
||||
|
||||
@@ -30,42 +27,12 @@ def dns_query_a(hostname):
|
||||
:return: 查询结果
|
||||
"""
|
||||
resolver = dns_resolver()
|
||||
return resolver.query(hostname, 'A')
|
||||
|
||||
|
||||
def aiodns_resolver():
|
||||
"""
|
||||
异步dns解析器
|
||||
"""
|
||||
return aiodns.DNSResolver(nameservers=config.resolver_nameservers,
|
||||
timeout=config.resolver_timeout)
|
||||
|
||||
|
||||
async def aiodns_query_a(hostname, semaphore=None):
|
||||
"""
|
||||
异步查询A记录
|
||||
|
||||
:param str hostname: 主机名
|
||||
:param semaphore: 并发查询数量
|
||||
:return: 主机名或查询结果或查询异常
|
||||
"""
|
||||
if semaphore is None:
|
||||
resolver = aiodns_resolver()
|
||||
try:
|
||||
answers = await resolver.query(hostname, 'A')
|
||||
except BaseException as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
answers = None
|
||||
return hostname, answers
|
||||
else:
|
||||
async with semaphore:
|
||||
resolver = aiodns_resolver()
|
||||
try:
|
||||
answers = await resolver.query(hostname, 'A')
|
||||
except BaseException as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
answers = None
|
||||
return hostname, answers
|
||||
try:
|
||||
answer = resolver.query(hostname, 'A')
|
||||
except BaseException as exception:
|
||||
logger.log('TRACE', exception.args)
|
||||
answer = exception
|
||||
return answer
|
||||
|
||||
|
||||
def resolve_callback(future, index, datas):
|
||||
@@ -76,18 +43,17 @@ def resolve_callback(future, index, datas):
|
||||
:param datas: 结果集
|
||||
"""
|
||||
try:
|
||||
result = future.result()
|
||||
answer = future.result()
|
||||
except BaseException as e:
|
||||
datas[index]['ips'] = str(e.args)
|
||||
datas[index]['valid'] = 0
|
||||
else:
|
||||
if isinstance(result, tuple):
|
||||
_, answers = result
|
||||
if answers:
|
||||
ips = {record.host for record in answers}
|
||||
datas[index]['ips'] = str(ips)
|
||||
else:
|
||||
datas[index]['ips'] = 'No answers'
|
||||
if isinstance(answer, dns.resolver.Answer):
|
||||
ips = {item.address for item in answer}
|
||||
datas[index]['ips'] = str(ips)[1:-1]
|
||||
else:
|
||||
datas[index]['ips'] = 'Something error'
|
||||
datas[index]['valid'] = 0
|
||||
|
||||
|
||||
async def bulk_query_a(datas):
|
||||
@@ -99,11 +65,11 @@ async def bulk_query_a(datas):
|
||||
"""
|
||||
logger.log('INFOR', '正在异步查询子域的A记录')
|
||||
tasks = []
|
||||
semaphore = asyncio.Semaphore(config.limit_resolve_conn)
|
||||
# semaphore = asyncio.Semaphore(config.limit_resolve_conn)
|
||||
for i, data in enumerate(datas):
|
||||
if not data.get('ips'):
|
||||
subdomain = data.get('subdomain')
|
||||
task = asyncio.ensure_future(aiodns_query_a(subdomain, semaphore))
|
||||
task = asyncio.ensure_future(dns_query_a(subdomain))
|
||||
task.add_done_callback(functools.partial(resolve_callback,
|
||||
index=i,
|
||||
datas=datas)) # 回调
|
||||
@@ -115,10 +81,7 @@ async def bulk_query_a(datas):
|
||||
desc='Progress',
|
||||
smoothing=1.0,
|
||||
ncols=True):
|
||||
try:
|
||||
await future
|
||||
except:
|
||||
pass
|
||||
await future
|
||||
# await asyncio.wait(tasks) # 等待所有task完成
|
||||
logger.log('INFOR', '完成异步查询子域的A记录')
|
||||
return datas
|
||||
|
||||
@@ -137,7 +137,7 @@ def get_semaphore():
|
||||
"""
|
||||
system = platform.system()
|
||||
if system == 'Windows':
|
||||
return 300
|
||||
return 800
|
||||
elif system == 'Linux':
|
||||
return 800
|
||||
elif system == 'Darwin':
|
||||
@@ -207,3 +207,29 @@ def check_response(method, resp):
|
||||
else:
|
||||
logger.log('ALERT', msg)
|
||||
return False
|
||||
|
||||
|
||||
def mark_subdomain(old_data, new_data):
|
||||
"""
|
||||
标记新增子域并返回新的数据集
|
||||
|
||||
:param old_data: 之前数据集
|
||||
:param new_data: 现在数据集
|
||||
:return: 已标记的新的数据集
|
||||
"""
|
||||
# 第一次收集子域的情况
|
||||
if not old_data:
|
||||
for index, item in enumerate(new_data):
|
||||
item['new'] = 1
|
||||
new_data[index] = item
|
||||
return new_data
|
||||
# 非第一次收集子域的情况
|
||||
old_subdomains = {item.get('subdomain') for item in old_data}
|
||||
for index, item in enumerate(new_data):
|
||||
subdomain = item.get('subdomain')
|
||||
if subdomain in old_subdomains:
|
||||
item['new'] = 0
|
||||
else:
|
||||
item['new'] = 1
|
||||
new_data[index] = item
|
||||
return new_data
|
||||
|
||||
+29
-30
@@ -31,7 +31,7 @@ enable_verify_subdomain = True # 验证子域有效性(默认True)
|
||||
enable_wildcard_check = True # 开启泛解析检测 会去掉泛解析的子域
|
||||
# 爆破时使用的进程数(根据系统中CPU数量情况设置 不宜大于CPU数量 默认为系统中的CPU数量)
|
||||
brute_process_num = os.cpu_count()
|
||||
brute_coroutine_num = 128 # 爆破时每个进程下的协程数(不宜大于1000)
|
||||
brute_coroutine_num = 64 # 爆破时每个进程下的协程数(不宜大于500)
|
||||
# 爆破所使用的字典路径 默认data/subdomains.txt
|
||||
brute_wordlist_path = data_storage_path.joinpath('subnames.txt')
|
||||
brute_task_segment = 500
|
||||
@@ -82,35 +82,34 @@ resolver_nameservers = [
|
||||
] # 指定查询的DNS域名服务器
|
||||
resolver_timeout = 5.0 # 解析超时时间
|
||||
resolver_lifetime = 30.0 # 解析存活时间
|
||||
limit_resolve_conn = 50 # 限制同一时间解析的数量(默认50)
|
||||
limit_resolve_conn = 500 # 限制同一时间解析的数量(默认500)
|
||||
|
||||
# http探测设置
|
||||
small_ports = {80, 443}
|
||||
medium_ports = {80, 443, 8000, 8080, 8443} # 默认使用
|
||||
large_ports = {80, 81, 443, 591, 2082, 2087, 2095, 2096, 3000, 8000, 8001,
|
||||
8008, 8080, 8083, 8443, 8834, 8888}
|
||||
xlarge_ports = {80, 81, 300, 443, 591, 593, 832, 981, 1010, 1311, 2082,
|
||||
2087, 2095, 2096, 2480, 3000, 3128, 3333, 4243, 4567, 4711,
|
||||
4712, 4993, 5000, 5104, 5108, 5800, 6543, 7000, 7396, 7474,
|
||||
8000, 8001, 8008, 8014, 8042, 8069, 8080, 8081, 8088, 8090,
|
||||
8091, 8118, 8123, 8172, 8222, 8243, 8280, 8281, 8333, 8443,
|
||||
8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080, 9090,
|
||||
9091, 9200, 9443, 9800, 9981, 12443, 16080, 18091, 18092,
|
||||
20720, 28017}
|
||||
ports = {'small': small_ports, 'medium': medium_ports,
|
||||
'large': large_ports, 'xlarge': xlarge_ports}
|
||||
# 请求端口探测设置
|
||||
default_ports = {80} # 默认使用
|
||||
small_ports = {80, 443, 8000, 8080, 8443}
|
||||
medium_ports = {80, 81, 443, 591, 2082, 2087, 2095, 2096, 3000, 8000, 8001,
|
||||
8008, 8080, 8083, 8443, 8834, 8888}
|
||||
large_ports = {80, 81, 300, 443, 591, 593, 832, 888, 981, 1010, 1311, 2082,
|
||||
2087, 2095, 2096, 2480, 3000, 3128, 3333, 4243, 4567, 4711,
|
||||
4712, 4993, 5000, 5104, 5108, 5800, 6543, 7000, 7396, 7474,
|
||||
8000, 8001, 8008, 8014, 8042, 8069, 8080, 8081, 8088, 8090,
|
||||
8091, 8016, 8118, 8123, 8172, 8222, 8243, 8280, 8281, 8333,
|
||||
8443, 8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080,
|
||||
9090, 9091, 9200, 9443, 9800, 9981, 12443, 16080, 18091, 18092,
|
||||
20720, 28017} # 可以在这里面添加端口
|
||||
ports = {'default': default_ports, 'small': small_ports,
|
||||
'medium': medium_ports, 'large': large_ports}
|
||||
verify_ssl = False
|
||||
# aiohttp 支持 HTTP/HTTPS形式的代理
|
||||
get_proxy = None # proxy="http://user:pass@some.proxy.com"
|
||||
get_timeout = 120 # http请求探测总超时时间 None或者0则表示不检测超时
|
||||
get_timeout = 60 # http请求探测总超时时间 None或者0则表示不检测超时
|
||||
get_redirects = True # 允许请求跳转
|
||||
fake_header = True # 使用伪造请求头
|
||||
# 限制同一时间打开的连接数(默认None，根据系统不同设置，Windows系统400 其他系统800)
|
||||
limit_open_conn = None
|
||||
limit_open_conn = 200
|
||||
# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
|
||||
limit_per_host = 0 # 默认0表示不限制
|
||||
|
||||
|
||||
# 模块API配置
|
||||
# Censys可以免费注册获取API：https://censys.io/api
|
||||
censys_api_id = ''
|
||||
@@ -158,8 +157,8 @@ virustotal_api_key = ''
|
||||
zoomeye_api_username = ''
|
||||
zoomeye_api_password = ''
|
||||
|
||||
# Certdb可以免费注册获取API: https://spyse.com/
|
||||
certdb_api_token = ''
|
||||
# Spyse可以免费注册获取API: https://spyse.com/
|
||||
spyse_api_token = ''
|
||||
|
||||
# https://www.circl.lu/services/passive-dns/
|
||||
circl_api_username = ''
|
||||
@@ -193,16 +192,16 @@ subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
|
||||
# 日志配置
|
||||
# 终端日志输出格式
|
||||
stdout_fmt = '<cyan>{time:HH:mm:ss,SSS}</cyan> ' \
|
||||
'[<level>{level: <5}</level>] ' \
|
||||
'<blue>{module}</blue>:<cyan>{line}</cyan> - ' \
|
||||
'<level>{message}</level>'
|
||||
'[<level>{level: <5}</level>] ' \
|
||||
'<blue>{module}</blue>:<cyan>{line}</cyan> - ' \
|
||||
'<level>{message}</level>'
|
||||
# 日志文件记录格式
|
||||
logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
|
||||
'[<level>{level: <5}</level>] ' \
|
||||
'<cyan>{process.name}({process.id})</cyan>:' \
|
||||
'<cyan>{thread.name: <10}({thread.id: <5})</cyan> | ' \
|
||||
'<blue>{module}</blue>.<blue>{function}</blue>:' \
|
||||
'<blue>{line}</blue> - <level>{message}</level>'
|
||||
'[<level>{level: <5}</level>] ' \
|
||||
'<cyan>{process.name}({process.id})</cyan>:' \
|
||||
'<cyan>{thread.name: <10}({thread.id: <5})</cyan> | ' \
|
||||
'<blue>{module}</blue>.<blue>{function}</blue>:' \
|
||||
'<blue>{line}</blue> - <level>{message}</level>'
|
||||
|
||||
log_path = result_save_path.joinpath('oneforall.log')
|
||||
|
||||
|
||||
@@ -1,102 +1,102 @@
|
||||
[
|
||||
"_afpovertcp._tcp.",
|
||||
"_aix._tcp.",
|
||||
"_autodiscover._tcp.",
|
||||
"_caldav._tcp.",
|
||||
"_certificates._tcp.",
|
||||
"_client._smtp.",
|
||||
"_cmp._tcp.",
|
||||
"_crls._tcp.",
|
||||
"_crl._tcp.",
|
||||
"_finger._tcp.",
|
||||
"_ftp._tcp.",
|
||||
"_gc._tcp.",
|
||||
"_h323be._tcp.",
|
||||
"_h323be._udp.",
|
||||
"_h323cs._tcp.",
|
||||
"_h323cs._udp.",
|
||||
"_h323ls._tcp.",
|
||||
"_h323ls._udp.",
|
||||
"_h323rs._tcp.",
|
||||
"_hkps._tcp.",
|
||||
"_hkp._tcp.",
|
||||
"_http._tcp.",
|
||||
"_iax.udp.",
|
||||
"_imaps._tcp.",
|
||||
"_imap._tcp.",
|
||||
"_jabber-client._tcp.",
|
||||
"_jabber-client._udp.",
|
||||
"_jabber._tcp.",
|
||||
"_jabber._udp.",
|
||||
"_kerberos-adm._tcp.",
|
||||
"_kerberos._tcp.",
|
||||
"_kerberos._tcp.dc._msdcs.",
|
||||
"_kerberos._udp.",
|
||||
"_kpasswd._tcp.",
|
||||
"_kpasswd._udp.",
|
||||
"_ldap._tcp.",
|
||||
"_ldap._tcp.dc._msdcs.",
|
||||
"_ldap._tcp.gc._msdcs.",
|
||||
"_ldap._tcp.pdc._msdcs.",
|
||||
"_msdcs.",
|
||||
"_mysqlsrv._tcp.",
|
||||
"_nntp._tcp.",
|
||||
"_ntp._udp.",
|
||||
"_ocsp._tcp.",
|
||||
"_pgpkeys._tcp.",
|
||||
"_pgprevokations._tcp.",
|
||||
"_PKIXREP._tcp.",
|
||||
"_pop3s._tcp.",
|
||||
"_pop3._tcp.",
|
||||
"_sipfederationtls._tcp.",
|
||||
"_sipinternal._tcp.",
|
||||
"_sipinternaltls._tcp.",
|
||||
"_sips._tcp.",
|
||||
"_sip._tcp.",
|
||||
"_sip._tls.",
|
||||
"_sip._udp.",
|
||||
"_smtp._tcp.",
|
||||
"_ssh._tcp.",
|
||||
"_stun._tcp.",
|
||||
"_stun._udp.",
|
||||
"_svcp._tcp.",
|
||||
"_tcp.",
|
||||
"_telnet._tcp.",
|
||||
"_test._tcp.",
|
||||
"_tls.",
|
||||
"_udp.",
|
||||
"_vlmcs._tcp.",
|
||||
"_vlmcs._udp.",
|
||||
"_whois._tcp.",
|
||||
"_wpad._tcp.",
|
||||
"_xmpp-client._tcp.",
|
||||
"_xmpp-client._udp.",
|
||||
"_xmpp-server._tcp.",
|
||||
"_xmpp-server._udp.",
|
||||
"_https._tcp.",
|
||||
"_imap.tcp.",
|
||||
"_kerberos.tcp.dc._msdcs.",
|
||||
"_ldap._tcp.ForestDNSZones.",
|
||||
"_submission._tcp.",
|
||||
"_caldavs._tcp.",
|
||||
"_carddav._tcp.",
|
||||
"_carddavs._tcp.",
|
||||
"_x-puppet._tcp.",
|
||||
"_x-puppet-ca._tcp.",
|
||||
"_domainkey.",
|
||||
"_pkixrep._tcp.",
|
||||
"_cisco-phone-http.",
|
||||
"_cisco-phone-tftp.",
|
||||
"_cisco-uds._tcp.",
|
||||
"_ciscowtp._tcp.",
|
||||
"_collab-edge._tls.",
|
||||
"_cuplogin._tcp.",
|
||||
"_client._smtp._tcp.",
|
||||
"_sftp._tcp.",
|
||||
"_h323rs._udp.",
|
||||
"_sql._tcp.",
|
||||
"_sip._tcp.internal.",
|
||||
"_snmp._udp.",
|
||||
"_rdp._tcp.",
|
||||
"_xmpp-server._udp."
|
||||
]
|
||||
[
|
||||
"_afpovertcp._tcp.",
|
||||
"_aix._tcp.",
|
||||
"_autodiscover._tcp.",
|
||||
"_caldav._tcp.",
|
||||
"_certificates._tcp.",
|
||||
"_client._smtp.",
|
||||
"_cmp._tcp.",
|
||||
"_crls._tcp.",
|
||||
"_crl._tcp.",
|
||||
"_finger._tcp.",
|
||||
"_ftp._tcp.",
|
||||
"_gc._tcp.",
|
||||
"_h323be._tcp.",
|
||||
"_h323be._udp.",
|
||||
"_h323cs._tcp.",
|
||||
"_h323cs._udp.",
|
||||
"_h323ls._tcp.",
|
||||
"_h323ls._udp.",
|
||||
"_h323rs._tcp.",
|
||||
"_hkps._tcp.",
|
||||
"_hkp._tcp.",
|
||||
"_http._tcp.",
|
||||
"_iax.udp.",
|
||||
"_imaps._tcp.",
|
||||
"_imap._tcp.",
|
||||
"_jabber-client._tcp.",
|
||||
"_jabber-client._udp.",
|
||||
"_jabber._tcp.",
|
||||
"_jabber._udp.",
|
||||
"_kerberos-adm._tcp.",
|
||||
"_kerberos._tcp.",
|
||||
"_kerberos._tcp.dc._msdcs.",
|
||||
"_kerberos._udp.",
|
||||
"_kpasswd._tcp.",
|
||||
"_kpasswd._udp.",
|
||||
"_ldap._tcp.",
|
||||
"_ldap._tcp.dc._msdcs.",
|
||||
"_ldap._tcp.gc._msdcs.",
|
||||
"_ldap._tcp.pdc._msdcs.",
|
||||
"_msdcs.",
|
||||
"_mysqlsrv._tcp.",
|
||||
"_nntp._tcp.",
|
||||
"_ntp._udp.",
|
||||
"_ocsp._tcp.",
|
||||
"_pgpkeys._tcp.",
|
||||
"_pgprevokations._tcp.",
|
||||
"_PKIXREP._tcp.",
|
||||
"_pop3s._tcp.",
|
||||
"_pop3._tcp.",
|
||||
"_sipfederationtls._tcp.",
|
||||
"_sipinternal._tcp.",
|
||||
"_sipinternaltls._tcp.",
|
||||
"_sips._tcp.",
|
||||
"_sip._tcp.",
|
||||
"_sip._tls.",
|
||||
"_sip._udp.",
|
||||
"_smtp._tcp.",
|
||||
"_ssh._tcp.",
|
||||
"_stun._tcp.",
|
||||
"_stun._udp.",
|
||||
"_svcp._tcp.",
|
||||
"_tcp.",
|
||||
"_telnet._tcp.",
|
||||
"_test._tcp.",
|
||||
"_tls.",
|
||||
"_udp.",
|
||||
"_vlmcs._tcp.",
|
||||
"_vlmcs._udp.",
|
||||
"_whois._tcp.",
|
||||
"_wpad._tcp.",
|
||||
"_xmpp-client._tcp.",
|
||||
"_xmpp-client._udp.",
|
||||
"_xmpp-server._tcp.",
|
||||
"_xmpp-server._udp.",
|
||||
"_https._tcp.",
|
||||
"_imap.tcp.",
|
||||
"_kerberos.tcp.dc._msdcs.",
|
||||
"_ldap._tcp.ForestDNSZones.",
|
||||
"_submission._tcp.",
|
||||
"_caldavs._tcp.",
|
||||
"_carddav._tcp.",
|
||||
"_carddavs._tcp.",
|
||||
"_x-puppet._tcp.",
|
||||
"_x-puppet-ca._tcp.",
|
||||
"_domainkey.",
|
||||
"_pkixrep._tcp.",
|
||||
"_cisco-phone-http.",
|
||||
"_cisco-phone-tftp.",
|
||||
"_cisco-uds._tcp.",
|
||||
"_ciscowtp._tcp.",
|
||||
"_collab-edge._tls.",
|
||||
"_cuplogin._tcp.",
|
||||
"_client._smtp._tcp.",
|
||||
"_sftp._tcp.",
|
||||
"_h323rs._udp.",
|
||||
"_sql._tcp.",
|
||||
"_sip._tcp.internal.",
|
||||
"_snmp._udp.",
|
||||
"_rdp._tcp.",
|
||||
"_xmpp-server._udp."
|
||||
]
|
||||
+3001
-1
File diff suppressed because it is too large
Load Diff
@@ -30,19 +30,14 @@ def export(table, db=None, valid=None, dpath=None, format='csv', show=False):
|
||||
:param str table: 要导出的表
|
||||
:param str db: 要导出的数据库路径(默认为results/result.sqlite3)
|
||||
:param int valid: 导出子域的有效性(默认None)
|
||||
:param str format: 导出格式(默认xls)
|
||||
:param str format: 导出格式(默认csv)
|
||||
:param str dpath: 导出目录(默认None)
|
||||
:param bool show: 终端显示导出数据(默认False)
|
||||
"""
|
||||
format = utils.check_format(format)
|
||||
dpath = utils.check_dpath(dpath)
|
||||
database = Database(db)
|
||||
if valid is None:
|
||||
rows = database.get_data(table)
|
||||
elif isinstance(valid, int):
|
||||
rows = database.get_subdomain(table, valid)
|
||||
else:
|
||||
rows = database.get_data(table) # 意外情况导出全部子域
|
||||
rows = database.export_data(table, valid) # 意外情况导出全部子域
|
||||
if show:
|
||||
print(rows.dataset)
|
||||
if format == 'txt':
|
||||
@@ -50,7 +45,7 @@ def export(table, db=None, valid=None, dpath=None, format='csv', show=False):
|
||||
else:
|
||||
data = rows.export(format)
|
||||
database.close()
|
||||
fpath = dpath.joinpath(f'{table}.{format}')
|
||||
fpath = dpath.joinpath(f'{table}_subdomain.{format}')
|
||||
utils.save_data(fpath, data)
|
||||
|
||||
|
||||
|
||||
@@ -24,19 +24,19 @@ class CensysAPI(Query):
|
||||
data = {
|
||||
'query': f'parsed.names: {self.domain}',
|
||||
'page': 1,
|
||||
'fields': ['parsed.subject_dn'],
|
||||
'fields': ['parsed.subject_dn', 'parsed.names'],
|
||||
'flatten': True}
|
||||
resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
|
||||
if not resp:
|
||||
return
|
||||
data = resp.json()
|
||||
status = data.get('status')
|
||||
json = resp.json()
|
||||
status = json.get('status')
|
||||
if status != 'ok':
|
||||
logger.log('ALERT', status)
|
||||
return
|
||||
subdomains = self.match(self.domain, str(data))
|
||||
subdomains = self.match(self.domain, str(json))
|
||||
self.subdomains = self.subdomains.union(subdomains)
|
||||
pages = data.get('metadata').get('pages')
|
||||
pages = json.get('metadata').get('pages')
|
||||
for page in range(2, pages + 1):
|
||||
time.sleep(self.delay)
|
||||
data['page'] = page
|
||||
|
||||
+67
-67
@@ -1,67 +1,67 @@
|
||||
import time
|
||||
import config
|
||||
from common import utils
|
||||
from common.query import Query
|
||||
|
||||
|
||||
class CertDBAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Certificate'
|
||||
self.source = 'CertDBAPIQuery'
|
||||
self.addr = 'https://api.spyse.com/v1/subdomains'
|
||||
self.token = config.certdb_api_token
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
page_num = 1
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'domain': self.domain,
|
||||
'api_token': self.token,
|
||||
'page': page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
json = resp.json()
|
||||
subdomains = utils.match_subdomain(self.domain, str(json))
|
||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
# 合并搜索子域名搜索结果
|
||||
self.subdomains = self.subdomains.union(subdomains)
|
||||
page_num += 1
|
||||
# 默认每次查询最多返回30条 当前条数小于30条说明已经查完
|
||||
if json.get('count') < 30:
|
||||
break
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.check(self.token):
|
||||
return
|
||||
self.begin()
|
||||
self.query()
|
||||
self.finish()
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
|
||||
|
||||
def do(domain): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
query = CertDBAPI(domain)
|
||||
query.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
do('example.com')
|
||||
import time
|
||||
import config
|
||||
from common import utils
|
||||
from common.query import Query
|
||||
|
||||
|
||||
class SpyseAPI(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Certificate'
|
||||
self.source = 'CertDBAPIQuery'
|
||||
self.addr = 'https://api.spyse.com/v1/subdomains'
|
||||
self.token = config.spyse_api_token
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
page_num = 1
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'domain': self.domain,
|
||||
'api_token': self.token,
|
||||
'page': page_num}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
json = resp.json()
|
||||
subdomains = utils.match_subdomain(self.domain, str(json))
|
||||
if not subdomains: # 搜索没有发现子域名则停止搜索
|
||||
break
|
||||
# 合并搜索子域名搜索结果
|
||||
self.subdomains = self.subdomains.union(subdomains)
|
||||
page_num += 1
|
||||
# 默认每次查询最多返回30条 当前条数小于30条说明已经查完
|
||||
if json.get('count') < 30:
|
||||
break
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
if not self.check(self.token):
|
||||
return
|
||||
self.begin()
|
||||
self.query()
|
||||
self.finish()
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
|
||||
|
||||
def do(domain): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
query = SpyseAPI(domain)
|
||||
query.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
do('example.com')
|
||||
@@ -34,7 +34,7 @@ class CheckAXFR(Module):
|
||||
"""
|
||||
logger.log('DEBUG', f'尝试对{self.domain}的域名服务器{server}进行域传送')
|
||||
try:
|
||||
xfr = dns.query.xfr(server, self.domain, timeout=30.0)
|
||||
xfr = dns.query.xfr(where=server, zone=self.domain)
|
||||
zone = dns.zone.from_xfr(xfr)
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', str(e))
|
||||
@@ -93,4 +93,4 @@ def do(domain): # 统一入口名字 方便多线程调用
|
||||
|
||||
if __name__ == '__main__':
|
||||
do('ZoneTransfer.me')
|
||||
do('example.com')
|
||||
# do('example.com')
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import time
|
||||
from common.query import Query
|
||||
|
||||
|
||||
class CeBaidu(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'CeBaiduQuery'
|
||||
self.addr = 'http://ce.baidu.com/index/getRelatedSites'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'site_address': self.domain}
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
subdomains = self.match(self.domain, str(resp.json()))
|
||||
# 合并搜索子域名搜索结果
|
||||
self.subdomains = self.subdomains.union(subdomains)
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
self.begin()
|
||||
self.query()
|
||||
self.finish()
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
|
||||
|
||||
def do(domain): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
query = CeBaidu(domain)
|
||||
query.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
do('example.com')
|
||||
@@ -1,92 +0,0 @@
|
||||
import time
|
||||
import random
|
||||
import cloudscraper
|
||||
from bs4 import BeautifulSoup
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class DNSdb(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'Dataset'
|
||||
self.source = 'DNSdbQuery'
|
||||
self.addr = 'http://www.dnsdb.org/'
|
||||
self.url = f'{self.addr}{self.domain}/'
|
||||
|
||||
def get_tokens(self):
|
||||
"""
|
||||
绕过cloudFlare验证并获取taken
|
||||
|
||||
:return: 绕过失败返回None 成功返回tokens
|
||||
"""
|
||||
scraper = cloudscraper.create_scraper()
|
||||
scraper.interpreter = 'js2py'
|
||||
scraper.proxies = self.get_proxy(self.source)
|
||||
scraper.timeout = 10
|
||||
try:
|
||||
tokens = scraper.get_tokens(self.url)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e.args)
|
||||
return None
|
||||
if len(tokens) != 2:
|
||||
return None
|
||||
return tokens
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
tokens = self.get_tokens()
|
||||
if not tokens:
|
||||
logger.log('ALERT', f'{self.source}模块绕过cloudFlare检查失败')
|
||||
return False
|
||||
self.cookie = tokens[0]
|
||||
self.header = {'User-Agent': tokens[1]}
|
||||
self.timeout = 10
|
||||
resp = self.get(self.url)
|
||||
if not resp:
|
||||
return
|
||||
if 'index' in resp.text:
|
||||
soup = BeautifulSoup(resp.text, features='lxml')
|
||||
base = self.addr+self.domain
|
||||
urls = list(map(lambda a: base + '/' + a.get('href'),
|
||||
soup.find_all('a')))
|
||||
urls = urls[:-1] # idn域名暂时不考虑
|
||||
for url in urls:
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
return
|
||||
subdomains = self.match(self.domain, resp.text)
|
||||
# 合并搜索子域名搜索结果
|
||||
self.subdomains = self.subdomains.union(subdomains)
|
||||
else:
|
||||
subdomains = self.match(self.domain, resp.text)
|
||||
# 合并搜索子域名搜索结果
|
||||
self.subdomains = self.subdomains.union(subdomains)
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
self.begin()
|
||||
self.query()
|
||||
self.finish()
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
|
||||
|
||||
def do(domain): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
query = DNSdb(domain)
|
||||
query.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
do('example.com')
|
||||
@@ -1,5 +1,6 @@
|
||||
import config
|
||||
from common.query import Query
|
||||
from config import logger
|
||||
|
||||
|
||||
class IPv4InfoAPI(Query):
|
||||
@@ -26,14 +27,18 @@ class IPv4InfoAPI(Query):
|
||||
return
|
||||
if resp.status_code != 200:
|
||||
break # 请求不正常通常网络是有问题，不再继续请求下去
|
||||
data = resp.json()
|
||||
subdomains = self.match(self.domain, str(data))
|
||||
try:
|
||||
json = resp.json()
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
break
|
||||
subdomains = self.match(self.domain, str(json))
|
||||
if not subdomains:
|
||||
break
|
||||
# 合并搜索子域名搜索结果
|
||||
self.subdomains = self.subdomains.union(subdomains)
|
||||
# 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
|
||||
subdomains = data.get('Subdomains')
|
||||
subdomains = json.get('Subdomains')
|
||||
if subdomains:
|
||||
# ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
|
||||
if len(subdomains) < 300:
|
||||
|
||||
@@ -4,69 +4,54 @@
|
||||
通过枚举域名常见的SRV记录并做查询来发现子域
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import aiodns
|
||||
import queue
|
||||
import threading
|
||||
|
||||
from common import utils
|
||||
from common import resolve
|
||||
from common.module import Module
|
||||
from config import data_storage_path, logger, resolver_nameservers
|
||||
from config import data_storage_path, logger
|
||||
|
||||
|
||||
class BruteSRV(Module):
|
||||
def __init__(self, domain: str):
|
||||
def __init__(self, domain):
|
||||
Module.__init__(self)
|
||||
self.domain = self.register(domain)
|
||||
self.module = 'dnsquery'
|
||||
self.source = "BruteSRV"
|
||||
self.loop = asyncio.new_event_loop()
|
||||
self.nameservers = resolver_nameservers
|
||||
self.resolver = aiodns.DNSResolver(self.nameservers, self.loop)
|
||||
self.thread_num = 10
|
||||
self.names_que = queue.Queue()
|
||||
self.answers_que = queue.Queue()
|
||||
|
||||
async def query(self, name):
|
||||
"""
|
||||
查询域名的SRV记录
|
||||
:param str name: SRV记录
|
||||
:return: 查询结果
|
||||
"""
|
||||
logger.log('TRACE', f'尝试查询{name}的SRV记录')
|
||||
try:
|
||||
answers = await self.resolver.query(name, 'SRV')
|
||||
except Exception as e:
|
||||
logger.log('TRACE', e)
|
||||
logger.log('TRACE', f'查询{name}的SRV记录失败')
|
||||
return None
|
||||
else:
|
||||
logger.log('TRACE', f'查询{name}的SRV记录成功')
|
||||
return answers
|
||||
def gen_names(self):
|
||||
path = data_storage_path.joinpath('srv_prefixes.json')
|
||||
with open(path) as file:
|
||||
prefixes = json.load(file)
|
||||
names = map(lambda prefix: prefix + self.domain, prefixes)
|
||||
|
||||
for name in names:
|
||||
self.names_que.put(name)
|
||||
|
||||
def brute(self):
|
||||
"""
|
||||
枚举域名的SRV记录
|
||||
"""
|
||||
names_path = data_storage_path.joinpath('srv_names.json')
|
||||
with open(names_path) as fp:
|
||||
names_dict = json.load(fp)
|
||||
query_map = map(lambda name: name + self.domain, names_dict)
|
||||
self.gen_names()
|
||||
|
||||
tasks = []
|
||||
for query in query_map:
|
||||
tasks.append(self.query(query))
|
||||
task_group = asyncio.gather(*tasks, loop=self.loop)
|
||||
self.loop.run_until_complete(asyncio.gather(task_group))
|
||||
self.loop.close()
|
||||
results = task_group.result()
|
||||
for result in results:
|
||||
if result:
|
||||
for answer in result:
|
||||
subdomains = utils.match_subdomain(self.domain, answer.host)
|
||||
if subdomains:
|
||||
self.subdomains = self.subdomains.union(subdomains)
|
||||
else:
|
||||
logger.log('DEBUG', f'{answer.host}不是{self.domain}的子域')
|
||||
if not len(self.subdomains):
|
||||
logger.log('DEBUG', f'没有找到{self.domain}的SRV记录')
|
||||
for i in range(self.thread_num):
|
||||
thread = BruteThread(self.names_que, self.answers_que)
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
self.names_que.join()
|
||||
|
||||
while not self.answers_que.empty():
|
||||
answer = self.answers_que.get()
|
||||
if answer is not None:
|
||||
for item in answer:
|
||||
subdomains = utils.match_subdomain(self.domain, str(item))
|
||||
self.subdomains = self.subdomains.union(subdomains)
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
@@ -80,6 +65,38 @@ class BruteSRV(Module):
|
||||
self.save_db()
|
||||
|
||||
|
||||
class BruteThread(threading.Thread):
|
||||
def __init__(self, names_que, answers_que):
|
||||
threading.Thread.__init__(self)
|
||||
self.names_que = names_que
|
||||
self.answers_que = answers_que
|
||||
self.resolver = resolve.dns_resolver()
|
||||
|
||||
def query(self, name):
|
||||
"""
|
||||
查询域名的SRV记录
|
||||
:param str name: SRV记录
|
||||
:return: 查询结果
|
||||
"""
|
||||
logger.log('TRACE', f'尝试查询{name}的SRV记录')
|
||||
try:
|
||||
answer = self.resolver.query(name, 'SRV')
|
||||
except Exception as exception:
|
||||
logger.log('TRACE', exception.args)
|
||||
logger.log('TRACE', f'查询{name}的SRV记录失败')
|
||||
return None
|
||||
else:
|
||||
logger.log('TRACE', f'查询{name}的SRV记录成功')
|
||||
return answer
|
||||
|
||||
def run(self):
|
||||
while True:
|
||||
name = self.names_que.get()
|
||||
answer = self.query(name)
|
||||
self.answers_que.put(answer)
|
||||
self.names_que.task_done()
|
||||
|
||||
|
||||
def do(domain): # 统一入口名字 方便多线程调用
|
||||
"""
|
||||
类统一调用入口
|
||||
@@ -91,5 +108,5 @@ def do(domain): # 统一入口名字 方便多线程调用
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
do('example.com')
|
||||
do('zonetransfer.me')
|
||||
# do('example.com')
|
||||
|
||||
@@ -21,9 +21,11 @@ class VirusTotalAPI(Query):
|
||||
resp = self.get(self.addr, params)
|
||||
if not resp:
|
||||
return
|
||||
data = resp.json()
|
||||
subdomains = set(data.get('subdomains'))
|
||||
self.subdomains = self.subdomains.union(subdomains)
|
||||
json = resp.json()
|
||||
data = json.get('subdomains')
|
||||
if data:
|
||||
subdomains = set(data)
|
||||
self.subdomains = self.subdomains.union(subdomains)
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
|
||||
@@ -80,7 +80,6 @@ class Github(Search):
|
||||
break
|
||||
soup = BeautifulSoup(resp.text, 'lxml')
|
||||
subdomains = self.match(self.domain, soup.text)
|
||||
print(subdomains)
|
||||
self.subdomains = self.subdomains.union(subdomains)
|
||||
if not subdomains:
|
||||
break
|
||||
|
||||
@@ -23,7 +23,7 @@ class ZoomEyeAPI(Search):
|
||||
data = {'username': self.user, 'password': self.pwd}
|
||||
resp = self.post(url=url, json=data)
|
||||
if not resp:
|
||||
logger.log('FETAL', f'登录失败无法获取{self.source}的访问token')
|
||||
logger.log('FATAL', f'登录失败无法获取{self.source}的访问token')
|
||||
return
|
||||
data = resp.json()
|
||||
if resp.status_code == 200:
|
||||
|
||||
+37
-24
@@ -28,7 +28,7 @@ blue = '\033[01;34m'
|
||||
red = '\033[1;31m'
|
||||
end = '\033[0m'
|
||||
|
||||
version = white + '{' + red + 'v0.0.6#dev' + white + '}'
|
||||
version = white + '{' + red + 'v0.0.8#dev' + white + '}'
|
||||
|
||||
banner = f"""{yellow}
|
||||
___ _ _
|
||||
@@ -42,7 +42,7 @@ class OneForAll(object):
|
||||
"""
|
||||
OneForAll是一款功能强大的子域收集工具
|
||||
|
||||
Version: 0.0.6
|
||||
Version: 0.0.8
|
||||
Project: https://git.io/fjHT1
|
||||
|
||||
Example:
|
||||
@@ -59,25 +59,25 @@ class OneForAll(object):
|
||||
Note:
|
||||
参数valid可选值1，0，None分别表示导出有效，无效，全部子域
|
||||
参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
|
||||
参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
|
||||
参数port可选值有'default', 'small', 'medium', 'large', 详见config.py配置
|
||||
参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
|
||||
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
|
||||
|
||||
:param str target: 单个域名或者每行一个域名的文件路径(必需参数)
|
||||
:param bool brute: 使用爆破模块(默认False)
|
||||
:param bool verify: 验证子域有效性(默认True)
|
||||
:param str port: 请求验证的端口范围(默认medium)
|
||||
:param str port: 请求验证子域的端口范围(默认只探测80端口)
|
||||
:param int valid: 导出子域的有效性(默认1)
|
||||
:param str format: 导出格式(默认csv)
|
||||
:param bool show: 终端显示导出数据(默认False)
|
||||
"""
|
||||
def __init__(self, target, brute=None, verify=None, port='medium', valid=1,
|
||||
format='csv', takeover=True, show=False):
|
||||
def __init__(self, target, brute=None, verify=None, port='default',
|
||||
valid=1, format='csv', takeover=True, show=False):
|
||||
self.target = target
|
||||
self.port = port
|
||||
self.domains = set()
|
||||
self.domain = str()
|
||||
self.datas = list()
|
||||
self.data = list()
|
||||
self.brute = brute
|
||||
self.verify = verify
|
||||
self.takeover = takeover
|
||||
@@ -90,7 +90,8 @@ class OneForAll(object):
|
||||
self.brute = config.enable_brute_module
|
||||
if self.verify is None:
|
||||
self.verify = config.enable_verify_subdomain
|
||||
rename_table = self.domain + '_last'
|
||||
old_table = self.domain + '_last'
|
||||
new_table = self.domain + '_now'
|
||||
collect = Collect(self.domain, export=False)
|
||||
collect.run()
|
||||
if self.brute:
|
||||
@@ -102,49 +103,61 @@ class OneForAll(object):
|
||||
db.copy_table(self.domain, self.domain+'_ori')
|
||||
db.remove_invalid(self.domain)
|
||||
db.deduplicate_subdomain(self.domain)
|
||||
|
||||
old_data = []
|
||||
# 非第一次收集子域的情况时数据库预处理
|
||||
if db.exist_table(new_table):
|
||||
db.drop_table(old_table) # 如果存在上次收集结果表就先删除
|
||||
db.rename_table(new_table, old_table) # 新表重命名为旧表
|
||||
old_data = db.get_data(old_table).as_dict()
|
||||
|
||||
# 不验证子域的情况
|
||||
if not self.verify:
|
||||
# 数据库导出
|
||||
self.valid = None
|
||||
dbexport.export(self.domain, valid=self.valid, format=self.format,
|
||||
show=self.show)
|
||||
db.drop_table(rename_table)
|
||||
db.rename_table(self.domain, rename_table)
|
||||
dbexport.export(self.domain, valid=self.valid,
|
||||
format=self.format, show=self.show)
|
||||
db.drop_table(new_table)
|
||||
db.rename_table(self.domain, new_table)
|
||||
return
|
||||
# 开始验证子域工作
|
||||
self.datas = db.get_data(self.domain).as_dict()
|
||||
self.data = db.get_data(self.domain).as_dict()
|
||||
|
||||
# 标记新发现子域
|
||||
self.data = utils.mark_subdomain(old_data, self.data)
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
# 解析域名地址
|
||||
task = resolve.bulk_query_a(self.datas)
|
||||
self.datas = loop.run_until_complete(task)
|
||||
task = resolve.bulk_query_a(self.data)
|
||||
self.data = loop.run_until_complete(task)
|
||||
|
||||
# 保存解析结果
|
||||
resolve_table = self.domain + '_res'
|
||||
db.drop_table(resolve_table)
|
||||
db.create_table(resolve_table)
|
||||
db.save_db(resolve_table, self.datas, 'resolve')
|
||||
db.save_db(resolve_table, self.data, 'resolve')
|
||||
|
||||
# 请求域名地址
|
||||
task = request.bulk_get_request(self.datas, self.port)
|
||||
self.datas = loop.run_until_complete(task)
|
||||
task = request.bulk_get_request(self.data, self.port)
|
||||
self.data = loop.run_until_complete(task)
|
||||
# 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
|
||||
loop.run_until_complete(asyncio.sleep(0.25))
|
||||
|
||||
db.clear_table(self.domain)
|
||||
db.save_db(self.domain, self.datas)
|
||||
db.save_db(self.domain, self.data)
|
||||
|
||||
# 数据库导出
|
||||
dbexport.export(self.domain, valid=self.valid, format=self.format,
|
||||
show=self.show)
|
||||
db.drop_table(rename_table)
|
||||
db.rename_table(self.domain, rename_table)
|
||||
dbexport.export(self.domain, valid=self.valid,
|
||||
format=self.format, show=self.show)
|
||||
db.drop_table(new_table)
|
||||
db.rename_table(self.domain, new_table)
|
||||
db.close()
|
||||
# 子域接管检查
|
||||
|
||||
if self.takeover:
|
||||
subdomains = set(map(lambda x: x.get('subdomain'), self.datas))
|
||||
subdomains = set(map(lambda x: x.get('subdomain'), self.data))
|
||||
takeover = Takeover(subdomains)
|
||||
takeover.run()
|
||||
|
||||
|
||||
@@ -34,8 +34,8 @@ def get_cname(subdomain):
|
||||
resolver = resolve.dns_resolver()
|
||||
try:
|
||||
answers = resolver.query(subdomain, 'CNAME')
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
except Exception as exception:
|
||||
logger.log('TRACE', exception.args)
|
||||
return None
|
||||
for answer in answers:
|
||||
return answer.to_text() # 一个子域只有一个CNAME记录
|
||||
@@ -58,7 +58,7 @@ class Takeover(Module):
|
||||
'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
|
||||
参数dpath为None默认使用OneForAll结果目录
|
||||
|
||||
:param str target: 单个子域或者每行一个子域的文件路径(必需参数)
|
||||
:param any target: 单个子域或者每行一个子域的文件路径(必需参数)
|
||||
:param int thread: 线程数(默认100)
|
||||
:param str format: 导出格式(默认csv)
|
||||
:param str dpath: 导出目录(默认None)
|
||||
@@ -83,12 +83,13 @@ class Takeover(Module):
|
||||
data = str(self.results)
|
||||
else:
|
||||
data = self.results.export(self.format)
|
||||
fpath = self.dpath.joinpath(f'takeover.{self.format}')
|
||||
ts = int(time.time())
|
||||
fpath = self.dpath.joinpath(f'takeover_{ts}.{self.format}')
|
||||
utils.save_data(fpath, data)
|
||||
|
||||
def compare(self, subdomain, cname, responses):
|
||||
domain_resp = self.get('http://' + subdomain, check=False)
|
||||
cname_resp = self.get('http://'+cname, check=False)
|
||||
cname_resp = self.get('http://' + cname, check=False)
|
||||
if domain_resp is None or cname_resp is None:
|
||||
return
|
||||
|
||||
|
||||
+18
-20
@@ -1,21 +1,19 @@
|
||||
aiodns==2.0.0
|
||||
aiohttp==3.5.4
|
||||
aiohttp==3.6.2
|
||||
aiomultiprocess==0.6.1
|
||||
asn1crypto==0.24.0
|
||||
async-timeout==3.0.1
|
||||
attrs==19.1.0
|
||||
attrs==19.3.0
|
||||
backports.csv==1.0.7
|
||||
beautifulsoup4==4.8.0
|
||||
beautifulsoup4==4.8.1
|
||||
Brotli==1.0.7
|
||||
brotlipy==0.7.0
|
||||
bs4==0.0.1
|
||||
cchardet==2.1.4
|
||||
certifi==2019.6.16
|
||||
cffi==1.12.3
|
||||
certifi==2019.9.11
|
||||
cffi==1.13.1
|
||||
chardet==3.0.4
|
||||
cloudscraper==1.1.40
|
||||
cloudscraper==1.2.5
|
||||
colorama==0.4.1
|
||||
cryptography==2.7
|
||||
cryptography==2.8
|
||||
defusedxml==0.6.0
|
||||
dnspython==1.16.0
|
||||
docopt==0.6.2
|
||||
@@ -26,31 +24,31 @@ idna==2.8
|
||||
jdcal==1.4.1
|
||||
Js2Py==0.66
|
||||
loguru==0.3.2
|
||||
lxml==4.4.1
|
||||
lxml==4.3.5
|
||||
MarkupPy==1.14
|
||||
multidict==4.5.2
|
||||
odfpy==1.4.0
|
||||
openpyxl==2.4.11
|
||||
pycares==3.0.0
|
||||
pycparser==2.19
|
||||
pyjsparser==2.7.1
|
||||
pyOpenSSL==19.0.0
|
||||
PySocks==1.7.0
|
||||
pytz==2019.2
|
||||
PySocks==1.7.1
|
||||
pytz==2019.3
|
||||
PyYAML==5.1.2
|
||||
records==0.5.3
|
||||
requests==2.22.0
|
||||
requests-file==1.4.3
|
||||
requests-toolbelt==0.9.1
|
||||
six==1.12.0
|
||||
soupsieve==1.9.3
|
||||
SQLAlchemy==1.3.7
|
||||
tablib==0.13.0
|
||||
soupsieve==1.9.4
|
||||
SQLAlchemy==1.3.10
|
||||
tablib==0.14.0
|
||||
termcolor==1.1.0
|
||||
tldextract==2.2.1
|
||||
tqdm==4.34.0
|
||||
tldextract==2.2.2
|
||||
tqdm==4.36.1
|
||||
tzlocal==2.0.0
|
||||
urllib3==1.25.3
|
||||
urllib3==1.25.6
|
||||
win32-setctime==1.0.1
|
||||
xlrd==1.2.0
|
||||
xlwt==1.3.0
|
||||
yarl==1.3.0
|
||||
yarl==1.3.0
|
||||
Reference in New Issue
Block a user