mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-26 04:47:48 +08:00
Merge remote-tracking branch 'origin/master'
This commit is contained in:
Jing Ling
@@ -36,7 +36,7 @@ A clear and concise description of the actual results (optional, such as any err
|
||||
Screenshot of complete OneForAll execution process (recommended upload)
|
||||
|
||||
**Log upload**
|
||||
Upload oneforall.log log files (it is recommended to upload logs in case of complex problems)
|
||||
Upload oneforall.log files (it is recommended to upload logs in case of complex problems)
|
||||
|
||||
**Supplementary information**
|
||||
Some other supplementary notes about bug
|
||||
|
||||
Generated
+121
-69
@@ -18,11 +18,11 @@
|
||||
"default": {
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7",
|
||||
"sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8",
|
||||
"sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c"
|
||||
"sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35",
|
||||
"sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25",
|
||||
"sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"
|
||||
],
|
||||
"version": "==4.9.1"
|
||||
"version": "==4.9.3"
|
||||
},
|
||||
"bs4": {
|
||||
"hashes": [
|
||||
@@ -33,33 +33,33 @@
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
|
||||
"sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
|
||||
"sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
|
||||
"sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
|
||||
],
|
||||
"version": "==2020.6.20"
|
||||
"version": "==2020.12.5"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
|
||||
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
|
||||
"sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
|
||||
"sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
|
||||
],
|
||||
"version": "==3.0.4"
|
||||
"version": "==4.0.0"
|
||||
},
|
||||
"colorama": {
|
||||
"hashes": [
|
||||
"sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff",
|
||||
"sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"
|
||||
"sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b",
|
||||
"sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"
|
||||
],
|
||||
"markers": "sys_platform == 'win32'",
|
||||
"version": "==0.4.3"
|
||||
"version": "==0.4.4"
|
||||
},
|
||||
"dnspython": {
|
||||
"hashes": [
|
||||
"sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7",
|
||||
"sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d"
|
||||
"sha256:95d12f6ef0317118d2a1a6fc49aac65ffec7eb8087474158f42f26a639135216",
|
||||
"sha256:e4a87f0b573201a0f3727fa18a516b055fd1107e0e5477cded4a2de497df1dd4"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.0.0"
|
||||
"version": "==2.1.0"
|
||||
},
|
||||
"exrex": {
|
||||
"hashes": [
|
||||
@@ -70,10 +70,10 @@
|
||||
},
|
||||
"fire": {
|
||||
"hashes": [
|
||||
"sha256:9736a16227c3d469e5d2d296bce5b4d8fa8d7851e953bda327a455fc2994307f"
|
||||
"sha256:c5e2b8763699d1142393a46d0e3e790c5eb2f0706082df8f647878842c216a62"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.3.1"
|
||||
"version": "==0.4.0"
|
||||
},
|
||||
"future": {
|
||||
"hashes": [
|
||||
@@ -81,6 +81,55 @@
|
||||
],
|
||||
"version": "==0.18.2"
|
||||
},
|
||||
"greenlet": {
|
||||
"hashes": [
|
||||
"sha256:0a77691f0080c9da8dfc81e23f4e3cffa5accf0f5b56478951016d7cfead9196",
|
||||
"sha256:0ddd77586553e3daf439aa88b6642c5f252f7ef79a39271c25b1d4bf1b7cbb85",
|
||||
"sha256:111cfd92d78f2af0bc7317452bd93a477128af6327332ebf3c2be7df99566683",
|
||||
"sha256:122c63ba795fdba4fc19c744df6277d9cfd913ed53d1a286f12189a0265316dd",
|
||||
"sha256:181300f826625b7fd1182205b830642926f52bd8cdb08b34574c9d5b2b1813f7",
|
||||
"sha256:1a1ada42a1fd2607d232ae11a7b3195735edaa49ea787a6d9e6a53afaf6f3476",
|
||||
"sha256:1bb80c71de788b36cefb0c3bb6bfab306ba75073dbde2829c858dc3ad70f867c",
|
||||
"sha256:1d1d4473ecb1c1d31ce8fd8d91e4da1b1f64d425c1dc965edc4ed2a63cfa67b2",
|
||||
"sha256:292e801fcb3a0b3a12d8c603c7cf340659ea27fd73c98683e75800d9fd8f704c",
|
||||
"sha256:2c65320774a8cd5fdb6e117c13afa91c4707548282464a18cf80243cf976b3e6",
|
||||
"sha256:4365eccd68e72564c776418c53ce3c5af402bc526fe0653722bc89efd85bf12d",
|
||||
"sha256:5352c15c1d91d22902582e891f27728d8dac3bd5e0ee565b6a9f575355e6d92f",
|
||||
"sha256:58ca0f078d1c135ecf1879d50711f925ee238fe773dfe44e206d7d126f5bc664",
|
||||
"sha256:5d4030b04061fdf4cbc446008e238e44936d77a04b2b32f804688ad64197953c",
|
||||
"sha256:5d69bbd9547d3bc49f8a545db7a0bd69f407badd2ff0f6e1a163680b5841d2b0",
|
||||
"sha256:5f297cb343114b33a13755032ecf7109b07b9a0020e841d1c3cedff6602cc139",
|
||||
"sha256:62afad6e5fd70f34d773ffcbb7c22657e1d46d7fd7c95a43361de979f0a45aef",
|
||||
"sha256:647ba1df86d025f5a34043451d7c4a9f05f240bee06277a524daad11f997d1e7",
|
||||
"sha256:719e169c79255816cdcf6dccd9ed2d089a72a9f6c42273aae12d55e8d35bdcf8",
|
||||
"sha256:7cd5a237f241f2764324396e06298b5dee0df580cf06ef4ada0ff9bff851286c",
|
||||
"sha256:875d4c60a6299f55df1c3bb870ebe6dcb7db28c165ab9ea6cdc5d5af36bb33ce",
|
||||
"sha256:90b6a25841488cf2cb1c8623a53e6879573010a669455046df5f029d93db51b7",
|
||||
"sha256:94620ed996a7632723a424bccb84b07e7b861ab7bb06a5aeb041c111dd723d36",
|
||||
"sha256:b5f1b333015d53d4b381745f5de842f19fe59728b65f0fbb662dafbe2018c3a5",
|
||||
"sha256:c5b22b31c947ad8b6964d4ed66776bcae986f73669ba50620162ba7c832a6b6a",
|
||||
"sha256:c93d1a71c3fe222308939b2e516c07f35a849c5047f0197442a4d6fbcb4128ee",
|
||||
"sha256:cdb90267650c1edb54459cdb51dab865f6c6594c3a47ebd441bc493360c7af70",
|
||||
"sha256:cfd06e0f0cc8db2a854137bd79154b61ecd940dce96fad0cba23fe31de0b793c",
|
||||
"sha256:d3789c1c394944084b5e57c192889985a9f23bd985f6d15728c745d380318128",
|
||||
"sha256:da7d09ad0f24270b20f77d56934e196e982af0d0a2446120cb772be4e060e1a2",
|
||||
"sha256:df3e83323268594fa9755480a442cabfe8d82b21aba815a71acf1bb6c1776218",
|
||||
"sha256:df8053867c831b2643b2c489fe1d62049a98566b1646b194cc815f13e27b90df",
|
||||
"sha256:e1128e022d8dce375362e063754e129750323b67454cac5600008aad9f54139e",
|
||||
"sha256:e6e9fdaf6c90d02b95e6b0709aeb1aba5affbbb9ccaea5502f8638e4323206be",
|
||||
"sha256:eac8803c9ad1817ce3d8d15d1bb82c2da3feda6bee1153eec5c58fa6e5d3f770",
|
||||
"sha256:eb333b90036358a0e2c57373f72e7648d7207b76ef0bd00a4f7daad1f79f5203",
|
||||
"sha256:ed1d1351f05e795a527abc04a0d82e9aecd3bdf9f46662c36ff47b0b00ecaf06",
|
||||
"sha256:f3dc68272990849132d6698f7dc6df2ab62a88b0d36e54702a8fd16c0490e44f",
|
||||
"sha256:f59eded163d9752fd49978e0bab7a1ff21b1b8d25c05f0995d140cc08ac83379",
|
||||
"sha256:f5e2d36c86c7b03c94b8459c3bd2c9fe2c7dab4b258b8885617d44a22e453fb7",
|
||||
"sha256:f6f65bf54215e4ebf6b01e4bb94c49180a589573df643735107056f7a910275b",
|
||||
"sha256:f8450d5ef759dbe59f84f2c9f77491bb3d3c44bc1a573746daf086e70b14c243",
|
||||
"sha256:f97d83049715fd9dec7911860ecf0e17b48d8725de01e45de07d8ac0bd5bc378"
|
||||
],
|
||||
"markers": "python_version >= '3'",
|
||||
"version": "==1.0.0"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
|
||||
@@ -107,11 +156,11 @@
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
|
||||
"sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
|
||||
"sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
|
||||
"sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.24.0"
|
||||
"version": "==2.25.1"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
@@ -122,56 +171,59 @@
|
||||
},
|
||||
"soupsieve": {
|
||||
"hashes": [
|
||||
"sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
|
||||
"sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
|
||||
"sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc",
|
||||
"sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b"
|
||||
],
|
||||
"version": "==2.0.1"
|
||||
"markers": "python_version >= '3.0'",
|
||||
"version": "==2.2.1"
|
||||
},
|
||||
"sqlalchemy": {
|
||||
"hashes": [
|
||||
"sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb",
|
||||
"sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804",
|
||||
"sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6",
|
||||
"sha256:26c5ca9d09f0e21b8671a32f7d83caad5be1f6ff45eef5ec2f6fd0db85fc5dc0",
|
||||
"sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe",
|
||||
"sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de",
|
||||
"sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36",
|
||||
"sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e",
|
||||
"sha256:465c999ef30b1c7525f81330184121521418a67189053bcf585824d833c05b66",
|
||||
"sha256:51064ee7938526bab92acd049d41a1dc797422256086b39c08bafeffb9d304c6",
|
||||
"sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc",
|
||||
"sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d",
|
||||
"sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce",
|
||||
"sha256:6557af9e0d23f46b8cd56f8af08eaac72d2e3c632ac8d5cf4e20215a8dca7cea",
|
||||
"sha256:73a40d4fcd35fdedce07b5885905753d5d4edf413fbe53544dd871f27d48bd4f",
|
||||
"sha256:8280f9dae4adb5889ce0bb3ec6a541bf05434db5f9ab7673078c00713d148365",
|
||||
"sha256:83469ad15262402b0e0974e612546bc0b05f379b5aa9072ebf66d0f8fef16bea",
|
||||
"sha256:860d0fe234922fd5552b7f807fbb039e3e7ca58c18c8d38aa0d0a95ddf4f6c23",
|
||||
"sha256:883c9fb62cebd1e7126dd683222b3b919657590c3e2db33bdc50ebbad53e0338",
|
||||
"sha256:8afcb6f4064d234a43fea108859942d9795c4060ed0fbd9082b0f280181a15c1",
|
||||
"sha256:96f51489ac187f4bab588cf51f9ff2d40b6d170ac9a4270ffaed535c8404256b",
|
||||
"sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e",
|
||||
"sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba",
|
||||
"sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02",
|
||||
"sha256:b6ff91356354b7ff3bd208adcf875056d3d886ed7cef90c571aef2ab8a554b12",
|
||||
"sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86",
|
||||
"sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d",
|
||||
"sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7",
|
||||
"sha256:e49947d583fe4d29af528677e4f0aa21f5e535ca2ae69c48270ebebd0d8843c0",
|
||||
"sha256:eb1d71643e4154398b02e88a42fc8b29db8c44ce4134cf0f4474bfc5cb5d4dac",
|
||||
"sha256:f2e8a9c0c8813a468aa659a01af6592f71cd30237ec27c4cc0683f089f90dcfc",
|
||||
"sha256:fe7fe11019fc3e6600819775a7d55abc5446dda07e9795f5954fdbf8a49e1c37"
|
||||
"sha256:02b039e0e7e6de2f15ea2d2de3995e31a170e700ec0b37b4eded662171711d19",
|
||||
"sha256:08943201a1e3c6238e48f4d5d56c27ea1e1b39d3d9f36a9d81fc3cfb0e1b83bd",
|
||||
"sha256:0ee0054d4a598d2920cae14bcbd33e200e02c5e3b47b902627f8cf5d4c9a2a4b",
|
||||
"sha256:11e7a86209f69273e75d2dd64b06c0c2660e39cd942fce2170515c404ed7358a",
|
||||
"sha256:1294f05916c044631fd626a4866326bbfbd17f62bd37510d000afaef4b35bd74",
|
||||
"sha256:2f11b5783933bff55291ca06496124347627d211ff2e509e846af1c35de0a3fb",
|
||||
"sha256:301d0cd6ef1dc73b607748183da857e712d6f743de8d92b1e1f8facfb0ba2aa2",
|
||||
"sha256:344b58b4b4193b72e8b768a51ef6eb5a4c948ce313a0f23e2ea081e71ce8ac0e",
|
||||
"sha256:44e11a06168782b6d485daef197783366ce7ab0d5eea0066c899ae06cef47bbc",
|
||||
"sha256:45b091ccbf94374ed14abde17e9a04522b0493a17282eaaf4383efdd413f5243",
|
||||
"sha256:48540072f43b3c080159ec1f24a4b014c0ee83d3b73795399974aa358a8cf71b",
|
||||
"sha256:4df07161897191ed8d4a0cfc92425c81296160e5c5f76c9256716d3085172883",
|
||||
"sha256:4f7ce3bfdab6520554af4a5b1df4513d45388624d015ba4d921daf48ce1d6503",
|
||||
"sha256:5361e25181b9872d6906c8c9be7dc05cb0a0951d71ee59ee5a71c1deb301b8a8",
|
||||
"sha256:6f8fdad2f335d2f3ca2f3ee3b01404f7abcf519b03de2c510f1f42d16e39ffb4",
|
||||
"sha256:70a1387396ea5b3022539b560c287daf79403d8b4b365f89b56d660e625a4457",
|
||||
"sha256:7481f9c2c832a3bf37c80bee44d91ac9938b815cc06f7e795b976e300914aab9",
|
||||
"sha256:7c0c7bb49167ac738ca6ee6e7f94a9988a7e4e261d8da335341e8c8c8f3b2e9b",
|
||||
"sha256:7de84feb31af3d8fdf819cac2042928d0b60d3cb16f49c4b2f48d88db46e79f6",
|
||||
"sha256:7f5087104c3c5af11ea59e49ae66c33ca98b14a47d3796ae97498fca53f84aef",
|
||||
"sha256:81badd7d3e0e6aba70a5d1b50fabe8112e9835a6fdb0684054c3fe5378ce0d01",
|
||||
"sha256:82f11b679df91275788be6734dd4a9dfa29bac67b85326992609f62b05bdab37",
|
||||
"sha256:8301ecf3e819eb5dbc171e84654ff60872807775301a55fe35b0ab2ba3742031",
|
||||
"sha256:8d6a9feb5efd2fdab25c6d5a0a5589fed9d789f5ec57ec12263fd0e60ce1dea6",
|
||||
"sha256:915d4fa08776c0252dc5a34fa15c6490f66f411ea1ac9492022f98875d6baf20",
|
||||
"sha256:94040a92b6676f9ffdab6c6b479b3554b927a635c90698c761960b266b04fc88",
|
||||
"sha256:a08027ae84efc563f0f2f341dda572eadebeca38c0ae028a009988f27e9e6230",
|
||||
"sha256:a103294583383660d9e06dbd82037dc8e94c184bdcb27b2be44ae4457dafc6b4",
|
||||
"sha256:c22bfac8d3b955cdb13f0fcd6343156bf56d925196cf7d9ab9ce9f61d3f1e11c",
|
||||
"sha256:c3810ebcf1d42c532c8f5c3f442c705d94442a27a32f2df5344f0857306ab321",
|
||||
"sha256:ee4ddc904fb6414b5118af5b8d45e428aac2ccda01326b2ba2fe4354b0d8d1ae",
|
||||
"sha256:f16801795f1ffe9472360589a04301018c79e4582a85e68067275bb4f765e4e2",
|
||||
"sha256:f62c57ceadedeb8e7b98b48ac4d684bf2b0f73b9d882fed3ca260d9aedf6403f",
|
||||
"sha256:fbb0fda1c574975807aceb0e2332e0ecfe9e5656c191ed482c1a5eafe7a33823"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.3.19"
|
||||
"version": "==1.4.5"
|
||||
},
|
||||
"tenacity": {
|
||||
"hashes": [
|
||||
"sha256:29ae90e7faf488a8628432154bb34ace1cca58244c6ea399fd33f066ac71339a",
|
||||
"sha256:5a5d3dcd46381abe8b4f82b5736b8726fd3160c6c7161f53f8af7f1eb9b82173"
|
||||
"sha256:5bd16ef5d3b985647fe28dfa6f695d343aa26479a04e8792b9d3c8f49e361ae1",
|
||||
"sha256:a0ce48587271515db7d3a5e700df9ae69cce98c4b57c23a4886da15243603dd8"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==6.2.0"
|
||||
"version": "==7.0.0"
|
||||
},
|
||||
"termcolor": {
|
||||
"hashes": [
|
||||
@@ -181,11 +233,11 @@
|
||||
},
|
||||
"tqdm": {
|
||||
"hashes": [
|
||||
"sha256:8f3c5815e3b5e20bc40463fa6b42a352178859692a68ffaa469706e6d38342a5",
|
||||
"sha256:faf9c671bd3fad5ebaeee366949d969dca2b2be32c872a7092a1e1a9048d105b"
|
||||
"sha256:9fdf349068d047d4cfbe24862c425883af1db29bcddf4b0eeb2524f6fbdb23c7",
|
||||
"sha256:d666ae29164da3e517fcf125e41d4fe96e5bb375cd87ff9763f6b38b5592fe33"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.49.0"
|
||||
"version": "==4.59.0"
|
||||
},
|
||||
"treelib": {
|
||||
"hashes": [
|
||||
@@ -196,18 +248,18 @@
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a",
|
||||
"sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"
|
||||
"sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
|
||||
"sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
|
||||
],
|
||||
"version": "==1.25.10"
|
||||
"version": "==1.26.4"
|
||||
},
|
||||
"win32-setctime": {
|
||||
"hashes": [
|
||||
"sha256:02b4c5959ca0b195f45c98115826c6e8a630b7cf648e724feaab1a5aa6250640",
|
||||
"sha256:47aa7c43548c1fc0a4f026d1944b748b37036df116c7c4cf908e82638d854313"
|
||||
"sha256:4e88556c32fdf47f64165a2180ba4552f8bb32c1103a2fafd05723a0bd42bd4b",
|
||||
"sha256:dc925662de0a6eb987f0b01f599c01a8236cb8c62831c22d9cada09ad958243e"
|
||||
],
|
||||
"markers": "sys_platform == 'win32'",
|
||||
"version": "==1.0.2"
|
||||
"version": "==1.0.3"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/releases)
|
||||
[](https://github.com/shmilylty/OneForAll/releases)
|
||||
|
||||
👊**OneForAll是一款功能强大的子域收集工具** 📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md)
|
||||
|
||||
@@ -88,7 +88,7 @@ config
|
||||
|
||||
```shell
|
||||
docker pull shmilylty/oneforall
|
||||
docker run -it --rm -v ~/results:/OneForAll/results -v ~/.config:/OneForAll/config oneforall --target example.com run
|
||||
docker run -it --rm -v ~/results:/OneForAll/results -v ~/.config:/OneForAll/config shmilylty/oneforall --target example.com run
|
||||
```
|
||||
参数直接加在指令末尾，结果会输出在本地目录`~/results`，如需保存到其他位置，可以自行修改
|
||||
</details>
|
||||
@@ -97,8 +97,6 @@ docker run -it --rm -v ~/results:/OneForAll/results -v ~/.config:/OneForAll/conf
|
||||
<details>
|
||||
<summary><b>✨使用演示</b></summary>
|
||||
|
||||
如果你的主机不在中国，请把 [setting](https://github.com/shmilylty/OneForAll/blob/master/config/setting.py#L46) 中`brute_nameservers_path`选项的`cn_nameservers.txt`修改为`nameservers.txt`。
|
||||
|
||||
如果你是通过pip3安装的依赖则使用以下命令运行示例：
|
||||
```bash
|
||||
python3 oneforall.py --target example.com run
|
||||
@@ -170,7 +168,7 @@ DESCRIPTION
|
||||
python3 oneforall.py --target example.com --valid None run
|
||||
python3 oneforall.py --target example.com --brute True run
|
||||
python3 oneforall.py --target example.com --port small run
|
||||
python3 oneforall.py --target example.com --format csv run
|
||||
python3 oneforall.py --target example.com --fmt csv run
|
||||
python3 oneforall.py --target example.com --dns False run
|
||||
python3 oneforall.py --target example.com --req False run
|
||||
python3 oneforall.py --target example.com --takeover False run
|
||||
@@ -179,7 +177,7 @@ DESCRIPTION
|
||||
Note:
|
||||
参数alive可选值True，False分别表示导出存活，全部子域结果
|
||||
参数port可选值有'default', 'small', 'large', 详见config.py配置
|
||||
参数format可选格式有 'csv','json'
|
||||
参数fmt可选格式有 'csv','json'
|
||||
参数path默认None使用OneForAll结果目录生成路径
|
||||
|
||||
ARGUMENTS
|
||||
@@ -199,7 +197,7 @@ FLAGS
|
||||
请求验证子域的端口范围(默认只探测80端口)
|
||||
--valid=VALID
|
||||
只导出存活的子域结果(默认False)
|
||||
--format=FORMAT
|
||||
--fmt=FMT
|
||||
结果保存格式(默认csv)
|
||||
--path=PATH
|
||||
结果保存路径(默认None)
|
||||
@@ -235,6 +233,8 @@ FLAGS
|
||||
7. 利用搜索引擎发现子域（目前有18个模块：`ask`, `baidu`, `bing`, `bing_api`, `duckduckgo`, `exalead`, `fofa_api`, `gitee`, `github`, `github_api`, `google`, `google_api`, `shodan_api`, `so`, `sogou`, `yahoo`, `yandex`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
|
||||
* **支持子域爆破**，该模块有常规的字典爆破，也有自定义的fuzz模式，支持批量爆破和递归爆破，自动判断泛解析并处理。
|
||||
* **支持子域验证**，默认开启子域验证，自动解析子域DNS，自动请求子域获取title和banner，并综合判断子域存活情况。
|
||||
* **支持子域爬取**，根据已有的子域，请求子域响应体以及响应体里的JS，从中再次发现新的子域。
|
||||
* **支持子域置换**，根据已有的子域，使用子域替换技术再次发现新的子域。
|
||||
* **支持子域接管**，默认开启子域接管风险检查，支持子域自动接管（目前只有Github，有待完善），支持批量检查。
|
||||
* **处理功能强大**，发现的子域结果支持自动去除，自动DNS解析，HTTP请求探测，自动筛选出有效子域，拓展子域的Banner信息，最终支持的导出格式有`txt`, `csv`, `json`。
|
||||
* **速度极快**，[收集模块](https://github.com/shmilylty/OneForAll/tree/master/collect.py)使用多线程调用，[爆破模块](https://github.com/shmilylty/OneForAll/tree/master/brute.py)使用[massdns](https://github.com/blechschmidt/massdns)，DNS解析速度每秒可解析350000以上个域名，子域验证中DNS解析和HTTP请求使用异步多协程，多线程检查[子域接管](https://github.com/shmilylty/OneForAll/tree/master/takeover.py)风险。
|
||||
@@ -269,8 +269,6 @@ FLAGS
|
||||
## ⌛后续计划
|
||||
|
||||
- [ ] 各模块持续优化和完善
|
||||
- [x] 子域监控（标记每次新发现的子域）
|
||||
- [x] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
|
||||
- [ ] 操作强大交互人性的前端界面实现
|
||||
|
||||
更多信息请参阅[后续开发计划](https://github.com/shmilylty/OneForAll/tree/master/docs/todo.md)。
|
||||
|
||||
@@ -18,141 +18,14 @@ import tenacity
|
||||
from dns.exception import Timeout
|
||||
from dns.resolver import NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers
|
||||
|
||||
import dbexport
|
||||
import export
|
||||
from common import utils
|
||||
from common import similarity
|
||||
from config import settings
|
||||
from common.module import Module
|
||||
from modules import wildcard
|
||||
from config.log import logger
|
||||
|
||||
|
||||
def config_resolver(nameservers):
|
||||
"""
|
||||
配置DNS解析器
|
||||
|
||||
:param nameservers: 名称解析服务器地址
|
||||
"""
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.nameservers = nameservers
|
||||
resolver.rotate = True # 随机使用NS
|
||||
resolver.cache = None # 不使用DNS缓存
|
||||
return resolver
|
||||
|
||||
|
||||
def gen_random_subdomains(domain, count):
|
||||
"""
|
||||
生成指定数量的随机子域域名列表
|
||||
|
||||
:param domain: 主域
|
||||
:param count: 数量
|
||||
"""
|
||||
subdomains = set()
|
||||
if count < 1:
|
||||
return subdomains
|
||||
for _ in range(count):
|
||||
token = secrets.token_hex(4)
|
||||
subdomains.add(f'{token}.{domain}')
|
||||
return subdomains
|
||||
|
||||
|
||||
def query_a_record(subdomain, resolver):
|
||||
"""
|
||||
查询子域A记录
|
||||
|
||||
:param subdomain: 子域
|
||||
:param resolver: DNS解析器
|
||||
"""
|
||||
try:
|
||||
answer = resolver.query(subdomain, 'A')
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', f'Query {subdomain} wildcard dns record error')
|
||||
logger.log('DEBUG', e.args)
|
||||
return False
|
||||
if answer.rrset is None:
|
||||
return False
|
||||
ttl = answer.ttl
|
||||
name = answer.name
|
||||
ips = {item.address for item in answer}
|
||||
logger.log('ALERT', f'{subdomain} resolve to: {name} '
|
||||
f'IP: {ips} TTL: {ttl}')
|
||||
return True
|
||||
|
||||
|
||||
def all_resolve_success(subdomains):
|
||||
"""
|
||||
判断是否所有子域都解析成功
|
||||
|
||||
:param subdomains: 子域列表
|
||||
"""
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.cache = None # 不使用DNS缓存
|
||||
status = set()
|
||||
for subdomain in subdomains:
|
||||
status.add(query_a_record(subdomain, resolver))
|
||||
return all(status)
|
||||
|
||||
|
||||
def all_request_success(subdomains):
|
||||
"""
|
||||
判断是否所有子域都请求成功
|
||||
|
||||
:param subdomains: 子域列表
|
||||
"""
|
||||
result = list()
|
||||
for subdomain in subdomains:
|
||||
url = f'http://{subdomain}'
|
||||
resp = utils.get_url_resp(url)
|
||||
if resp:
|
||||
logger.log('ALERT', f'Request: {url} Status: {resp.status_code} '
|
||||
f'Size: {len(resp.content)}')
|
||||
result.append(resp.text)
|
||||
else:
|
||||
result.append(resp)
|
||||
return all(result), result
|
||||
|
||||
|
||||
def any_similar_html(resp_list):
|
||||
"""
|
||||
判断是否有一组HTML页面结构相似
|
||||
|
||||
:param resp_list: 响应HTML页面
|
||||
"""
|
||||
html_doc1, html_doc2, html_doc3 = resp_list
|
||||
if similarity.is_similar(html_doc1, html_doc2):
|
||||
return True
|
||||
if similarity.is_similar(html_doc1, html_doc3):
|
||||
return True
|
||||
if similarity.is_similar(html_doc2, html_doc3):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def detect_wildcard(domain):
|
||||
"""
|
||||
Detect use wildcard dns record or not
|
||||
|
||||
:param str domain: domain
|
||||
:return bool use wildcard dns record or not
|
||||
"""
|
||||
logger.log('INFOR', f'Detecting {domain} use wildcard dns record or not')
|
||||
random_subdomains = gen_random_subdomains(domain, 3)
|
||||
if not all_resolve_success(random_subdomains):
|
||||
return False
|
||||
is_all_success, all_request_resp = all_request_success(random_subdomains)
|
||||
if not is_all_success:
|
||||
return True
|
||||
return any_similar_html(all_request_resp)
|
||||
|
||||
|
||||
def is_enable_wildcard(domain):
|
||||
is_enable = detect_wildcard(domain)
|
||||
if is_enable:
|
||||
logger.log('ALERT', f'The domain {domain} enables wildcard')
|
||||
else:
|
||||
logger.log('ALERT', f'The domain {domain} disables wildcard')
|
||||
return is_enable
|
||||
|
||||
|
||||
def gen_subdomains(expression, path):
|
||||
"""
|
||||
Generate subdomains
|
||||
@@ -261,106 +134,6 @@ def query_domain_ns(domain):
|
||||
return ns
|
||||
|
||||
|
||||
@tenacity.retry(stop=tenacity.stop_after_attempt(2))
|
||||
def get_wildcard_record(domain, resolver):
|
||||
logger.log('INFOR', f"Query {domain} 's wildcard dns record "
|
||||
f"in authoritative name server")
|
||||
try:
|
||||
answer = resolver.query(domain, 'A')
|
||||
# 如果查询随机域名A记录时抛出Timeout异常则重新查询
|
||||
except Timeout as e:
|
||||
logger.log('ALERT', f'Query timeout, retrying')
|
||||
logger.log('DEBUG', e.args)
|
||||
raise tenacity.TryAgain
|
||||
except (NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers) as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
logger.log('DEBUG', f'{domain} dont have A record on authoritative name server')
|
||||
return None, None
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e.args)
|
||||
logger.log('ERROR', f'Query {domain} wildcard dns record in '
|
||||
f'authoritative name server error')
|
||||
exit(1)
|
||||
else:
|
||||
if answer.rrset is None:
|
||||
logger.log('DEBUG', f'No record of query result')
|
||||
return None, None
|
||||
name = answer.name
|
||||
ip = {item.address for item in answer}
|
||||
ttl = answer.ttl
|
||||
logger.log('INFOR', f'{domain} results on authoritative name server: {name} '
|
||||
f'IP: {ip} TTL: {ttl}')
|
||||
return ip, ttl
|
||||
|
||||
|
||||
def collect_wildcard_record(domain, authoritative_ns):
|
||||
logger.log('INFOR', f'Collecting wildcard dns record for {domain}')
|
||||
if not authoritative_ns:
|
||||
return list(), int()
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.nameservers = authoritative_ns # 使用权威名称服务器
|
||||
resolver.rotate = True # 随机使用NS
|
||||
resolver.cache = None # 不使用DNS缓存
|
||||
ips = set()
|
||||
ttl = int()
|
||||
ttls_check = list()
|
||||
ips_stat = dict()
|
||||
ips_check = list()
|
||||
while True:
|
||||
token = secrets.token_hex(4)
|
||||
random_subdomain = f'{token}.{domain}'
|
||||
try:
|
||||
ip, ttl = get_wildcard_record(random_subdomain, resolver)
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
logger.log('ALERT', f'Multiple query errors,'
|
||||
f'try to query a new random subdomain')
|
||||
continue
|
||||
# 每5次查询检查结果列表 如果都没结果则结束查询
|
||||
ips_check.append(ip)
|
||||
ttls_check.append(ttl)
|
||||
if len(ips_check) == 5:
|
||||
if not any(ips_check):
|
||||
logger.log('ALERT', 'The query ends because there are '
|
||||
'no results for 5 consecutive queries.')
|
||||
break
|
||||
ips_check = list()
|
||||
if len(ttls_check) == 5 and len(set(ttls_check)) == 5:
|
||||
logger.log('ALERT', 'The query ends because there are '
|
||||
'5 different TTL results for 5 consecutive queries.')
|
||||
ips, ttl = set(), int()
|
||||
break
|
||||
if ip is None:
|
||||
continue
|
||||
ips.update(ip)
|
||||
# 统计每个泛解析IP出现次数
|
||||
for addr in ip:
|
||||
count = ips_stat.setdefault(addr, 0)
|
||||
ips_stat[addr] = count + 1
|
||||
# 筛选出出现次数2次以上的IP地址
|
||||
addrs = list()
|
||||
for addr, times in ips_stat.items():
|
||||
if times >= 2:
|
||||
addrs.append(addr)
|
||||
# 大部分的IP地址出现次数大于2次停止收集泛解析IP记录
|
||||
if len(addrs) / len(ips) >= 0.8:
|
||||
break
|
||||
logger.log('DEBUG', f'Collected the wildcard dns record of {domain}\n{ips}\n{ttl}')
|
||||
return ips, ttl
|
||||
|
||||
|
||||
def get_nameservers_path(enable_wildcard, ns_ip_list):
|
||||
path = settings.brute_nameservers_path
|
||||
if not enable_wildcard:
|
||||
return path
|
||||
if not ns_ip_list:
|
||||
return path
|
||||
path = settings.authoritative_dns_path
|
||||
ns_data = '\n'.join(ns_ip_list)
|
||||
utils.save_data(path, ns_data)
|
||||
return path
|
||||
|
||||
|
||||
def check_dict():
|
||||
if not settings.enable_check_dict:
|
||||
return
|
||||
@@ -375,7 +148,7 @@ def check_dict():
|
||||
exit(0)
|
||||
|
||||
|
||||
def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
|
||||
def gen_result_infos(items, infos, subdomains, appear_times, wc_ips, wc_ttl):
|
||||
qname = items.get('name')[:-1] # 去除最右边的`.`点号
|
||||
reason = items.get('status')
|
||||
resolver = items.get('resolver')
|
||||
@@ -384,27 +157,27 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
|
||||
info = dict()
|
||||
cnames = list()
|
||||
ips = list()
|
||||
public = list()
|
||||
times = list()
|
||||
ip_times = list()
|
||||
cname_times = list()
|
||||
ttls = list()
|
||||
is_valid_flags = list()
|
||||
have_a_record = False
|
||||
for answer in answers:
|
||||
if answer.get('type') != 'A':
|
||||
logger.log('TRACE', f'The query result of {qname} has no A record\n{answer}')
|
||||
continue
|
||||
logger.log('TRACE', f'The query result of {qname} no A record\n{answer}')
|
||||
have_a_record = True
|
||||
ttl = answer.get('ttl')
|
||||
ttls.append(ttl)
|
||||
cname = answer.get('name')[:-1]
|
||||
cnames.append(cname) # 去除最右边的`.`点号
|
||||
name = answer.get('name') # 去除最右边的`.`点号
|
||||
cname = name[:-1].lower() # 去除最右边的`.`点号
|
||||
cnames.append(cname)
|
||||
cname_num = appear_times.get(cname)
|
||||
cname_times.append(cname_num)
|
||||
ip = answer.get('data')
|
||||
ips.append(ip)
|
||||
public.append(utils.ip_is_public(ip))
|
||||
num = ip_times.get(ip)
|
||||
times.append(num)
|
||||
isvalid, reason = is_valid_subdomain(ip, ttl, num, wc_ips, wc_ttl, cname)
|
||||
ip_num = appear_times.get(ip)
|
||||
ip_times.append(ip_num)
|
||||
isvalid, reason = wildcard.is_valid_subdomain(ip, ip_num, cname, cname_num, ttl, wc_ttl, wc_ips)
|
||||
logger.log('TRACE', f'{ip} effective: {isvalid} reason: {reason}')
|
||||
is_valid_flags.append(isvalid)
|
||||
if not have_a_record:
|
||||
@@ -416,18 +189,17 @@ def gen_result_infos(items, infos, subdomains, ip_times, wc_ips, wc_ttl):
|
||||
info['ttl'] = ttls
|
||||
info['cname'] = cnames
|
||||
info['ip'] = ips
|
||||
info['public'] = public
|
||||
info['times'] = times
|
||||
info['ip_times'] = ip_times
|
||||
info['cname_times'] = cname_times
|
||||
info['resolver'] = resolver
|
||||
infos[qname] = info
|
||||
subdomains.append(qname)
|
||||
return infos, subdomains
|
||||
|
||||
|
||||
def stat_ip_times(result_paths):
|
||||
logger.log('INFOR', f'Counting IP')
|
||||
def stat_appear_times(result_path):
|
||||
logger.log('INFOR', f'Counting IP cname appear times')
|
||||
times = dict()
|
||||
for result_path in result_paths:
|
||||
logger.log('DEBUG', f'Reading {result_path}')
|
||||
with open(result_path) as fd:
|
||||
for line in fd:
|
||||
@@ -450,16 +222,26 @@ def stat_ip_times(result_paths):
|
||||
if answer.get('type') == 'A':
|
||||
ip = answer.get('data')
|
||||
# 取值 如果是首次出现的IP集合 出现次数先赋值0
|
||||
value = times.setdefault(ip, 0)
|
||||
times[ip] = value + 1
|
||||
value_one = times.setdefault(ip, 0)
|
||||
times[ip] = value_one + 1
|
||||
name = answer.get('data')
|
||||
cname = name[:-1].lower() # 去除最右边的`.`点号
|
||||
# 取值 如果是首次出现的IP集合 出现次数先赋值0
|
||||
value_two = times.setdefault(cname, 0)
|
||||
times[cname] = value_two + 1
|
||||
if answer.get('type') == 'CNAME':
|
||||
name = answer.get('data')
|
||||
cname = name[:-1].lower() # 去除最右边的`.`点号
|
||||
# 取值 如果是首次出现的IP集合 出现次数先赋值0
|
||||
value_three = times.setdefault(cname, 0)
|
||||
times[cname] = value_three + 1
|
||||
return times
|
||||
|
||||
|
||||
def deal_output(output_paths, ip_times, wildcard_ips, wildcard_ttl):
|
||||
def deal_output(output_path, appear_times, wildcard_ips, wildcard_ttl):
|
||||
logger.log('INFOR', f'Processing result')
|
||||
infos = dict() # 用来记录所有域名有关信息
|
||||
subdomains = list() # 用来保存所有通过有效性检查的子域
|
||||
for output_path in output_paths:
|
||||
logger.log('DEBUG', f'Processing {output_path}')
|
||||
with open(output_path) as fd:
|
||||
for line in fd:
|
||||
@@ -481,67 +263,22 @@ def deal_output(output_paths, ip_times, wildcard_ips, wildcard_ttl):
|
||||
logger.log('TRACE', f'Processing {line}, {qname} no response')
|
||||
continue
|
||||
infos, subdomains = gen_result_infos(items, infos, subdomains,
|
||||
ip_times, wildcard_ips, wildcard_ttl)
|
||||
appear_times, wildcard_ips,
|
||||
wildcard_ttl)
|
||||
return infos, subdomains
|
||||
|
||||
|
||||
def check_by_compare(ip, ttl, wc_ips, wc_ttl):
|
||||
"""
|
||||
Use TTL comparison to detect wildcard dns record
|
||||
|
||||
:param set ip: A record IP address set
|
||||
:param int ttl: A record TTL value
|
||||
:param set wc_ips: wildcard dns record IP address set
|
||||
:param int wc_ttl: wildcard dns record TTL value
|
||||
:return bool: result
|
||||
"""
|
||||
# Reference：http://sh3ll.me/archives/201704041222.txt
|
||||
if ip not in wc_ips:
|
||||
return False # 子域IP不在泛解析IP集合则不是泛解析
|
||||
if ttl != wc_ttl and ttl % 60 == 0 and wc_ttl % 60 == 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def check_ip_times(times):
|
||||
"""
|
||||
Use IP address times to determine wildcard or not
|
||||
|
||||
:param times: IP address times
|
||||
:return bool: result
|
||||
"""
|
||||
if times > settings.ip_appear_maximum:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_valid_subdomain(ip, ttl, times, wc_ips, wc_ttl, cname):
|
||||
ip_blacklist = settings.brute_ip_blacklist
|
||||
cname_blacklist = settings.brute_cname_blacklist
|
||||
if cname in cname_blacklist:
|
||||
return 0, 'cname blacklist' # 有些泛解析会统一解析到一个cname上
|
||||
if ip in ip_blacklist: # 解析ip在黑名单ip则为非法子域
|
||||
return 0, 'IP blacklist'
|
||||
if all([wc_ips, wc_ttl]): # 有泛解析记录才进行对比
|
||||
if check_by_compare(ip, ttl, wc_ips, wc_ttl):
|
||||
return 0, 'IP wildcard'
|
||||
if check_ip_times(times):
|
||||
return 0, 'IP exceeded'
|
||||
return 1, 'OK'
|
||||
|
||||
|
||||
def save_brute_dict(dict_path, dict_set):
|
||||
dict_data = '\n'.join(dict_set)
|
||||
if not utils.save_data(dict_path, dict_data):
|
||||
if not utils.save_to_file(dict_path, dict_data):
|
||||
logger.log('FATAL', 'Saving dictionary error')
|
||||
exit(1)
|
||||
|
||||
|
||||
def delete_file(dict_path, output_paths):
|
||||
def delete_file(dict_path, output_path):
|
||||
if settings.delete_generated_dict:
|
||||
dict_path.unlink()
|
||||
if settings.delete_massdns_result:
|
||||
for output_path in output_paths:
|
||||
output_path.unlink()
|
||||
|
||||
|
||||
@@ -559,13 +296,12 @@ class Brute(Module):
|
||||
brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
|
||||
|
||||
Note:
|
||||
--format csv/json (result format)
|
||||
--fmt csv/json (result format)
|
||||
--path Result path (default None, automatically generated)
|
||||
|
||||
|
||||
:param str target: One domain (target or targets must be provided)
|
||||
:param str targets: File path of one domain per line
|
||||
:param int process: Number of processes (default 1)
|
||||
:param int concurrent: Number of concurrent (default 2000)
|
||||
:param bool word: Use word mode generate dictionary (default False)
|
||||
:param str wordlist: Dictionary path used in word mode (default use ./config/default.py)
|
||||
@@ -578,19 +314,18 @@ class Brute(Module):
|
||||
:param str rule: Specify the regexp rules used in fuzz mode (required if use fuzz mode)
|
||||
:param str fuzzlist: Dictionary path used in fuzz mode (default use ./config/default.py)
|
||||
:param bool export: Export the results (default True)
|
||||
:param str format: Result format (default csv)
|
||||
:param str fmt: Result format (default csv)
|
||||
:param str path: Result directory (default None)
|
||||
"""
|
||||
def __init__(self, target=None, targets=None, process=None, concurrent=None,
|
||||
word=False, wordlist=None, recursive=False, depth=None, nextlist=None,
|
||||
fuzz=False, place=None, rule=None, fuzzlist=None, export=True,
|
||||
alive=True, format='csv', path=None):
|
||||
def __init__(self, target=None, targets=None, concurrent=None,
|
||||
word=False, wordlist=None, recursive=False, depth=None,
|
||||
nextlist=None, fuzz=False, place=None, rule=None, fuzzlist=None,
|
||||
export=True, alive=True, fmt='csv', path=None):
|
||||
Module.__init__(self)
|
||||
self.module = 'Brute'
|
||||
self.source = 'Brute'
|
||||
self.target = target
|
||||
self.targets = targets
|
||||
self.process_num = process or utils.get_process_num()
|
||||
self.concurrent_num = concurrent or settings.brute_concurrent_num
|
||||
self.word = word
|
||||
self.wordlist = wordlist or settings.brute_wordlist_path
|
||||
@@ -603,15 +338,15 @@ class Brute(Module):
|
||||
self.fuzzlist = fuzzlist or settings.fuzz_list
|
||||
self.export = export
|
||||
self.alive = alive
|
||||
self.format = format
|
||||
self.fmt = fmt
|
||||
self.path = path
|
||||
self.bulk = False # 是否是批量爆破场景
|
||||
self.domains = list() # 待爆破的所有域名集合
|
||||
self.domain = str() # 当前正在进行爆破的域名
|
||||
self.ips_times = dict() # IP集合出现次数
|
||||
self.enable_wildcard = False # 当前域名是否使用泛解析
|
||||
self.check_env = True
|
||||
self.enable_wildcard = None # 当前域名是否使用泛解析
|
||||
self.quite = False
|
||||
self.in_china = None
|
||||
|
||||
def gen_brute_dict(self, domain):
|
||||
logger.log('INFOR', f'Generating dictionary for {domain}')
|
||||
@@ -668,6 +403,13 @@ class Brute(Module):
|
||||
logger.log('FATAL', f'Incorrect domain for fuzz')
|
||||
exit(1)
|
||||
|
||||
def init_dict_path(self):
|
||||
data_dir = settings.data_storage_dir
|
||||
if self.wordlist is None:
|
||||
self.wordlist = settings.brute_wordlist_path or data_dir.joinpath('subnames.txt')
|
||||
if self.recursive_nextlist is None:
|
||||
self.recursive_nextlist = settings.recursive_nextlist_path or data_dir.joinpath('subnames_next.txt')
|
||||
|
||||
def main(self, domain):
|
||||
start = time.time()
|
||||
logger.log('INFOR', f'Blasting {domain} ')
|
||||
@@ -682,12 +424,12 @@ class Brute(Module):
|
||||
wildcard_ttl = int() # 泛解析TTL整型值
|
||||
ns_list = query_domain_ns(self.domain)
|
||||
ns_ip_list = query_domain_ns_a(ns_list) # DNS权威名称服务器对应A记录列表
|
||||
self.enable_wildcard = is_enable_wildcard(domain)
|
||||
if self.enable_wildcard is None:
|
||||
self.enable_wildcard = wildcard.detect_wildcard(domain)
|
||||
|
||||
if self.enable_wildcard:
|
||||
wildcard_ips, wildcard_ttl = collect_wildcard_record(domain,
|
||||
ns_ip_list)
|
||||
ns_path = get_nameservers_path(self.enable_wildcard, ns_ip_list)
|
||||
wildcard_ips, wildcard_ttl = wildcard.collect_wildcard_record(domain, ns_ip_list)
|
||||
ns_path = utils.get_ns_path(self.in_china, self.enable_wildcard, ns_ip_list)
|
||||
|
||||
dict_set = self.gen_brute_dict(domain)
|
||||
|
||||
@@ -704,20 +446,11 @@ class Brute(Module):
|
||||
logger.log('INFOR', f'Running massdns to brute subdomains')
|
||||
utils.call_massdns(massdns_path, dict_path, ns_path, output_path,
|
||||
log_path, quiet_mode=self.quite,
|
||||
process_num=self.process_num,
|
||||
concurrent_num=self.concurrent_num)
|
||||
output_paths = []
|
||||
if self.process_num == 1:
|
||||
output_paths.append(output_path)
|
||||
else:
|
||||
for i in range(self.process_num):
|
||||
output_name = f'resolved_result_{domain}_{timestring}.json{i}'
|
||||
output_path = temp_dir.joinpath(output_name)
|
||||
output_paths.append(output_path)
|
||||
ip_times = stat_ip_times(output_paths)
|
||||
self.infos, self.subdomains = deal_output(output_paths, ip_times,
|
||||
appear_times = stat_appear_times(output_path)
|
||||
self.infos, self.subdomains = deal_output(output_path, appear_times,
|
||||
wildcard_ips, wildcard_ttl)
|
||||
delete_file(dict_path, output_paths)
|
||||
delete_file(dict_path, output_path)
|
||||
end = time.time()
|
||||
self.elapse = round(end - start, 1)
|
||||
logger.log('ALERT', f'{self.source} module takes {self.elapse} seconds, '
|
||||
@@ -730,12 +463,13 @@ class Brute(Module):
|
||||
|
||||
def run(self):
|
||||
logger.log('INFOR', f'Start running {self.source} module')
|
||||
if self.check_env:
|
||||
utils.check_env()
|
||||
if self.in_china is None:
|
||||
_, self.in_china = utils.get_net_env()
|
||||
self.domains = utils.get_domains(self.target, self.targets)
|
||||
for self.domain in self.domains:
|
||||
self.results = list() # 置空
|
||||
all_subdomains = list()
|
||||
self.init_dict_path()
|
||||
self.check_brute_params()
|
||||
if self.recursive_brute:
|
||||
logger.log('INFOR', f'Start recursively brute the 1 layer subdomain'
|
||||
@@ -761,16 +495,15 @@ class Brute(Module):
|
||||
|
||||
logger.log('INFOR', f'Finished {self.source} module to brute {self.domain}')
|
||||
if not self.path:
|
||||
name = f'{self.domain}_brute_result.{self.format}'
|
||||
name = f'{self.domain}_brute_result.{self.fmt}'
|
||||
self.path = settings.result_save_dir.joinpath(name)
|
||||
# 数据库导出
|
||||
if self.export:
|
||||
dbexport.export(self.domain,
|
||||
type='table',
|
||||
export.export_data(self.domain,
|
||||
alive=self.alive,
|
||||
limit='resolve',
|
||||
path=self.path,
|
||||
format=self.format)
|
||||
fmt=self.fmt)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
+55
-27
@@ -57,7 +57,6 @@ class Database(object):
|
||||
f'alive int,'
|
||||
f'request int,'
|
||||
f'resolve int,'
|
||||
f'new int,'
|
||||
f'url text,'
|
||||
f'subdomain text,'
|
||||
f'port int,'
|
||||
@@ -73,7 +72,8 @@ class Database(object):
|
||||
f'header text,'
|
||||
f'history text,'
|
||||
f'response text,'
|
||||
f'times text,'
|
||||
f'ip_times text,'
|
||||
f'cname_times text,'
|
||||
f'ttl text,'
|
||||
f'cidr text,'
|
||||
f'asn text,'
|
||||
@@ -86,6 +86,20 @@ class Database(object):
|
||||
f'elapse float,'
|
||||
f'find int)')
|
||||
|
||||
def insert_table(self, table_name, result):
|
||||
table_name = table_name.replace('.', '_')
|
||||
self.conn.query(
|
||||
f'insert into "{table_name}" '
|
||||
f'(id, alive, resolve, request, url, subdomain, port, level,'
|
||||
f'cname, ip, public, cdn, status, reason, title, banner, header,'
|
||||
f'history, response, ip_times, cname_times, ttl, cidr, asn, org,'
|
||||
f'addr, isp, resolver, module, source, elapse, find) '
|
||||
f'values (:id, :alive, :resolve, :request, :url,'
|
||||
f':subdomain, :port, :level, :cname, :ip, :public, :cdn,'
|
||||
f':status, :reason, :title, :banner, :header, :history, :response,'
|
||||
f':ip_times, :cname_times, :ttl, :cidr, :asn, :org, :addr, :isp,'
|
||||
f':resolver, :module, :source, :elapse, :find)', **result)
|
||||
|
||||
def save_db(self, table_name, results, module_name=None):
|
||||
"""
|
||||
Save the results of each module in the database
|
||||
@@ -101,15 +115,15 @@ class Database(object):
|
||||
try:
|
||||
self.conn.bulk_query(
|
||||
f'insert into "{table_name}" '
|
||||
f'(id, alive, resolve, request, new, url, subdomain, port, level,'
|
||||
f'cname, ip, public, cdn, status, reason, title, banner, header,'
|
||||
f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,'
|
||||
f'module, source, elapse, find) '
|
||||
f'values (:id, :alive, :resolve, :request, :new, :url,'
|
||||
f'(id, alive, resolve, request, url, subdomain, port, level, '
|
||||
f'cname, ip, public, cdn, status, reason, title, banner, header, '
|
||||
f'history, response, ip_times, cname_times, ttl, cidr, asn, org, '
|
||||
f'addr, isp, resolver, module, source, elapse, find) '
|
||||
f'values (:id, :alive, :resolve, :request, :url, '
|
||||
f':subdomain, :port, :level, :cname, :ip, :public, :cdn,'
|
||||
f':status, :reason, :title, :banner, :header, :history, :response,'
|
||||
f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,'
|
||||
f':source, :elapse, :find)', results)
|
||||
f':status, :reason, :title, :banner, :header, :history, :response, '
|
||||
f':ip_times, :cname_times, :ttl, :cidr, :asn, :org, :addr, :isp, '
|
||||
f':resolver, :module, :source, :elapse, :find)', results)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
|
||||
@@ -178,7 +192,7 @@ class Database(object):
|
||||
|
||||
def deduplicate_subdomain(self, table_name):
|
||||
"""
|
||||
Deduplicates of subdomains in the table
|
||||
Deduplicate subdomains in the table
|
||||
|
||||
:param str table_name: table name
|
||||
"""
|
||||
@@ -199,17 +213,6 @@ class Database(object):
|
||||
self.query(f'delete from "{table_name}" where '
|
||||
f'subdomain is null or resolve == 0')
|
||||
|
||||
def deal_table(self, deal_table_name, backup_table_name):
|
||||
"""
|
||||
Process the table when the collection task is complete
|
||||
|
||||
:param str deal_table_name: Pending table name
|
||||
:param str backup_table_name: Table name for backup
|
||||
"""
|
||||
self.copy_table(deal_table_name, backup_table_name)
|
||||
self.remove_invalid(deal_table_name)
|
||||
self.deduplicate_subdomain(deal_table_name)
|
||||
|
||||
def get_data(self, table_name):
|
||||
"""
|
||||
Get all the data in the table
|
||||
@@ -229,18 +232,43 @@ class Database(object):
|
||||
:param str limit: limit value
|
||||
"""
|
||||
table_name = table_name.replace('.', '_')
|
||||
query = f'select id, new, alive, request, resolve, url, subdomain, level,' \
|
||||
sql = f'select id, alive, request, resolve, url, subdomain, level,' \
|
||||
f'cname, ip, public, cdn, port, status, reason, title, banner,' \
|
||||
f'cidr, asn, org, addr, isp, source from "{table_name}"'
|
||||
f'cidr, asn, org, addr, isp, source from "{table_name}" '
|
||||
if alive and limit:
|
||||
if limit in ['resolve', 'request']:
|
||||
where = f' where {limit} = 1'
|
||||
query += where
|
||||
sql += where
|
||||
elif alive:
|
||||
where = f' where alive = 1'
|
||||
query += where
|
||||
sql += where
|
||||
sql += ' order by subdomain'
|
||||
logger.log('TRACE', f'Get the data from {table_name} table')
|
||||
return self.query(query)
|
||||
return self.query(sql)
|
||||
|
||||
def count_alive(self, table_name):
|
||||
table_name = table_name.replace('.', '_')
|
||||
sql = f'select count() from "{table_name}" where alive = 1'
|
||||
return self.query(sql)
|
||||
|
||||
def get_resp_by_url(self, table_name, url):
|
||||
table_name = table_name.replace('.', '_')
|
||||
sql = f'select response from "{table_name}" where url = "{url}"'
|
||||
logger.log('TRACE', f'Get response data from {url}')
|
||||
return self.query(sql).scalar()
|
||||
|
||||
def get_data_by_fields(self, table_name, fields):
|
||||
table_name = table_name.replace('.', '_')
|
||||
field_str = ', '.join(fields)
|
||||
sql = f'select {field_str} from "{table_name}"'
|
||||
logger.log('TRACE', f'Get specified field data {fields} from {table_name} table')
|
||||
return self.query(sql)
|
||||
|
||||
def update_data_by_url(self, table_name, info, url):
|
||||
table_name = table_name.replace('.', '_')
|
||||
field_str = ', '.join(map(lambda kv: f'{kv[0]} = "{kv[1]}"', info.items()))
|
||||
sql = f'update "{table_name}" set {field_str} where url = "{url}"'
|
||||
return self.query(sql)
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
|
||||
+9
-9
@@ -275,7 +275,6 @@ class Module(object):
|
||||
'alive': None,
|
||||
'request': None,
|
||||
'resolve': None,
|
||||
'new': None,
|
||||
'url': None,
|
||||
'subdomain': None,
|
||||
'port': None,
|
||||
@@ -291,7 +290,8 @@ class Module(object):
|
||||
'header': None,
|
||||
'history': None,
|
||||
'response': None,
|
||||
'times': None,
|
||||
'ip_times': None,
|
||||
'cname_times': None,
|
||||
'ttl': None,
|
||||
'cidr': None,
|
||||
'asn': None,
|
||||
@@ -313,27 +313,26 @@ class Module(object):
|
||||
info = dict()
|
||||
cname = info.get('cname')
|
||||
ip = info.get('ip')
|
||||
times = info.get('times')
|
||||
ip_times = info.get('ip_times')
|
||||
cname_times = info.get('cname_times')
|
||||
ttl = info.get('ttl')
|
||||
public = info.get('public')
|
||||
if isinstance(cname, list):
|
||||
cname = ','.join(cname)
|
||||
ip = ','.join(ip)
|
||||
times = ','.join([str(num) for num in times])
|
||||
ip_times = ','.join([str(num) for num in ip_times])
|
||||
cname_times = ','.join([str(num) for num in cname_times])
|
||||
ttl = ','.join([str(num) for num in ttl])
|
||||
public = ','.join([str(num) for num in public])
|
||||
result = {'id': None,
|
||||
'alive': info.get('alive'),
|
||||
'request': info.get('request'),
|
||||
'resolve': info.get('resolve'),
|
||||
'new': None,
|
||||
'url': url,
|
||||
'subdomain': subdomain,
|
||||
'port': 80,
|
||||
'level': level,
|
||||
'cname': cname,
|
||||
'ip': ip,
|
||||
'public': public,
|
||||
'public': info.get('public'),
|
||||
'cdn': info.get('cdn'),
|
||||
'status': None,
|
||||
'reason': info.get('reason'),
|
||||
@@ -342,7 +341,8 @@ class Module(object):
|
||||
'header': None,
|
||||
'history': None,
|
||||
'response': None,
|
||||
'times': times,
|
||||
'ip_times': ip_times,
|
||||
'cname_times': cname_times,
|
||||
'ttl': ttl,
|
||||
'cidr': info.get('cidr'),
|
||||
'asn': info.get('asn'),
|
||||
|
||||
+57
-54
@@ -1,7 +1,6 @@
|
||||
import json
|
||||
from threading import Thread
|
||||
from queue import Queue
|
||||
from operator import attrgetter
|
||||
|
||||
import tqdm
|
||||
import requests
|
||||
@@ -9,6 +8,7 @@ from bs4 import BeautifulSoup
|
||||
|
||||
from common import utils
|
||||
from config.log import logger
|
||||
from common.database import Database
|
||||
from config import settings
|
||||
|
||||
|
||||
@@ -127,7 +127,7 @@ def get_progress_bar(total):
|
||||
return bar
|
||||
|
||||
|
||||
def get(url, resp_list, session):
|
||||
def get_resp(url, session):
|
||||
timeout = settings.request_timeout_second
|
||||
redirect = settings.request_allow_redirect
|
||||
proxy = utils.get_proxy()
|
||||
@@ -136,13 +136,14 @@ def get(url, resp_list, session):
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
resp = e
|
||||
resp_list.append((url, resp))
|
||||
return resp
|
||||
|
||||
|
||||
def request(urls_queue, resp_list, session):
|
||||
def request(urls_queue, resp_queue, session):
|
||||
while not urls_queue.empty():
|
||||
url = urls_queue.get()
|
||||
get(url, resp_list, session)
|
||||
index, url = urls_queue.get()
|
||||
resp = get_resp(url, session)
|
||||
resp_queue.put((index, resp))
|
||||
urls_queue.task_done()
|
||||
|
||||
|
||||
@@ -168,31 +169,6 @@ def get_session():
|
||||
return session
|
||||
|
||||
|
||||
def bulk_request(urls):
|
||||
logger.log('INFOR', 'Requesting urls in bulk')
|
||||
resp_list = list()
|
||||
urls_queue = Queue()
|
||||
for url in urls:
|
||||
urls_queue.put(url)
|
||||
total = len(urls)
|
||||
session = get_session()
|
||||
thread_count = req_thread_count()
|
||||
bar = get_progress_bar(total)
|
||||
|
||||
progress_thread = Thread(target=progress, name='ProgressThread',
|
||||
args=(bar, total, urls_queue), daemon=True)
|
||||
progress_thread.start()
|
||||
|
||||
for i in range(thread_count):
|
||||
request_thread = Thread(target=request, name=f'RequestThread-{i}',
|
||||
args=(urls_queue, resp_list, session), daemon=True)
|
||||
request_thread.start()
|
||||
|
||||
urls_queue.join()
|
||||
|
||||
return resp_list
|
||||
|
||||
|
||||
def gen_new_info(info, resp):
|
||||
if isinstance(resp, Exception):
|
||||
info['reason'] = str(resp.args)
|
||||
@@ -220,13 +196,54 @@ def gen_new_info(info, resp):
|
||||
return info
|
||||
|
||||
|
||||
def gen_new_data(data, resp_list):
|
||||
new_data = list()
|
||||
for url, resp in resp_list:
|
||||
for info in data:
|
||||
if info.get('url') == url:
|
||||
new_data.append(gen_new_info(info, resp))
|
||||
return new_data
|
||||
def save(name, total, req_data, resp_queue):
|
||||
db = Database()
|
||||
db.create_table(name)
|
||||
i = 0
|
||||
while True:
|
||||
if not resp_queue.empty():
|
||||
i += 1
|
||||
index, resp = resp_queue.get()
|
||||
old_info = req_data[index]
|
||||
new_info = gen_new_info(old_info, resp)
|
||||
db.insert_table(name, new_info)
|
||||
resp_queue.task_done()
|
||||
if i >= total: # 得存入完所有请求结果才能结束
|
||||
break
|
||||
db.close()
|
||||
|
||||
|
||||
def bulk_request(domain, req_data, ret=False):
|
||||
logger.log('INFOR', 'Requesting urls in bulk')
|
||||
resp_queue = Queue()
|
||||
urls_queue = Queue()
|
||||
task_count = len(req_data)
|
||||
for index, info in enumerate(req_data):
|
||||
url = info.get('url')
|
||||
urls_queue.put((index, url))
|
||||
session = get_session()
|
||||
thread_count = req_thread_count()
|
||||
if task_count <= thread_count:
|
||||
# 如果请求任务数很小不用创建很多线程了
|
||||
thread_count = task_count
|
||||
bar = get_progress_bar(task_count)
|
||||
|
||||
progress_thread = Thread(target=progress, name='ProgressThread',
|
||||
args=(bar, task_count, urls_queue), daemon=True)
|
||||
progress_thread.start()
|
||||
|
||||
for i in range(thread_count):
|
||||
request_thread = Thread(target=request, name=f'RequestThread-{i}',
|
||||
args=(urls_queue, resp_queue, session), daemon=True)
|
||||
request_thread.start()
|
||||
if ret:
|
||||
urls_queue.join()
|
||||
return resp_queue
|
||||
save_thread = Thread(target=save, name=f'SaveThread',
|
||||
args=(domain, task_count, req_data, resp_queue), daemon=True)
|
||||
save_thread.start()
|
||||
urls_queue.join()
|
||||
save_thread.join()
|
||||
|
||||
|
||||
def run_request(domain, data, port):
|
||||
@@ -242,20 +259,6 @@ def run_request(domain, data, port):
|
||||
data = utils.set_id_none(data)
|
||||
ports = get_port_seq(port)
|
||||
req_data, req_urls = gen_req_data(data, ports)
|
||||
resp_list = bulk_request(req_urls)
|
||||
new_data = gen_new_data(req_data, resp_list)
|
||||
count = utils.count_alive(new_data)
|
||||
bulk_request(domain, req_data)
|
||||
count = utils.count_alive(domain)
|
||||
logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
|
||||
sorted_data = utils.sort_by_subdomain(new_data)
|
||||
return sorted_data
|
||||
|
||||
|
||||
def save_db(name, data):
|
||||
"""
|
||||
Save request results to database
|
||||
|
||||
:param str name: table name
|
||||
:param list data: data to be saved
|
||||
"""
|
||||
logger.log('INFOR', f'Saving requested results')
|
||||
utils.save_db(name, data, 'request')
|
||||
|
||||
+17
-38
@@ -4,12 +4,6 @@ import json
|
||||
from config.log import logger
|
||||
from config import settings
|
||||
from common import utils
|
||||
from common.ipasn import IPAsnInfo
|
||||
from common.ipreg import IpRegData
|
||||
|
||||
|
||||
ip_asn = IPAsnInfo()
|
||||
ip_reg = IpRegData()
|
||||
|
||||
|
||||
def filter_subdomain(data):
|
||||
@@ -41,19 +35,20 @@ def update_data(data, infos):
|
||||
if not infos:
|
||||
logger.log('ALERT', f'No valid resolved result')
|
||||
return data
|
||||
new_data = list()
|
||||
for index, items in enumerate(data):
|
||||
if items.get('ip'):
|
||||
new_data.append(items)
|
||||
continue
|
||||
subdomain = items.get('subdomain')
|
||||
record = infos.get(subdomain)
|
||||
if record:
|
||||
items.update(record)
|
||||
new_data.append(items)
|
||||
else:
|
||||
items['resolve'] = 0
|
||||
items['alive'] = 0
|
||||
items['reason'] = 'NoResult'
|
||||
data[index] = items
|
||||
return data
|
||||
subdomain = items.get('subdomain')
|
||||
logger.log('DEBUG', f'{subdomain} resolution has no result')
|
||||
return new_data
|
||||
|
||||
|
||||
def save_db(name, data):
|
||||
@@ -64,57 +59,40 @@ def save_db(name, data):
|
||||
:param list data: data to be saved
|
||||
"""
|
||||
logger.log('INFOR', f'Saving resolved results')
|
||||
utils.save_db(name, data, 'resolve')
|
||||
utils.save_to_db(name, data, 'resolve')
|
||||
|
||||
|
||||
def save_subdomains(save_path, subdomain_list):
|
||||
logger.log('DEBUG', f'Saving resolved subdomain')
|
||||
subdomain_data = '\n'.join(subdomain_list)
|
||||
if not utils.save_data(save_path, subdomain_data):
|
||||
if not utils.save_to_file(save_path, subdomain_data):
|
||||
logger.log('FATAL', 'Save resolved subdomain error')
|
||||
exit(1)
|
||||
|
||||
|
||||
def gen_infos(data, qname, info, infos):
|
||||
flag = False
|
||||
cname = list()
|
||||
cnames = list()
|
||||
ips = list()
|
||||
public = list()
|
||||
ttl = list()
|
||||
cidr = list()
|
||||
asn = list()
|
||||
org = list()
|
||||
addr = list()
|
||||
isp = list()
|
||||
answers = data.get('answers')
|
||||
for answer in answers:
|
||||
if answer.get('type') == 'A':
|
||||
flag = True
|
||||
cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号
|
||||
name = answer.get('name')
|
||||
cname = name[:-1].lower() # 去除最右边的`.`点号
|
||||
cnames.append(cname)
|
||||
ip = answer.get('data')
|
||||
ips.append(ip)
|
||||
ttl.append(str(answer.get('ttl')))
|
||||
public.append(str(utils.ip_is_public(ip)))
|
||||
asn_info = ip_asn.find(ip)
|
||||
cidr.append(asn_info.get('cidr'))
|
||||
asn.append(asn_info.get('asn'))
|
||||
org.append(asn_info.get('org'))
|
||||
ip_info = ip_reg.query(ip)
|
||||
addr.append(ip_info.get('addr'))
|
||||
isp.append(ip_info.get('isp'))
|
||||
info['resolve'] = 1
|
||||
info['reason'] = 'OK'
|
||||
info['cname'] = ','.join(cname)
|
||||
info['cname'] = ','.join(cnames)
|
||||
info['ip'] = ','.join(ips)
|
||||
info['public'] = ','.join(public)
|
||||
info['ttl'] = ','.join(ttl)
|
||||
info['cidr'] = ','.join(cidr)
|
||||
info['asn'] = ','.join(asn)
|
||||
info['org'] = ','.join(org)
|
||||
info['addr'] = ','.join(addr)
|
||||
info['isp'] = ','.join(isp)
|
||||
infos[qname] = info
|
||||
if not flag:
|
||||
logger.log('DEBUG', f'Resolving {qname} have not a record')
|
||||
info['alive'] = 0
|
||||
info['resolve'] = 0
|
||||
info['reason'] = 'NoARecord'
|
||||
@@ -139,9 +117,11 @@ def deal_output(output_path):
|
||||
qname = items.get('name')[:-1] # 去除最右边的`.`点号
|
||||
status = items.get('status')
|
||||
if status != 'NOERROR':
|
||||
logger.log('DEBUG', f'Resolving {qname}: {status}')
|
||||
continue
|
||||
data = items.get('data')
|
||||
if 'answers' not in data:
|
||||
logger.log('DEBUG', f'Resolving {qname} have not any answers')
|
||||
info['alive'] = 0
|
||||
info['resolve'] = 0
|
||||
info['reason'] = 'NoAnswer'
|
||||
@@ -181,8 +161,7 @@ def run_resolve(domain, data):
|
||||
output_name = f'resolved_result_{domain}_{timestring}.json'
|
||||
output_path = temp_dir.joinpath(output_name)
|
||||
log_path = result_dir.joinpath('massdns.log')
|
||||
|
||||
ns_path = settings.brute_nameservers_path
|
||||
ns_path = utils.get_ns_path()
|
||||
|
||||
logger.log('INFOR', f'Running massdns to resolve subdomains')
|
||||
utils.call_massdns(massdns_path, save_path, ns_path,
|
||||
|
||||
+102
-157
@@ -10,10 +10,10 @@ import platform
|
||||
import subprocess
|
||||
from urllib.parse import scheme_chars
|
||||
from ipaddress import IPv4Address, ip_address
|
||||
from distutils.version import LooseVersion
|
||||
from pathlib import Path
|
||||
from stat import S_IXUSR
|
||||
|
||||
import dns
|
||||
import requests
|
||||
import tenacity
|
||||
from dns.resolver import Resolver
|
||||
@@ -54,7 +54,7 @@ def gen_fake_header():
|
||||
"""
|
||||
Generate fake request headers
|
||||
"""
|
||||
headers = settings.request_default_headers
|
||||
headers = settings.request_default_headers.copy()
|
||||
if not isinstance(headers, dict):
|
||||
headers = dict()
|
||||
if settings.enable_random_ua:
|
||||
@@ -163,7 +163,6 @@ def get_domains(target, targets=None):
|
||||
domains = list(target_domains.union(targets_domains))
|
||||
if targets_domains:
|
||||
domains = sorted(domains, key=targets_domains.index) # 按照targets原本的index排序
|
||||
logger.log('INFOR', f'Get {len(domains)} domains')
|
||||
if not domains:
|
||||
logger.log('ERROR', f'Did not get a valid domain name')
|
||||
logger.log('DEBUG', f'The obtained domains \n{domains}')
|
||||
@@ -176,16 +175,16 @@ def check_dir(dir_path):
|
||||
dir_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def check_path(path, name, format):
|
||||
def check_path(path, name, fmt):
|
||||
"""
|
||||
检查结果输出目录路径
|
||||
|
||||
:param path: 保存路径
|
||||
:param name: 导出名字
|
||||
:param format: 保存格式
|
||||
:param fmt: 保存格式
|
||||
:return: 保存路径
|
||||
"""
|
||||
filename = f'{name}.{format}'
|
||||
filename = f'{name}.{fmt}'
|
||||
default_path = settings.result_save_dir.joinpath(filename)
|
||||
if isinstance(path, str):
|
||||
path = repr(path).replace('\\', '/') # 将路径中的反斜杠替换为正斜杠
|
||||
@@ -204,19 +203,18 @@ def check_path(path, name, format):
|
||||
return path
|
||||
|
||||
|
||||
def check_format(format, count):
|
||||
def check_format(fmt):
|
||||
"""
|
||||
检查导出格式
|
||||
|
||||
:param format: 传入的导出格式
|
||||
:param count: 数量
|
||||
:param fmt: 传入的导出格式
|
||||
:return: 导出格式
|
||||
"""
|
||||
formats = ['csv', 'json', ]
|
||||
if format in formats:
|
||||
return format
|
||||
if fmt in formats:
|
||||
return fmt
|
||||
else:
|
||||
logger.log('ALERT', f'Does not support {format} format')
|
||||
logger.log('ALERT', f'Does not support {fmt} format')
|
||||
logger.log('ALERT', 'So use csv format by default')
|
||||
return 'csv'
|
||||
|
||||
@@ -226,7 +224,7 @@ def load_json(path):
|
||||
return json.load(fp)
|
||||
|
||||
|
||||
def save_db(name, data, module):
|
||||
def save_to_db(name, data, module):
|
||||
"""
|
||||
Save request results to database
|
||||
|
||||
@@ -241,7 +239,7 @@ def save_db(name, data, module):
|
||||
db.close()
|
||||
|
||||
|
||||
def save_data(path, data):
|
||||
def save_to_file(path, data):
|
||||
"""
|
||||
保存数据到文件
|
||||
|
||||
@@ -262,21 +260,6 @@ def save_data(path, data):
|
||||
return False
|
||||
|
||||
|
||||
def remove_data(path):
|
||||
"""
|
||||
删除保存数据的文件
|
||||
|
||||
:param path: 路径
|
||||
:return: 删除成功与否
|
||||
"""
|
||||
try:
|
||||
path.unlink()
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e.args)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def check_response(method, resp):
|
||||
"""
|
||||
检查响应 输出非正常响应返回json的信息
|
||||
@@ -333,8 +316,8 @@ def remove_invalid_string(string):
|
||||
return re.sub(r'[\000-\010]|[\013-\014]|[\016-\037]', r'', string)
|
||||
|
||||
|
||||
def export_all_results(path, name, format, datas):
|
||||
path = check_path(path, name, format)
|
||||
def export_all_results(path, name, fmt, datas):
|
||||
path = check_path(path, name, fmt)
|
||||
logger.log('ALERT', f'The subdomain result for all main domains: {path}')
|
||||
row_list = list()
|
||||
for row in datas:
|
||||
@@ -346,8 +329,8 @@ def export_all_results(path, name, format, datas):
|
||||
values = row.values()
|
||||
row_list.append(Record(keys, values))
|
||||
rows = RecordCollection(iter(row_list))
|
||||
content = rows.export(format)
|
||||
save_data(path, content)
|
||||
content = rows.export(fmt)
|
||||
save_to_file(path, content)
|
||||
|
||||
|
||||
def export_all_subdomains(alive, path, name, datas):
|
||||
@@ -363,22 +346,22 @@ def export_all_subdomains(alive, path, name, datas):
|
||||
else:
|
||||
subdomains.add(subdomain)
|
||||
data = '\n'.join(subdomains)
|
||||
save_data(path, data)
|
||||
save_to_file(path, data)
|
||||
|
||||
|
||||
def export_all(alive, format, path, datas):
|
||||
def export_all(alive, fmt, path, datas):
|
||||
"""
|
||||
将所有结果数据导出
|
||||
|
||||
:param bool alive: 只导出存活子域结果
|
||||
:param str format: 导出文件格式
|
||||
:param str fmt: 导出文件格式
|
||||
:param str path: 导出文件路径
|
||||
:param list datas: 待导出的结果数据
|
||||
"""
|
||||
format = check_format(format, len(datas))
|
||||
fmt = check_format(fmt)
|
||||
timestamp = get_timestring()
|
||||
name = f'all_subdomain_result_{timestamp}'
|
||||
export_all_results(path, name, format, datas)
|
||||
export_all_results(path, name, fmt, datas)
|
||||
export_all_subdomains(alive, path, name, datas)
|
||||
|
||||
|
||||
@@ -430,10 +413,18 @@ def python_version():
|
||||
return sys.version
|
||||
|
||||
|
||||
def count_alive(data):
|
||||
def calc_alive(data):
|
||||
return len(list(filter(lambda item: item.get('alive') == 1, data)))
|
||||
|
||||
|
||||
def count_alive(name):
|
||||
db = Database()
|
||||
result = db.count_alive(name)
|
||||
count = result.scalar()
|
||||
db.close()
|
||||
return count
|
||||
|
||||
|
||||
def get_subdomains(data):
|
||||
return set(map(lambda item: item.get('subdomain'), data))
|
||||
|
||||
@@ -485,16 +476,8 @@ def ip_is_public(ip_str):
|
||||
return 1
|
||||
|
||||
|
||||
def get_process_num():
|
||||
process_num = settings.brute_process_num
|
||||
if isinstance(process_num, int):
|
||||
return min(os.cpu_count(), process_num)
|
||||
else:
|
||||
return 1
|
||||
|
||||
|
||||
def get_request_count():
|
||||
return 32
|
||||
return os.cpu_count() * 16
|
||||
|
||||
|
||||
def uniq_dict_list(dict_list):
|
||||
@@ -509,50 +492,54 @@ def delete_file(*paths):
|
||||
logger.log('ERROR', e.args)
|
||||
|
||||
|
||||
@tenacity.retry(stop=tenacity.stop_after_attempt(3))
|
||||
@tenacity.retry(stop=tenacity.stop_after_attempt(3),
|
||||
wait=tenacity.wait_fixed(2))
|
||||
def check_net():
|
||||
logger.log('INFOR', 'Checking Internet environment')
|
||||
urls = ['http://www.baidu.com', 'http://www.bing.com',
|
||||
'http://www.apple.com', 'http://www.microsoft.com']
|
||||
urls = ['http://ip-api.com/json/']
|
||||
url = random.choice(urls)
|
||||
logger.log('INFOR', f'Trying to access {url}')
|
||||
header = {'User_Agent': 'curl'}
|
||||
timeout = settings.request_timeout_second
|
||||
verify = settings.request_ssl_verify
|
||||
logger.log('DEBUG', f'Trying to access {url}')
|
||||
session = requests.Session()
|
||||
session.trust_env = False
|
||||
try:
|
||||
rsp = session.get(url, proxies=get_proxy())
|
||||
rsp = session.get(url, headers=header, timeout=timeout, verify=verify)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e.args)
|
||||
logger.log('ALERT', 'Can not access Internet, retrying')
|
||||
raise tenacity.TryAgain
|
||||
if rsp.status_code != 200:
|
||||
logger.log('ALERT', f'{rsp.request.method} {rsp.request.url} '
|
||||
f'{rsp.status_code} {rsp.reason}')
|
||||
logger.log('ALERT', 'Can not access Internet normally, retrying')
|
||||
raise tenacity.TryAgain
|
||||
logger.log('INFOR', 'Access to Internet OK')
|
||||
logger.log('ALERT', 'Unable to access Internet, retrying...')
|
||||
raise e
|
||||
logger.log('DEBUG', 'Access to Internet OK')
|
||||
country = rsp.json().get('country').lower()
|
||||
if country in ['cn', 'china']:
|
||||
logger.log('DEBUG', f'The computer is located in China')
|
||||
return True, True
|
||||
else:
|
||||
logger.log('DEBUG', f'The computer is not located in China')
|
||||
return True, False
|
||||
|
||||
|
||||
def check_pre():
|
||||
def check_dep():
|
||||
logger.log('INFOR', 'Checking dependent environment')
|
||||
implementation = platform.python_implementation()
|
||||
version = platform.python_version()
|
||||
if implementation != 'CPython':
|
||||
logger.log('FATAL', f'OneForAll only passed the test under CPython')
|
||||
exit(1)
|
||||
if version < '3.6':
|
||||
if LooseVersion(version) < LooseVersion('3.6'):
|
||||
logger.log('FATAL', 'OneForAll requires Python 3.6 or higher')
|
||||
exit(1)
|
||||
|
||||
|
||||
def check_env():
|
||||
logger.log('INFOR', 'Checking the environment')
|
||||
def get_net_env():
|
||||
logger.log('INFOR', 'Checking network environment')
|
||||
try:
|
||||
check_net()
|
||||
result = check_net()
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
logger.log('FATAL', 'Can not access Internet')
|
||||
exit(1)
|
||||
check_pre()
|
||||
logger.log('ALERT', 'Please check your network environment.')
|
||||
return False, None
|
||||
return result
|
||||
|
||||
|
||||
def check_version(local):
|
||||
@@ -570,7 +557,7 @@ def check_version(local):
|
||||
resp_json = resp.json()
|
||||
latest = resp_json['tag_name']
|
||||
except Exception as e:
|
||||
logger.log('ERROR', 'An error occurred while checking the latest version')
|
||||
logger.log('ALERT', 'An error occurred while checking the latest version')
|
||||
logger.log('DEBUG', e.args)
|
||||
return
|
||||
if latest > local:
|
||||
@@ -619,11 +606,11 @@ def get_massdns_path(massdns_dir):
|
||||
machine = platform.machine().lower()
|
||||
name = f'massdns_{system}_{machine}'
|
||||
if system == 'windows':
|
||||
name = name + '.exe'
|
||||
name = f'massdns.exe'
|
||||
if machine == 'amd64':
|
||||
massdns_dir = massdns_dir.joinpath('windows', 'x64')
|
||||
else:
|
||||
massdns_dir = massdns_dir.joinpath('windows', 'x84')
|
||||
massdns_dir = massdns_dir.joinpath('windows', 'x86')
|
||||
path = massdns_dir.joinpath(name)
|
||||
path.chmod(S_IXUSR)
|
||||
if not path.exists():
|
||||
@@ -736,90 +723,6 @@ def sort_by_subdomain(data):
|
||||
return sorted(data, key=lambda item: item.get('subdomain'))
|
||||
|
||||
|
||||
def ping(host, path):
|
||||
param = '-n' if platform.system().lower() == 'windows' else '-c'
|
||||
command = ['ping', param, '5', host]
|
||||
with open(path, "w") as f:
|
||||
return subprocess.call(command, stdout=f, stderr=f)
|
||||
|
||||
|
||||
def ping_avg_time(nameserver):
|
||||
check_dir(settings.temp_save_dir)
|
||||
temp_path = settings.temp_save_dir.joinpath('ping')
|
||||
ping(nameserver, path=temp_path)
|
||||
with open(temp_path, 'r') as f:
|
||||
text = f.read()
|
||||
if '100.0% packet loss' in text or '100% packet loss' in text or '100% 丢失' in text:
|
||||
logger.log('ALERT', f'100.0% packet loss, ping {nameserver} failed.')
|
||||
return None
|
||||
elif platform.system() in ('Darwin', 'Linux'):
|
||||
try:
|
||||
avg_time = re.findall(r'(?:min/avg/max/.+ )(?:\d+\.\d+)/(\d+\.\d+)/', text)[0]
|
||||
logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.')
|
||||
except IndexError:
|
||||
return None
|
||||
return avg_time
|
||||
elif platform.system() == 'Windows':
|
||||
try:
|
||||
avg_time = re.findall(r'(?:Average|平均).+(\d.?)ms', text)[0]
|
||||
logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.')
|
||||
except IndexError:
|
||||
return None
|
||||
return avg_time
|
||||
else:
|
||||
logger.log('ALERT', f'{text}')
|
||||
return None
|
||||
|
||||
|
||||
def auto_select_nameserver():
|
||||
logger.log('INFOR', f'Ping test start, to select nameservers.')
|
||||
avg_time1 = ping_avg_time('114.114.114.114')
|
||||
avg_time2 = ping_avg_time('8.8.8.8')
|
||||
if avg_time1 and avg_time2:
|
||||
if avg_time1 < avg_time2:
|
||||
change_nameservers_file('cn')
|
||||
logger.log('INFOR', f'Ping test finished, use cn nameservers.')
|
||||
else:
|
||||
change_nameservers_file('common')
|
||||
logger.log('INFOR', f'Ping test finished, use common nameservers.')
|
||||
elif avg_time1 and not avg_time2:
|
||||
change_nameservers_file('cn')
|
||||
logger.log('INFOR', f'Ping test finished, use cn nameservers.')
|
||||
elif not avg_time1 and avg_time1:
|
||||
change_nameservers_file('common')
|
||||
logger.log('INFOR', f'Ping test finished, use common nameservers.')
|
||||
elif not avg_time1 and not avg_time1:
|
||||
change_nameservers_file('default')
|
||||
logger.log('INFOR', f'Ping test finished, use default nameservers.')
|
||||
return
|
||||
|
||||
|
||||
def change_nameservers_file(option):
|
||||
text = ''
|
||||
if option == 'cn':
|
||||
with open(settings.data_storage_dir.joinpath('cn_nameservers.txt'), 'r') as f:
|
||||
text = f.read()
|
||||
elif option == 'common':
|
||||
with open(settings.data_storage_dir.joinpath('common_nameservers.txt'), 'r') as f:
|
||||
text = f.read()
|
||||
elif option == 'default':
|
||||
for n in default_nameserver():
|
||||
text = '\n'.join(n)
|
||||
with open(settings.data_storage_dir.joinpath('nameservers.txt'), 'w') as f:
|
||||
f.write(text)
|
||||
return
|
||||
|
||||
|
||||
def default_nameserver():
|
||||
try:
|
||||
resolver = dns.resolver.Resolver()
|
||||
return resolver.nameservers
|
||||
except dns.resolver.NoResolverConfiguration:
|
||||
logger.log('ERROR', 'Resolver configuration could not be read '
|
||||
'or specified no nameservers.')
|
||||
exit(1)
|
||||
|
||||
|
||||
def looks_like_ip(maybe_ip):
|
||||
"""Does the given str look like an IP address?"""
|
||||
if not maybe_ip[0].isdigit():
|
||||
@@ -833,3 +736,45 @@ def looks_like_ip(maybe_ip):
|
||||
return True
|
||||
except socket.error:
|
||||
return False
|
||||
|
||||
|
||||
def deal_data(domain):
|
||||
db = Database()
|
||||
db.remove_invalid(domain)
|
||||
db.deduplicate_subdomain(domain)
|
||||
db.close()
|
||||
|
||||
|
||||
def get_data(domain):
|
||||
db = Database()
|
||||
data = db.get_data(domain).as_dict()
|
||||
db.close()
|
||||
return data
|
||||
|
||||
|
||||
def clear_data(domain):
|
||||
db = Database()
|
||||
db.drop_table(domain)
|
||||
db.close()
|
||||
|
||||
|
||||
def get_ns_path(in_china=None, enable_wildcard=None, ns_ip_list=None):
|
||||
data_dir = settings.data_storage_dir
|
||||
path = data_dir.joinpath('nameservers.txt')
|
||||
if in_china:
|
||||
path = data_dir.joinpath('nameservers_cn.txt')
|
||||
if not enable_wildcard:
|
||||
return path
|
||||
if not ns_ip_list:
|
||||
return path
|
||||
path = settings.authoritative_dns_path
|
||||
ns_data = '\n'.join(ns_ip_list)
|
||||
save_to_file(path, ns_data)
|
||||
return path
|
||||
|
||||
|
||||
def init_table(domain):
|
||||
db = Database()
|
||||
db.drop_table(domain)
|
||||
db.create_table(domain)
|
||||
db.close()
|
||||
|
||||
+18
-20
@@ -23,16 +23,15 @@ enable_brute_module = True # 使用爆破模块(默认True)
|
||||
enable_dns_resolve = True # 使用DNS解析子域(默认True)
|
||||
enable_http_request = True # 使用HTTP请求子域(默认True)
|
||||
enable_finder_module = True # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True)
|
||||
enable_altdns_module = False # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
|
||||
enable_cdn_check = True # 开启cdn检查模块(默认True)
|
||||
enable_altdns_module = True # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
|
||||
enable_enrich_module = True # 开启enrich模块，开启会富化出信息，如ip的cdn，cidr，asn，org，addr和isp等信息
|
||||
enable_banner_identify = True # 开启WEB指纹识别模块(默认True)
|
||||
enable_takeover_check = False # 开启子域接管风险检查(默认False)
|
||||
# 参数可选值有'small', 'medium', 'large'
|
||||
http_request_port = 'small' # HTTP请求子域(默认'small'，探测80,443端口)
|
||||
# 参数可选值有 'small', 'medium', 'large'
|
||||
http_request_port = 'small' # HTTP请求子域(默认 'small'，探测80,443端口)
|
||||
# 参数可选值True，False分别表示导出存活，全部子域结果
|
||||
result_export_alive = False # 只导出存活的子域结果(默认False)
|
||||
# 参数可选格式有'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
|
||||
# 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
|
||||
# 参数可选格式有 'csv', 'json'
|
||||
result_save_format = 'csv' # 子域结果保存文件格式(默认csv)
|
||||
# 参数path默认None使用OneForAll结果目录自动生成路径
|
||||
result_save_path = None # 子域结果保存文件路径(默认None)
|
||||
@@ -42,8 +41,7 @@ save_module_result = False # 保存各模块发现结果为json文件(默认Fal
|
||||
enable_all_module = True # 启用所有收集模块(默认True)
|
||||
enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_module才能生效
|
||||
# 只使用ask和baidu搜索引擎收集子域的示例
|
||||
# enable_partial_module = [('modules.search', 'ask')
|
||||
# ('modules.search', 'baidu')]
|
||||
# enable_partial_module = ['modules.search.ask', 'modules.search.baidu']
|
||||
module_thread_timeout = 90.0 # 每个收集模块线程超时时间(默认90秒)
|
||||
|
||||
# 爆破模块设置
|
||||
@@ -51,22 +49,17 @@ enable_wildcard_check = True # 开启泛解析检测(默认True)
|
||||
enable_wildcard_deal = True # 开启泛解析处理(默认True)
|
||||
brute_massdns_path = None # 默认None自动选择 如需填写请填写绝对路径
|
||||
brute_status_format = 'ansi' # 爆破时状态输出格式（默认asni，可选json）
|
||||
# 爆破时使用的进程数(根据计算机中CPU数量情况设置 不宜大于逻辑CPU个数)
|
||||
brute_process_num = 1 # 默认1
|
||||
brute_concurrent_num = 2000 # 并发查询数量(默认2000，最大推荐10000)
|
||||
brute_socket_num = 1 # 爆破时每个进程下的socket数量
|
||||
brute_resolve_num = 15 # 解析失败时尝试换名称服务器重查次数
|
||||
# 爆破所使用的字典路径 默认data/subdomains.txt
|
||||
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
|
||||
# 爆破所使用的字典路径 默认data/cn_nameservers.txt
|
||||
# 如果你不在中国请改为nameservers.txt
|
||||
brute_nameservers_path = data_storage_dir.joinpath('cn_nameservers.txt')
|
||||
# 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径)
|
||||
brute_wordlist_path = None
|
||||
# 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询
|
||||
authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt')
|
||||
enable_recursive_brute = False # 是否使用递归爆破(默认False)
|
||||
brute_recursive_depth = 2 # 递归爆破深度(默认2层)
|
||||
# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt
|
||||
recursive_nextlist_path = data_storage_dir.joinpath('next_subnames.txt')
|
||||
# 爆破下一层子域所使用的字典路径(默认None则使用data/subnames_next.txt，自定义字典请使用绝对路径)
|
||||
recursive_nextlist_path = None
|
||||
enable_check_dict = False # 是否开启字典配置检查提示(默认False)
|
||||
delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True)
|
||||
delete_massdns_result = True # 是否删除爆破时massdns输出的解析结果 (默认True)
|
||||
@@ -80,7 +73,12 @@ brute_ip_blacklist = {'0.0.0.0', '0.0.0.1'} # IP黑名单 子域解析到IP黑
|
||||
ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标记为非法(泛解析)子域
|
||||
|
||||
# altdns模块设置
|
||||
enable_fast_alt = True # 是否开启快速置换(默认True，只使用部分置换规则)
|
||||
altdns_increase_num = True
|
||||
altdns_decrease_num = True
|
||||
altdns_replace_word = False
|
||||
altdns_insert_word = False
|
||||
altdns_add_word = False
|
||||
|
||||
|
||||
# banner识别模块设置
|
||||
banner_process_number = 4 # 识别进程数量(默认4)
|
||||
@@ -99,8 +97,8 @@ request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
||||
|
||||
|
||||
# 请求设置
|
||||
request_thread_count = None # 请求线程数量(默认None，则根据内存大小设置)
|
||||
request_timeout_second = (3.05, 27) # 请求超时秒数(默认connect timout推荐略大于3秒，read秒)
|
||||
request_thread_count = None # 请求线程数量(默认None，则根据情况自动设置)
|
||||
request_timeout_second = (13, 27) # 请求超时秒数(默认connect timout推荐略大于3秒)
|
||||
request_ssl_verify = False # 请求SSL验证(默认False)
|
||||
request_allow_redirect = True # 请求允许重定向(默认True)
|
||||
request_redirect_limit = 10 # 请求跳转限制(默认10次)
|
||||
|
||||
+34
-15
@@ -1,6 +1,6 @@
|
||||
# coding=utf-8
|
||||
"""
|
||||
OneForAll配置
|
||||
OneForAll自定义配置
|
||||
"""
|
||||
|
||||
import pathlib
|
||||
@@ -15,11 +15,12 @@ enable_brute_module = True # 使用爆破模块(默认True)
|
||||
enable_dns_resolve = True # 使用DNS解析子域(默认True)
|
||||
enable_http_request = True # 使用HTTP请求子域(默认True)
|
||||
enable_finder_module = True # 开启finder模块,开启会从响应体和JS中再次发现子域(默认True)
|
||||
enable_altdns_module = True # 开启altdns模块,开启会利用置换技术重组子域再次发现新子域(默认True)
|
||||
enable_cdn_check = True # 开启cdn检查模块(默认True)
|
||||
enable_banner_identify = True # 开启WEB指纹识别模块(默认True)
|
||||
enable_takeover_check = False # 开启子域接管风险检查(默认False)
|
||||
# 参数可选值有'small', 'medium', 'large'
|
||||
http_request_port = 'small' # HTTP请求子域(默认'small'，探测80,443端口)
|
||||
# HTTP请求子域的端口范围 参数可选值有 'small', 'medium', 'large'
|
||||
http_request_port = 'small' # 请求端口范围(默认 'small'，表示请求子域的80,443端口)
|
||||
# 参数可选值True，False分别表示导出存活，全部子域结果
|
||||
result_export_alive = False # 只导出存活的子域结果(默认False)
|
||||
result_save_format = 'csv' # 子域结果保存文件格式(默认csv)
|
||||
@@ -31,23 +32,16 @@ save_module_result = False # 保存各模块发现结果为json文件(默认Fal
|
||||
enable_all_module = True # 启用所有收集模块(默认True)
|
||||
enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_module才能生效
|
||||
# 只使用ask和baidu搜索引擎收集子域的示例
|
||||
# enable_partial_module = [('modules.search', 'ask')
|
||||
# ('modules.search', 'baidu')]
|
||||
# enable_partial_module = ['modules.search.ask', 'modules.search.baidu']
|
||||
|
||||
# 爆破模块设置
|
||||
brute_concurrent_num = 2000 # 爆破时并发查询数量(默认2000，最大推荐10000)
|
||||
# 爆破所使用的字典路径 默认data/subdomains.txt
|
||||
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
|
||||
# 爆破所使用的DNS服务器路径 默认data/cn_nameservers.txt 如果你不在中国请改为nameservers.txt
|
||||
# DNS resolve server file path default data/nameservers.txt
|
||||
# If your computer's location are not in China, change `cn_nameservers.txt` to `nameservers.txt` plz.
|
||||
brute_nameservers_path = data_storage_dir.joinpath('cn_nameservers.txt')
|
||||
# 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询
|
||||
authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt')
|
||||
# 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径)
|
||||
brute_wordlist_path = None
|
||||
enable_recursive_brute = False # 是否使用递归爆破(默认False)
|
||||
brute_recursive_depth = 2 # 递归爆破深度(默认2层)
|
||||
# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt
|
||||
recursive_nextlist_path = data_storage_dir.joinpath('next_subnames.txt')
|
||||
# 爆破下一层子域所使用的字典路径(默认None则使用data/subnames_next.txt，自定义字典请使用绝对路径)
|
||||
recursive_nextlist_path = None
|
||||
enable_check_dict = False # 是否开启字典配置检查提示(默认False)
|
||||
delete_generated_dict = True # 是否删除爆破时临时生成的字典(默认True)
|
||||
# 是否删除爆破时massdns输出的解析结果 (默认True)
|
||||
@@ -63,6 +57,7 @@ brute_ip_blacklist = {'0.0.0.0', '0.0.0.1'} # IP黑名单 子域解析到IP黑
|
||||
# CNAME黑名单 子域解析到CNAME黑名单则标记为非法子域
|
||||
brute_cname_blacklist = {'nonexist.sdo.com', 'shop.taobao.com'}
|
||||
ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标记为非法(泛解析)子域
|
||||
cname_appear_maximum = 50 # 多个子域解析到同一cname次数超过50次则标记为非法(泛解析)子域
|
||||
|
||||
# 代理设置
|
||||
enable_request_proxy = False # 是否使用代理(全局开关)
|
||||
@@ -76,6 +71,30 @@ request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
||||
# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
||||
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
||||
|
||||
|
||||
# 请求设置
|
||||
request_thread_count = None # 请求线程数量(默认None，则根据情况自动设置)
|
||||
request_timeout_second = (13, 27) # 请求超时秒数(默认connect timout推荐略大于3秒)
|
||||
request_ssl_verify = False # 请求SSL验证(默认False)
|
||||
request_allow_redirect = True # 请求允许重定向(默认True)
|
||||
request_redirect_limit = 10 # 请求跳转限制(默认10次)
|
||||
# 默认请求头 可以在headers里添加自定义请求头
|
||||
request_default_headers = {
|
||||
'Accept': 'text/html,application/xhtml+xml,'
|
||||
'application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
|
||||
'Cache-Control': 'max-age=0',
|
||||
'DNT': '1',
|
||||
'Referer': 'https://www.google.com/',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||||
'(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'X-Forwarded-For': '127.0.0.1'
|
||||
}
|
||||
enable_random_ua = True # 使用随机UA(默认True，开启可以覆盖request_default_headers的UA)
|
||||
|
||||
|
||||
# 搜索模块设置
|
||||
enable_recursive_search = False # 递归搜索子域
|
||||
search_recursive_times = 2 # 递归搜索层数
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
47.254.51.88
|
||||
163.177.156.225
|
||||
161.117.97.232
|
||||
218.98.58.194
|
||||
117.91.188.195
|
||||
14.17.109.84
|
||||
58.52.135.164
|
||||
172.96.125.3
|
||||
106.38.197.52
|
||||
163.177.156.225
|
||||
117.91.188.196
|
||||
218.98.58.194
|
||||
58.52.135.165
|
||||
172.96.125.3
|
||||
106.38.197.48
|
||||
47.254.51.88
|
||||
161.117.97.232
|
||||
14.17.109.85
|
||||
172.96.125.3
|
||||
14.17.109.83
|
||||
163.177.156.225
|
||||
161.117.97.232
|
||||
218.98.58.194
|
||||
47.254.51.88
|
||||
58.52.135.163
|
||||
117.91.188.194
|
||||
106.38.197.52
|
||||
+132
-132
@@ -1,134 +1,134 @@
|
||||
[
|
||||
"10576",
|
||||
"10762",
|
||||
"11748",
|
||||
"131099",
|
||||
"132601",
|
||||
"133496",
|
||||
"134409",
|
||||
"135295",
|
||||
"136764",
|
||||
"137187",
|
||||
"13777",
|
||||
"13890",
|
||||
"14103",
|
||||
"14520",
|
||||
"17132",
|
||||
"199251",
|
||||
"200013",
|
||||
"200325",
|
||||
"200856",
|
||||
"201263",
|
||||
"202294",
|
||||
"203075",
|
||||
"203139",
|
||||
"204248",
|
||||
"204286",
|
||||
"204545",
|
||||
"206227",
|
||||
"206734",
|
||||
"206848",
|
||||
"206986",
|
||||
"207158",
|
||||
"208559",
|
||||
"209403",
|
||||
"21030",
|
||||
"21257",
|
||||
"23327",
|
||||
"23393",
|
||||
"23637",
|
||||
"23794",
|
||||
"24997",
|
||||
"26492",
|
||||
"268843",
|
||||
"28709",
|
||||
"29264",
|
||||
"30282",
|
||||
"30637",
|
||||
"328126",
|
||||
"36408",
|
||||
"38107",
|
||||
"397192",
|
||||
"40366",
|
||||
"43303",
|
||||
"44907",
|
||||
"46071",
|
||||
"46177",
|
||||
"47542",
|
||||
"49287",
|
||||
"49689",
|
||||
"51286",
|
||||
"55082",
|
||||
"55254",
|
||||
"56636",
|
||||
"57363",
|
||||
"58127",
|
||||
"59730",
|
||||
"59776",
|
||||
"60068",
|
||||
"60626",
|
||||
"60922",
|
||||
"61107",
|
||||
"61159",
|
||||
"62026",
|
||||
"62229",
|
||||
"63062",
|
||||
"64232",
|
||||
"8868",
|
||||
"9053",
|
||||
"55770",
|
||||
"49846",
|
||||
"49249",
|
||||
"48163",
|
||||
"45700",
|
||||
"43639",
|
||||
"39836",
|
||||
"393560",
|
||||
"393234",
|
||||
"36183",
|
||||
"35994",
|
||||
"35993",
|
||||
"35204",
|
||||
"34850",
|
||||
"34164",
|
||||
"33905",
|
||||
"32787",
|
||||
"31377",
|
||||
"31110",
|
||||
"31109",
|
||||
"31108",
|
||||
"31107",
|
||||
"30675",
|
||||
"24319",
|
||||
"23903",
|
||||
"23455",
|
||||
"23454",
|
||||
"22207",
|
||||
"21399",
|
||||
"21357",
|
||||
"21342",
|
||||
"20940",
|
||||
"20189",
|
||||
"18717",
|
||||
"18680",
|
||||
"17334",
|
||||
"16702",
|
||||
"16625",
|
||||
"12222",
|
||||
"209101",
|
||||
"201585",
|
||||
"135429",
|
||||
"395747",
|
||||
"394536",
|
||||
"209242",
|
||||
"203898",
|
||||
"202623",
|
||||
"14789",
|
||||
"133877",
|
||||
"13335",
|
||||
"132892",
|
||||
"21859",
|
||||
"6185",
|
||||
"47823",
|
||||
"4134"
|
||||
"AS10576",
|
||||
"AS10762",
|
||||
"AS11748",
|
||||
"AS131099",
|
||||
"AS132601",
|
||||
"AS133496",
|
||||
"AS134409",
|
||||
"AS135295",
|
||||
"AS136764",
|
||||
"AS137187",
|
||||
"AS13777",
|
||||
"AS13890",
|
||||
"AS14103",
|
||||
"AS14520",
|
||||
"AS17132",
|
||||
"AS199251",
|
||||
"AS200013",
|
||||
"AS200325",
|
||||
"AS200856",
|
||||
"AS201263",
|
||||
"AS202294",
|
||||
"AS203075",
|
||||
"AS203139",
|
||||
"AS204248",
|
||||
"AS204286",
|
||||
"AS204545",
|
||||
"AS206227",
|
||||
"AS206734",
|
||||
"AS206848",
|
||||
"AS206986",
|
||||
"AS207158",
|
||||
"AS208559",
|
||||
"AS209403",
|
||||
"AS21030",
|
||||
"AS21257",
|
||||
"AS23327",
|
||||
"AS23393",
|
||||
"AS23637",
|
||||
"AS23794",
|
||||
"AS24997",
|
||||
"AS26492",
|
||||
"AS268843",
|
||||
"AS28709",
|
||||
"AS29264",
|
||||
"AS30282",
|
||||
"AS30637",
|
||||
"AS328126",
|
||||
"AS36408",
|
||||
"AS38107",
|
||||
"AS397192",
|
||||
"AS40366",
|
||||
"AS43303",
|
||||
"AS44907",
|
||||
"AS46071",
|
||||
"AS46177",
|
||||
"AS47542",
|
||||
"AS49287",
|
||||
"AS49689",
|
||||
"AS51286",
|
||||
"AS55082",
|
||||
"AS55254",
|
||||
"AS56636",
|
||||
"AS57363",
|
||||
"AS58127",
|
||||
"AS59730",
|
||||
"AS59776",
|
||||
"AS60068",
|
||||
"AS60626",
|
||||
"AS60922",
|
||||
"AS61107",
|
||||
"AS61159",
|
||||
"AS62026",
|
||||
"AS62229",
|
||||
"AS63062",
|
||||
"AS64232",
|
||||
"AS8868",
|
||||
"AS9053",
|
||||
"AS55770",
|
||||
"AS49846",
|
||||
"AS49249",
|
||||
"AS48163",
|
||||
"AS45700",
|
||||
"AS43639",
|
||||
"AS39836",
|
||||
"AS393560",
|
||||
"AS393234",
|
||||
"AS36183",
|
||||
"AS35994",
|
||||
"AS35993",
|
||||
"AS35204",
|
||||
"AS34850",
|
||||
"AS34164",
|
||||
"AS33905",
|
||||
"AS32787",
|
||||
"AS31377",
|
||||
"AS31110",
|
||||
"AS31109",
|
||||
"AS31108",
|
||||
"AS31107",
|
||||
"AS30675",
|
||||
"AS24319",
|
||||
"AS23903",
|
||||
"AS23455",
|
||||
"AS23454",
|
||||
"AS22207",
|
||||
"AS21399",
|
||||
"AS21357",
|
||||
"AS21342",
|
||||
"AS20940",
|
||||
"AS20189",
|
||||
"AS18717",
|
||||
"AS18680",
|
||||
"AS17334",
|
||||
"AS16702",
|
||||
"AS16625",
|
||||
"AS12222",
|
||||
"AS209101",
|
||||
"AS201585",
|
||||
"AS135429",
|
||||
"AS395747",
|
||||
"AS394536",
|
||||
"AS209242",
|
||||
"AS203898",
|
||||
"AS202623",
|
||||
"AS14789",
|
||||
"AS133877",
|
||||
"AS13335",
|
||||
"AS132892",
|
||||
"AS21859",
|
||||
"AS6185",
|
||||
"AS47823",
|
||||
"AS4134"
|
||||
]
|
||||
@@ -120,9 +120,7 @@
|
||||
"edgekey": "Akamai",
|
||||
"fastly": "Fastly",
|
||||
"chinacache": "ChinaCache",
|
||||
"edgekey": "Akamai",
|
||||
"akamai": "Akamai",
|
||||
"fastly": "Fastly",
|
||||
"edgecast": "EdgeCast",
|
||||
"azioncdn": "Azion",
|
||||
"cachefly": "CacheFly",
|
||||
@@ -135,7 +133,6 @@
|
||||
"cloudflare": "CloudFlare",
|
||||
"hwcdn": "HighWinds",
|
||||
"kxcdn": "KeyCDN",
|
||||
"awsdns": "KeyCDN",
|
||||
"fpbns": "Level3",
|
||||
"footprint": "Level3",
|
||||
"llnwd": "LimeLight",
|
||||
@@ -155,33 +152,20 @@
|
||||
"cdnsun": "CDN SUN",
|
||||
"cdnvideo": "CDN Video",
|
||||
"clients.turbobytes.net": "TurboBytes",
|
||||
"clients.turbobytes.net": "TurboBytes",
|
||||
"turbobytes-cdn.com": "TurboBytes",
|
||||
"afxcdn.net": "afxcdn.net",
|
||||
"akamai.net": "Akamai",
|
||||
"akamaiedge.net": "Akamai",
|
||||
"akadns.net": "Akamai",
|
||||
"akamaitechnologies.com": "Akamai",
|
||||
"gslb.tbcache.com": "Alimama",
|
||||
"cloudfront.net": "Amazon Cloudfront",
|
||||
"anankecdn.com.br": "Ananke",
|
||||
"att-dsa.net": "AT&T",
|
||||
"azioncdn.net": "Azion",
|
||||
"belugacdn.com": "BelugaCDN",
|
||||
"bluehatnetwork.com": "Blue Hat Network",
|
||||
"systemcdn.net": "EdgeCast",
|
||||
"cachefly.net": "Cachefly",
|
||||
"cdn77.net": "CDN77",
|
||||
"cdn77.org": "CDN77",
|
||||
"panthercdn.com": "CDNetworks",
|
||||
"cdngc.net": "CDNetworks",
|
||||
"gccdn.net": "CDNetworks",
|
||||
"gccdn.cn": "CDNetworks",
|
||||
"cdnify.io": "CDNify",
|
||||
"ccgslb.com": "ChinaCache",
|
||||
"ccgslb.net": "ChinaCache",
|
||||
"c3cache.net": "ChinaCache",
|
||||
"chinacache.net": "ChinaCache",
|
||||
"cncssr.chinacache.net": "ChinaCache",
|
||||
"c3cdn.net": "ChinaCache",
|
||||
"lxdns.com": "ChinaNetCenter",
|
||||
@@ -189,11 +173,9 @@
|
||||
"mwcloudcdn.com": "QUANTIL/ChinaNetCenter",
|
||||
"cloudflare.com": "Cloudflare",
|
||||
"cloudflare.net": "Cloudflare",
|
||||
"edgecastcdn.net": "EdgeCast",
|
||||
"adn.": "EdgeCast",
|
||||
"wac.": "EdgeCast",
|
||||
"wpc.": "EdgeCast",
|
||||
"fastly.net": "Fastly",
|
||||
"fastlylb.net": "Fastly",
|
||||
"google.": "Google",
|
||||
"googlesyndication.": "Google",
|
||||
@@ -201,17 +183,11 @@
|
||||
"googleusercontent.com": "Google",
|
||||
"l.doubleclick.net": "Google",
|
||||
"hiberniacdn.com": "Hibernia",
|
||||
"hwcdn.net": "Highwinds",
|
||||
"incapdns.net": "Incapsula",
|
||||
"inscname.net": "Instartlogic",
|
||||
"insnw.net": "Instartlogic",
|
||||
"internapcdn.net": "Internap",
|
||||
"kxcdn.com": "KeyCDN",
|
||||
"lswcdn.net": "LeaseWeb CDN",
|
||||
"footprint.net": "Level3",
|
||||
"llnwd.net": "Limelight",
|
||||
"lldns.net": "Limelight",
|
||||
"netdna-cdn.com": "MaxCDN",
|
||||
"netdna-ssl.com": "MaxCDN",
|
||||
"netdna.com": "MaxCDN",
|
||||
"stackpathdns.com": "StackPath",
|
||||
@@ -231,5 +207,9 @@
|
||||
"yimg.": "Yahoo",
|
||||
"zenedge.net": "Zenedge",
|
||||
"cdnsun.net.": "CDNsun",
|
||||
"pilidns.com": "QiNiu"
|
||||
"pilidns.com": "QiNiu",
|
||||
"cdngslb.com": "AliCDN Global",
|
||||
"ialicdn.com": "AliCDN",
|
||||
"alivecdn.com": "AliCDN",
|
||||
"myalicdn.com": "AliCDN"
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
8.8.8.8
|
||||
8.8.4.4
|
||||
9.9.9.9
|
||||
9.9.9.10
|
||||
149.112.112.112
|
||||
4.2.2.1
|
||||
4.2.2.2
|
||||
4.2.2.3
|
||||
4.2.2.4
|
||||
4.2.2.5
|
||||
4.2.2.6
|
||||
1.1.1.1
|
||||
1.0.0.1
|
||||
1.0.0.2
|
||||
1.0.0.3
|
||||
1.0.0.19
|
||||
208.67.222.222
|
||||
208.67.220.220
|
||||
8.26.56.26
|
||||
8.20.247.20
|
||||
84.200.69.80
|
||||
84.200.70.40
|
||||
185.228.168.9
|
||||
185.228.169.9
|
||||
64.6.64.6
|
||||
64.6.65.6
|
||||
198.101.242.72
|
||||
23.253.163.53
|
||||
176.103.130.130
|
||||
176.103.130.131
|
||||
@@ -245,5 +245,10 @@
|
||||
"name":"readme",
|
||||
"cname":["readme.io"],
|
||||
"response":["Project doesnt exist... yet!"]
|
||||
},
|
||||
{
|
||||
"name":"alibaba_oss",
|
||||
"cname":["aliyuncs.com"],
|
||||
"response":["NoSuchBucket", "The specified bucket does not exist."]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
8.8.8.8
|
||||
8.8.4.4
|
||||
9.9.9.9
|
||||
9.9.9.10
|
||||
149.112.112.112
|
||||
4.2.2.1
|
||||
4.2.2.2
|
||||
4.2.2.3
|
||||
4.2.2.4
|
||||
4.2.2.5
|
||||
4.2.2.6
|
||||
1.1.1.1
|
||||
1.0.0.1
|
||||
1.0.0.2
|
||||
1.0.0.3
|
||||
1.0.0.19
|
||||
208.67.222.222
|
||||
208.67.220.220
|
||||
8.26.56.26
|
||||
8.20.247.20
|
||||
84.200.69.80
|
||||
84.200.70.40
|
||||
185.228.168.9
|
||||
185.228.169.9
|
||||
64.6.64.6
|
||||
64.6.65.6
|
||||
198.101.242.72
|
||||
23.253.163.53
|
||||
176.103.130.130
|
||||
176.103.130.131
|
||||
Binary file not shown.
-81
@@ -1,81 +0,0 @@
|
||||
#!/usr/bin/python3
|
||||
# coding=utf-8
|
||||
|
||||
"""
|
||||
OneForAll export from database module
|
||||
|
||||
:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
|
||||
:license: GNU General Public License v3.0, see LICENSE for more details.
|
||||
"""
|
||||
|
||||
import fire
|
||||
|
||||
from common import utils
|
||||
from common.database import Database
|
||||
from config.log import logger
|
||||
|
||||
|
||||
def export(target, type='target', db=None, alive=False, limit=None, path=None, format='csv', show=False):
|
||||
"""
|
||||
OneForAll export from database module
|
||||
|
||||
Example:
|
||||
python3 dbexport.py --target name --format csv --dir= ./result.csv
|
||||
python3 dbexport.py --db result.db --target name --show False
|
||||
python3 dbexport.py --target table_name --tb True --show False
|
||||
|
||||
Note:
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--path Result directory (default directory is ./results)
|
||||
|
||||
:param str target: Table to be exported
|
||||
:param str type: Type of target
|
||||
:param str db: Database path to be exported (default ./results/result.sqlite3)
|
||||
:param bool alive: Only export the results of alive subdomains (default False)
|
||||
:param str limit: Export limit (default None)
|
||||
:param str format: Result format (default csv)
|
||||
:param str path: Result directory (default None)
|
||||
:param bool show: Displays the exported data in terminal (default False)
|
||||
"""
|
||||
|
||||
if type == 'target':
|
||||
database = Database(db)
|
||||
domains = utils.get_domains(target)
|
||||
datas = []
|
||||
if domains:
|
||||
for domain in domains:
|
||||
table_name = domain_to_table(domain)
|
||||
rows = database.export_data(table_name, alive, limit)
|
||||
if rows is None:
|
||||
continue
|
||||
data = export_data(format, path, rows, show, table_name, target)
|
||||
datas.extend(data)
|
||||
database.close()
|
||||
if len(domains) > 1:
|
||||
utils.export_all(alive, format, path, datas)
|
||||
elif type == 'table':
|
||||
database = Database(db)
|
||||
rows = database.export_data(target, alive, limit)
|
||||
data = export_data(format, path, rows, show, target, target)
|
||||
database.close()
|
||||
return data
|
||||
|
||||
|
||||
def export_data(format, path, rows, show, table_name, target):
|
||||
format = utils.check_format(format, len(rows))
|
||||
path = utils.check_path(path, target, format)
|
||||
if show:
|
||||
print(rows.dataset)
|
||||
data = rows.export(format)
|
||||
utils.save_data(path, data)
|
||||
logger.log('ALERT', f'The subdomain result for {table_name}: {path}')
|
||||
data = rows.as_dict()
|
||||
return data, format, path
|
||||
|
||||
|
||||
def domain_to_table(table):
|
||||
return table.replace('.', '_') + "_now_result"
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
fire.Fire(export)
|
||||
@@ -8,6 +8,31 @@ OneForAll遵守[语义化版本格式](https://semver.org/)。
|
||||
# Unreleased
|
||||
|
||||
# Released
|
||||
## [0.4.3](https://github.com/shmilylty/oneforall/releases/tag/v0.4.3) - 2020-11-29
|
||||
- 修复了已知问题
|
||||
- 更新了文档
|
||||
|
||||
## [0.4.2](https://github.com/shmilylty/oneforall/releases/tag/v0.4.2) - 2020-11-23
|
||||
- 添加了数据表初始化处理流程，修复了#163中出现的问题。
|
||||
|
||||
## [0.4.1](https://github.com/shmilylty/oneforall/releases/tag/v0.4.1) - 2020-11-18
|
||||
- 修复了数字开头主域（如58.com）出现数据库报错的问题
|
||||
|
||||
## [0.4.0](https://github.com/shmilylty/oneforall/releases/tag/v0.4.0) - 2020-11-18
|
||||
- 重构了子域请求模块，解决了内存占用过大问题
|
||||
- 新增了子域置换模块，能从现有的子域发现更多新子域
|
||||
- 新增了数据富化模块，富化出更多有用的信息
|
||||
- 新增了finder模块，能从响应体和JS及跳转历史收集子域
|
||||
- 重构了泛解析探测，泛解析探测更加准确
|
||||
- 实现了配置插拔式设计
|
||||
- 实现了版本更新检查、运行环境检查、网络环境检查
|
||||
- 优化了子域爆破模块
|
||||
- 优化了泛解析处理
|
||||
- 优化了子域字典
|
||||
- 删除和优化了部分收集模块
|
||||
- 修复了一些反馈的bug
|
||||
- 更新了文档
|
||||
|
||||
## [0.3.0](https://github.com/shmilylty/oneforall/releases/tag/v0.3.0) - 2020-05-13
|
||||
- 重构了项目目录结构
|
||||
- 修改了输出显示为英文
|
||||
|
||||
+24
-24
@@ -1,15 +1,15 @@
|
||||
# 收集模块说明 #
|
||||
|
||||
如果要使用通过API收集子域的模块请先到[api.py](../oneforall/config/api.py)配置相关信息，大多平台的API都是可以注册账号免费获取的。
|
||||
如果要使用通过API收集子域的模块请先到[api.py](../config/api.py)配置相关信息，大多平台的API都是可以注册账号免费获取的。
|
||||
|
||||
如果你指定使用某些模块可以在[api.py](../oneforall/config/api.py)中设置：
|
||||
如果你指定使用某些模块可以在[api.py](../config/api.py)中设置：
|
||||
|
||||
```python
|
||||
enable_all_module = False # 不开启所有模块
|
||||
enable_partial_module = [('modules.search', 'ask'), ('modules.search', 'baidu')] # 只使用ask和baidu搜索引擎收集子域
|
||||
```
|
||||
|
||||
如果你指定使用某些模块使用代理可以在[api.py](../oneforall/config/api.py)中设置：
|
||||
如果你指定使用某些模块使用代理可以在[api.py](../config/api.py)中设置：
|
||||
|
||||
```python
|
||||
enable_proxy = True # 使用代理
|
||||
@@ -23,12 +23,12 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch'] # 只代理GoogleQuery和As
|
||||
|
||||
| 模块名称 | 是否需要代理 | 是否需要API | 其他说明 |
|
||||
| ----------- | ------------ | ----------- | -------------------------------------------------- |
|
||||
| censys_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| censys_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| certspotter | 否 | 否 | |
|
||||
| crtsh | 否 | 否 | |
|
||||
| entrust | 否 | 否 | |
|
||||
| google | 是 | 否 | |
|
||||
| spyse_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| spyse_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
|
||||
|
||||
2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，后续会添加检查NSEC记录，NSEC3记录等模块）
|
||||
@@ -52,25 +52,25 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch'] # 只代理GoogleQuery和As
|
||||
|
||||
| 模块名称 | 是否需要代理 | 是否需要API | 其他说明 |
|
||||
| ------------------ | ------------ | ----------- | -------------------------------------------------- |
|
||||
| binaryedge_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| binaryedge_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| bufferover | 否 | 否 | |
|
||||
| cebaidu | 否 | 否 | |
|
||||
| chinaz | 否 | 否 | |
|
||||
| chinaz_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| circl_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| cloudflare_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| chinaz_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| circl_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| cloudflare_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| dnsdb | 否 | 否 | |
|
||||
| dnsdb_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| dnsdb_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| dnsdumpster | 否 | 否 | |
|
||||
| hackertarget | 否 | 否 | |
|
||||
| ip138 | 否 | 否 | |
|
||||
| ipv4info | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| ipv4info | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| netcraft | 否 | 否 | |
|
||||
| passivedns_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| ptrarchive | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| riddler | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| passivedns_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| ptrarchive | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| riddler | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| robtex | 否 | 否 | |
|
||||
| securitytrails_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| securitytrails_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| sitedossier | 否 | 否 | |
|
||||
| threatcrowd | 否 | 否 | |
|
||||
| ximcx | 否 | 否 | |
|
||||
@@ -84,11 +84,11 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch'] # 只代理GoogleQuery和As
|
||||
| 模块名称 | 是否需要代理 | 是否需要API | 其他说明 |
|
||||
| -------------- | ------------ | ----------- | ------------------------------------------------- |
|
||||
| alienvault | 否 | 否 | |
|
||||
| riskiq_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| threatbook_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| riskiq_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| threatbook_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| threatminer | 否 | 否 | |
|
||||
| virustotal | 否 | 否 | |
|
||||
| virustotal_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| virustotal_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
7. 利用搜索引擎发现子域（目前有16个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`）
|
||||
|
||||
除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
|
||||
@@ -98,17 +98,17 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch'] # 只代理GoogleQuery和As
|
||||
| ask | 是 | 否 | |
|
||||
| baidu | 否 | 否 | |
|
||||
| bing | 否 | 否 | |
|
||||
| bing_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| bing_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| duckduckgo | 是 | 否 | |
|
||||
| exalead | 否，最好使用国外代理。 | 否 | |
|
||||
| fofa_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| fofa_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| gitee | 否 | 否 | |
|
||||
| github | 否 | 否 | 在[api.py](../oneforall/config/api.py)设置Github邮件名和密码。 |
|
||||
| github | 否 | 否 | 在[api.py](../config/api.py)设置Github邮件名和密码。 |
|
||||
| google | 是 | 否 | |
|
||||
| google_api | 是 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| shodan_api | 否，最好使用国外代理。 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| google_api | 是 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| shodan_api | 否，最好使用国外代理。 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
| so | 否 | 否 | |
|
||||
| sogou | 否 | 否 | |
|
||||
| yahoo | 是 | 否 | |
|
||||
| yandex | 是 | 否 | |
|
||||
| zoomeye_api | 否 | 是 | API使用和申请见[api.py](../oneforall/config/api.py) |
|
||||
| zoomeye_api | 否 | 是 | API使用和申请见[api.py](../config/api.py) |
|
||||
|
||||
@@ -47,11 +47,11 @@ D:.
|
||||
|
|
||||
+---data 存放一些所需数据
|
||||
| authoritative_dns.txt 临时存放开启了泛解析域名的权威DNS名称服务器IP地址
|
||||
| big_subnames.txt 子域爆破超大字典
|
||||
| cn_nameservers.txt 中国主流名称服务器IP地址
|
||||
| subnames_big.7z 子域爆破超大字典
|
||||
| nameservers_cn.txt 中国主流名称服务器IP地址
|
||||
| fingerprints.json 检查子域接管风险的指纹
|
||||
| nameservers.txt 全球主流名称服务器IP地址
|
||||
| next_subnames.txt 下一层子域字典
|
||||
| subnames_next.txt 下一层子域字典
|
||||
| public_suffix_list.dat 顶级域名后缀
|
||||
| srv_prefixes.json 常见SRV记录前缀名
|
||||
| subnames.txt 子域爆破常见字典
|
||||
|
||||
+12
-12
@@ -5,7 +5,7 @@
|
||||
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/releases)
|
||||
[](https://github.com/shmilylty/OneForAll/releases)
|
||||
|
||||
👊**OneForAll is a powerful subdomain integration tool** 📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
|
||||
|
||||
@@ -89,8 +89,6 @@ Result will be saved in `~/results`.
|
||||
<details>
|
||||
<summary><b>✨OneForAll usage</b></summary>
|
||||
|
||||
If your computer are not in China, change [setting](https://github.com/shmilylty/OneForAll/blob/master/config/setting.py#L46) `brute_nameservers_path` param `cn_nameservers.txt` to `nameservers.txt` plz.
|
||||
|
||||
If you are use pip3, run the following command:
|
||||
|
||||
```bash
|
||||
@@ -125,7 +123,7 @@ Let's take the command `python3 oneforall.py --target example.com run` as an exa
|
||||
|
||||
`example_com_now_result` table stores the collection results of the current subdomains. Usually using this table is enough.
|
||||
|
||||
For more information, please see [Field explanation](./docs/field.md).
|
||||
For more information, please see [Field explanation](../field.md).
|
||||
|
||||
</details>
|
||||
|
||||
@@ -164,7 +162,7 @@ DESCRIPTION
|
||||
python3 oneforall.py --target example.com --alive False run
|
||||
python3 oneforall.py --target example.com --brute True run
|
||||
python3 oneforall.py --target example.com --port medium run
|
||||
python3 oneforall.py --target example.com --format csv run
|
||||
python3 oneforall.py --target example.com --fmt csv run
|
||||
python3 oneforall.py --target example.com --dns False run
|
||||
python3 oneforall.py --target example.com --req False run
|
||||
python3 oneforall.py --target example.com --takeover False run
|
||||
@@ -173,7 +171,7 @@ DESCRIPTION
|
||||
Note:
|
||||
--alive True/False Only export alive subdomains or not (default False)
|
||||
--port default/small/large See details in ./config/setting.py(default port 80)
|
||||
--format csv/json (result format)
|
||||
--fmt csv/json (result format)
|
||||
--path Result directory (default directory is ./results)
|
||||
|
||||
ARGUMENTS
|
||||
@@ -193,7 +191,7 @@ FLAGS
|
||||
The port range request to the subdomains (default port 80)
|
||||
--alive=ALIVE
|
||||
Only export alive subdomains (default False)
|
||||
--format=FORMAT
|
||||
--fmt=FMT
|
||||
Result format (default csv)
|
||||
--path=PATH
|
||||
Result directory (default None)
|
||||
@@ -223,7 +221,7 @@ Problems with other tools
|
||||
|
||||
In order to solve the above problems, OneForAll born! As its name, OneForAll is committed to becoming the only one subdomain integration tool you need. We hope that one day OneForAll can be called "probably the best subdomain tool"
|
||||
|
||||
At present, OneForAll is under development, there must be a lot of problems and areas for improvement. Welcome to submit [Issues](https://github.com/shmilylty/OneForAll/issues) or [PR](https://github.com/shmilylty/OneForAll/pulls), If you like, star please✨. You can contact me through QQ group [**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771) or twitter [tweet](https://twitter.com/shmilylty) to me: 👨👨👦👦.
|
||||
At present, OneForAll is under development, there must be a lot of problems and areas for improvement. Welcome to submit [Issues](https://github.com/shmilylty/OneForAll/issues) or [PR](https://github.com/shmilylty/OneForAll/pulls), If you like, star please✨. You can contact me through QQ group [**824414244**](https://shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771) or twitter [tweet](https://twitter.com/shmilylty) to me: 👨👨👦👦.
|
||||
|
||||
## 👍Features
|
||||
|
||||
@@ -236,7 +234,9 @@ At present, OneForAll is under development, there must be a lot of problems and
|
||||
6. Use 6 threat intelligence modules: `alienvault`, `riskiq_ api`, `threatbook_ api`, `threatkeeper `, `virustotal`, `virustotal_ api`, which need to be added and improved.
|
||||
7. Use 16 search engines modules: `ask`, `baidu`, `bing`, `bing_api`, `fofa_api`, `gitee`, `github_api`, `google`, `google_api`, `shodan_api`, `so`, `sogou`, `yahoo`, `yandex`, `zoomeye_api`, except for special search engines. General search engines support automatic exclusion of search, full search and recursive search.
|
||||
* **Support subdomain brute force**, can use dictionary mode or custom fuzz mode. Supports bulk brute and recursive brute, and automatically determine wildcard or not and processing.
|
||||
* **Support subdmain verification**, default enable, automatically resolve DNS, request subdomain to obtain response, and determine subdomain alive or not.
|
||||
* **Support subdomain verification**, default enable, automatically resolve DNS, request subdomain to obtain response, and determine subdomain alive or not.
|
||||
* **Support subdomain crawling**, according to the existing subdomains, the response body of the request subdomain and the JS in the response body can be found again from the new subdomain.
|
||||
* **Support subdomain replacement**, according to the existing subdomain, use subdomain replacement technology to discover new subdomains again.
|
||||
* **Support subdomain takeover**, default enable, supports bulk inspection, and automatic takeover subdomain (only Github, remains to be improved at present).
|
||||
* **Powerful processing feature**, support automatic deduplicate, DNS resolve, HTTP request, filter valid subdomains and information for subdomains. Supported export formats: `txt`, `csv`, `json`.
|
||||
* **Very fast**, [collection module](https://github.com/shmilylty/OneForAll/tree/master/collect.py) uses multi-threading, [brute module](https://github.com/shmilylty/OneForAll/tree/master/brute.py) uses [MassDNS](https://github.com/blechschmidt/massdns), MassDNS is capable of resolving over 350,000 names per second using publicly available resolvers. DNS resolve and HTTP requests use async-coroutine. [subdomain takeover](https://github.com/shmilylty/OneForAll/tree/master/takeover.py) uses multi-threading.
|
||||
@@ -270,8 +270,6 @@ The project uses [SemVer](https://semver.org/) for version management, and you c
|
||||
## ⌛Follow-up plan
|
||||
|
||||
- [ ] Continuous optimize and improve of each module
|
||||
- [x] Subdomain monitoring (mark newly discovered subdomain)
|
||||
- [x] Subdomain collection crawler (collect subdomains from static files such as JS)
|
||||
- [ ] Implementation of front-end interface for powerful interaction
|
||||
|
||||
For more details, read [todo.md](https://github.com/shmilylty/OneForAll/tree/master/docs/todo.md).
|
||||
@@ -285,7 +283,7 @@ Very warmly welcome all people to make OneForAll better together!
|
||||
* **[Jing Ling](https://github.com/shmilylty)**
|
||||
* Core developer
|
||||
|
||||
You can view all contributors and their contributions in the [contributor documentation](https://github.com/shmilylty/OneForAll/tree/master/docs/contributors.md)) and thank them for making OneForAll more powerful and useful.
|
||||
You can view all contributors and their contributions in the [contributor documentation](https://github.com/shmilylty/OneForAll/tree/master/docs/contributors.md) and thank them for making OneForAll more powerful and useful.
|
||||
|
||||
## 📄License
|
||||
|
||||
@@ -297,6 +295,8 @@ Thanks to the various subdomain collection projects of online open source!
|
||||
|
||||
Thanks ace of [A-Team](https://github.com/QAX-A-Team) for their enthusiastic and unselfish answers!
|
||||
|
||||
Developed with drive and [PyCharm](https://www.jetbrains.com/pycharm/)!
|
||||
|
||||
## 📜Disclaimer
|
||||
|
||||
This tool can only be used in the safety construction of enterprises with sufficient legal authorization.
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/python3
|
||||
# coding=utf-8
|
||||
|
||||
"""
|
||||
OneForAll export from database module
|
||||
|
||||
:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
|
||||
:license: GNU General Public License v3.0, see LICENSE for more details.
|
||||
"""
|
||||
|
||||
import fire
|
||||
|
||||
from common import utils
|
||||
from common.database import Database
|
||||
from config.log import logger
|
||||
|
||||
|
||||
def export_data(target, db=None, alive=False, limit=None, path=None, fmt='csv', show=False):
|
||||
"""
|
||||
OneForAll export from database module
|
||||
|
||||
Example:
|
||||
python3 export.py --target name --fmt csv --dir= ./result.csv
|
||||
python3 export.py --target name --tb True --show False
|
||||
python3 export.py --db result.db --target name --show False
|
||||
|
||||
Note:
|
||||
--fmt csv/json (result format)
|
||||
--path Result directory (default directory is ./results)
|
||||
|
||||
:param str target: Table to be exported
|
||||
:param str db: Database path to be exported (default ./results/result.sqlite3)
|
||||
:param bool alive: Only export the results of alive subdomains (default False)
|
||||
:param str limit: Export limit (default None)
|
||||
:param str fmt: Result format (default csv)
|
||||
:param str path: Result directory (default None)
|
||||
:param bool show: Displays the exported data in terminal (default False)
|
||||
"""
|
||||
|
||||
database = Database(db)
|
||||
domains = utils.get_domains(target)
|
||||
datas = list()
|
||||
if domains:
|
||||
for domain in domains:
|
||||
table_name = domain.replace('.', '_')
|
||||
rows = database.export_data(table_name, alive, limit)
|
||||
if rows is None:
|
||||
continue
|
||||
data, _, _ = do_export(fmt, path, rows, show, domain, target)
|
||||
datas.extend(data)
|
||||
database.close()
|
||||
if len(domains) > 1:
|
||||
utils.export_all(alive, fmt, path, datas)
|
||||
return datas
|
||||
|
||||
|
||||
def do_export(fmt, path, rows, show, domain, target):
|
||||
fmt = utils.check_format(fmt)
|
||||
path = utils.check_path(path, target, fmt)
|
||||
if show:
|
||||
print(rows.dataset)
|
||||
data = rows.export(fmt)
|
||||
utils.save_to_file(path, data)
|
||||
logger.log('ALERT', f'The subdomain result for {domain}: {path}')
|
||||
data = rows.as_dict()
|
||||
return data, fmt, path
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
fire.Fire(export_data)
|
||||
+24
-9
@@ -7,6 +7,7 @@ import itertools
|
||||
|
||||
from config import settings
|
||||
|
||||
from modules import wildcard
|
||||
from common import utils
|
||||
from common import resolve
|
||||
from common import request
|
||||
@@ -75,14 +76,16 @@ class Altdns(Module):
|
||||
# test1.example.com -> test2.example.com, test3.example.com, ...
|
||||
# test01.example.com -> test02.example.com, test03.example.com, ...
|
||||
|
||||
count = 0
|
||||
digits = re.findall(r'\d{1,3}', subname)
|
||||
|
||||
for d in digits:
|
||||
for m in range(self.num_count):
|
||||
replacement = str(int(d) + 1 + m).zfill(len(d))
|
||||
tmp_domain = subname.replace(d, replacement)
|
||||
new_domain = f'{tmp_domain}.{self.domain}'
|
||||
self.new_subdomains.add(new_domain)
|
||||
count += 1
|
||||
logger.log('DEBUG', f'The increase_num generated {count} subdomains')
|
||||
|
||||
def decrease_num(self, subname):
|
||||
"""
|
||||
@@ -94,8 +97,8 @@ class Altdns(Module):
|
||||
# test4.example.com -> test3.example.com, test2.example.com, ...
|
||||
# test04.example.com -> test03.example.com, test02.example.com, ...
|
||||
|
||||
count = 0
|
||||
digits = re.findall(r'\d{1,3}', subname)
|
||||
|
||||
for d in digits:
|
||||
for m in range(self.num_count):
|
||||
new_digit = (int(d) - 1 - m)
|
||||
@@ -106,6 +109,8 @@ class Altdns(Module):
|
||||
tmp_domain = subname.replace(d, replacement)
|
||||
new_domain = f'{tmp_domain}.{self.domain}'
|
||||
self.new_subdomains.add(new_domain)
|
||||
count += 1
|
||||
logger.log('DEBUG', f'The decrease_num generated {count} subdomains')
|
||||
|
||||
def insert_word(self, parts):
|
||||
"""
|
||||
@@ -118,12 +123,15 @@ class Altdns(Module):
|
||||
# test.1.foo.WORD.example.com,
|
||||
# ...
|
||||
|
||||
count = 0
|
||||
for word in self.words:
|
||||
for index in range(len(parts)):
|
||||
tmp_parts = parts.copy()
|
||||
tmp_parts.insert(index, word)
|
||||
new_domain = '.'.join(tmp_parts)
|
||||
self.new_subdomains.add(new_domain)
|
||||
count += 1
|
||||
logger.log('DEBUG', f'The insert_word generated {count} subdomains')
|
||||
|
||||
def add_word(self, subnames):
|
||||
"""
|
||||
@@ -131,6 +139,7 @@ class Altdns(Module):
|
||||
append existing content with `-WORD`
|
||||
"""
|
||||
|
||||
count = 0
|
||||
for word in self.words:
|
||||
for index, name in enumerate(subnames):
|
||||
# Prepend with `-`
|
||||
@@ -146,6 +155,8 @@ class Altdns(Module):
|
||||
tmp_subnames[index] = f'{name}-{word}'
|
||||
new_subname = '.'.join(tmp_subnames + [self.domain])
|
||||
self.new_subdomains.add(new_subname)
|
||||
count += 1
|
||||
logger.log('DEBUG', f'The add_word generated {count} subdomains')
|
||||
|
||||
def replace_word(self, subname):
|
||||
"""
|
||||
@@ -158,6 +169,7 @@ class Altdns(Module):
|
||||
# WORD4.1.foo.example.com,
|
||||
# ..
|
||||
|
||||
count = 0
|
||||
for word in self.words:
|
||||
if word not in subname:
|
||||
continue
|
||||
@@ -167,16 +179,22 @@ class Altdns(Module):
|
||||
new_subname = subname.replace(word, word_alt)
|
||||
new_subdomain = f'{new_subname}.{self.domain}'
|
||||
self.new_subdomains.add(new_subdomain)
|
||||
count += 1
|
||||
logger.log('DEBUG', f'The replace_word generated {count} subdomains')
|
||||
|
||||
def gen_new_subdomains(self):
|
||||
for subdomain in self.now_subdomains:
|
||||
subname, parts = split_domain(subdomain)
|
||||
subnames = subname.split('.')
|
||||
if settings.altdns_increase_num:
|
||||
self.increase_num(subname)
|
||||
if settings.altdns_decrease_num:
|
||||
self.decrease_num(subname)
|
||||
if settings.altdns_replace_word:
|
||||
self.replace_word(subname)
|
||||
if not settings.enable_fast_alt:
|
||||
if settings.altdns_insert_word:
|
||||
self.insert_word(parts)
|
||||
if settings.altdns_add_word:
|
||||
self.add_word(subnames)
|
||||
count = len(self.new_subdomains)
|
||||
logger.log('DEBUG', f'The altdns module generated {count} subdomains')
|
||||
@@ -193,9 +211,6 @@ class Altdns(Module):
|
||||
self.end = time.time()
|
||||
self.elapse = round(self.end - self.start, 1)
|
||||
self.gen_result()
|
||||
temp_data = resolve.run_resolve(self.domain, self.results)
|
||||
fina_data = request.run_request(self.domain, temp_data, port)
|
||||
data = data + fina_data
|
||||
logger.log('INFOR', f'Saving altdns results')
|
||||
utils.save_db(self.domain, data, 'altdns')
|
||||
return data
|
||||
resolved_data = resolve.run_resolve(self.domain, self.results)
|
||||
valid_data = wildcard.deal_wildcard(resolved_data) # 强制开启泛解析处理
|
||||
request.run_request(self.domain, valid_data, port)
|
||||
|
||||
@@ -9,7 +9,7 @@ class CensysAPI(Query):
|
||||
self.domain = domain
|
||||
self.module = 'Certificate'
|
||||
self.source = "CensysAPIQuery"
|
||||
self.addr = 'https://www.censys.io/api/v1/search/certificates'
|
||||
self.addr = 'https://search.censys.io/api/v1/search/certificates'
|
||||
self.id = settings.censys_api_id
|
||||
self.secret = settings.censys_api_secret
|
||||
self.delay = 3.0 # Censys 接口查询速率限制 最快2.5秒查1次
|
||||
|
||||
@@ -26,6 +26,9 @@ class NSEC(Check):
|
||||
self.subdomains.update(subdomains)
|
||||
if subdomain == self.domain: # 当查出子域为主域 说明完成了一个循环 不再继续查询
|
||||
break
|
||||
if domain != self.domain: # 防止出现wwdmas.cn 000.000.wwdmas.cn 000.000.000.wwdmas.cn情况
|
||||
if domain.split('.')[0] == subdomain.split('.')[0]:
|
||||
break
|
||||
domain = subdomain
|
||||
return self.subdomains
|
||||
|
||||
|
||||
+3
-2
@@ -23,7 +23,7 @@ class Collect(object):
|
||||
module_path = settings.module_dir.joinpath(module)
|
||||
for path in module_path.rglob('*.py'):
|
||||
import_module = f'modules.{module}.{path.stem}'
|
||||
self.modules.append([import_module, path.stem])
|
||||
self.modules.append(import_module)
|
||||
else:
|
||||
self.modules = settings.enable_partial_module
|
||||
|
||||
@@ -31,7 +31,8 @@ class Collect(object):
|
||||
"""
|
||||
Import do function
|
||||
"""
|
||||
for module, name in self.modules:
|
||||
for module in self.modules:
|
||||
name = module.split('.')[-1]
|
||||
import_object = importlib.import_module(module)
|
||||
func = getattr(import_object, 'run')
|
||||
self.collect_funcs.append([func, name])
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
from common.query import Query
|
||||
|
||||
|
||||
class DNSdumpster(Query):
|
||||
class DNSDumpster(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Dataset'
|
||||
self.source = "DNSdumpsterQuery"
|
||||
self.source = "DNSDumpsterQuery"
|
||||
self.addr = 'https://dnsdumpster.com/'
|
||||
|
||||
def query(self):
|
||||
@@ -21,7 +21,8 @@ class DNSdumpster(Query):
|
||||
return
|
||||
self.cookie = resp.cookies
|
||||
data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'),
|
||||
'targetip': self.domain}
|
||||
'targetip': self.domain,
|
||||
'user':'free'}
|
||||
resp = self.post(self.addr, data)
|
||||
self.subdomains = self.collect_subdomains(resp)
|
||||
|
||||
@@ -43,10 +44,9 @@ def run(domain):
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
query = DNSdumpster(domain)
|
||||
query = DNSDumpster(domain)
|
||||
query.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
run('example.com')
|
||||
run('mi.com')
|
||||
|
||||
@@ -12,39 +12,21 @@ class NetCraft(Query):
|
||||
self.domain = domain
|
||||
self.module = 'Dataset'
|
||||
self.source = 'NetCraftQuery'
|
||||
self.init = 'https://searchdns.netcraft.com/'
|
||||
self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains'
|
||||
self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains&position=limited'
|
||||
self.page_num = 1
|
||||
self.per_page_num = 20
|
||||
|
||||
def bypass_verification(self):
|
||||
"""
|
||||
绕过NetCraft的JS验证
|
||||
"""
|
||||
self.header = self.get_header() # Netcraft会检查User-Agent
|
||||
resp = self.get(self.init)
|
||||
if not resp:
|
||||
return False
|
||||
self.cookie = resp.cookies
|
||||
cookie_value = self.cookie['netcraft_js_verification_challenge']
|
||||
cookie_encode = parse.unquote(cookie_value).encode('utf-8')
|
||||
verify_taken = hashlib.sha1(cookie_encode).hexdigest()
|
||||
self.cookie['netcraft_js_verification_response'] = verify_taken
|
||||
return True
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
if not self.bypass_verification():
|
||||
return
|
||||
self.header = self.get_header() # NetCraft会检查User-Agent
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
last = ''
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'restriction': 'site ends with',
|
||||
'host': '.' + self.domain,
|
||||
params = {'host': '*.' + self.domain,
|
||||
'from': self.page_num}
|
||||
resp = self.get(self.addr + last, params)
|
||||
subdomains = self.match_subdomains(resp)
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
from config.log import logger
|
||||
from common.query import Query
|
||||
|
||||
|
||||
class PhoneBook(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Dataset'
|
||||
self.source = 'PhoneBookQuery'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
self.header.update({'Referer': 'https://phonebook.cz/',
|
||||
'Origin': 'https://phonebook.cz'})
|
||||
addr = 'https://public.intelx.io/phonebook/search'
|
||||
key = 'd7d1ed06-f0c5-49d4-a9ca-a167e6d2ffab'
|
||||
url = f'{addr}?k={key}'
|
||||
data = {"term": self.domain, "maxresults": 10000,
|
||||
"media": 0, "target": 1,
|
||||
"terminate": [], "timeout": 20}
|
||||
resp = self.post(url, json=data)
|
||||
if not resp:
|
||||
return
|
||||
json = resp.json()
|
||||
ids = json.get('id')
|
||||
if not ids:
|
||||
logger.log('ALERT', f'Get PhoneBook id fail')
|
||||
return
|
||||
url = f'{addr}/result?k={key}&id={ids}&limit=10000'
|
||||
resp = self.get(url)
|
||||
self.subdomains = self.collect_subdomains(resp)
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
self.begin()
|
||||
self.query()
|
||||
self.finish()
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
|
||||
|
||||
def run(domain):
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
query = PhoneBook(domain)
|
||||
query.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run('freebuf.com')
|
||||
@@ -17,7 +17,7 @@ class Robtex(Query):
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
base_addr = 'https://freeapi.robtex.com/pdns/'
|
||||
base_addr = 'https://freeapi.robtex.com/pdns'
|
||||
url = f'{base_addr}/forward/{self.domain}'
|
||||
resp = self.get(url)
|
||||
if not resp:
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
import time
|
||||
from config.log import logger
|
||||
from common.query import Query
|
||||
|
||||
|
||||
class WZPCQuery(Query):
|
||||
def __init__(self, domain):
|
||||
Query.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Dataset'
|
||||
self.source = 'WZPCQuery'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
|
||||
base_addr = 'http://114.55.181.28/check_web/' \
|
||||
'databaseInfo_mainSearch.action'
|
||||
page_num = 1
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'isSearch': 'true', 'searchType': 'url',
|
||||
'term': self.domain, 'pageNo': page_num}
|
||||
try:
|
||||
resp = self.get(base_addr, params)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e.args)
|
||||
break
|
||||
if not resp:
|
||||
break
|
||||
subdomains = self.match_subdomains(resp.text)
|
||||
if not subdomains: # 没有发现子域名则停止查询
|
||||
break
|
||||
self.subdomains.update(subdomains)
|
||||
if not subdomains:
|
||||
break
|
||||
if page_num > 10:
|
||||
break
|
||||
page_num += 1
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
self.begin()
|
||||
self.query()
|
||||
self.finish()
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
|
||||
|
||||
def run(domain):
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
query = WZPCQuery(domain)
|
||||
query.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run('sc.gov.cn')
|
||||
run('bkzy.org')
|
||||
@@ -0,0 +1,72 @@
|
||||
from modules import iscdn
|
||||
from common import utils
|
||||
from common.database import Database
|
||||
from common.ipasn import IPAsnInfo
|
||||
from common.ipreg import IpRegData
|
||||
|
||||
|
||||
def get_ips(info):
|
||||
ip = info.get('ip')
|
||||
if not ip:
|
||||
return None
|
||||
ips = ip.split(',')
|
||||
return ips
|
||||
|
||||
|
||||
def enrich_info(data):
|
||||
ip_asn = IPAsnInfo()
|
||||
ip_reg = IpRegData()
|
||||
for index, info in enumerate(data):
|
||||
ips = get_ips(info)
|
||||
if not ips:
|
||||
continue
|
||||
public = list()
|
||||
cidr = list()
|
||||
asn = list()
|
||||
org = list()
|
||||
addr = list()
|
||||
isp = list()
|
||||
for ip in ips:
|
||||
public.append(str(utils.ip_is_public(ip)))
|
||||
asn_info = ip_asn.find(ip)
|
||||
cidr.append(asn_info.get('cidr'))
|
||||
asn.append(asn_info.get('asn'))
|
||||
org.append(asn_info.get('org'))
|
||||
ip_info = ip_reg.query(ip)
|
||||
addr.append(ip_info.get('addr'))
|
||||
isp.append(ip_info.get('isp'))
|
||||
data[index]['public'] = ','.join(public)
|
||||
data[index]['cidr'] = ','.join(cidr)
|
||||
data[index]['asn'] = ','.join(asn)
|
||||
data[index]['org'] = ','.join(org)
|
||||
data[index]['addr'] = ','.join(addr)
|
||||
data[index]['isp'] = ','.join(isp)
|
||||
return data
|
||||
|
||||
|
||||
class Enrich(object):
|
||||
def __init__(self, domain):
|
||||
self.domain = domain
|
||||
|
||||
def get_data(self):
|
||||
db = Database()
|
||||
fields = ['url', 'cname', 'ip', 'public', 'cdn', 'header',
|
||||
'cidr', 'asn', 'org', 'addr', 'isp']
|
||||
results = db.get_data_by_fields(self.domain, fields)
|
||||
return results.as_dict()
|
||||
|
||||
def save_db(self, data):
|
||||
db = Database()
|
||||
for info in data:
|
||||
url = info.pop('url')
|
||||
info.pop('cname')
|
||||
info.pop('ip')
|
||||
info.pop('header')
|
||||
db.update_data_by_url(self.domain, info, url)
|
||||
db.close()
|
||||
|
||||
def run(self):
|
||||
data = self.get_data()
|
||||
data = enrich_info(data)
|
||||
data = iscdn.do_check(data)
|
||||
self.save_db(data)
|
||||
+15
-10
@@ -7,6 +7,7 @@ from common import utils
|
||||
from common import resolve
|
||||
from common import request
|
||||
from common.module import Module
|
||||
from common.database import Database
|
||||
from config import settings
|
||||
from config.log import logger
|
||||
|
||||
@@ -25,16 +26,11 @@ class Finder(Module):
|
||||
new_subdomains = found_subdomains - existing_subdomains
|
||||
if not len(new_subdomains):
|
||||
self.finish() # 未发现新的子域就直接返回
|
||||
return data
|
||||
self.subdomains = new_subdomains
|
||||
self.finish()
|
||||
self.gen_result()
|
||||
temp_data = resolve.run_resolve(domain, self.results)
|
||||
fina_data = request.run_request(domain, temp_data, port)
|
||||
data = data + fina_data
|
||||
logger.log('INFOR', f'Saving finder results')
|
||||
utils.save_db(domain, data, 'finder')
|
||||
return data
|
||||
resolved_data = resolve.run_resolve(domain, self.results)
|
||||
request.run_request(domain, resolved_data, port)
|
||||
|
||||
|
||||
file_path = settings.data_storage_dir.joinpath('common_js_library.json')
|
||||
@@ -177,22 +173,31 @@ def find_js_urls(domain, req_url, rsp_html):
|
||||
return js_urls
|
||||
|
||||
|
||||
def convert_to_dict(url_list):
|
||||
url_dict = []
|
||||
for url in url_list:
|
||||
url_dict.append({'url': url})
|
||||
return url_dict
|
||||
|
||||
def find_subdomains(domain, data):
|
||||
subdomains = set()
|
||||
js_urls = set()
|
||||
db = Database()
|
||||
for infos in data:
|
||||
jump_history = infos.get('history')
|
||||
req_url = infos.get('url')
|
||||
subdomains.update(find_in_history(domain, req_url, jump_history))
|
||||
rsp_html = infos.get('response')
|
||||
rsp_html = db.get_resp_by_url(domain, req_url)
|
||||
if not rsp_html:
|
||||
logger.log('DEBUG', f'an abnormal response occurred in the request {req_url}')
|
||||
continue
|
||||
subdomains.update(find_in_resp(domain, req_url, rsp_html))
|
||||
js_urls.update(find_js_urls(domain, req_url, rsp_html))
|
||||
|
||||
resp_data = request.bulk_request(js_urls)
|
||||
for _, resp in resp_data:
|
||||
req_data = convert_to_dict(js_urls)
|
||||
resp_data = request.bulk_request(domain, req_data, ret=True)
|
||||
while not resp_data.empty():
|
||||
_, resp = resp_data.get()
|
||||
if not isinstance(resp, Response):
|
||||
continue
|
||||
text = utils.decode_resp_text(resp)
|
||||
|
||||
@@ -7,7 +7,7 @@ class ThreatMiner(Query):
|
||||
self.domain = domain
|
||||
self.module = 'Intelligence'
|
||||
self.source = 'ThreatMinerQuery'
|
||||
self.addr = 'https://www.threatminer.org/getData.php'
|
||||
self.addr = 'https://api.threatminer.org/v2/domain.php'
|
||||
|
||||
def query(self):
|
||||
"""
|
||||
@@ -15,8 +15,7 @@ class ThreatMiner(Query):
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'e': 'subdomains_container',
|
||||
'q': self.domain, 't': 0, 'rt': 10}
|
||||
params = {'q': self.domain, 'rt': 5}
|
||||
resp = self.get(self.addr, params)
|
||||
self.subdomains = self.collect_subdomains(resp)
|
||||
|
||||
|
||||
+18
-16
@@ -3,6 +3,7 @@ import ipaddress
|
||||
|
||||
from config import settings
|
||||
from common import utils
|
||||
from common.database import Database
|
||||
from config.log import logger
|
||||
|
||||
data_dir = settings.data_storage_dir
|
||||
@@ -18,6 +19,8 @@ cdn_header_key = utils.load_json(data_dir.joinpath('cdn_header_keys.json'))
|
||||
|
||||
|
||||
def check_cname_keyword(cname):
|
||||
if not cname:
|
||||
return False
|
||||
names = cname.lower().split(',')
|
||||
for name in names:
|
||||
for keyword in cdn_cname_keyword.keys():
|
||||
@@ -26,14 +29,22 @@ def check_cname_keyword(cname):
|
||||
|
||||
|
||||
def check_header_key(header):
|
||||
if isinstance(header, str):
|
||||
header = json.loads(header)
|
||||
if isinstance(header, dict):
|
||||
header = set(map(lambda x: x.lower(), header.keys()))
|
||||
for key in cdn_header_key:
|
||||
if key in header:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def check_cdn_cidr(content):
|
||||
ips = set(content.split(','))
|
||||
def check_cdn_cidr(ips):
|
||||
if isinstance(ips, str):
|
||||
ips = set(ips.split(','))
|
||||
else:
|
||||
return False
|
||||
for ip in ips:
|
||||
try:
|
||||
ip = ipaddress.ip_address(ip)
|
||||
@@ -46,39 +57,30 @@ def check_cdn_cidr(content):
|
||||
|
||||
|
||||
def check_cdn_asn(asn):
|
||||
if str(asn) in cdn_asn_list:
|
||||
if isinstance(asn, str):
|
||||
if asn in cdn_asn_list:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def check_cdn(data):
|
||||
logger.log('DEBUG', f'Start cdn check module')
|
||||
def do_check(data):
|
||||
logger.log('DEBUG', f'Checking cdn')
|
||||
for index, item in enumerate(data):
|
||||
cname = item.get('cname')
|
||||
if cname:
|
||||
if check_cname_keyword(cname):
|
||||
data[index]['cdn'] = 1
|
||||
continue
|
||||
header = item.get('header')
|
||||
if header:
|
||||
header = json.loads(header)
|
||||
if check_header_key(header):
|
||||
data[index]['cdn'] = 1
|
||||
continue
|
||||
ip = item.get('ip')
|
||||
if ip:
|
||||
if check_cdn_cidr(ip):
|
||||
data[index]['cdn'] = 1
|
||||
continue
|
||||
asn = item.get('asn')
|
||||
if asn:
|
||||
asn = asn[2:] # 去除AS
|
||||
if check_cdn_asn(asn):
|
||||
data[index]['cdn'] = 1
|
||||
continue
|
||||
data[index]['cdn'] = 0
|
||||
return data
|
||||
|
||||
|
||||
def save_db(name, data):
|
||||
logger.log('DEBUG', f'Saving cdn check results')
|
||||
utils.save_db(name, data, 'cdn')
|
||||
|
||||
@@ -57,7 +57,7 @@ class Baidu(Search):
|
||||
self.subdomains.update(subdomains)
|
||||
self.page_num += self.per_page_num
|
||||
# 搜索页面没有出现下一页时停止搜索
|
||||
if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:
|
||||
if f'&pn={self.page_num}&' not in resp.text:
|
||||
break
|
||||
if self.page_num >= self.limit_num: # 搜索条数限制
|
||||
break
|
||||
|
||||
@@ -19,9 +19,9 @@ class Bing(Search):
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
"""
|
||||
self.page_num = 0 # 二次搜索重新置0
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
self.page_num = 0 # 二次搜索重新置0
|
||||
resp = self.get(self.init)
|
||||
if not resp:
|
||||
return
|
||||
|
||||
@@ -21,7 +21,7 @@ class FoFa(Search):
|
||||
发送搜索请求并做子域匹配
|
||||
"""
|
||||
self.page_num = 1
|
||||
subdomain_encode = f'domain={self.domain} || cert={self.domain}'.encode('utf-8')
|
||||
subdomain_encode = f'domain="{self.domain}"'.encode('utf-8')
|
||||
query_data = base64.b64encode(subdomain_encode)
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
@@ -32,7 +32,7 @@ class FoFa(Search):
|
||||
'qbase64': query_data,
|
||||
'page': self.page_num,
|
||||
'full': 'true',
|
||||
'size': 5000}
|
||||
'size': 1000}
|
||||
resp = self.get(self.addr, query)
|
||||
if not resp:
|
||||
return
|
||||
@@ -42,7 +42,7 @@ class FoFa(Search):
|
||||
break
|
||||
self.subdomains.update(subdomains)
|
||||
size = resp_json.get('size')
|
||||
if size < 5000:
|
||||
if size < 1000:
|
||||
break
|
||||
self.page_num += 1
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ class Gitee(Search):
|
||||
self.module = 'Search'
|
||||
self.addr = 'https://search.gitee.com/'
|
||||
self.domain = domain
|
||||
self.header = self.get_header()
|
||||
|
||||
def search(self):
|
||||
"""
|
||||
@@ -20,6 +19,8 @@ class Gitee(Search):
|
||||
page_num = 1
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
params = {'pageno': page_num, 'q': self.domain, 'type': 'code'}
|
||||
try:
|
||||
resp = self.get(self.addr, params=params)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import requests
|
||||
import time
|
||||
|
||||
from config import settings
|
||||
from common.search import Search
|
||||
from config.log import logger
|
||||
@@ -11,53 +12,30 @@ class GithubAPI(Search):
|
||||
self.module = 'Search'
|
||||
self.addr = 'https://api.github.com/search/code'
|
||||
self.domain = domain
|
||||
self.session = requests.Session()
|
||||
self.session.trust_env = False
|
||||
self.auth_url = 'https://api.github.com'
|
||||
self.delay = 5
|
||||
self.token = settings.github_api_token
|
||||
|
||||
def auth_github(self):
|
||||
"""
|
||||
github api 认证
|
||||
|
||||
:return: 认证失败返回False 成功返回True
|
||||
"""
|
||||
self.session.headers.update({'Authorization': 'token ' + self.token})
|
||||
try:
|
||||
resp = self.session.get(self.auth_url)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e.args)
|
||||
return False
|
||||
if resp.status_code != 200:
|
||||
resp_json = resp.json()
|
||||
msg = resp_json.get('message')
|
||||
logger.log('ERROR', msg)
|
||||
return False
|
||||
return True
|
||||
|
||||
def search(self):
|
||||
"""
|
||||
向接口查询子域并做子域匹配
|
||||
"""
|
||||
self.session.headers = self.get_header()
|
||||
self.session.proxies = self.get_proxy(self.source)
|
||||
self.session.verify = self.verify
|
||||
self.session.headers.update(
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
self.header.update(
|
||||
{'Accept': 'application/vnd.github.v3.text-match+json'})
|
||||
|
||||
if not self.auth_github():
|
||||
logger.log('ERROR', f'{self.source} module login failed')
|
||||
return
|
||||
page = 1
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
params = {'q': self.domain, 'per_page': 100,
|
||||
'page': page, 'sort': 'indexed'}
|
||||
'page': page, 'sort': 'indexed',
|
||||
'access_token': self.token}
|
||||
try:
|
||||
resp = self.session.get(self.addr, params=params)
|
||||
resp = self.get(self.addr, params=params)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e.args)
|
||||
break
|
||||
if resp.status_code != 200:
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.log('ERROR', f'{self.source} module query failed')
|
||||
break
|
||||
subdomains = self.match_subdomains(resp)
|
||||
@@ -103,4 +81,4 @@ def run(domain):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run('example.com')
|
||||
run('freebuf.com')
|
||||
|
||||
@@ -21,13 +21,14 @@ class Yahoo(Search):
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(self.init)
|
||||
if not resp:
|
||||
return
|
||||
self.cookie = resp.cookies # 获取cookie Yahoo在搜索时需要带上cookie
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:.' + domain + filtered_subdomain
|
||||
params = {'p': query, 'b': self.page_num, 'pz': self.per_page_num}
|
||||
|
||||
@@ -20,6 +20,8 @@ class Yandex(Search):
|
||||
:param str domain: 域名
|
||||
:param str filtered_subdomain: 过滤的子域
|
||||
"""
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
self.page_num = 0 # 二次搜索重新置0
|
||||
resp = self.get(self.init)
|
||||
if not resp:
|
||||
@@ -27,7 +29,6 @@ class Yandex(Search):
|
||||
self.cookie = resp.cookies # 获取cookie
|
||||
while True:
|
||||
time.sleep(self.delay)
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
query = 'site:.' + domain + filtered_subdomain
|
||||
params = {'text': query, 'p': self.page_num,
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
import secrets
|
||||
|
||||
import tenacity
|
||||
from dns.exception import Timeout
|
||||
from dns.resolver import NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers
|
||||
|
||||
from common import utils
|
||||
from config import settings
|
||||
from common import similarity
|
||||
from config.log import logger
|
||||
|
||||
|
||||
def gen_random_subdomains(domain, count):
|
||||
"""
|
||||
生成指定数量的随机子域域名列表
|
||||
|
||||
:param domain: 主域
|
||||
:param count: 数量
|
||||
"""
|
||||
subdomains = set()
|
||||
if count < 1:
|
||||
return subdomains
|
||||
for _ in range(count):
|
||||
token = secrets.token_hex(4)
|
||||
subdomains.add(f'{token}.{domain}')
|
||||
return subdomains
|
||||
|
||||
|
||||
def query_a_record(subdomain, resolver):
|
||||
"""
|
||||
查询子域A记录
|
||||
|
||||
:param subdomain: 子域
|
||||
:param resolver: DNS解析器
|
||||
"""
|
||||
try:
|
||||
answer = resolver.query(subdomain, 'A')
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', f'Query {subdomain} wildcard dns record error')
|
||||
logger.log('DEBUG', e.args)
|
||||
return False
|
||||
if answer.rrset is None:
|
||||
return False
|
||||
ttl = answer.ttl
|
||||
name = answer.name
|
||||
ips = {item.address for item in answer}
|
||||
logger.log('ALERT', f'{subdomain} resolve to: {name} '
|
||||
f'IP: {ips} TTL: {ttl}')
|
||||
return True
|
||||
|
||||
|
||||
def all_resolve_success(subdomains):
|
||||
"""
|
||||
判断是否所有子域都解析成功
|
||||
|
||||
:param subdomains: 子域列表
|
||||
"""
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.cache = None # 不使用DNS缓存
|
||||
status = set()
|
||||
for subdomain in subdomains:
|
||||
status.add(query_a_record(subdomain, resolver))
|
||||
return all(status)
|
||||
|
||||
|
||||
def all_request_success(subdomains):
|
||||
"""
|
||||
判断是否所有子域都请求成功
|
||||
|
||||
:param subdomains: 子域列表
|
||||
"""
|
||||
result = list()
|
||||
for subdomain in subdomains:
|
||||
url = f'http://{subdomain}'
|
||||
resp = utils.get_url_resp(url)
|
||||
if resp:
|
||||
logger.log('ALERT', f'Request: {url} Status: {resp.status_code} '
|
||||
f'Size: {len(resp.content)}')
|
||||
result.append(resp.text)
|
||||
else:
|
||||
result.append(resp)
|
||||
return all(result), result
|
||||
|
||||
|
||||
def any_similar_html(resp_list):
|
||||
"""
|
||||
判断是否有一组HTML页面结构相似
|
||||
|
||||
:param resp_list: 响应HTML页面
|
||||
"""
|
||||
html_doc1, html_doc2, html_doc3 = resp_list
|
||||
if similarity.is_similar(html_doc1, html_doc2):
|
||||
return True
|
||||
if similarity.is_similar(html_doc1, html_doc3):
|
||||
return True
|
||||
if similarity.is_similar(html_doc2, html_doc3):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def to_detect_wildcard(domain):
|
||||
"""
|
||||
Detect use wildcard dns record or not
|
||||
|
||||
:param str domain: domain
|
||||
:return bool use wildcard dns record or not
|
||||
"""
|
||||
logger.log('INFOR', f'Detecting {domain} use wildcard dns record or not')
|
||||
random_subdomains = gen_random_subdomains(domain, 3)
|
||||
if not all_resolve_success(random_subdomains):
|
||||
return False
|
||||
is_all_success, all_request_resp = all_request_success(random_subdomains)
|
||||
if not is_all_success:
|
||||
return True
|
||||
return any_similar_html(all_request_resp)
|
||||
|
||||
|
||||
def detect_wildcard(domain):
|
||||
is_enable = to_detect_wildcard(domain)
|
||||
if is_enable:
|
||||
logger.log('ALERT', f'The domain {domain} enables wildcard')
|
||||
else:
|
||||
logger.log('ALERT', f'The domain {domain} disables wildcard')
|
||||
return is_enable
|
||||
|
||||
|
||||
@tenacity.retry(stop=tenacity.stop_after_attempt(2))
|
||||
def get_wildcard_record(domain, resolver):
|
||||
logger.log('INFOR', f"Query {domain} 's wildcard dns record "
|
||||
f"in authoritative name server")
|
||||
try:
|
||||
answer = resolver.query(domain, 'A')
|
||||
# 如果查询随机域名A记录时抛出Timeout异常则重新查询
|
||||
except Timeout as e:
|
||||
logger.log('ALERT', f'Query timeout, retrying')
|
||||
logger.log('DEBUG', e.args)
|
||||
raise e
|
||||
except (NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers) as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
logger.log('DEBUG', f'{domain} dont have A record on authoritative name server')
|
||||
return None, None
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e.args)
|
||||
logger.log('ERROR', f'Query {domain} wildcard dns record in '
|
||||
f'authoritative name server error')
|
||||
exit(1)
|
||||
else:
|
||||
if answer.rrset is None:
|
||||
logger.log('DEBUG', f'No record of query result')
|
||||
return None, None
|
||||
name = answer.name
|
||||
ip = {item.address for item in answer}
|
||||
ttl = answer.ttl
|
||||
logger.log('INFOR', f'{domain} results on authoritative name server: {name} '
|
||||
f'IP: {ip} TTL: {ttl}')
|
||||
return ip, ttl
|
||||
|
||||
|
||||
def collect_wildcard_record(domain, authoritative_ns):
|
||||
logger.log('INFOR', f'Collecting wildcard dns record for {domain}')
|
||||
if not authoritative_ns:
|
||||
return list(), int()
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.nameservers = authoritative_ns # 使用权威名称服务器
|
||||
resolver.rotate = True # 随机使用NS
|
||||
resolver.cache = None # 不使用DNS缓存
|
||||
ips = set()
|
||||
ttl = int()
|
||||
ttls_check = list()
|
||||
ips_stat = dict()
|
||||
ips_check = list()
|
||||
while True:
|
||||
token = secrets.token_hex(4)
|
||||
random_subdomain = f'{token}.{domain}'
|
||||
try:
|
||||
ip, ttl = get_wildcard_record(random_subdomain, resolver)
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
logger.log('ALERT', f'Multiple query errors,'
|
||||
f'try to query a new random subdomain')
|
||||
continue
|
||||
# 每5次查询检查结果列表 如果都没结果则结束查询
|
||||
ips_check.append(ip)
|
||||
ttls_check.append(ttl)
|
||||
if len(ips_check) == 5:
|
||||
if not any(ips_check):
|
||||
logger.log('ALERT', 'The query ends because there are '
|
||||
'no results for 5 consecutive queries.')
|
||||
break
|
||||
ips_check = list()
|
||||
if len(ttls_check) == 5 and len(set(ttls_check)) == 5:
|
||||
logger.log('ALERT', 'The query ends because there are '
|
||||
'5 different TTL results for 5 consecutive queries.')
|
||||
ips, ttl = set(), int()
|
||||
break
|
||||
if ip is None:
|
||||
continue
|
||||
ips.update(ip)
|
||||
# 统计每个泛解析IP出现次数
|
||||
for addr in ip:
|
||||
count = ips_stat.setdefault(addr, 0)
|
||||
ips_stat[addr] = count + 1
|
||||
# 筛选出出现次数2次以上的IP地址
|
||||
addrs = list()
|
||||
for addr, times in ips_stat.items():
|
||||
if times >= 2:
|
||||
addrs.append(addr)
|
||||
# 大部分的IP地址出现次数大于2次停止收集泛解析IP记录
|
||||
if len(addrs) / len(ips) >= 0.8:
|
||||
break
|
||||
logger.log('DEBUG', f'Collected the wildcard dns record of {domain}\n{ips}\n{ttl}')
|
||||
return ips, ttl
|
||||
|
||||
|
||||
def check_by_compare(ip, ttl, wc_ips, wc_ttl):
|
||||
"""
|
||||
Use TTL comparison to detect wildcard dns record
|
||||
|
||||
:param set ip: A record IP address set
|
||||
:param int ttl: A record TTL value
|
||||
:param set wc_ips: wildcard dns record IP address set
|
||||
:param int wc_ttl: wildcard dns record TTL value
|
||||
:return bool: result
|
||||
"""
|
||||
# Reference：http://sh3ll.me/archives/201704041222.txt
|
||||
if ip not in wc_ips:
|
||||
return False # 子域IP不在泛解析IP集合则不是泛解析
|
||||
if ttl != wc_ttl and ttl % 60 == 0 and wc_ttl % 60 == 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def check_ip_times(times):
|
||||
"""
|
||||
Use IP address times to determine wildcard or not
|
||||
|
||||
:param times: IP address times
|
||||
:return bool: result
|
||||
"""
|
||||
if times > settings.ip_appear_maximum:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def check_cname_times(times):
|
||||
"""
|
||||
Use cname times to determine wildcard or not
|
||||
|
||||
:param times: cname times
|
||||
:return bool: result
|
||||
"""
|
||||
if times > settings.cname_appear_maximum:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_valid_subdomain(ip=None, ip_num=None, cname=None, cname_num=None,
|
||||
ttl=None, wc_ttl=None, wc_ips=None):
|
||||
ip_blacklist = settings.brute_ip_blacklist
|
||||
cname_blacklist = settings.brute_cname_blacklist
|
||||
if cname and cname in cname_blacklist:
|
||||
return 0, 'cname blacklist' # 有些泛解析会统一解析到一个cname上
|
||||
if ip and ip in ip_blacklist: # 解析ip在黑名单ip则为非法子域
|
||||
return 0, 'IP blacklist'
|
||||
if all([wc_ips, wc_ttl]): # 有泛解析记录才进行对比
|
||||
if check_by_compare(ip, ttl, wc_ips, wc_ttl):
|
||||
return 0, 'IP wildcard'
|
||||
if ip_num and check_ip_times(ip_num):
|
||||
return 0, 'IP exceeded'
|
||||
if cname_num and check_cname_times(cname_num):
|
||||
return 0, 'cname exceeded'
|
||||
return 1, 'OK'
|
||||
|
||||
|
||||
def stat_times(data):
|
||||
times = dict()
|
||||
for info in data:
|
||||
ip_str = info.get('ip')
|
||||
if isinstance(ip_str, str):
|
||||
ips = ip_str.split(',')
|
||||
for ip in ips:
|
||||
value_one = times.setdefault(ip, 0)
|
||||
times[ip] = value_one + 1
|
||||
cname_str = info.get('cname')
|
||||
if isinstance(cname_str, str):
|
||||
cnames = cname_str.split(',')
|
||||
for cname in cnames:
|
||||
value_two = times.setdefault(cname, 0)
|
||||
times[cname] = value_two + 1
|
||||
return times
|
||||
|
||||
|
||||
def check_valid_subdomain(appear_times, info):
|
||||
ip_str = info.get('ip')
|
||||
if ip_str:
|
||||
ips = ip_str.split(',')
|
||||
for ip in ips:
|
||||
ip_num = appear_times.get(ip)
|
||||
isvalid, reason = is_valid_subdomain(ip=ip, ip_num=ip_num)
|
||||
if not isvalid:
|
||||
return False, reason
|
||||
cname_str = info.get('cname')
|
||||
if cname_str:
|
||||
cnames = cname_str.split(',')
|
||||
for cname in cnames:
|
||||
cname_num = appear_times.get(cname)
|
||||
isvalid, reason = is_valid_subdomain(cname=cname, cname_num=cname_num)
|
||||
if not isvalid:
|
||||
return False, reason
|
||||
return True, 'OK'
|
||||
|
||||
|
||||
def deal_wildcard(data):
|
||||
new_data = list()
|
||||
appear_times = stat_times(data)
|
||||
for info in data:
|
||||
subdomain = info.get('subdomain')
|
||||
isvalid, reason = check_valid_subdomain(appear_times, info)
|
||||
logger.log('DEBUG', f'{subdomain} is {isvalid} subdomain reason because {reason}')
|
||||
if isvalid:
|
||||
new_data.append(info)
|
||||
return new_data
|
||||
+63
-89
@@ -12,15 +12,15 @@ import fire
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
import dbexport
|
||||
import export
|
||||
from brute import Brute
|
||||
from common import utils, resolve, request
|
||||
from common.database import Database
|
||||
from modules.collect import Collect
|
||||
from modules.srv import BruteSRV
|
||||
from modules.finder import Finder
|
||||
from modules.altdns import Altdns
|
||||
from modules import iscdn
|
||||
from modules.enrich import Enrich
|
||||
from modules import wildcard
|
||||
from config import settings
|
||||
from config.log import logger
|
||||
from takeover import Takeover
|
||||
@@ -32,7 +32,7 @@ blue = '\033[01;34m'
|
||||
red = '\033[1;31m'
|
||||
end = '\033[0m'
|
||||
|
||||
version = 'v0.3.0'
|
||||
version = 'v0.4.3'
|
||||
message = white + '{' + red + version + ' #dev' + white + '}'
|
||||
|
||||
oneforall_banner = f"""
|
||||
@@ -60,7 +60,7 @@ class OneForAll(object):
|
||||
python3 oneforall.py --target example.com --alive False run
|
||||
python3 oneforall.py --target example.com --brute False run
|
||||
python3 oneforall.py --target example.com --port medium run
|
||||
python3 oneforall.py --target example.com --format csv run
|
||||
python3 oneforall.py --target example.com --fmt csv run
|
||||
python3 oneforall.py --target example.com --dns False run
|
||||
python3 oneforall.py --target example.com --req False run
|
||||
python3 oneforall.py --target example.com --takeover False run
|
||||
@@ -68,7 +68,7 @@ class OneForAll(object):
|
||||
|
||||
Note:
|
||||
--port small/medium/large See details in ./config/setting.py(default small)
|
||||
--format csv/json (result format)
|
||||
--fmt csv/json (result format)
|
||||
--path Result path (default None, automatically generated)
|
||||
|
||||
:param str target: One domain (target or targets must be provided)
|
||||
@@ -78,12 +78,12 @@ class OneForAll(object):
|
||||
:param bool req: HTTP request subdomains (default True)
|
||||
:param str port: The port range to request (default small port is 80,443)
|
||||
:param bool alive: Only export alive subdomains (default False)
|
||||
:param str format: Result format (default csv)
|
||||
:param str fmt: Result format (default csv)
|
||||
:param str path: Result path (default None, automatically generated)
|
||||
:param bool takeover: Scan subdomain takeover (default False)
|
||||
"""
|
||||
def __init__(self, target=None, targets=None, brute=None, dns=None, req=None,
|
||||
port=None, alive=None, format=None, path=None, takeover=None):
|
||||
port=None, alive=None, fmt=None, path=None, takeover=None):
|
||||
self.target = target
|
||||
self.targets = targets
|
||||
self.brute = brute
|
||||
@@ -91,17 +91,16 @@ class OneForAll(object):
|
||||
self.req = req
|
||||
self.port = port
|
||||
self.alive = alive
|
||||
self.format = format
|
||||
self.fmt = fmt
|
||||
self.path = path
|
||||
self.takeover = takeover
|
||||
self.domain = str() # The domain currently being collected
|
||||
self.domains = set() # All domains that are to be collected
|
||||
self.data = list() # The subdomain results of the current domain
|
||||
self.datas = list() # All subdomain results of the domain
|
||||
self.old_table = str() # The table name of the last result
|
||||
self.new_table = str() # The table name of the current result
|
||||
self.origin_table = str() # The table name of the origin result
|
||||
self.resolve_table = str() # The table name of the resolute result
|
||||
self.in_china = None
|
||||
self.access_internet = False
|
||||
self.enable_wildcard = False
|
||||
|
||||
def config_param(self):
|
||||
"""
|
||||
@@ -119,8 +118,8 @@ class OneForAll(object):
|
||||
self.port = settings.http_request_port
|
||||
if self.alive is None:
|
||||
self.alive = bool(settings.result_export_alive)
|
||||
if self.format is None:
|
||||
self.format = settings.result_save_format
|
||||
if self.fmt is None:
|
||||
self.fmt = settings.result_save_format
|
||||
if self.path is None:
|
||||
self.path = settings.result_save_path
|
||||
|
||||
@@ -132,49 +131,14 @@ class OneForAll(object):
|
||||
logger.log('FATAL', 'You must provide either target or targets parameter')
|
||||
exit(1)
|
||||
|
||||
def export(self, table):
|
||||
def export_data(self):
|
||||
"""
|
||||
Export data from the database and do some follow-up processing
|
||||
Export data from the database
|
||||
|
||||
:param table: table name
|
||||
:return: export data
|
||||
:return: exported data
|
||||
:rtype: list
|
||||
"""
|
||||
db = Database()
|
||||
data = dbexport.export(table, type='table', alive=self.alive, format=self.format)
|
||||
db.drop_table(self.new_table)
|
||||
db.rename_table(self.domain, self.new_table)
|
||||
db.close()
|
||||
return data
|
||||
|
||||
def deal_db(self):
|
||||
"""
|
||||
Process the data when the collection task is completed
|
||||
"""
|
||||
db = Database()
|
||||
db.deal_table(self.domain, self.origin_table)
|
||||
db.close()
|
||||
|
||||
def mark(self):
|
||||
"""
|
||||
Mark the new discovered subdomain
|
||||
|
||||
:return: marked data
|
||||
:rtype: list
|
||||
"""
|
||||
db = Database()
|
||||
old_data = list()
|
||||
now_data = db.get_data(self.domain).as_dict()
|
||||
# Database pre-processing when it is not the first time to collect this subdomain
|
||||
if db.exist_table(self.new_table):
|
||||
# If there is the last collection result table, delete it first
|
||||
db.drop_table(self.old_table)
|
||||
# Rename the new table to the old table
|
||||
db.rename_table(self.new_table, self.old_table)
|
||||
old_data = db.get_data(self.old_table).as_dict()
|
||||
db.close()
|
||||
marked_data = utils.mark_subdomain(old_data, now_data)
|
||||
return marked_data
|
||||
return export.export_data(self.domain, alive=self.alive, fmt=self.fmt, path=self.path)
|
||||
|
||||
def main(self):
|
||||
"""
|
||||
@@ -183,10 +147,13 @@ class OneForAll(object):
|
||||
:return: subdomain results
|
||||
:rtype: list
|
||||
"""
|
||||
self.old_table = self.domain + '_old_result'
|
||||
self.new_table = self.domain + '_now_result'
|
||||
self.origin_table = self.domain + '_origin_result'
|
||||
self.resolve_table = self.domain + '_resolve_result'
|
||||
utils.init_table(self.domain)
|
||||
|
||||
if not self.access_internet:
|
||||
logger.log('ALERT', 'Because it cannot access the Internet, '
|
||||
'OneForAll will not execute the subdomain collection module!')
|
||||
if self.access_internet:
|
||||
self.enable_wildcard = wildcard.detect_wildcard(self.domain)
|
||||
|
||||
collect = Collect(self.domain)
|
||||
collect.run()
|
||||
@@ -198,54 +165,58 @@ class OneForAll(object):
|
||||
# Due to there will be a large number of dns resolution requests,
|
||||
# may cause other network tasks to be error
|
||||
brute = Brute(self.domain, word=True, export=False)
|
||||
brute.check_env = False
|
||||
brute.enable_wildcard = self.enable_wildcard
|
||||
brute.in_china = self.in_china
|
||||
brute.quite = True
|
||||
brute.run()
|
||||
|
||||
# Database processing
|
||||
self.deal_db()
|
||||
# Mark the new discovered subdomain
|
||||
self.data = self.mark()
|
||||
|
||||
utils.deal_data(self.domain)
|
||||
# Export results without resolve
|
||||
if not self.dns:
|
||||
return self.export(self.domain)
|
||||
self.data = self.export_data()
|
||||
self.datas.extend(self.data)
|
||||
return self.data
|
||||
|
||||
self.data = utils.get_data(self.domain)
|
||||
|
||||
# Resolve subdomains
|
||||
utils.clear_data(self.domain)
|
||||
self.data = resolve.run_resolve(self.domain, self.data)
|
||||
# Save resolve results
|
||||
resolve.save_db(self.resolve_table, self.data)
|
||||
resolve.save_db(self.domain, self.data)
|
||||
|
||||
# Export results without HTTP request
|
||||
if not self.req:
|
||||
return self.export(self.resolve_table)
|
||||
self.data = self.export_data()
|
||||
self.datas.extend(self.data)
|
||||
return self.data
|
||||
|
||||
if self.enable_wildcard:
|
||||
# deal wildcard
|
||||
self.data = wildcard.deal_wildcard(self.data)
|
||||
|
||||
# HTTP request
|
||||
self.data = request.run_request(self.domain, self.data, self.port)
|
||||
# Save HTTP request result
|
||||
request.save_db(self.domain, self.data)
|
||||
utils.clear_data(self.domain)
|
||||
request.run_request(self.domain, self.data, self.port)
|
||||
|
||||
# Finder module
|
||||
if settings.enable_finder_module:
|
||||
finder = Finder()
|
||||
self.data = finder.run(self.domain, self.data, self.port)
|
||||
finder.run(self.domain, self.data, self.port)
|
||||
|
||||
# altdns module
|
||||
if settings.enable_altdns_module:
|
||||
finder = Altdns(self.domain)
|
||||
self.data = finder.run(self.data, self.port)
|
||||
altdns = Altdns(self.domain)
|
||||
altdns.run(self.data, self.port)
|
||||
|
||||
# check cdn module
|
||||
if settings.enable_cdn_check:
|
||||
self.data = iscdn.check_cdn(self.data)
|
||||
iscdn.save_db(self.domain, self.data)
|
||||
# Information enrichment module
|
||||
if settings.enable_enrich_module:
|
||||
enrich = Enrich(self.domain)
|
||||
enrich.run()
|
||||
|
||||
# Add the final result list to the total data list
|
||||
self.data = self.export_data()
|
||||
self.datas.extend(self.data)
|
||||
|
||||
# Export
|
||||
self.export(self.domain)
|
||||
|
||||
# Scan subdomain takeover
|
||||
if self.takeover:
|
||||
subdomains = utils.get_subdomains(self.data)
|
||||
@@ -263,23 +234,26 @@ class OneForAll(object):
|
||||
print(oneforall_banner)
|
||||
dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
print(f'[*] Starting OneForAll @ {dt}\n')
|
||||
utils.check_env()
|
||||
utils.auto_select_nameserver()
|
||||
if settings.enable_check_version:
|
||||
utils.check_version(version)
|
||||
logger.log('DEBUG', 'Python ' + utils.python_version())
|
||||
logger.log('DEBUG', 'OneForAll ' + version)
|
||||
utils.check_dep()
|
||||
self.access_internet, self.in_china = utils.get_net_env()
|
||||
if self.access_internet and settings.enable_check_version:
|
||||
utils.check_version(version)
|
||||
logger.log('INFOR', 'Start running OneForAll')
|
||||
self.config_param()
|
||||
self.check_param()
|
||||
self.domains = utils.get_domains(self.target, self.targets)
|
||||
if self.domains:
|
||||
count = len(self.domains)
|
||||
logger.log('INFOR', f'Got {count} domains')
|
||||
if not count:
|
||||
logger.log('FATAL', 'Failed to obtain domain')
|
||||
exit(1)
|
||||
for domain in self.domains:
|
||||
self.domain = utils.get_main_domain(domain)
|
||||
self.main()
|
||||
utils.export_all(self.alive, self.format, self.path, self.datas)
|
||||
else:
|
||||
logger.log('FATAL', 'Failed to obtain domain')
|
||||
if count > 1:
|
||||
utils.export_all(self.alive, self.fmt, self.path, self.datas)
|
||||
logger.log('INFOR', 'Finished OneForAll')
|
||||
|
||||
@staticmethod
|
||||
|
||||
+14
-15
@@ -1,23 +1,22 @@
|
||||
-i https://mirrors.aliyun.com/pypi/simple/
|
||||
beautifulsoup4==4.9.1
|
||||
beautifulsoup4==4.9.3
|
||||
bs4==0.0.1
|
||||
certifi==2020.6.20
|
||||
chardet==3.0.4
|
||||
colorama==0.4.3 ; sys_platform == 'win32'
|
||||
dnspython==2.0.0
|
||||
certifi==2020.12.5
|
||||
chardet==4.0.0
|
||||
colorama==0.4.4
|
||||
dnspython==2.1.0
|
||||
exrex==0.10.5
|
||||
fire==0.3.1
|
||||
fire==0.4.0
|
||||
future==0.18.2
|
||||
idna==2.10
|
||||
loguru==0.5.3
|
||||
pysocks==1.7.1
|
||||
requests==2.24.0
|
||||
PySocks==1.7.1
|
||||
requests==2.25.1
|
||||
six==1.15.0
|
||||
soupsieve==2.0.1
|
||||
sqlalchemy==1.3.19
|
||||
tenacity==6.2.0
|
||||
soupsieve==2.2.1
|
||||
SQLAlchemy==1.3.22
|
||||
tenacity==7.0.0
|
||||
termcolor==1.1.0
|
||||
tqdm==4.49.0
|
||||
tqdm==4.59.0
|
||||
treelib==1.6.1
|
||||
urllib3==1.25.10
|
||||
win32-setctime==1.0.2 ; sys_platform == 'win32'
|
||||
urllib3==1.26.4
|
||||
win32-setctime==1.0.3
|
||||
|
||||
+17
-17
@@ -45,21 +45,21 @@ class Takeover(Module):
|
||||
OneForAll subdomain takeover module
|
||||
|
||||
Example:
|
||||
python3 takeover.py --target www.example.com --format csv run
|
||||
python3 takeover.py --target www.example.com --fmt csv run
|
||||
python3 takeover.py --targets ./subdomains.txt --thread 10 run
|
||||
|
||||
Note:
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--fmt txt/csv/json (result format)
|
||||
--path Result directory (default directory is ./results)
|
||||
|
||||
:param str target: One domain (target or targets must be provided)
|
||||
:param str targets: File path of one domain per line
|
||||
:param int thread: threads number (default 20)
|
||||
:param str format: Result format (default csv)
|
||||
:param str fmt: Result format (default csv)
|
||||
:param str path: Result directory (default None)
|
||||
"""
|
||||
|
||||
def __init__(self, target=None, targets=None, thread=20, path=None, format='csv'):
|
||||
def __init__(self, target=None, targets=None, thread=20, path=None, fmt='csv'):
|
||||
Module.__init__(self)
|
||||
self.subdomains = set()
|
||||
self.module = 'Check'
|
||||
@@ -68,19 +68,19 @@ class Takeover(Module):
|
||||
self.targets = targets
|
||||
self.thread = thread
|
||||
self.path = path
|
||||
self.format = format
|
||||
self.fmt = fmt
|
||||
self.fingerprints = None
|
||||
self.subdomainq = Queue()
|
||||
self.queue = Queue() # subdomain queue
|
||||
self.cnames = list()
|
||||
self.results = Dataset()
|
||||
|
||||
def save(self):
|
||||
logger.log('DEBUG', 'Saving results')
|
||||
if self.format == 'txt':
|
||||
if self.fmt == 'txt':
|
||||
data = str(self.results)
|
||||
else:
|
||||
data = self.results.export(self.format)
|
||||
utils.save_data(self.path, data)
|
||||
data = self.results.export(self.fmt)
|
||||
utils.save_to_file(self.path, data)
|
||||
|
||||
def compare(self, subdomain, cname, responses):
|
||||
domain_resp = self.get('http://' + subdomain, check=False, ignore=True)
|
||||
@@ -107,10 +107,10 @@ class Takeover(Module):
|
||||
self.compare(subdomain, cname, responses)
|
||||
|
||||
def check(self):
|
||||
while not self.subdomainq.empty(): # 保证域名队列遍历结束后能退出线程
|
||||
subdomain = self.subdomainq.get() # 从队列中获取域名
|
||||
while not self.queue.empty(): # 保证域名队列遍历结束后能退出线程
|
||||
subdomain = self.queue.get() # 从队列中获取域名
|
||||
self.worker(subdomain)
|
||||
self.subdomainq.task_done()
|
||||
self.queue.task_done()
|
||||
|
||||
def progress(self):
|
||||
bar = tqdm()
|
||||
@@ -118,7 +118,7 @@ class Takeover(Module):
|
||||
bar.desc = 'Check Progress'
|
||||
bar.ncols = 80
|
||||
while True:
|
||||
done = bar.total - self.subdomainq.qsize()
|
||||
done = bar.total - self.queue.qsize()
|
||||
bar.n = done
|
||||
bar.update()
|
||||
if done == bar.total: # 完成队列中所有子域的检查退出
|
||||
@@ -131,17 +131,17 @@ class Takeover(Module):
|
||||
self.subdomains = self.targets
|
||||
else:
|
||||
self.subdomains = utils.get_domains(self.target, self.targets)
|
||||
self.format = utils.check_format(self.format, len(self.subdomains))
|
||||
self.fmt = utils.check_format(self.fmt)
|
||||
timestamp = utils.get_timestamp()
|
||||
name = f'takeover_check_result_{timestamp}'
|
||||
self.path = utils.check_path(self.path, name, self.format)
|
||||
self.path = utils.check_path(self.path, name, self.fmt)
|
||||
if self.subdomains:
|
||||
logger.log('INFOR', f'Checking subdomain takeover')
|
||||
self.fingerprints = get_fingerprint()
|
||||
self.results.headers = ['subdomain', 'cname']
|
||||
# 创建待检查的子域队列
|
||||
for domain in self.subdomains:
|
||||
self.subdomainq.put(domain)
|
||||
self.queue.put(domain)
|
||||
# 进度线程
|
||||
progress_thread = Thread(target=self.progress, name='ProgressThread',
|
||||
daemon=True)
|
||||
@@ -152,7 +152,7 @@ class Takeover(Module):
|
||||
daemon=True)
|
||||
check_thread.start()
|
||||
|
||||
self.subdomainq.join()
|
||||
self.queue.join()
|
||||
self.save()
|
||||
else:
|
||||
logger.log('FATAL', f'Failed to obtain domain')
|
||||
|
||||
@@ -6,18 +6,18 @@ Example
|
||||
"""
|
||||
|
||||
from oneforall import OneForAll
|
||||
from dbexport import export
|
||||
|
||||
|
||||
def oneforall(target):
|
||||
test = OneForAll(target=target)
|
||||
def oneforall(domain):
|
||||
test = OneForAll(target=domain)
|
||||
test.dns = True
|
||||
test.brute = True
|
||||
test.req = True
|
||||
test.takeover = True
|
||||
test.run()
|
||||
results = test.datas
|
||||
print(results)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
TARGET = 'freebuf.com'
|
||||
oneforall(target=TARGET)
|
||||
export(target=TARGET)
|
||||
oneforall('freebuf.com')
|
||||
|
||||
BIN
Binary file not shown.
Reference in New Issue
Block a user