v0.4.4

Del Instapage Takeover

.github/ISSUE_TEMPLATE/bug_report.md

@@ -36,7 +36,7 @@ A clear and concise description of the actual results (optional, such as any err
 Screenshot of complete OneForAll execution process (recommended upload)
 
 **Log upload**
-Upload oneforall.log log files (it is recommended to upload logs in case of complex problems)
+Upload oneforall.log files (it is recommended to upload logs in case of complex problems)
 
 **Supplementary information**
 Some other supplementary notes about bug

Pipfile.lock

@@ -33,17 +33,17 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
-                "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
+                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
+                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
             ],
-            "version": "==2020.6.20"
+            "version": "==2020.12.5"
         },
         "chardet": {
             "hashes": [
-                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
-                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+                "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
+                "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
             ],
-            "version": "==3.0.4"
+            "version": "==4.0.0"
         },
         "colorama": {
             "hashes": [
@@ -55,11 +55,11 @@
         },
         "dnspython": {
             "hashes": [
-                "sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7",
-                "sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d"
+                "sha256:95d12f6ef0317118d2a1a6fc49aac65ffec7eb8087474158f42f26a639135216",
+                "sha256:e4a87f0b573201a0f3727fa18a516b055fd1107e0e5477cded4a2de497df1dd4"
             ],
             "index": "pypi",
-            "version": "==2.0.0"
+            "version": "==2.1.0"
         },
         "exrex": {
             "hashes": [
@@ -70,24 +70,71 @@
         },
         "fire": {
             "hashes": [
-                "sha256:9736a16227c3d469e5d2d296bce5b4d8fa8d7851e953bda327a455fc2994307f"
+                "sha256:c5e2b8763699d1142393a46d0e3e790c5eb2f0706082df8f647878842c216a62"
             ],
             "index": "pypi",
-            "version": "==0.3.1"
+            "version": "==0.4.0"
         },
         "future": {
             "hashes": [
                 "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
             ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==0.18.2"
         },
+        "greenlet": {
+            "hashes": [
+                "sha256:0a77691f0080c9da8dfc81e23f4e3cffa5accf0f5b56478951016d7cfead9196",
+                "sha256:0ddd77586553e3daf439aa88b6642c5f252f7ef79a39271c25b1d4bf1b7cbb85",
+                "sha256:111cfd92d78f2af0bc7317452bd93a477128af6327332ebf3c2be7df99566683",
+                "sha256:122c63ba795fdba4fc19c744df6277d9cfd913ed53d1a286f12189a0265316dd",
+                "sha256:181300f826625b7fd1182205b830642926f52bd8cdb08b34574c9d5b2b1813f7",
+                "sha256:1a1ada42a1fd2607d232ae11a7b3195735edaa49ea787a6d9e6a53afaf6f3476",
+                "sha256:1bb80c71de788b36cefb0c3bb6bfab306ba75073dbde2829c858dc3ad70f867c",
+                "sha256:1d1d4473ecb1c1d31ce8fd8d91e4da1b1f64d425c1dc965edc4ed2a63cfa67b2",
+                "sha256:292e801fcb3a0b3a12d8c603c7cf340659ea27fd73c98683e75800d9fd8f704c",
+                "sha256:2c65320774a8cd5fdb6e117c13afa91c4707548282464a18cf80243cf976b3e6",
+                "sha256:4365eccd68e72564c776418c53ce3c5af402bc526fe0653722bc89efd85bf12d",
+                "sha256:5352c15c1d91d22902582e891f27728d8dac3bd5e0ee565b6a9f575355e6d92f",
+                "sha256:58ca0f078d1c135ecf1879d50711f925ee238fe773dfe44e206d7d126f5bc664",
+                "sha256:5d4030b04061fdf4cbc446008e238e44936d77a04b2b32f804688ad64197953c",
+                "sha256:5d69bbd9547d3bc49f8a545db7a0bd69f407badd2ff0f6e1a163680b5841d2b0",
+                "sha256:5f297cb343114b33a13755032ecf7109b07b9a0020e841d1c3cedff6602cc139",
+                "sha256:62afad6e5fd70f34d773ffcbb7c22657e1d46d7fd7c95a43361de979f0a45aef",
+                "sha256:647ba1df86d025f5a34043451d7c4a9f05f240bee06277a524daad11f997d1e7",
+                "sha256:719e169c79255816cdcf6dccd9ed2d089a72a9f6c42273aae12d55e8d35bdcf8",
+                "sha256:7cd5a237f241f2764324396e06298b5dee0df580cf06ef4ada0ff9bff851286c",
+                "sha256:875d4c60a6299f55df1c3bb870ebe6dcb7db28c165ab9ea6cdc5d5af36bb33ce",
+                "sha256:90b6a25841488cf2cb1c8623a53e6879573010a669455046df5f029d93db51b7",
+                "sha256:94620ed996a7632723a424bccb84b07e7b861ab7bb06a5aeb041c111dd723d36",
+                "sha256:b5f1b333015d53d4b381745f5de842f19fe59728b65f0fbb662dafbe2018c3a5",
+                "sha256:c5b22b31c947ad8b6964d4ed66776bcae986f73669ba50620162ba7c832a6b6a",
+                "sha256:c93d1a71c3fe222308939b2e516c07f35a849c5047f0197442a4d6fbcb4128ee",
+                "sha256:cdb90267650c1edb54459cdb51dab865f6c6594c3a47ebd441bc493360c7af70",
+                "sha256:cfd06e0f0cc8db2a854137bd79154b61ecd940dce96fad0cba23fe31de0b793c",
+                "sha256:d3789c1c394944084b5e57c192889985a9f23bd985f6d15728c745d380318128",
+                "sha256:da7d09ad0f24270b20f77d56934e196e982af0d0a2446120cb772be4e060e1a2",
+                "sha256:df3e83323268594fa9755480a442cabfe8d82b21aba815a71acf1bb6c1776218",
+                "sha256:df8053867c831b2643b2c489fe1d62049a98566b1646b194cc815f13e27b90df",
+                "sha256:e1128e022d8dce375362e063754e129750323b67454cac5600008aad9f54139e",
+                "sha256:e6e9fdaf6c90d02b95e6b0709aeb1aba5affbbb9ccaea5502f8638e4323206be",
+                "sha256:eac8803c9ad1817ce3d8d15d1bb82c2da3feda6bee1153eec5c58fa6e5d3f770",
+                "sha256:eb333b90036358a0e2c57373f72e7648d7207b76ef0bd00a4f7daad1f79f5203",
+                "sha256:ed1d1351f05e795a527abc04a0d82e9aecd3bdf9f46662c36ff47b0b00ecaf06",
+                "sha256:f3dc68272990849132d6698f7dc6df2ab62a88b0d36e54702a8fd16c0490e44f",
+                "sha256:f59eded163d9752fd49978e0bab7a1ff21b1b8d25c05f0995d140cc08ac83379",
+                "sha256:f5e2d36c86c7b03c94b8459c3bd2c9fe2c7dab4b258b8885617d44a22e453fb7",
+                "sha256:f6f65bf54215e4ebf6b01e4bb94c49180a589573df643735107056f7a910275b",
+                "sha256:f8450d5ef759dbe59f84f2c9f77491bb3d3c44bc1a573746daf086e70b14c243",
+                "sha256:f97d83049715fd9dec7911860ecf0e17b48d8725de01e45de07d8ac0bd5bc378"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==1.0.0"
+        },
         "idna": {
             "hashes": [
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
         "loguru": {
@@ -109,79 +156,74 @@
         },
         "requests": {
             "hashes": [
-                "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
-                "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
+                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
+                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
             ],
             "index": "pypi",
-            "version": "==2.24.0"
+            "version": "==2.25.1"
         },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
                 "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.15.0"
         },
         "soupsieve": {
             "hashes": [
-                "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
-                "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
+                "sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc",
+                "sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b"
             ],
             "markers": "python_version >= '3.0'",
-            "version": "==2.0.1"
+            "version": "==2.2.1"
         },
         "sqlalchemy": {
             "hashes": [
-                "sha256:009e8388d4d551a2107632921320886650b46332f61dc935e70c8bcf37d8e0d6",
-                "sha256:0157c269701d88f5faf1fa0e4560e4d814f210c01a5b55df3cab95e9346a8bcc",
-                "sha256:0a92745bb1ebbcb3985ed7bda379b94627f0edbc6c82e9e4bac4fb5647ae609a",
-                "sha256:0cca1844ba870e81c03633a99aa3dc62256fb96323431a5dec7d4e503c26372d",
-                "sha256:166917a729b9226decff29416f212c516227c2eb8a9c9f920d69ced24e30109f",
-                "sha256:1f5f369202912be72fdf9a8f25067a5ece31a2b38507bb869306f173336348da",
-                "sha256:2909dffe5c9a615b7e6c92d1ac2d31e3026dc436440a4f750f4749d114d88ceb",
-                "sha256:2b5dafed97f778e9901b79cc01b88d39c605e0545b4541f2551a2fd785adc15b",
-                "sha256:2e9bd5b23bba8ae8ce4219c9333974ff5e103c857d9ff0e4b73dc4cb244c7d86",
-                "sha256:3aa6d45e149a16aa1f0c46816397e12313d5e37f22205c26e06975e150ffcf2a",
-                "sha256:4bdbdb8ca577c6c366d15791747c1de6ab14529115a2eb52774240c412a7b403",
-                "sha256:53fd857c6c8ffc0aa6a5a3a2619f6a74247e42ec9e46b836a8ffa4abe7aab327",
-                "sha256:5cdfe54c1e37279dc70d92815464b77cd8ee30725adc9350f06074f91dbfeed2",
-                "sha256:5d92c18458a4aa27497a986038d5d797b5279268a2de303cd00910658e8d149c",
-                "sha256:632b32183c0cb0053194a4085c304bc2320e5299f77e3024556fa2aa395c2a8b",
-                "sha256:7c735c7a6db8ee9554a3935e741cf288f7dcbe8706320251eb38c412e6a4281d",
-                "sha256:7cd40cb4bc50d9e87b3540b23df6e6b24821ba7e1f305c1492b0806c33dbdbec",
-                "sha256:84f0ac4a09971536b38cc5d515d6add7926a7e13baa25135a1dbb6afa351a376",
-                "sha256:8dcbf377529a9af167cbfc5b8acec0fadd7c2357fc282a1494c222d3abfc9629",
-                "sha256:950f0e17ffba7a7ceb0dd056567bc5ade22a11a75920b0e8298865dc28c0eff6",
-                "sha256:9e379674728f43a0cd95c423ac0e95262500f9bfd81d33b999daa8ea1756d162",
-                "sha256:b15002b9788ffe84e42baffc334739d3b68008a973d65fad0a410ca5d0531980",
-                "sha256:b6f036ecc017ec2e2cc2a40615b41850dc7aaaea6a932628c0afc73ab98ba3fb",
-                "sha256:bad73f9888d30f9e1d57ac8829f8a12091bdee4949b91db279569774a866a18e",
-                "sha256:bbc58fca72ce45a64bb02b87f73df58e29848b693869e58bd890b2ddbb42d83b",
-                "sha256:bca4d367a725694dae3dfdc86cf1d1622b9f414e70bd19651f5ac4fb3aa96d61",
-                "sha256:be41d5de7a8e241864189b7530ca4aaf56a5204332caa70555c2d96379e18079",
-                "sha256:bf53d8dddfc3e53a5bda65f7f4aa40fae306843641e3e8e701c18a5609471edf",
-                "sha256:c092fe282de83d48e64d306b4bce03114859cdbfe19bf8a978a78a0d44ddadb1",
-                "sha256:c3ab23ee9674336654bf9cac30eb75ac6acb9150dc4b1391bec533a7a4126471",
-                "sha256:ce64a44c867d128ab8e675f587aae7f61bd2db836a3c4ba522d884cd7c298a77",
-                "sha256:d05cef4a164b44ffda58200efcb22355350979e000828479971ebca49b82ddb1",
-                "sha256:d2f25c7f410338d31666d7ddedfa67570900e248b940d186b48461bd4e5569a1",
-                "sha256:d3b709d64b5cf064972b3763b47139e4a0dc4ae28a36437757f7663f67b99710",
-                "sha256:e32e3455db14602b6117f0f422f46bc297a3853ae2c322ecd1e2c4c04daf6ed5",
-                "sha256:ed53209b5f0f383acb49a927179fa51a6e2259878e164273ebc6815f3a752465",
-                "sha256:f605f348f4e6a2ba00acb3399c71d213b92f27f2383fc4abebf7a37368c12142",
-                "sha256:fcdb3755a7c355bc29df1b5e6fb8226d5c8b90551d202d69d0076a8a5649d68b"
+                "sha256:02b039e0e7e6de2f15ea2d2de3995e31a170e700ec0b37b4eded662171711d19",
+                "sha256:08943201a1e3c6238e48f4d5d56c27ea1e1b39d3d9f36a9d81fc3cfb0e1b83bd",
+                "sha256:0ee0054d4a598d2920cae14bcbd33e200e02c5e3b47b902627f8cf5d4c9a2a4b",
+                "sha256:11e7a86209f69273e75d2dd64b06c0c2660e39cd942fce2170515c404ed7358a",
+                "sha256:1294f05916c044631fd626a4866326bbfbd17f62bd37510d000afaef4b35bd74",
+                "sha256:2f11b5783933bff55291ca06496124347627d211ff2e509e846af1c35de0a3fb",
+                "sha256:301d0cd6ef1dc73b607748183da857e712d6f743de8d92b1e1f8facfb0ba2aa2",
+                "sha256:344b58b4b4193b72e8b768a51ef6eb5a4c948ce313a0f23e2ea081e71ce8ac0e",
+                "sha256:44e11a06168782b6d485daef197783366ce7ab0d5eea0066c899ae06cef47bbc",
+                "sha256:45b091ccbf94374ed14abde17e9a04522b0493a17282eaaf4383efdd413f5243",
+                "sha256:48540072f43b3c080159ec1f24a4b014c0ee83d3b73795399974aa358a8cf71b",
+                "sha256:4df07161897191ed8d4a0cfc92425c81296160e5c5f76c9256716d3085172883",
+                "sha256:4f7ce3bfdab6520554af4a5b1df4513d45388624d015ba4d921daf48ce1d6503",
+                "sha256:5361e25181b9872d6906c8c9be7dc05cb0a0951d71ee59ee5a71c1deb301b8a8",
+                "sha256:6f8fdad2f335d2f3ca2f3ee3b01404f7abcf519b03de2c510f1f42d16e39ffb4",
+                "sha256:70a1387396ea5b3022539b560c287daf79403d8b4b365f89b56d660e625a4457",
+                "sha256:7481f9c2c832a3bf37c80bee44d91ac9938b815cc06f7e795b976e300914aab9",
+                "sha256:7c0c7bb49167ac738ca6ee6e7f94a9988a7e4e261d8da335341e8c8c8f3b2e9b",
+                "sha256:7de84feb31af3d8fdf819cac2042928d0b60d3cb16f49c4b2f48d88db46e79f6",
+                "sha256:7f5087104c3c5af11ea59e49ae66c33ca98b14a47d3796ae97498fca53f84aef",
+                "sha256:81badd7d3e0e6aba70a5d1b50fabe8112e9835a6fdb0684054c3fe5378ce0d01",
+                "sha256:82f11b679df91275788be6734dd4a9dfa29bac67b85326992609f62b05bdab37",
+                "sha256:8301ecf3e819eb5dbc171e84654ff60872807775301a55fe35b0ab2ba3742031",
+                "sha256:8d6a9feb5efd2fdab25c6d5a0a5589fed9d789f5ec57ec12263fd0e60ce1dea6",
+                "sha256:915d4fa08776c0252dc5a34fa15c6490f66f411ea1ac9492022f98875d6baf20",
+                "sha256:94040a92b6676f9ffdab6c6b479b3554b927a635c90698c761960b266b04fc88",
+                "sha256:a08027ae84efc563f0f2f341dda572eadebeca38c0ae028a009988f27e9e6230",
+                "sha256:a103294583383660d9e06dbd82037dc8e94c184bdcb27b2be44ae4457dafc6b4",
+                "sha256:c22bfac8d3b955cdb13f0fcd6343156bf56d925196cf7d9ab9ce9f61d3f1e11c",
+                "sha256:c3810ebcf1d42c532c8f5c3f442c705d94442a27a32f2df5344f0857306ab321",
+                "sha256:ee4ddc904fb6414b5118af5b8d45e428aac2ccda01326b2ba2fe4354b0d8d1ae",
+                "sha256:f16801795f1ffe9472360589a04301018c79e4582a85e68067275bb4f765e4e2",
+                "sha256:f62c57ceadedeb8e7b98b48ac4d684bf2b0f73b9d882fed3ca260d9aedf6403f",
+                "sha256:fbb0fda1c574975807aceb0e2332e0ecfe9e5656c191ed482c1a5eafe7a33823"
             ],
             "index": "pypi",
-            "version": "==1.3.20"
+            "version": "==1.4.5"
         },
         "tenacity": {
             "hashes": [
-                "sha256:29ae90e7faf488a8628432154bb34ace1cca58244c6ea399fd33f066ac71339a",
-                "sha256:5a5d3dcd46381abe8b4f82b5736b8726fd3160c6c7161f53f8af7f1eb9b82173"
+                "sha256:5bd16ef5d3b985647fe28dfa6f695d343aa26479a04e8792b9d3c8f49e361ae1",
+                "sha256:a0ce48587271515db7d3a5e700df9ae69cce98c4b57c23a4886da15243603dd8"
             ],
             "index": "pypi",
-            "version": "==6.2.0"
+            "version": "==7.0.0"
         },
         "termcolor": {
             "hashes": [
@@ -191,11 +233,11 @@
         },
         "tqdm": {
             "hashes": [
-                "sha256:9ad44aaf0fc3697c06f6e05c7cf025dd66bc7bcb7613c66d85f4464c47ac8fad",
-                "sha256:ef54779f1c09f346b2b5a8e5c61f96fbcb639929e640e59f8cf810794f406432"
+                "sha256:9fdf349068d047d4cfbe24862c425883af1db29bcddf4b0eeb2524f6fbdb23c7",
+                "sha256:d666ae29164da3e517fcf125e41d4fe96e5bb375cd87ff9763f6b38b5592fe33"
             ],
             "index": "pypi",
-            "version": "==4.51.0"
+            "version": "==4.59.0"
         },
         "treelib": {
             "hashes": [
@@ -206,11 +248,10 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2",
-                "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e"
+                "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
+                "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
-            "version": "==1.25.11"
+            "version": "==1.26.4"
         },
         "win32-setctime": {
             "hashes": [

README.md

@@ -4,8 +4,8 @@
 [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
-[![python](https://img.shields.io/badge/python-3.6|3.7|3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.4.3-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/python-3.6+-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/release-v0.4.4-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
 
 👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md)
 
@@ -88,7 +88,7 @@ config
 
 ```shell
 docker pull shmilylty/oneforall
-docker run -it --rm -v ~/results:/OneForAll/results -v ~/.config:/OneForAll/config oneforall --target example.com run
+docker run -it --rm -v ~/results:/OneForAll/results -v ~/.config:/OneForAll/config shmilylty/oneforall --target example.com run
 ```
 参数直接加在指令末尾，结果会输出在本地目录`~/results`，如需保存到其他位置，可以自行修改
 </details>

brute.py

@@ -10,13 +10,9 @@ OneForAll subdomain brute module
 import gc
 import json
 import time
-import secrets
 
 import exrex
 import fire
-import tenacity
-from dns.exception import Timeout
-from dns.resolver import NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers
 
 import export
 from common import utils

common/request.py

@@ -208,7 +208,7 @@ def save(name, total, req_data, resp_queue):
             new_info = gen_new_info(old_info, resp)
             db.insert_table(name, new_info)
             resp_queue.task_done()
-        if i >= total:
+        if i >= total:  # 得存入完所有请求结果才能结束
             break
     db.close()
 

common/resolve.py

@@ -38,6 +38,7 @@ def update_data(data, infos):
     new_data = list()
     for index, items in enumerate(data):
         if items.get('ip'):
+            new_data.append(items)
             continue
         subdomain = items.get('subdomain')
         record = infos.get(subdomain)

common/utils.py

@@ -1,3 +1,4 @@
+import os
 import re
 import sys
 import time
@@ -53,7 +54,7 @@ def gen_fake_header():
     """
     Generate fake request headers
     """
-    headers = settings.request_default_headers
+    headers = settings.request_default_headers.copy()
     if not isinstance(headers, dict):
         headers = dict()
     if settings.enable_random_ua:
@@ -191,7 +192,7 @@ def check_path(path, name, fmt):
     else:
         path = default_path
     path = Path(path)
-    if not path.suffix:  # 输入是目录的情况
+    if path.is_dir():  # 输入是目录的情况
         path = path.joinpath(filename)
     parent_dir = path.parent
     if not parent_dir.exists():
@@ -476,7 +477,7 @@ def ip_is_public(ip_str):
 
 
 def get_request_count():
-    return 32
+    return os.cpu_count() * 16
 
 
 def uniq_dict_list(dict_list):
@@ -494,7 +495,7 @@ def delete_file(*paths):
 @tenacity.retry(stop=tenacity.stop_after_attempt(3),
                 wait=tenacity.wait_fixed(2))
 def check_net():
-    urls = ['http://ipinfo.io/json', 'http://ipconfig.io/json']
+    urls = ['http://ip-api.com/json/']
     url = random.choice(urls)
     header = {'User_Agent': 'curl'}
     timeout = settings.request_timeout_second
@@ -605,11 +606,11 @@ def get_massdns_path(massdns_dir):
     machine = platform.machine().lower()
     name = f'massdns_{system}_{machine}'
     if system == 'windows':
-        name = name + '.exe'
+        name = f'massdns.exe'
         if machine == 'amd64':
             massdns_dir = massdns_dir.joinpath('windows', 'x64')
         else:
-            massdns_dir = massdns_dir.joinpath('windows', 'x84')
+            massdns_dir = massdns_dir.joinpath('windows', 'x86')
     path = massdns_dir.joinpath(name)
     path.chmod(S_IXUSR)
     if not path.exists():

config/default.py

@@ -31,8 +31,7 @@ enable_takeover_check = False  # 开启子域接管风险检查(默认False)
 http_request_port = 'small'  # HTTP请求子域(默认 'small'，探测80,443端口)
 # 参数可选值True，False分别表示导出存活，全部子域结果
 result_export_alive = False  # 只导出存活的子域结果(默认False)
-# 参数可选格式有'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
-# 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+# 参数可选格式有 'csv', 'json'
 result_save_format = 'csv'  # 子域结果保存文件格式(默认csv)
 # 参数path默认None使用OneForAll结果目录自动生成路径
 result_save_path = None  # 子域结果保存文件路径(默认None)
@@ -42,8 +41,7 @@ save_module_result = False  # 保存各模块发现结果为json文件(默认Fal
 enable_all_module = True  # 启用所有收集模块(默认True)
 enable_partial_module = []  # 启用部分收集模块 必须禁用enable_all_module才能生效
 # 只使用ask和baidu搜索引擎收集子域的示例
-# enable_partial_module = [('modules.search', 'ask')
-#                          ('modules.search', 'baidu')]
+# enable_partial_module = ['modules.search.ask', 'modules.search.baidu']
 module_thread_timeout = 90.0  # 每个收集模块线程超时时间(默认90秒)
 
 # 爆破模块设置
@@ -99,8 +97,8 @@ request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
 
 
 # 请求设置
-request_thread_count = None  # 请求线程数量(默认None，则根据内存大小设置)
-request_timeout_second = (3.05, 27)  # 请求超时秒数(默认connect timout推荐略大于3秒，read秒)
+request_thread_count = None  # 请求线程数量(默认None，则根据情况自动设置)
+request_timeout_second = (13, 27)  # 请求超时秒数(默认connect timout推荐略大于3秒)
 request_ssl_verify = False  # 请求SSL验证(默认False)
 request_allow_redirect = True  # 请求允许重定向(默认True)
 request_redirect_limit = 10  # 请求跳转限制(默认10次)
@@ -242,3 +240,9 @@ github_api_token = ''
 
 # obtain Cloudflare API key from https://dash.cloudflare.com/profile/api-tokens
 cloudflare_api_token = ''
+
+# https://hunter.qianxin.com/home/userInfo
+hunter_api_key = ''
+
+# https://api-docs.fullhunt.io/
+fullhunt_api_key = ''

config/setting.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 """
-OneForAll配置
+OneForAll自定义配置
 """
 
 import pathlib
@@ -32,8 +32,7 @@ save_module_result = False  # 保存各模块发现结果为json文件(默认Fal
 enable_all_module = True  # 启用所有收集模块(默认True)
 enable_partial_module = []  # 启用部分收集模块 必须禁用enable_all_module才能生效
 # 只使用ask和baidu搜索引擎收集子域的示例
-# enable_partial_module = [('modules.search', 'ask')
-#                          ('modules.search', 'baidu')]
+# enable_partial_module = ['modules.search.ask', 'modules.search.baidu']
 
 # 爆破模块设置
 brute_concurrent_num = 2000  # 爆破时并发查询数量(默认2000，最大推荐10000)
@@ -72,6 +71,30 @@ request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
 # request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
 #                        'https': 'socks5h://127.0.0.1:10808'}]  # 代理池
 
+
+# 请求设置
+request_thread_count = None  # 请求线程数量(默认None，则根据情况自动设置)
+request_timeout_second = (13, 27)  # 请求超时秒数(默认connect timout推荐略大于3秒)
+request_ssl_verify = False  # 请求SSL验证(默认False)
+request_allow_redirect = True  # 请求允许重定向(默认True)
+request_redirect_limit = 10  # 请求跳转限制(默认10次)
+# 默认请求头 可以在headers里添加自定义请求头
+request_default_headers = {
+    'Accept': 'text/html,application/xhtml+xml,'
+              'application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Encoding': 'gzip, deflate',
+    'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+    'Cache-Control': 'max-age=0',
+    'DNT': '1',
+    'Referer': 'https://www.google.com/',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                  '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Upgrade-Insecure-Requests': '1',
+    'X-Forwarded-For': '127.0.0.1'
+}
+enable_random_ua = True  # 使用随机UA(默认True，开启可以覆盖request_default_headers的UA)
+
+
 # 搜索模块设置
 enable_recursive_search = False  # 递归搜索子域
 search_recursive_times = 2  # 递归搜索层数

data/authoritative_dns.txt

@@ -0,0 +1,27 @@
+47.254.51.88
+163.177.156.225
+161.117.97.232
+218.98.58.194
+117.91.188.195
+14.17.109.84
+58.52.135.164
+172.96.125.3
+106.38.197.52
+163.177.156.225
+117.91.188.196
+218.98.58.194
+58.52.135.165
+172.96.125.3
+106.38.197.48
+47.254.51.88
+161.117.97.232
+14.17.109.85
+172.96.125.3
+14.17.109.83
+163.177.156.225
+161.117.97.232
+218.98.58.194
+47.254.51.88
+58.52.135.163
+117.91.188.194
+106.38.197.52

data/cdn_cname_keywords.json

@@ -120,9 +120,7 @@
     "edgekey": "Akamai",
     "fastly": "Fastly",
     "chinacache": "ChinaCache",
-    "edgekey": "Akamai",
     "akamai": "Akamai",
-    "fastly": "Fastly",
     "edgecast": "EdgeCast",
     "azioncdn": "Azion",
     "cachefly": "CacheFly",
@@ -135,7 +133,6 @@
     "cloudflare": "CloudFlare",
     "hwcdn": "HighWinds",
     "kxcdn": "KeyCDN",
-    "awsdns": "KeyCDN",
     "fpbns": "Level3",
     "footprint": "Level3",
     "llnwd": "LimeLight",
@@ -155,33 +152,20 @@
     "cdnsun": "CDN SUN",
     "cdnvideo": "CDN Video",
     "clients.turbobytes.net": "TurboBytes",
-    "clients.turbobytes.net": "TurboBytes",
     "turbobytes-cdn.com": "TurboBytes",
     "afxcdn.net": "afxcdn.net",
-    "akamai.net": "Akamai",
     "akamaiedge.net": "Akamai",
-    "akadns.net": "Akamai",
     "akamaitechnologies.com": "Akamai",
     "gslb.tbcache.com": "Alimama",
-    "cloudfront.net": "Amazon Cloudfront",
-    "anankecdn.com.br": "Ananke",
     "att-dsa.net": "AT&T",
-    "azioncdn.net": "Azion",
     "belugacdn.com": "BelugaCDN",
     "bluehatnetwork.com": "Blue Hat Network",
     "systemcdn.net": "EdgeCast",
-    "cachefly.net": "Cachefly",
-    "cdn77.net": "CDN77",
-    "cdn77.org": "CDN77",
     "panthercdn.com": "CDNetworks",
     "cdngc.net": "CDNetworks",
     "gccdn.net": "CDNetworks",
     "gccdn.cn": "CDNetworks",
-    "cdnify.io": "CDNify",
-    "ccgslb.com": "ChinaCache",
-    "ccgslb.net": "ChinaCache",
     "c3cache.net": "ChinaCache",
-    "chinacache.net": "ChinaCache",
     "cncssr.chinacache.net": "ChinaCache",
     "c3cdn.net": "ChinaCache",
     "lxdns.com": "ChinaNetCenter",
@@ -189,11 +173,9 @@
     "mwcloudcdn.com": "QUANTIL/ChinaNetCenter",
     "cloudflare.com": "Cloudflare",
     "cloudflare.net": "Cloudflare",
-    "edgecastcdn.net": "EdgeCast",
     "adn.": "EdgeCast",
     "wac.": "EdgeCast",
     "wpc.": "EdgeCast",
-    "fastly.net": "Fastly",
     "fastlylb.net": "Fastly",
     "google.": "Google",
     "googlesyndication.": "Google",
@@ -201,17 +183,11 @@
     "googleusercontent.com": "Google",
     "l.doubleclick.net": "Google",
     "hiberniacdn.com": "Hibernia",
-    "hwcdn.net": "Highwinds",
-    "incapdns.net": "Incapsula",
     "inscname.net": "Instartlogic",
     "insnw.net": "Instartlogic",
-    "internapcdn.net": "Internap",
-    "kxcdn.com": "KeyCDN",
     "lswcdn.net": "LeaseWeb CDN",
-    "footprint.net": "Level3",
     "llnwd.net": "Limelight",
     "lldns.net": "Limelight",
-    "netdna-cdn.com": "MaxCDN",
     "netdna-ssl.com": "MaxCDN",
     "netdna.com": "MaxCDN",
     "stackpathdns.com": "StackPath",

data/fingerprints.json

@@ -24,11 +24,6 @@
         "cname":["myshopify.com"],
         "response":["Sorry, this shop is currently unavailable.", "Only one step left!"]
     },
-    {
-        "name":"instapage",
-        "cname":["pageserve.co", "secure.pageserve.co", "https://instapage.com/"],
-        "response":["Looks Like You're Lost","The page you're looking for is no longer available."]
-    },
     {
         "name":"desk",
         "cname":["desk.com"],
@@ -245,5 +240,10 @@
         "name":"readme",
         "cname":["readme.io"],
         "response":["Project doesnt exist... yet!"]
+    },
+    {
+        "name":"alibaba_oss",
+        "cname":["aliyuncs.com"],
+        "response":["NoSuchBucket", "The specified bucket does not exist."]
     }
 ]

docs/changes.md

@@ -8,6 +8,11 @@ OneForAll遵守[语义化版本格式](https://semver.org/)。
 # Unreleased
 
 # Released
+## [0.4.4](https://github.com/shmilylty/oneforall/releases/tag/v0.4.4) - 2022-07-03
+- 修复了多个已知问题
+- 添加了多个查询接口
+- 添加对M1芯片的Mac支持
+
 ## [0.4.3](https://github.com/shmilylty/oneforall/releases/tag/v0.4.3) - 2020-11-29
 - 修复了已知问题
 - 更新了文档

docs/collection_modules.md

@@ -1,15 +1,15 @@
 #  收集模块说明 #
 
-如果要使用通过API收集子域的模块请先到[api.py](../oneforall/config/api.py)配置相关信息，大多平台的API都是可以注册账号免费获取的。
+如果要使用通过API收集子域的模块请先到[api.py](../config/api.py)配置相关信息，大多平台的API都是可以注册账号免费获取的。
 
-如果你指定使用某些模块可以在[api.py](../oneforall/config/api.py)中设置：
+如果你指定使用某些模块可以在[api.py](../config/api.py)中设置：
 
 ```python
 enable_all_module = False  # 不开启所有模块
 enable_partial_module = [('modules.search', 'ask'), ('modules.search', 'baidu')]  # 只使用ask和baidu搜索引擎收集子域
 ```
 
-如果你指定使用某些模块使用代理可以在[api.py](../oneforall/config/api.py)中设置：
+如果你指定使用某些模块使用代理可以在[api.py](../config/api.py)中设置：
 
 ```python
 enable_proxy = True  # 使用代理
@@ -23,12 +23,12 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
     
       | 模块名称    | 是否需要代理 | 是否需要API | 其他说明                                           |
       | ----------- | ------------ | ----------- | -------------------------------------------------- |
-      | censys_api  | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | censys_api  | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | certspotter | 否           | 否          |                                                    |
       | crtsh       | 否           | 否          |                                                    |
       | entrust     | 否           | 否          |                                                    |
       | google      | 是           | 否          |                                                    |
-      | spyse_api   | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | spyse_api   | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
 
 
   2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，后续会添加检查NSEC记录，NSEC3记录等模块）
@@ -52,25 +52,25 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
 
       | 模块名称           | 是否需要代理 | 是否需要API | 其他说明                                           |
       | ------------------ | ------------ | ----------- | -------------------------------------------------- |
-      | binaryedge_api     | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | binaryedge_api     | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | bufferover         | 否           | 否          |                                                    |
       | cebaidu            | 否           | 否          |                                                    |
       | chinaz             | 否           | 否          |                                                    |
-      | chinaz_api         | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
-      | circl_api          | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
-      | cloudflare_api     | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | chinaz_api         | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
+      | circl_api          | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
+      | cloudflare_api     | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | dnsdb              | 否           | 否          |                                                    |
-      | dnsdb_api          | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | dnsdb_api          | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | dnsdumpster        | 否           | 否          |                                                    |
       | hackertarget       | 否           | 否          |                                                    |
       | ip138              | 否           | 否          |                                                    |
-      | ipv4info           | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | ipv4info           | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | netcraft           | 否           | 否          |                                                    |
-      | passivedns_api     | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
-      | ptrarchive         | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
-      | riddler            | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | passivedns_api     | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
+      | ptrarchive         | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
+      | riddler            | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | robtex             | 否           | 否          |                                                    |
-      | securitytrails_api | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | securitytrails_api | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | sitedossier        | 否           | 否          |                                                    |
       | threatcrowd        | 否           | 否          |                                                    |
       | ximcx              | 否           | 否          |                                                    |
@@ -84,11 +84,11 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
       | 模块名称       | 是否需要代理 | 是否需要API | 其他说明                                           |
       | -------------- | ------------ | ----------- | ------------------------------------------------- |
       | alienvault     | 否           | 否          |                                                    |
-      | riskiq_api     | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py)         |
-      | threatbook_api | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py)         |
+      | riskiq_api     | 否           | 是          | API使用和申请见[api.py](../config/api.py)         |
+      | threatbook_api | 否           | 是          | API使用和申请见[api.py](../config/api.py)         |
       | threatminer    | 否           | 否          |                                                    |
       | virustotal     | 否           | 否          |                                                    |
-      | virustotal_api | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py)         |
+      | virustotal_api | 否           | 是          | API使用和申请见[api.py](../config/api.py)         |
   7. 利用搜索引擎发现子域（目前有16个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`）
 
      除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
@@ -98,17 +98,17 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
      | ask         | 是                     | 否          |                                                             |
      | baidu       | 否                     | 否          |                                                             |
      | bing        | 否                     | 否          |                                                             |
-     | bing_api    | 否                     | 是          | API使用和申请见[api.py](../oneforall/config/api.py)                  |
+     | bing_api    | 否                     | 是          | API使用和申请见[api.py](../config/api.py)                  |
      | duckduckgo  | 是                     | 否          |                                                             |
      | exalead     | 否，最好使用国外代理。    | 否          |                                                             |
-     | fofa_api    | 否                     | 是          | API使用和申请见[api.py](../oneforall/config/api.py)                  |
+     | fofa_api    | 否                     | 是          | API使用和申请见[api.py](../config/api.py)                  |
      | gitee       | 否                     | 否          |                                                             |
-     | github      | 否                     | 否          | 在[api.py](../oneforall/config/api.py)设置Github邮件名和密码。        |
+     | github      | 否                     | 否          | 在[api.py](../config/api.py)设置Github邮件名和密码。        |
      | google      | 是                     | 否          |                                                             |
-     | google_api  | 是                     | 是          | API使用和申请见[api.py](../oneforall/config/api.py)                  |
-     | shodan_api  | 否，最好使用国外代理。    | 是          | API使用和申请见[api.py](../oneforall/config/api.py)                  |
+     | google_api  | 是                     | 是          | API使用和申请见[api.py](../config/api.py)                  |
+     | shodan_api  | 否，最好使用国外代理。    | 是          | API使用和申请见[api.py](../config/api.py)                  |
      | so          | 否                     | 否          |                                                             |
      | sogou       | 否                     | 否          |                                                             |
      | yahoo       | 是                     | 否          |                                                             |
      | yandex      | 是                     | 否          |                                                             |
-     | zoomeye_api | 否                     | 是          | API使用和申请见[api.py](../oneforall/config/api.py)                  |
+     | zoomeye_api | 否                     | 是          | API使用和申请见[api.py](../config/api.py)                  |

docs/en-us/README.md

@@ -4,8 +4,8 @@
 [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
-[![python](https://img.shields.io/badge/python-3.6|3.7|3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.4.3-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/python-3.6+-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/release-v0.4.4-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
 
 👊**OneForAll is a powerful subdomain integration tool**  📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
 
@@ -123,7 +123,7 @@ Let's take the command `python3 oneforall.py --target example.com run` as an exa
 
 `example_com_now_result` table stores the collection results of the current subdomains. Usually using this table is enough.
 
-For more information, please see [Field explanation](./docs/field.md).
+For more information, please see [Field explanation](../field.md).
 
 </details>
 
@@ -221,7 +221,7 @@ Problems with other tools
 
 In order to solve the above problems, OneForAll born! As its name, OneForAll is committed to becoming the only one subdomain integration tool you need. We hope that one day OneForAll can be called "probably the best subdomain tool"
 
-At present, OneForAll is under development, there must be a lot of problems and areas for improvement. Welcome to submit [Issues](https://github.com/shmilylty/OneForAll/issues) or [PR](https://github.com/shmilylty/OneForAll/pulls), If you like, star please✨. You can contact me through QQ group [**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771) or twitter [tweet](https://twitter.com/shmilylty) to me: 👨‍👨‍👦‍👦.
+At present, OneForAll is under development, there must be a lot of problems and areas for improvement. Welcome to submit [Issues](https://github.com/shmilylty/OneForAll/issues) or [PR](https://github.com/shmilylty/OneForAll/pulls), If you like, star please✨. You can contact me through QQ group [**824414244**](https://shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771) or twitter [tweet](https://twitter.com/shmilylty) to me: 👨‍👨‍👦‍👦.
 
 ## 👍Features
 
@@ -283,7 +283,7 @@ Very warmly welcome all people to make OneForAll better together!
 * **[Jing Ling](https://github.com/shmilylty)**
   * Core developer
 
-You can view all contributors and their contributions in the [contributor documentation](https://github.com/shmilylty/OneForAll/tree/master/docs/contributors.md)) and thank them for making OneForAll more powerful and useful.
+You can view all contributors and their contributions in the [contributor documentation](https://github.com/shmilylty/OneForAll/tree/master/docs/contributors.md) and thank them for making OneForAll more powerful and useful.
 
 ## 📄License
 
@@ -295,6 +295,8 @@ Thanks to the various subdomain collection projects of online open source!
 
 Thanks ace of [A-Team](https://github.com/QAX-A-Team) for their enthusiastic and unselfish answers!
 
+Developed with drive and [PyCharm](https://www.jetbrains.com/pycharm/)!
+
 ## 📜Disclaimer 
 
 This tool can only be used in the safety construction of enterprises with sufficient legal authorization.

modules/certificates/censys_api.py

@@ -9,7 +9,7 @@ class CensysAPI(Query):
         self.domain = domain
         self.module = 'Certificate'
         self.source = "CensysAPIQuery"
-        self.addr = 'https://www.censys.io/api/v1/search/certificates'
+        self.addr = 'https://search.censys.io/api/v1/search/certificates'
         self.id = settings.censys_api_id
         self.secret = settings.censys_api_secret
         self.delay = 3.0  # Censys 接口查询速率限制 最快2.5秒查1次

modules/check/nsec.py

@@ -26,6 +26,9 @@ class NSEC(Check):
                 self.subdomains.update(subdomains)
             if subdomain == self.domain:  # 当查出子域为主域 说明完成了一个循环 不再继续查询
                 break
+            if domain != self.domain:  # 防止出现wwdmas.cn 000.000.wwdmas.cn 000.000.000.wwdmas.cn情况
+                if domain.split('.')[0] == subdomain.split('.')[0]:
+                    break
             domain = subdomain
         return self.subdomains
 

modules/collect.py

@@ -23,7 +23,7 @@ class Collect(object):
                 module_path = settings.module_dir.joinpath(module)
                 for path in module_path.rglob('*.py'):
                     import_module = f'modules.{module}.{path.stem}'
-                    self.modules.append([import_module, path.stem])
+                    self.modules.append(import_module)
         else:
             self.modules = settings.enable_partial_module
 
@@ -31,7 +31,8 @@ class Collect(object):
         """
         Import do function
         """
-        for module, name in self.modules:
+        for module in self.modules:
+            name = module.split('.')[-1]
             import_object = importlib.import_module(module)
             func = getattr(import_object, 'run')
             self.collect_funcs.append([func, name])

modules/datasets/dnsdumpster.py

@@ -21,7 +21,8 @@ class DNSDumpster(Query):
             return
         self.cookie = resp.cookies
         data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'),
-                'targetip': self.domain}
+                'targetip': self.domain,
+                'user':'free'}
         resp = self.post(self.addr, data)
         self.subdomains = self.collect_subdomains(resp)
 

modules/datasets/fullhunt.py

@@ -0,0 +1,48 @@
+from config import settings
+from common.query import Query
+
+
+class FullHuntAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'FullHuntAPIQuery'
+        self.api = settings.fullhunt_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({'X-API-KEY': self.api})
+        self.proxy = self.get_proxy(self.source)
+        url = f'https://fullhunt.io/api/v1/domain/{self.domain}/subdomains'
+        resp = self.get(url)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+
+    """
+    query = FullHuntAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('qq.com')

modules/datasets/netcraft.py

@@ -12,39 +12,21 @@ class NetCraft(Query):
         self.domain = domain
         self.module = 'Dataset'
         self.source = 'NetCraftQuery'
-        self.init = 'https://searchdns.netcraft.com/'
-        self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains'
+        self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains&position=limited'
         self.page_num = 1
         self.per_page_num = 20
 
-    def bypass_verification(self):
-        """
-        绕过NetCraft的JS验证
-        """
-        self.header = self.get_header()  # Netcraft会检查User-Agent
-        resp = self.get(self.init)
-        if not resp:
-            return False
-        self.cookie = resp.cookies
-        cookie_value = self.cookie['netcraft_js_verification_challenge']
-        cookie_encode = parse.unquote(cookie_value).encode('utf-8')
-        verify_taken = hashlib.sha1(cookie_encode).hexdigest()
-        self.cookie['netcraft_js_verification_response'] = verify_taken
-        return True
-
     def query(self):
         """
         向接口查询子域并做子域匹配
         """
-        if not self.bypass_verification():
-            return
+        self.header = self.get_header()  # NetCraft会检查User-Agent
+        self.proxy = self.get_proxy(self.source)
         last = ''
         while True:
             time.sleep(self.delay)
-            self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
-            params = {'restriction': 'site ends with',
-                      'host': '.' + self.domain,
+            params = {'host': '*.' + self.domain,
                       'from': self.page_num}
             resp = self.get(self.addr + last, params)
             subdomains = self.match_subdomains(resp)

modules/datasets/wzpc.py

@@ -1,68 +0,0 @@
-import time
-from config.log import logger
-from common.query import Query
-
-
-class WZPCQuery(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = domain
-        self.module = 'Dataset'
-        self.source = 'WZPCQuery'
-
-    def query(self):
-        """
-        向接口查询子域并做子域匹配
-        """
-
-        base_addr = 'http://114.55.181.28/check_web/' \
-                    'databaseInfo_mainSearch.action'
-        page_num = 1
-        while True:
-            time.sleep(self.delay)
-            self.header = self.get_header()
-            self.proxy = self.get_proxy(self.source)
-            params = {'isSearch': 'true', 'searchType': 'url',
-                      'term': self.domain, 'pageNo': page_num}
-            try:
-                resp = self.get(base_addr, params)
-            except Exception as e:
-                logger.log('ERROR', e.args)
-                break
-            if not resp:
-                break
-            subdomains = self.match_subdomains(resp.text)
-            if not subdomains:  # 没有发现子域名则停止查询
-                break
-            self.subdomains.update(subdomains)
-            if not subdomains:
-                break
-            if page_num > 10:
-                break
-            page_num += 1
-
-    def run(self):
-        """
-        类执行入口
-        """
-        self.begin()
-        self.query()
-        self.finish()
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-
-
-def run(domain):
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    """
-    query = WZPCQuery(domain)
-    query.run()
-
-
-if __name__ == '__main__':
-    run('sc.gov.cn')
-    run('bkzy.org')

modules/intelligence/riskiq_api.py

@@ -8,7 +8,7 @@ class RiskIQ(Query):
         self.domain = domain
         self.module = 'Intelligence'
         self.source = 'RiskIQAPIQuery'
-        self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
+        self.addr = 'https://api.riskiq.net/pt/v2/enrichment/subdomains'
         self.user = settings.riskiq_api_username
         self.key = settings.riskiq_api_key
 
@@ -17,6 +17,7 @@ class RiskIQ(Query):
         向接口查询子域并做子域匹配
         """
         self.header = self.get_header()
+        self.header.update({'Accept': 'application/json'})
         self.proxy = self.get_proxy(self.source)
         params = {'query': self.domain}
         resp = self.get(url=self.addr,

modules/intelligence/threatminer.py

@@ -7,7 +7,7 @@ class ThreatMiner(Query):
         self.domain = domain
         self.module = 'Intelligence'
         self.source = 'ThreatMinerQuery'
-        self.addr = 'https://www.threatminer.org/getData.php'
+        self.addr = 'https://api.threatminer.org/v2/domain.php'
 
     def query(self):
         """
@@ -15,8 +15,7 @@ class ThreatMiner(Query):
         """
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
-        params = {'e': 'subdomains_container',
-                  'q': self.domain, 't': 0, 'rt': 10}
+        params = {'q': self.domain, 'rt': 5}
         resp = self.get(self.addr, params)
         self.subdomains = self.collect_subdomains(resp)
 

modules/intelligence/virustotal_api.py

@@ -28,6 +28,8 @@ class VirusTotalAPI(Query):
             self.subdomains.update(subdomains)
             data = resp.json()
             next_cursor = data.get('meta').get('cursor')
+            if not next_cursor:
+                break
 
     def run(self):
         """

modules/search/bing.py

@@ -19,9 +19,9 @@ class Bing(Search):
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
         """
-        self.page_num = 0  # 二次搜索重新置0
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
+        self.page_num = 0  # 二次搜索重新置0
         resp = self.get(self.init)
         if not resp:
             return

modules/search/bing_api.py

@@ -9,7 +9,7 @@ class BingAPI(Search):
         self.domain = domain
         self.module = 'Search'
         self.source = 'BingAPISearch'
-        self.addr = 'https://api.cognitive.microsoft.com/bing/v7.0/search'
+        self.addr = 'https://api.bing.microsoft.com/v7.0/search'
         self.id = settings.bing_api_id
         self.key = settings.bing_api_key
         self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数

modules/search/fofa_api.py

@@ -11,7 +11,7 @@ class FoFa(Search):
         self.domain = domain
         self.module = 'Search'
         self.source = 'FoFaAPISearch'
-        self.addr = 'https://fofa.so/api/v1/search/all'
+        self.addr = 'https://fofa.info/api/v1/search/all'
         self.delay = 1
         self.email = settings.fofa_api_email
         self.key = settings.fofa_api_key

modules/search/gitee.py

@@ -11,7 +11,6 @@ class Gitee(Search):
         self.module = 'Search'
         self.addr = 'https://search.gitee.com/'
         self.domain = domain
-        self.header = self.get_header()
 
     def search(self):
         """
@@ -20,6 +19,8 @@ class Gitee(Search):
         page_num = 1
         while True:
             time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
             params = {'pageno': page_num, 'q': self.domain, 'type': 'code'}
             try:
                 resp = self.get(self.addr, params=params)

modules/search/github_api.py

@@ -1,6 +1,5 @@
 import time
 
-import requests
 from config import settings
 from common.search import Search
 from config.log import logger
@@ -13,54 +12,32 @@ class GithubAPI(Search):
         self.module = 'Search'
         self.addr = 'https://api.github.com/search/code'
         self.domain = domain
-        self.session = requests.Session()
-        self.session.trust_env = False
-        self.auth_url = 'https://api.github.com'
+        self.delay = 5
         self.token = settings.github_api_token
 
-    def auth_github(self):
-        """
-        github api 认证
-
-        :return: 认证失败返回False 成功返回True
-        """
-        self.session.headers.update({'Authorization': 'token ' + self.token})
-        try:
-            resp = self.session.get(self.auth_url)
-        except Exception as e:
-            logger.log('ERROR', e.args)
-            return False
-        if resp.status_code != 200:
-            resp_json = resp.json()
-            msg = resp_json.get('message')
-            logger.log('ERROR', msg)
-            return False
-        return True
-
     def search(self):
         """
         向接口查询子域并做子域匹配
         """
-        self.session.headers = self.get_header()
-        self.session.proxies = self.get_proxy(self.source)
-        self.session.verify = self.verify
-        self.session.headers.update(
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        self.header.update(
             {'Accept': 'application/vnd.github.v3.text-match+json'})
+        self.header.update(
+            {'Authorization': 'token ' + self.token})
 
-        if not self.auth_github():
-            logger.log('ERROR', f'{self.source} module login failed')
-            return
         page = 1
         while True:
             time.sleep(self.delay)
             params = {'q': self.domain, 'per_page': 100,
-                      'page': page, 'sort': 'indexed'}
+                      'page': page, 'sort': 'indexed',
+                      'access_token': self.token}
             try:
-                resp = self.session.get(self.addr, params=params)
+                resp = self.get(self.addr, params=params)
             except Exception as e:
                 logger.log('ERROR', e.args)
                 break
-            if resp.status_code != 200:
+            if not resp or resp.status_code != 200:
                 logger.log('ERROR', f'{self.source} module query failed')
                 break
             subdomains = self.match_subdomains(resp)
@@ -106,4 +83,4 @@ def run(domain):
 
 
 if __name__ == '__main__':
-    run('example.com')
+    run('freebuf.com')

modules/search/hunter.py

@@ -0,0 +1,72 @@
+import base64
+import time
+
+from config import settings
+from common.search import Search
+
+
+class Hunter(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'HunterAPISearch'
+        self.addr = 'https://hunter.qianxin.com/openApi/search'
+        self.delay = 1
+        self.key = settings.hunter_api_key
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        self.page_num = 1
+        subdomain_encode = f'domain_suffix="{self.domain}"'.encode('utf-8')
+        query_data = base64.b64encode(subdomain_encode)
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = {'api-key': self.key,
+                     'search': query_data,
+                     'page': self.page_num,
+                     'page_size': 100,
+                     'is_web': 1}
+            resp = self.get(self.addr, query)
+            if not resp:
+                return
+            resp_json = resp.json()
+            subdomains = self.match_subdomains(resp)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains.update(subdomains)
+            total = resp_json.get('data').get('total')
+            if self.page_num * 100 >= int(total):
+                break
+            self.page_num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.key):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Hunter(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    run('freebuf.com')

modules/search/yahoo.py

@@ -21,13 +21,14 @@ class Yahoo(Search):
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
         """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
         resp = self.get(self.init)
         if not resp:
             return
         self.cookie = resp.cookies  # 获取cookie Yahoo在搜索时需要带上cookie
         while True:
             time.sleep(self.delay)
-            self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             query = 'site:.' + domain + filtered_subdomain
             params = {'p': query, 'b': self.page_num, 'pz': self.per_page_num}

modules/search/yandex.py

@@ -20,6 +20,8 @@ class Yandex(Search):
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
         """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
         self.page_num = 0  # 二次搜索重新置0
         resp = self.get(self.init)
         if not resp:
@@ -27,7 +29,6 @@ class Yandex(Search):
         self.cookie = resp.cookies  # 获取cookie
         while True:
             time.sleep(self.delay)
-            self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             query = 'site:.' + domain + filtered_subdomain
             params = {'text': query, 'p': self.page_num,

modules/wildcard.py

@@ -130,11 +130,11 @@ def get_wildcard_record(domain, resolver):
                         f"in authoritative name server")
     try:
         answer = resolver.query(domain, 'A')
-    # 如果查询随机域名A记录时抛出Timeout异常则重新查询
+
     except Timeout as e:
         logger.log('ALERT', f'Query timeout, retrying')
         logger.log('DEBUG', e.args)
-        raise e
+        return None, None
     except (NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers) as e:
         logger.log('DEBUG', e.args)
         logger.log('DEBUG', f'{domain} dont have A record on authoritative name server')

oneforall.py

@@ -32,7 +32,7 @@ blue = '\033[01;34m'
 red = '\033[1;31m'
 end = '\033[0m'
 
-version = 'v0.4.3'
+version = 'v0.4.4'
 message = white + '{' + red + version + ' #dev' + white + '}'
 
 oneforall_banner = f"""
@@ -138,7 +138,7 @@ class OneForAll(object):
         :return: exported data
         :rtype: list
         """
-        return export.export_data(self.domain, alive=self.alive, fmt=self.fmt)
+        return export.export_data(self.domain, alive=self.alive, fmt=self.fmt, path=self.path)
 
     def main(self):
         """
@@ -173,7 +173,9 @@ class OneForAll(object):
         utils.deal_data(self.domain)
         # Export results without resolve
         if not self.dns:
-            return self.export_data()
+            self.data = self.export_data()
+            self.datas.extend(self.data)
+            return self.data
 
         self.data = utils.get_data(self.domain)
 
@@ -185,7 +187,9 @@ class OneForAll(object):
 
         # Export results without HTTP request
         if not self.req:
-            return self.export_data()
+            self.data = self.export_data()
+            self.datas.extend(self.data)
+            return self.data
 
         if self.enable_wildcard:
             # deal wildcard
@@ -210,8 +214,8 @@ class OneForAll(object):
             enrich = Enrich(self.domain)
             enrich.run()
 
-        # Export
-        self.datas.extend(self.export_data())
+        self.data = self.export_data()
+        self.datas.extend(self.data)
 
         # Scan subdomain takeover
         if self.takeover:

requirements.txt

@@ -1,23 +1,22 @@
--i https://mirrors.aliyun.com/pypi/simple/
-beautifulsoup4==4.9.3
+beautifulsoup4==4.11.1
 bs4==0.0.1
-certifi==2020.6.20
-chardet==3.0.4
-colorama==0.4.4; sys_platform == 'win32'
-dnspython==2.0.0
+certifi==2022.06.15
+chardet==5.0.0
+colorama==0.4.4
+dnspython==2.2.1
 exrex==0.10.5
-fire==0.3.1
-future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-loguru==0.5.3
-pysocks==1.7.1
-requests==2.24.0
-six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-soupsieve==2.0.1; python_version >= '3.0'
-sqlalchemy==1.3.20
-tenacity==6.2.0
+fire==0.4.0
+future==0.18.2
+idna==3.3
+loguru==0.6.0
+PySocks==1.7.1
+requests==2.28.1
+six==1.16.0
+soupsieve==2.3.2
+SQLAlchemy==1.3.22
+tenacity==8.0.1
 termcolor==1.1.0
-tqdm==4.51.0
+tqdm==4.64.0
 treelib==1.6.1
-urllib3==1.25.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
-win32-setctime==1.0.3; sys_platform == 'win32'
+urllib3==1.26.9
+win32-setctime==1.1.0

takeover.py

@@ -70,7 +70,7 @@ class Takeover(Module):
         self.path = path
         self.fmt = fmt
         self.fingerprints = None
-        self.subdomainq = Queue()
+        self.queue = Queue()  # subdomain queue
         self.cnames = list()
         self.results = Dataset()
 
@@ -107,10 +107,10 @@ class Takeover(Module):
             self.compare(subdomain, cname, responses)
 
     def check(self):
-        while not self.subdomainq.empty():  # 保证域名队列遍历结束后能退出线程
-            subdomain = self.subdomainq.get()  # 从队列中获取域名
+        while not self.queue.empty():  # 保证域名队列遍历结束后能退出线程
+            subdomain = self.queue.get()  # 从队列中获取域名
             self.worker(subdomain)
-            self.subdomainq.task_done()
+            self.queue.task_done()
 
     def progress(self):
         bar = tqdm()
@@ -118,7 +118,7 @@ class Takeover(Module):
         bar.desc = 'Check Progress'
         bar.ncols = 80
         while True:
-            done = bar.total - self.subdomainq.qsize()
+            done = bar.total - self.queue.qsize()
             bar.n = done
             bar.update()
             if done == bar.total:  # 完成队列中所有子域的检查退出
@@ -141,7 +141,7 @@ class Takeover(Module):
             self.results.headers = ['subdomain', 'cname']
             # 创建待检查的子域队列
             for domain in self.subdomains:
-                self.subdomainq.put(domain)
+                self.queue.put(domain)
             # 进度线程
             progress_thread = Thread(target=self.progress, name='ProgressThread',
                                      daemon=True)
@@ -152,7 +152,7 @@ class Takeover(Module):
                                       daemon=True)
                 check_thread.start()
 
-            self.subdomainq.join()
+            self.queue.join()
             self.save()
         else:
             logger.log('FATAL', f'Failed to obtain domain')

test.py

@@ -10,10 +10,13 @@ from oneforall import OneForAll
 
 def oneforall(domain):
     test = OneForAll(target=domain)
+    test.dns = True
     test.brute = True
     test.req = True
     test.takeover = True
     test.run()
+    results = test.datas
+    print(results)
 
 
 if __name__ == '__main__':