重构泛解析探测

Pipfile

@@ -22,6 +22,7 @@ tablib = "*"
 psutil = "*"
 tenacity = "*"
 ip2location = "*"
+treelib = "*"
 
 [requires]
 python_version = "3.8"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "6556f720e7fb052c3f923f3e7986cc00c1cc230db729ad79685a127c02568c96"
+            "sha256": "78a5771767751025a636d651f53cce059223e1e045d351290fa5e7b413bc172b"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -43,10 +43,10 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
+                "sha256:0ef97238856430dcf9228e07f316aefc17e8939fc8507e18c6501b761ef1a42a",
-                "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
+                "sha256:2867b7b9f8326499ab5b0e2d12801fa5c98842d2cbd22b35112ae04bf85b4dff"
             ],
-            "version": "==19.3.0"
+            "version": "==20.1.0"
         },
         "beautifulsoup4": {
             "hashes": [
@@ -79,11 +79,11 @@
         },
         "cloudscraper": {
             "hashes": [
-                "sha256:1d4e158de3be7a4e651083f13a14a25b75853186e989d9d97db1a08056b7a0b6",
+                "sha256:793095bbc37aae84a5c38d9f66b56f3c83a00f0b1ff2d334f75a2b8f88b924af",
-                "sha256:c8c255aaa84e80d2cb5bf2c1ad1ffa000ce864595ca4cdb75ff7aab6f97befec"
+                "sha256:a7ae28b5dc4e4ef0789ef608c42a4382bf6d3e3dd6b8117c2e3633103131ae43"
             ],
             "index": "pypi",
-            "version": "==1.2.42"
+            "version": "==1.2.46"
         },
         "colorama": {
             "hashes": [
@@ -95,11 +95,11 @@
         },
         "dnspython": {
             "hashes": [
-                "sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01",
+                "sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7",
-                "sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d"
+                "sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d"
             ],
             "index": "pypi",
-            "version": "==1.16.0"
+            "version": "==2.0.0"
         },
         "docopt": {
             "hashes": [
@@ -127,12 +127,17 @@
             "index": "pypi",
             "version": "==0.3.1"
         },
+        "future": {
+            "hashes": [
+                "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
+            ],
+            "version": "==0.18.2"
+        },
         "idna": {
             "hashes": [
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
         "ip2location": {
@@ -178,7 +183,6 @@
                 "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255",
                 "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d"
             ],
-            "markers": "python_version >= '3.5'",
             "version": "==4.7.6"
         },
         "openpyxl": {
@@ -189,27 +193,26 @@
         },
         "psutil": {
             "hashes": [
-                "sha256:1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058",
+                "sha256:0ee3c36428f160d2d8fce3c583a0353e848abb7de9732c50cf3356dd49ad63f8",
-                "sha256:298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953",
+                "sha256:10512b46c95b02842c225f58fa00385c08fa00c68bac7da2d9a58ebe2c517498",
-                "sha256:60b86f327c198561f101a92be1995f9ae0399736b6eced8f24af41ec64fb88d4",
+                "sha256:4080869ed93cce662905b029a1770fe89c98787e543fa7347f075ade761b19d6",
-                "sha256:685ec16ca14d079455892f25bd124df26ff9137664af445563c1bd36629b5e0e",
+                "sha256:5e9d0f26d4194479a13d5f4b3798260c20cecf9ac9a461e718eb59ea520a360c",
-                "sha256:73f35ab66c6c7a9ce82ba44b1e9b1050be2a80cd4dcc3352cc108656b115c74f",
+                "sha256:66c18ca7680a31bf16ee22b1d21b6397869dda8059dbdb57d9f27efa6615f195",
-                "sha256:75e22717d4dbc7ca529ec5063000b2b294fc9a367f9c9ede1f65846c7955fd38",
+                "sha256:68d36986ded5dac7c2dcd42f2682af1db80d4bce3faa126a6145c1637e1b559f",
-                "sha256:a02f4ac50d4a23253b68233b07e7cdb567bd025b982d5cf0ee78296990c22d9e",
+                "sha256:90990af1c3c67195c44c9a889184f84f5b2320dce3ee3acbd054e3ba0b4a7beb",
-                "sha256:d008ddc00c6906ec80040d26dc2d3e3962109e40ad07fd8a12d0284ce5e0e4f8",
+                "sha256:a5b120bb3c0c71dfe27551f9da2f3209a8257a178ed6c628a819037a8df487f1",
-                "sha256:d84029b190c8a66a946e28b4d3934d2ca1528ec94764b180f7d6ea57b0e75e26",
+                "sha256:d8a82162f23c53b8525cf5f14a355f5d1eea86fa8edde27287dd3a98399e4fdf",
-                "sha256:e2d0c5b07c6fe5a87fa27b7855017edb0d52ee73b71e6ee368fae268605cc3f5",
+                "sha256:f2018461733b23f308c298653c8903d32aaad7873d25e1d228765e91ae42c3f2",
-                "sha256:f344ca230dd8e8d5eee16827596f1c22ec0876127c28e800d7ae20ed44c4b310"
+                "sha256:ff1977ba1a5f71f89166d5145c3da1cea89a0fdb044075a12c720ee9123ec818"
             ],
             "index": "pypi",
-            "version": "==5.7.0"
+            "version": "==5.7.2"
         },
         "pyparsing": {
             "hashes": [
                 "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
                 "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
             ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.4.7"
         },
         "pysocks": {
@@ -256,7 +259,6 @@
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
                 "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.15.0"
         },
         "soupsieve": {
@@ -264,42 +266,41 @@
                 "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
                 "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
             ],
-            "markers": "python_version >= '3.5'",
             "version": "==2.0.1"
         },
         "sqlalchemy": {
             "hashes": [
-                "sha256:0942a3a0df3f6131580eddd26d99071b48cfe5aaf3eab2783076fbc5a1c1882e",
+                "sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb",
-                "sha256:0ec575db1b54909750332c2e335c2bb11257883914a03bc5a3306a4488ecc772",
+                "sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804",
-                "sha256:109581ccc8915001e8037b73c29590e78ce74be49ca0a3630a23831f9e3ed6c7",
+                "sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6",
-                "sha256:16593fd748944726540cd20f7e83afec816c2ac96b082e26ae226e8f7e9688cf",
+                "sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe",
-                "sha256:427273b08efc16a85aa2b39892817e78e3ed074fcb89b2a51c4979bae7e7ba98",
+                "sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de",
-                "sha256:50c4ee32f0e1581828843267d8de35c3298e86ceecd5e9017dc45788be70a864",
+                "sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36",
-                "sha256:512a85c3c8c3995cc91af3e90f38f460da5d3cade8dc3a229c8e0879037547c9",
+                "sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e",
-                "sha256:57aa843b783179ab72e863512e14bdcba186641daf69e4e3a5761d705dcc35b1",
+                "sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc",
-                "sha256:621f58cd921cd71ba6215c42954ffaa8a918eecd8c535d97befa1a8acad986dd",
+                "sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d",
-                "sha256:6ac2558631a81b85e7fb7a44e5035347938b0a73f5fdc27a8566777d0792a6a4",
+                "sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce",
-                "sha256:716754d0b5490bdcf68e1e4925edc02ac07209883314ad01a137642ddb2056f1",
+                "sha256:6557af9e0d23f46b8cd56f8af08eaac72d2e3c632ac8d5cf4e20215a8dca7cea",
-                "sha256:736d41cfebedecc6f159fc4ac0769dc89528a989471dc1d378ba07d29a60ba1c",
+                "sha256:73a40d4fcd35fdedce07b5885905753d5d4edf413fbe53544dd871f27d48bd4f",
-                "sha256:8619b86cb68b185a778635be5b3e6018623c0761dde4df2f112896424aa27bd8",
+                "sha256:8280f9dae4adb5889ce0bb3ec6a541bf05434db5f9ab7673078c00713d148365",
-                "sha256:87fad64529cde4f1914a5b9c383628e1a8f9e3930304c09cf22c2ae118a1280e",
+                "sha256:83469ad15262402b0e0974e612546bc0b05f379b5aa9072ebf66d0f8fef16bea",
-                "sha256:89494df7f93b1836cae210c42864b292f9b31eeabca4810193761990dc689cce",
+                "sha256:860d0fe234922fd5552b7f807fbb039e3e7ca58c18c8d38aa0d0a95ddf4f6c23",
-                "sha256:8cac7bb373a5f1423e28de3fd5fc8063b9c8ffe8957dc1b1a59cb90453db6da1",
+                "sha256:883c9fb62cebd1e7126dd683222b3b919657590c3e2db33bdc50ebbad53e0338",
-                "sha256:8fd452dc3d49b3cc54483e033de6c006c304432e6f84b74d7b2c68afa2569ae5",
+                "sha256:8afcb6f4064d234a43fea108859942d9795c4060ed0fbd9082b0f280181a15c1",
-                "sha256:adad60eea2c4c2a1875eb6305a0b6e61a83163f8e233586a4d6a55221ef984fe",
+                "sha256:96f51489ac187f4bab588cf51f9ff2d40b6d170ac9a4270ffaed535c8404256b",
-                "sha256:c26f95e7609b821b5f08a72dab929baa0d685406b953efd7c89423a511d5c413",
+                "sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e",
-                "sha256:cbe1324ef52ff26ccde2cb84b8593c8bf930069dfc06c1e616f1bfd4e47f48a3",
+                "sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba",
-                "sha256:d05c4adae06bd0c7f696ae3ec8d993ed8ffcc4e11a76b1b35a5af8a099bd2284",
+                "sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02",
-                "sha256:d98bc827a1293ae767c8f2f18be3bb5151fd37ddcd7da2a5f9581baeeb7a3fa1",
+                "sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86",
-                "sha256:da2fb75f64792c1fc64c82313a00c728a7c301efe6a60b7a9fe35b16b4368ce7",
+                "sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d",
-                "sha256:e4624d7edb2576cd72bb83636cd71c8ce544d8e272f308bd80885056972ca299",
+                "sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7",
-                "sha256:e89e0d9e106f8a9180a4ca92a6adde60c58b1b0299e1b43bd5e0312f535fbf33",
+                "sha256:e49947d583fe4d29af528677e4f0aa21f5e535ca2ae69c48270ebebd0d8843c0",
-                "sha256:f11c2437fb5f812d020932119ba02d9e2bc29a6eca01a055233a8b449e3e1e7d",
+                "sha256:eb1d71643e4154398b02e88a42fc8b29db8c44ce4134cf0f4474bfc5cb5d4dac",
-                "sha256:f57be5673e12763dd400fea568608700a63ce1c6bd5bdbc3cc3a2c5fdb045274",
+                "sha256:f2e8a9c0c8813a468aa659a01af6592f71cd30237ec27c4cc0683f089f90dcfc",
-                "sha256:fc728ece3d5c772c196fd338a99798e7efac7a04f9cb6416299a3638ee9a94cd"
+                "sha256:fe7fe11019fc3e6600819775a7d55abc5446dda07e9795f5954fdbf8a49e1c37"
             ],
             "markers": "python_version >= '3.0'",
-            "version": "==1.3.18"
+            "version": "==1.3.19"
         },
         "tablib": {
             "hashes": [
@@ -325,27 +326,33 @@
         },
         "tldextract": {
             "hashes": [
-                "sha256:16b2f7e81d89c2a5a914d25bdbddd3932c31a6b510db886c3ce0764a195c0ee7",
+                "sha256:ab0e38977a129c72729476d5f8c85a8e1f8e49e9202e1db8dca76e95da7be9a8",
-                "sha256:9aa21a1f7827df4209e242ec4fc2293af5940ec730cde46ea80f66ed97bfc808"
+                "sha256:c2a8a392edf3ea6fa8be80930f04c3ac29e91fa604cb2139bdf6a37fc1e1ac6d"
             ],
             "index": "pypi",
-            "version": "==2.2.2"
+            "version": "==2.2.3"
         },
         "tqdm": {
             "hashes": [
-                "sha256:63ef7a6d3eb39f80d6b36e4867566b3d8e5f1fe3d6cb50c5e9ede2b3198ba7b7",
+                "sha256:1a336d2b829be50e46b84668691e0a2719f26c97c62846298dd5ae2937e4d5cf",
-                "sha256:7810e627bcf9d983a99d9ff8a0c09674400fd2927eddabeadf153c14a2ec8656"
+                "sha256:564d632ea2b9cb52979f7956e093e831c28d441c11751682f84c86fc46e4fd21"
             ],
             "index": "pypi",
-            "version": "==4.47.0"
+            "version": "==4.48.2"
+        },
+        "treelib": {
+            "hashes": [
+                "sha256:1cbfffb2d2b75ccac27d0200cee0507b6fbb0726e0afb9fae017ade5d2ce8788"
+            ],
+            "index": "pypi",
+            "version": "==1.6.1"
         },
         "urllib3": {
             "hashes": [
-                "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527",
+                "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a",
-                "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"
+                "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.25.10"
-            "version": "==1.25.9"
         },
         "win32-setctime": {
             "hashes": [
@@ -357,25 +364,25 @@
         },
         "yarl": {
             "hashes": [
-                "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce",
+                "sha256:040b237f58ff7d800e6e0fd89c8439b841f777dd99b4a9cca04d6935564b9409",
-                "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6",
+                "sha256:17668ec6722b1b7a3a05cc0167659f6c95b436d25a36c2d52db0eca7d3f72593",
-                "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce",
+                "sha256:3a584b28086bc93c888a6c2aa5c92ed1ae20932f078c46509a66dce9ea5533f2",
-                "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae",
+                "sha256:4439be27e4eee76c7632c2427ca5e73703151b22cae23e64adb243a9c2f565d8",
-                "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d",
+                "sha256:48e918b05850fffb070a496d2b5f97fc31d15d94ca33d3d08a4f86e26d4e7c5d",
-                "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f",
+                "sha256:9102b59e8337f9874638fcfc9ac3734a0cfadb100e47d55c20d0dc6087fb4692",
-                "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b",
+                "sha256:9b930776c0ae0c691776f4d2891ebc5362af86f152dd0da463a6614074cb1b02",
-                "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b",
+                "sha256:b3b9ad80f8b68519cc3372a6ca85ae02cc5a8807723ac366b53c0f089db19e4a",
-                "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb",
+                "sha256:bc2f976c0e918659f723401c4f834deb8a8e7798a71be4382e024bcc3f7e23a8",
-                "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462",
+                "sha256:c22c75b5f394f3d47105045ea551e08a3e804dc7e01b37800ca35b58f856c3d6",
-                "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea",
+                "sha256:c52ce2883dc193824989a9b97a76ca86ecd1fa7955b14f87bf367a61b6232511",
-                "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70",
+                "sha256:ce584af5de8830d8701b8979b18fcf450cef9a382b1a3c8ef189bedc408faf1e",
-                "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1",
+                "sha256:da456eeec17fa8aa4594d9a9f27c0b1060b6a75f2419fe0c00609587b2695f4a",
-                "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a",
+                "sha256:db6db0f45d2c63ddb1a9d18d1b9b22f308e52c83638c26b422d520a815c4b3fb",
-                "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b",
+                "sha256:df89642981b94e7db5596818499c4b2219028f2a528c9c37cc1de45bf2fd3a3f",
-                "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080",
+                "sha256:f18d68f2be6bf0e89f1521af2b1bb46e66ab0018faafa81d70f358153170a317",
-                "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2"
+                "sha256:f379b7f83f23fe12823085cd6b906edc49df969eb99757f58ff382349a3303c6"
             ],
-            "version": "==1.4.2"
+            "version": "==1.5.1"
         }
     },
     "develop": {}

brute.py

@@ -10,7 +10,6 @@ OneForAll subdomain brute module
 import gc
 import json
 import time
-import random
 import secrets
 
 import exrex
@@ -21,67 +20,137 @@ from dns.resolver import NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers
 
 import dbexport
 from common import utils
+from common import similarity
 from config import settings
 from common.module import Module
 from config.log import logger
 
 
-@tenacity.retry(stop=tenacity.stop_after_attempt(3))
+def config_resolver(nameservers):
-def do_query_a(domain, resolver):
+    """
+    配置DNS解析器
+
+    :param nameservers: 名称解析服务器地址
+    """
+    resolver = utils.dns_resolver()
+    resolver.nameservers = nameservers
+    resolver.rotate = True  # 随机使用NS
+    resolver.cache = None  # 不使用DNS缓存
+    return resolver
+
+
+def gen_random_subdomains(domain, count):
+    """
+    生成指定数量的随机子域域名列表
+
+    :param domain: 主域
+    :param count: 数量
+    """
+    subdomains = set()
+    if count < 1:
+        return subdomains
+    for _ in range(count):
+        token = secrets.token_hex(4)
+        subdomains.add(f'{token}.{domain}')
+    return subdomains
+
+
+def query_a_record(subdomain, resolver):
+    """
+    查询子域A记录
+
+    :param subdomain: 子域
+    :param resolver: DNS解析器
+    """
     try:
-        answer = resolver.query(domain, 'A')
+        answer = resolver.query(subdomain, 'A')
-    # If resolve random subdomain raise timeout error, try again
-    except Timeout as e:
-        logger.log('ALERT', f'DNS resolve timeout, retrying')
-        logger.log('DEBUG', e.args)
-        raise tenacity.TryAgain
-    # If resolve random subdomain raise NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers error
-    # It means that there is no A record of random subdomain and not use wildcard dns record
-    except (NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers) as e:
-        logger.log('DEBUG', e.args)
-        logger.log('INFOR', f'{domain} seems not use wildcard dns record')
-        return False
     except Exception as e:
-        logger.log('ALERT', f'Detect {domain} wildcard dns record error')
+        logger.log('DEBUG', f'Query {subdomain} wildcard dns record error')
-        logger.log('FATAL', e.args)
+        logger.log('DEBUG', e.args)
-        exit(1)
+        return False
-    else:
     if answer.rrset is None:
-            logger.log('ALERT', f'DNS resolve dont have result, retrying')
+        return False
-            raise tenacity.TryAgain
     ttl = answer.ttl
     name = answer.name
     ips = {item.address for item in answer}
-        logger.log('ALERT', f'{domain} use wildcard dns record')
+    logger.log('ALERT', f'{subdomain} resolve to: {name} '
-        logger.log('ALERT', f'{domain} resolve to: {name} '
                         f'IP: {ips} TTL: {ttl}')
     return True
 
 
-def detect_wildcard(domain, authoritative_ns):
+def all_resolve_success(subdomains):
+    """
+    判断是否所有子域都解析成功
+
+    :param subdomains: 子域列表
+    """
+    resolver = utils.dns_resolver()
+    resolver.cache = None  # 不使用DNS缓存
+    status = set()
+    for subdomain in subdomains:
+        status.add(query_a_record(subdomain, resolver))
+    return all(status)
+
+
+def all_request_success(subdomains):
+    """
+    判断是否所有子域都请求成功
+
+    :param subdomains: 子域列表
+    """
+    result = list()
+    for subdomain in subdomains:
+        url = f'http://{subdomain}'
+        resp = utils.get_url_resp(url)
+        if resp:
+            logger.log('ALERT', f'Request: {url} Status: {resp.status_code} '
+                                f'Size: {len(resp.content)}')
+            result.append(resp.text)
+        else:
+            result.append(resp)
+    return all(result), result
+
+
+def any_similar_html(resp_list):
+    """
+    判断是否有一组HTML页面结构相似
+
+    :param resp_list: 响应HTML页面
+    """
+    html_doc1, html_doc2, html_doc3 = resp_list
+    if similarity.is_similar(html_doc1, html_doc2):
+        return True
+    if similarity.is_similar(html_doc1, html_doc3):
+        return True
+    if similarity.is_similar(html_doc2, html_doc3):
+        return True
+    return False
+
+
+def detect_wildcard(domain):
     """
     Detect use wildcard dns record or not
 
     :param  str  domain:  domain
-    :param  list authoritative_ns: authoritative name server
     :return bool use wildcard dns record or not
     """
     logger.log('INFOR', f'Detecting {domain} use wildcard dns record or not')
-    token = secrets.token_hex(4)
+    random_subdomains = gen_random_subdomains(domain, 3)
-    random_subdomain = f'{token}.{domain}'
+    if not all_resolve_success(random_subdomains):
-    resolver = utils.dns_resolver()
-    resolver.nameservers = authoritative_ns
-    resolver.rotate = True
-    resolver.cache = None
-    try:
-        wildcard = do_query_a(random_subdomain, resolver)
-    except Exception as e:
-        logger.log('DEBUG', e.args)
-        logger.log('ALERT', f'Multiple detection errors, so temporarily {domain} '
-                            f'does not use wildcard dns record')
         return False
+    is_all_success, all_request_resp = all_request_success(random_subdomains)
+    if not is_all_success:
+        return True
+    return any_similar_html(all_request_resp)
+
+
+def is_enable_wildcard(domain):
+    is_enable = detect_wildcard(domain)
+    if is_enable:
+        logger.log('ALERT', f'The domain {domain} enables wildcard')
     else:
-        return wildcard
+        logger.log('ALERT', f'The domain {domain} disables wildcard')
+    return is_enable
 
 
 def gen_subdomains(expression, path):
@@ -229,9 +298,9 @@ def collect_wildcard_record(domain, authoritative_ns):
     if not authoritative_ns:
         return list(), int()
     resolver = utils.dns_resolver()
-    resolver.nameservers = authoritative_ns
+    resolver.nameservers = authoritative_ns  # 使用权威名称服务器
-    resolver.rotate = True
+    resolver.rotate = True  # 随机使用NS
-    resolver.cache = None
+    resolver.cache = None  # 不使用DNS缓存
     ips = set()
     ttl = int()
     ips_stat = dict()
@@ -596,7 +665,7 @@ class Brute(Module):
         wildcard_ttl = int()  # 泛解析TTL整型值
         ns_list = query_domain_ns(self.domain)
         ns_ip_list = query_domain_ns_a(ns_list)  # DNS权威名称服务器对应A记录列表
-        self.enable_wildcard = detect_wildcard(domain, ns_ip_list)
+        self.enable_wildcard = is_enable_wildcard(domain)
 
         if self.enable_wildcard:
             wildcard_ips, wildcard_ttl = collect_wildcard_record(domain,

common/similarity.py

@@ -0,0 +1,138 @@
+"""
+根据网页结构判断页面相似性(Determine page similarity based on HTML page structure)
+判断方法：根据网页的DOM树确定网页的模板特征向量，对模板特征向量计算网页结构相似性。
+来源地址：https://github.com/SPuerBRead/HTMLSimilarity
+计算参考: https://patents.google.com/patent/CN101694668B/zh
+"""
+from treelib import Tree
+from bs4 import BeautifulSoup
+import bs4
+
+
+class DOMTree(object):
+    def __init__(self, label, attrs):
+        self.label = label
+        self.attrs = attrs
+
+
+class HTMLParser(object):
+    def __init__(self, html):
+        self.dom_id = 1
+        self.dom_tree = Tree()
+        self.bs_html = BeautifulSoup(html, 'html.parser')
+
+    def get_dom_structure_tree(self):
+        for content in self.bs_html.contents:
+            if isinstance(content, bs4.element.Tag):
+                self.bs_html = content
+        self.recursive_descendants(self.bs_html, 1)
+        return self.dom_tree
+
+    def recursive_descendants(self, descendants, parent_id):
+        if self.dom_id == 1:
+            self.dom_tree.create_node(descendants.name, self.dom_id,
+                                      data=DOMTree(descendants.name, descendants.attrs))
+            self.dom_id = self.dom_id + 1
+        for child in descendants.contents:
+            if isinstance(child, bs4.element.Tag):
+                self.dom_tree.create_node(child.name, self.dom_id, parent_id,
+                                          data=DOMTree(child.name, child.attrs))
+                self.dom_id = self.dom_id + 1
+                self.recursive_descendants(child, self.dom_id - 1)
+
+
+class Converter(object):
+    def __init__(self, dom_tree, dimension):
+        self.dom_tree = dom_tree
+        self.node_info_list = []
+        self.dimension = dimension
+        self.initial_weight = 1
+        self.attenuation_ratio = 0.6
+        self.dom_eigenvector = {}.fromkeys(range(0, dimension), 0)
+
+    def get_eigenvector(self):
+        for node_id in range(1, self.dom_tree.size() + 1):
+            node = self.dom_tree.get_node(node_id)
+            node_feature = self.create_feature(node)
+            feature_hash = self.feature_hash(node_feature)
+            node_weight = self.calculate_weight(node, node_id, feature_hash)
+            self.construct_eigenvector(feature_hash, node_weight)
+        return self.dom_eigenvector
+
+    @staticmethod
+    def create_feature(node):
+        node_attr_list = []
+        node_feature = node.data.label + '|'
+        for attr in node.data.attrs.keys():
+            node_attr_list.append(attr + ':' + str(node.data.attrs[attr]))
+        node_feature += '|'.join(node_attr_list)
+        return node_feature
+
+    @staticmethod
+    def feature_hash(node_feature):
+        return abs(hash(node_feature)) % (10 ** 8)
+
+    def calculate_weight(self, node, node_id, feature_hash):
+        brother_node_count = 0
+        depth = self.dom_tree.depth(node)
+        for brother_node in self.dom_tree.siblings(node_id):
+            brother_node_feature = self.create_feature(brother_node)
+            brother_node_feature_hash = self.feature_hash(brother_node_feature)
+            if brother_node_feature_hash == feature_hash:
+                brother_node_count = brother_node_count + 1
+        if brother_node_count:
+            node_weight = self.initial_weight * self.attenuation_ratio ** depth \
+                          * self.attenuation_ratio ** brother_node_count
+        else:
+            node_weight = self.initial_weight * self.attenuation_ratio ** depth
+        return node_weight
+
+    def construct_eigenvector(self, feature_hash, node_weight):
+        feature_hash = feature_hash % self.dimension
+        self.dom_eigenvector[feature_hash] += node_weight
+
+
+def calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension):
+    a, b = 0, 0
+    for i in range(dimension):
+        a += dom1_eigenvector[i]-dom2_eigenvector[i]
+        if dom1_eigenvector[i] and dom2_eigenvector[i]:
+            b += dom1_eigenvector[i] + dom2_eigenvector[i]
+    pseudodistance = abs(a)/b
+    return pseudodistance
+
+
+def get_pseudodistance(html_doc1, html_doc2, dimension=5000):
+    """
+    获取html文档结构相似度
+
+    :param str html_doc1: html文档
+    :param str html_doc2: html文档
+    :param int dimension: 降维后的维数
+    :return 伪距离
+    """
+    hp1 = HTMLParser(html_doc1)
+    html_doc1_dom_tree = hp1.get_dom_structure_tree()
+    hp2 = HTMLParser(html_doc2)
+    html_doc2_dom_tree = hp2.get_dom_structure_tree()
+    converter = Converter(html_doc1_dom_tree, dimension)
+    dom1_eigenvector = converter.get_eigenvector()
+    converter = Converter(html_doc2_dom_tree, dimension)
+    dom2_eigenvector = converter.get_eigenvector()
+    return calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension)
+
+
+def is_similar(html_doc1, html_doc2, dimension=5000):
+    """
+    根据计算出的伪距离来判断是否html页面结构相似
+
+    :param str html_doc1: html文档
+    :param str html_doc2: html文档
+    :param int dimension: 降维后的维数
+    :return 是否相似（伪距离value<0.2时相似，value>0.2时不相似）
+    """
+    value = get_pseudodistance(html_doc1, html_doc2, dimension)
+    if value > 0.2:
+        return False
+    else:
+        return True

common/utils.py

@@ -741,3 +741,15 @@ def check_random_subdomain(subdomains):
         if subdomain:
             logger.log('ALERT', f'Please check whether {subdomain} is correct or not')
             return True
+
+
+def get_url_resp(url):
+    timeout = settings.request_timeout
+    verify = settings.request_verify
+    try:
+        resp = requests.get(url, params=None, timeout=timeout, verify=verify)
+    except Exception as e:
+        logger.log('DEBUG', f'Error request {url}')
+        logger.log('DEBUG', e.args)
+        return None
+    return resp