mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-25 20:37:48 +08:00
重构泛解析探测
This commit is contained in:
Jing Ling
@@ -22,6 +22,7 @@ tablib = "*"
|
||||
psutil = "*"
|
||||
tenacity = "*"
|
||||
ip2location = "*"
|
||||
treelib = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.8"
|
||||
|
||||
Generated
+91
-84
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "6556f720e7fb052c3f923f3e7986cc00c1cc230db729ad79685a127c02568c96"
|
||||
"sha256": "78a5771767751025a636d651f53cce059223e1e045d351290fa5e7b413bc172b"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
@@ -43,10 +43,10 @@
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
|
||||
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
|
||||
"sha256:0ef97238856430dcf9228e07f316aefc17e8939fc8507e18c6501b761ef1a42a",
|
||||
"sha256:2867b7b9f8326499ab5b0e2d12801fa5c98842d2cbd22b35112ae04bf85b4dff"
|
||||
],
|
||||
"version": "==19.3.0"
|
||||
"version": "==20.1.0"
|
||||
},
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
@@ -79,11 +79,11 @@
|
||||
},
|
||||
"cloudscraper": {
|
||||
"hashes": [
|
||||
"sha256:1d4e158de3be7a4e651083f13a14a25b75853186e989d9d97db1a08056b7a0b6",
|
||||
"sha256:c8c255aaa84e80d2cb5bf2c1ad1ffa000ce864595ca4cdb75ff7aab6f97befec"
|
||||
"sha256:793095bbc37aae84a5c38d9f66b56f3c83a00f0b1ff2d334f75a2b8f88b924af",
|
||||
"sha256:a7ae28b5dc4e4ef0789ef608c42a4382bf6d3e3dd6b8117c2e3633103131ae43"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.2.42"
|
||||
"version": "==1.2.46"
|
||||
},
|
||||
"colorama": {
|
||||
"hashes": [
|
||||
@@ -95,11 +95,11 @@
|
||||
},
|
||||
"dnspython": {
|
||||
"hashes": [
|
||||
"sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01",
|
||||
"sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d"
|
||||
"sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7",
|
||||
"sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.16.0"
|
||||
"version": "==2.0.0"
|
||||
},
|
||||
"docopt": {
|
||||
"hashes": [
|
||||
@@ -127,12 +127,17 @@
|
||||
"index": "pypi",
|
||||
"version": "==0.3.1"
|
||||
},
|
||||
"future": {
|
||||
"hashes": [
|
||||
"sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
|
||||
],
|
||||
"version": "==0.18.2"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
|
||||
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==2.10"
|
||||
},
|
||||
"ip2location": {
|
||||
@@ -178,7 +183,6 @@
|
||||
"sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255",
|
||||
"sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d"
|
||||
],
|
||||
"markers": "python_version >= '3.5'",
|
||||
"version": "==4.7.6"
|
||||
},
|
||||
"openpyxl": {
|
||||
@@ -189,27 +193,26 @@
|
||||
},
|
||||
"psutil": {
|
||||
"hashes": [
|
||||
"sha256:1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058",
|
||||
"sha256:298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953",
|
||||
"sha256:60b86f327c198561f101a92be1995f9ae0399736b6eced8f24af41ec64fb88d4",
|
||||
"sha256:685ec16ca14d079455892f25bd124df26ff9137664af445563c1bd36629b5e0e",
|
||||
"sha256:73f35ab66c6c7a9ce82ba44b1e9b1050be2a80cd4dcc3352cc108656b115c74f",
|
||||
"sha256:75e22717d4dbc7ca529ec5063000b2b294fc9a367f9c9ede1f65846c7955fd38",
|
||||
"sha256:a02f4ac50d4a23253b68233b07e7cdb567bd025b982d5cf0ee78296990c22d9e",
|
||||
"sha256:d008ddc00c6906ec80040d26dc2d3e3962109e40ad07fd8a12d0284ce5e0e4f8",
|
||||
"sha256:d84029b190c8a66a946e28b4d3934d2ca1528ec94764b180f7d6ea57b0e75e26",
|
||||
"sha256:e2d0c5b07c6fe5a87fa27b7855017edb0d52ee73b71e6ee368fae268605cc3f5",
|
||||
"sha256:f344ca230dd8e8d5eee16827596f1c22ec0876127c28e800d7ae20ed44c4b310"
|
||||
"sha256:0ee3c36428f160d2d8fce3c583a0353e848abb7de9732c50cf3356dd49ad63f8",
|
||||
"sha256:10512b46c95b02842c225f58fa00385c08fa00c68bac7da2d9a58ebe2c517498",
|
||||
"sha256:4080869ed93cce662905b029a1770fe89c98787e543fa7347f075ade761b19d6",
|
||||
"sha256:5e9d0f26d4194479a13d5f4b3798260c20cecf9ac9a461e718eb59ea520a360c",
|
||||
"sha256:66c18ca7680a31bf16ee22b1d21b6397869dda8059dbdb57d9f27efa6615f195",
|
||||
"sha256:68d36986ded5dac7c2dcd42f2682af1db80d4bce3faa126a6145c1637e1b559f",
|
||||
"sha256:90990af1c3c67195c44c9a889184f84f5b2320dce3ee3acbd054e3ba0b4a7beb",
|
||||
"sha256:a5b120bb3c0c71dfe27551f9da2f3209a8257a178ed6c628a819037a8df487f1",
|
||||
"sha256:d8a82162f23c53b8525cf5f14a355f5d1eea86fa8edde27287dd3a98399e4fdf",
|
||||
"sha256:f2018461733b23f308c298653c8903d32aaad7873d25e1d228765e91ae42c3f2",
|
||||
"sha256:ff1977ba1a5f71f89166d5145c3da1cea89a0fdb044075a12c720ee9123ec818"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==5.7.0"
|
||||
"version": "==5.7.2"
|
||||
},
|
||||
"pyparsing": {
|
||||
"hashes": [
|
||||
"sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
|
||||
"sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
|
||||
],
|
||||
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==2.4.7"
|
||||
},
|
||||
"pysocks": {
|
||||
@@ -256,7 +259,6 @@
|
||||
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
|
||||
"sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==1.15.0"
|
||||
},
|
||||
"soupsieve": {
|
||||
@@ -264,42 +266,41 @@
|
||||
"sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
|
||||
"sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
|
||||
],
|
||||
"markers": "python_version >= '3.5'",
|
||||
"version": "==2.0.1"
|
||||
},
|
||||
"sqlalchemy": {
|
||||
"hashes": [
|
||||
"sha256:0942a3a0df3f6131580eddd26d99071b48cfe5aaf3eab2783076fbc5a1c1882e",
|
||||
"sha256:0ec575db1b54909750332c2e335c2bb11257883914a03bc5a3306a4488ecc772",
|
||||
"sha256:109581ccc8915001e8037b73c29590e78ce74be49ca0a3630a23831f9e3ed6c7",
|
||||
"sha256:16593fd748944726540cd20f7e83afec816c2ac96b082e26ae226e8f7e9688cf",
|
||||
"sha256:427273b08efc16a85aa2b39892817e78e3ed074fcb89b2a51c4979bae7e7ba98",
|
||||
"sha256:50c4ee32f0e1581828843267d8de35c3298e86ceecd5e9017dc45788be70a864",
|
||||
"sha256:512a85c3c8c3995cc91af3e90f38f460da5d3cade8dc3a229c8e0879037547c9",
|
||||
"sha256:57aa843b783179ab72e863512e14bdcba186641daf69e4e3a5761d705dcc35b1",
|
||||
"sha256:621f58cd921cd71ba6215c42954ffaa8a918eecd8c535d97befa1a8acad986dd",
|
||||
"sha256:6ac2558631a81b85e7fb7a44e5035347938b0a73f5fdc27a8566777d0792a6a4",
|
||||
"sha256:716754d0b5490bdcf68e1e4925edc02ac07209883314ad01a137642ddb2056f1",
|
||||
"sha256:736d41cfebedecc6f159fc4ac0769dc89528a989471dc1d378ba07d29a60ba1c",
|
||||
"sha256:8619b86cb68b185a778635be5b3e6018623c0761dde4df2f112896424aa27bd8",
|
||||
"sha256:87fad64529cde4f1914a5b9c383628e1a8f9e3930304c09cf22c2ae118a1280e",
|
||||
"sha256:89494df7f93b1836cae210c42864b292f9b31eeabca4810193761990dc689cce",
|
||||
"sha256:8cac7bb373a5f1423e28de3fd5fc8063b9c8ffe8957dc1b1a59cb90453db6da1",
|
||||
"sha256:8fd452dc3d49b3cc54483e033de6c006c304432e6f84b74d7b2c68afa2569ae5",
|
||||
"sha256:adad60eea2c4c2a1875eb6305a0b6e61a83163f8e233586a4d6a55221ef984fe",
|
||||
"sha256:c26f95e7609b821b5f08a72dab929baa0d685406b953efd7c89423a511d5c413",
|
||||
"sha256:cbe1324ef52ff26ccde2cb84b8593c8bf930069dfc06c1e616f1bfd4e47f48a3",
|
||||
"sha256:d05c4adae06bd0c7f696ae3ec8d993ed8ffcc4e11a76b1b35a5af8a099bd2284",
|
||||
"sha256:d98bc827a1293ae767c8f2f18be3bb5151fd37ddcd7da2a5f9581baeeb7a3fa1",
|
||||
"sha256:da2fb75f64792c1fc64c82313a00c728a7c301efe6a60b7a9fe35b16b4368ce7",
|
||||
"sha256:e4624d7edb2576cd72bb83636cd71c8ce544d8e272f308bd80885056972ca299",
|
||||
"sha256:e89e0d9e106f8a9180a4ca92a6adde60c58b1b0299e1b43bd5e0312f535fbf33",
|
||||
"sha256:f11c2437fb5f812d020932119ba02d9e2bc29a6eca01a055233a8b449e3e1e7d",
|
||||
"sha256:f57be5673e12763dd400fea568608700a63ce1c6bd5bdbc3cc3a2c5fdb045274",
|
||||
"sha256:fc728ece3d5c772c196fd338a99798e7efac7a04f9cb6416299a3638ee9a94cd"
|
||||
"sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb",
|
||||
"sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804",
|
||||
"sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6",
|
||||
"sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe",
|
||||
"sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de",
|
||||
"sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36",
|
||||
"sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e",
|
||||
"sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc",
|
||||
"sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d",
|
||||
"sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce",
|
||||
"sha256:6557af9e0d23f46b8cd56f8af08eaac72d2e3c632ac8d5cf4e20215a8dca7cea",
|
||||
"sha256:73a40d4fcd35fdedce07b5885905753d5d4edf413fbe53544dd871f27d48bd4f",
|
||||
"sha256:8280f9dae4adb5889ce0bb3ec6a541bf05434db5f9ab7673078c00713d148365",
|
||||
"sha256:83469ad15262402b0e0974e612546bc0b05f379b5aa9072ebf66d0f8fef16bea",
|
||||
"sha256:860d0fe234922fd5552b7f807fbb039e3e7ca58c18c8d38aa0d0a95ddf4f6c23",
|
||||
"sha256:883c9fb62cebd1e7126dd683222b3b919657590c3e2db33bdc50ebbad53e0338",
|
||||
"sha256:8afcb6f4064d234a43fea108859942d9795c4060ed0fbd9082b0f280181a15c1",
|
||||
"sha256:96f51489ac187f4bab588cf51f9ff2d40b6d170ac9a4270ffaed535c8404256b",
|
||||
"sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e",
|
||||
"sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba",
|
||||
"sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02",
|
||||
"sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86",
|
||||
"sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d",
|
||||
"sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7",
|
||||
"sha256:e49947d583fe4d29af528677e4f0aa21f5e535ca2ae69c48270ebebd0d8843c0",
|
||||
"sha256:eb1d71643e4154398b02e88a42fc8b29db8c44ce4134cf0f4474bfc5cb5d4dac",
|
||||
"sha256:f2e8a9c0c8813a468aa659a01af6592f71cd30237ec27c4cc0683f089f90dcfc",
|
||||
"sha256:fe7fe11019fc3e6600819775a7d55abc5446dda07e9795f5954fdbf8a49e1c37"
|
||||
],
|
||||
"markers": "python_version >= '3.0'",
|
||||
"version": "==1.3.18"
|
||||
"version": "==1.3.19"
|
||||
},
|
||||
"tablib": {
|
||||
"hashes": [
|
||||
@@ -325,27 +326,33 @@
|
||||
},
|
||||
"tldextract": {
|
||||
"hashes": [
|
||||
"sha256:16b2f7e81d89c2a5a914d25bdbddd3932c31a6b510db886c3ce0764a195c0ee7",
|
||||
"sha256:9aa21a1f7827df4209e242ec4fc2293af5940ec730cde46ea80f66ed97bfc808"
|
||||
"sha256:ab0e38977a129c72729476d5f8c85a8e1f8e49e9202e1db8dca76e95da7be9a8",
|
||||
"sha256:c2a8a392edf3ea6fa8be80930f04c3ac29e91fa604cb2139bdf6a37fc1e1ac6d"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.2.2"
|
||||
"version": "==2.2.3"
|
||||
},
|
||||
"tqdm": {
|
||||
"hashes": [
|
||||
"sha256:63ef7a6d3eb39f80d6b36e4867566b3d8e5f1fe3d6cb50c5e9ede2b3198ba7b7",
|
||||
"sha256:7810e627bcf9d983a99d9ff8a0c09674400fd2927eddabeadf153c14a2ec8656"
|
||||
"sha256:1a336d2b829be50e46b84668691e0a2719f26c97c62846298dd5ae2937e4d5cf",
|
||||
"sha256:564d632ea2b9cb52979f7956e093e831c28d441c11751682f84c86fc46e4fd21"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.47.0"
|
||||
"version": "==4.48.2"
|
||||
},
|
||||
"treelib": {
|
||||
"hashes": [
|
||||
"sha256:1cbfffb2d2b75ccac27d0200cee0507b6fbb0726e0afb9fae017ade5d2ce8788"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.6.1"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527",
|
||||
"sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"
|
||||
"sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a",
|
||||
"sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
|
||||
"version": "==1.25.9"
|
||||
"version": "==1.25.10"
|
||||
},
|
||||
"win32-setctime": {
|
||||
"hashes": [
|
||||
@@ -357,25 +364,25 @@
|
||||
},
|
||||
"yarl": {
|
||||
"hashes": [
|
||||
"sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce",
|
||||
"sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6",
|
||||
"sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce",
|
||||
"sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae",
|
||||
"sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d",
|
||||
"sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f",
|
||||
"sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b",
|
||||
"sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b",
|
||||
"sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb",
|
||||
"sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462",
|
||||
"sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea",
|
||||
"sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70",
|
||||
"sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1",
|
||||
"sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a",
|
||||
"sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b",
|
||||
"sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080",
|
||||
"sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2"
|
||||
"sha256:040b237f58ff7d800e6e0fd89c8439b841f777dd99b4a9cca04d6935564b9409",
|
||||
"sha256:17668ec6722b1b7a3a05cc0167659f6c95b436d25a36c2d52db0eca7d3f72593",
|
||||
"sha256:3a584b28086bc93c888a6c2aa5c92ed1ae20932f078c46509a66dce9ea5533f2",
|
||||
"sha256:4439be27e4eee76c7632c2427ca5e73703151b22cae23e64adb243a9c2f565d8",
|
||||
"sha256:48e918b05850fffb070a496d2b5f97fc31d15d94ca33d3d08a4f86e26d4e7c5d",
|
||||
"sha256:9102b59e8337f9874638fcfc9ac3734a0cfadb100e47d55c20d0dc6087fb4692",
|
||||
"sha256:9b930776c0ae0c691776f4d2891ebc5362af86f152dd0da463a6614074cb1b02",
|
||||
"sha256:b3b9ad80f8b68519cc3372a6ca85ae02cc5a8807723ac366b53c0f089db19e4a",
|
||||
"sha256:bc2f976c0e918659f723401c4f834deb8a8e7798a71be4382e024bcc3f7e23a8",
|
||||
"sha256:c22c75b5f394f3d47105045ea551e08a3e804dc7e01b37800ca35b58f856c3d6",
|
||||
"sha256:c52ce2883dc193824989a9b97a76ca86ecd1fa7955b14f87bf367a61b6232511",
|
||||
"sha256:ce584af5de8830d8701b8979b18fcf450cef9a382b1a3c8ef189bedc408faf1e",
|
||||
"sha256:da456eeec17fa8aa4594d9a9f27c0b1060b6a75f2419fe0c00609587b2695f4a",
|
||||
"sha256:db6db0f45d2c63ddb1a9d18d1b9b22f308e52c83638c26b422d520a815c4b3fb",
|
||||
"sha256:df89642981b94e7db5596818499c4b2219028f2a528c9c37cc1de45bf2fd3a3f",
|
||||
"sha256:f18d68f2be6bf0e89f1521af2b1bb46e66ab0018faafa81d70f358153170a317",
|
||||
"sha256:f379b7f83f23fe12823085cd6b906edc49df969eb99757f58ff382349a3303c6"
|
||||
],
|
||||
"version": "==1.4.2"
|
||||
"version": "==1.5.1"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
|
||||
@@ -10,7 +10,6 @@ OneForAll subdomain brute module
|
||||
import gc
|
||||
import json
|
||||
import time
|
||||
import random
|
||||
import secrets
|
||||
|
||||
import exrex
|
||||
@@ -21,67 +20,137 @@ from dns.resolver import NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers
|
||||
|
||||
import dbexport
|
||||
from common import utils
|
||||
from common import similarity
|
||||
from config import settings
|
||||
from common.module import Module
|
||||
from config.log import logger
|
||||
|
||||
|
||||
@tenacity.retry(stop=tenacity.stop_after_attempt(3))
|
||||
def do_query_a(domain, resolver):
|
||||
def config_resolver(nameservers):
|
||||
"""
|
||||
配置DNS解析器
|
||||
|
||||
:param nameservers: 名称解析服务器地址
|
||||
"""
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.nameservers = nameservers
|
||||
resolver.rotate = True # 随机使用NS
|
||||
resolver.cache = None # 不使用DNS缓存
|
||||
return resolver
|
||||
|
||||
|
||||
def gen_random_subdomains(domain, count):
|
||||
"""
|
||||
生成指定数量的随机子域域名列表
|
||||
|
||||
:param domain: 主域
|
||||
:param count: 数量
|
||||
"""
|
||||
subdomains = set()
|
||||
if count < 1:
|
||||
return subdomains
|
||||
for _ in range(count):
|
||||
token = secrets.token_hex(4)
|
||||
subdomains.add(f'{token}.{domain}')
|
||||
return subdomains
|
||||
|
||||
|
||||
def query_a_record(subdomain, resolver):
|
||||
"""
|
||||
查询子域A记录
|
||||
|
||||
:param subdomain: 子域
|
||||
:param resolver: DNS解析器
|
||||
"""
|
||||
try:
|
||||
answer = resolver.query(domain, 'A')
|
||||
# If resolve random subdomain raise timeout error, try again
|
||||
except Timeout as e:
|
||||
logger.log('ALERT', f'DNS resolve timeout, retrying')
|
||||
logger.log('DEBUG', e.args)
|
||||
raise tenacity.TryAgain
|
||||
# If resolve random subdomain raise NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers error
|
||||
# It means that there is no A record of random subdomain and not use wildcard dns record
|
||||
except (NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers) as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
logger.log('INFOR', f'{domain} seems not use wildcard dns record')
|
||||
return False
|
||||
answer = resolver.query(subdomain, 'A')
|
||||
except Exception as e:
|
||||
logger.log('ALERT', f'Detect {domain} wildcard dns record error')
|
||||
logger.log('FATAL', e.args)
|
||||
exit(1)
|
||||
else:
|
||||
if answer.rrset is None:
|
||||
logger.log('ALERT', f'DNS resolve dont have result, retrying')
|
||||
raise tenacity.TryAgain
|
||||
ttl = answer.ttl
|
||||
name = answer.name
|
||||
ips = {item.address for item in answer}
|
||||
logger.log('ALERT', f'{domain} use wildcard dns record')
|
||||
logger.log('ALERT', f'{domain} resolve to: {name} '
|
||||
f'IP: {ips} TTL: {ttl}')
|
||||
logger.log('DEBUG', f'Query {subdomain} wildcard dns record error')
|
||||
logger.log('DEBUG', e.args)
|
||||
return False
|
||||
if answer.rrset is None:
|
||||
return False
|
||||
ttl = answer.ttl
|
||||
name = answer.name
|
||||
ips = {item.address for item in answer}
|
||||
logger.log('ALERT', f'{subdomain} resolve to: {name} '
|
||||
f'IP: {ips} TTL: {ttl}')
|
||||
return True
|
||||
|
||||
|
||||
def all_resolve_success(subdomains):
|
||||
"""
|
||||
判断是否所有子域都解析成功
|
||||
|
||||
:param subdomains: 子域列表
|
||||
"""
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.cache = None # 不使用DNS缓存
|
||||
status = set()
|
||||
for subdomain in subdomains:
|
||||
status.add(query_a_record(subdomain, resolver))
|
||||
return all(status)
|
||||
|
||||
|
||||
def all_request_success(subdomains):
|
||||
"""
|
||||
判断是否所有子域都请求成功
|
||||
|
||||
:param subdomains: 子域列表
|
||||
"""
|
||||
result = list()
|
||||
for subdomain in subdomains:
|
||||
url = f'http://{subdomain}'
|
||||
resp = utils.get_url_resp(url)
|
||||
if resp:
|
||||
logger.log('ALERT', f'Request: {url} Status: {resp.status_code} '
|
||||
f'Size: {len(resp.content)}')
|
||||
result.append(resp.text)
|
||||
else:
|
||||
result.append(resp)
|
||||
return all(result), result
|
||||
|
||||
|
||||
def any_similar_html(resp_list):
|
||||
"""
|
||||
判断是否有一组HTML页面结构相似
|
||||
|
||||
:param resp_list: 响应HTML页面
|
||||
"""
|
||||
html_doc1, html_doc2, html_doc3 = resp_list
|
||||
if similarity.is_similar(html_doc1, html_doc2):
|
||||
return True
|
||||
if similarity.is_similar(html_doc1, html_doc3):
|
||||
return True
|
||||
if similarity.is_similar(html_doc2, html_doc3):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def detect_wildcard(domain, authoritative_ns):
|
||||
def detect_wildcard(domain):
|
||||
"""
|
||||
Detect use wildcard dns record or not
|
||||
|
||||
:param str domain: domain
|
||||
:param list authoritative_ns: authoritative name server
|
||||
:return bool use wildcard dns record or not
|
||||
"""
|
||||
logger.log('INFOR', f'Detecting {domain} use wildcard dns record or not')
|
||||
token = secrets.token_hex(4)
|
||||
random_subdomain = f'{token}.{domain}'
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.nameservers = authoritative_ns
|
||||
resolver.rotate = True
|
||||
resolver.cache = None
|
||||
try:
|
||||
wildcard = do_query_a(random_subdomain, resolver)
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
logger.log('ALERT', f'Multiple detection errors, so temporarily {domain} '
|
||||
f'does not use wildcard dns record')
|
||||
random_subdomains = gen_random_subdomains(domain, 3)
|
||||
if not all_resolve_success(random_subdomains):
|
||||
return False
|
||||
is_all_success, all_request_resp = all_request_success(random_subdomains)
|
||||
if not is_all_success:
|
||||
return True
|
||||
return any_similar_html(all_request_resp)
|
||||
|
||||
|
||||
def is_enable_wildcard(domain):
|
||||
is_enable = detect_wildcard(domain)
|
||||
if is_enable:
|
||||
logger.log('ALERT', f'The domain {domain} enables wildcard')
|
||||
else:
|
||||
return wildcard
|
||||
logger.log('ALERT', f'The domain {domain} disables wildcard')
|
||||
return is_enable
|
||||
|
||||
|
||||
def gen_subdomains(expression, path):
|
||||
@@ -229,9 +298,9 @@ def collect_wildcard_record(domain, authoritative_ns):
|
||||
if not authoritative_ns:
|
||||
return list(), int()
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.nameservers = authoritative_ns
|
||||
resolver.rotate = True
|
||||
resolver.cache = None
|
||||
resolver.nameservers = authoritative_ns # 使用权威名称服务器
|
||||
resolver.rotate = True # 随机使用NS
|
||||
resolver.cache = None # 不使用DNS缓存
|
||||
ips = set()
|
||||
ttl = int()
|
||||
ips_stat = dict()
|
||||
@@ -596,7 +665,7 @@ class Brute(Module):
|
||||
wildcard_ttl = int() # 泛解析TTL整型值
|
||||
ns_list = query_domain_ns(self.domain)
|
||||
ns_ip_list = query_domain_ns_a(ns_list) # DNS权威名称服务器对应A记录列表
|
||||
self.enable_wildcard = detect_wildcard(domain, ns_ip_list)
|
||||
self.enable_wildcard = is_enable_wildcard(domain)
|
||||
|
||||
if self.enable_wildcard:
|
||||
wildcard_ips, wildcard_ttl = collect_wildcard_record(domain,
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"""
|
||||
根据网页结构判断页面相似性(Determine page similarity based on HTML page structure)
|
||||
判断方法：根据网页的DOM树确定网页的模板特征向量，对模板特征向量计算网页结构相似性。
|
||||
来源地址：https://github.com/SPuerBRead/HTMLSimilarity
|
||||
计算参考: https://patents.google.com/patent/CN101694668B/zh
|
||||
"""
|
||||
from treelib import Tree
|
||||
from bs4 import BeautifulSoup
|
||||
import bs4
|
||||
|
||||
|
||||
class DOMTree(object):
|
||||
def __init__(self, label, attrs):
|
||||
self.label = label
|
||||
self.attrs = attrs
|
||||
|
||||
|
||||
class HTMLParser(object):
|
||||
def __init__(self, html):
|
||||
self.dom_id = 1
|
||||
self.dom_tree = Tree()
|
||||
self.bs_html = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
def get_dom_structure_tree(self):
|
||||
for content in self.bs_html.contents:
|
||||
if isinstance(content, bs4.element.Tag):
|
||||
self.bs_html = content
|
||||
self.recursive_descendants(self.bs_html, 1)
|
||||
return self.dom_tree
|
||||
|
||||
def recursive_descendants(self, descendants, parent_id):
|
||||
if self.dom_id == 1:
|
||||
self.dom_tree.create_node(descendants.name, self.dom_id,
|
||||
data=DOMTree(descendants.name, descendants.attrs))
|
||||
self.dom_id = self.dom_id + 1
|
||||
for child in descendants.contents:
|
||||
if isinstance(child, bs4.element.Tag):
|
||||
self.dom_tree.create_node(child.name, self.dom_id, parent_id,
|
||||
data=DOMTree(child.name, child.attrs))
|
||||
self.dom_id = self.dom_id + 1
|
||||
self.recursive_descendants(child, self.dom_id - 1)
|
||||
|
||||
|
||||
class Converter(object):
|
||||
def __init__(self, dom_tree, dimension):
|
||||
self.dom_tree = dom_tree
|
||||
self.node_info_list = []
|
||||
self.dimension = dimension
|
||||
self.initial_weight = 1
|
||||
self.attenuation_ratio = 0.6
|
||||
self.dom_eigenvector = {}.fromkeys(range(0, dimension), 0)
|
||||
|
||||
def get_eigenvector(self):
|
||||
for node_id in range(1, self.dom_tree.size() + 1):
|
||||
node = self.dom_tree.get_node(node_id)
|
||||
node_feature = self.create_feature(node)
|
||||
feature_hash = self.feature_hash(node_feature)
|
||||
node_weight = self.calculate_weight(node, node_id, feature_hash)
|
||||
self.construct_eigenvector(feature_hash, node_weight)
|
||||
return self.dom_eigenvector
|
||||
|
||||
@staticmethod
|
||||
def create_feature(node):
|
||||
node_attr_list = []
|
||||
node_feature = node.data.label + '|'
|
||||
for attr in node.data.attrs.keys():
|
||||
node_attr_list.append(attr + ':' + str(node.data.attrs[attr]))
|
||||
node_feature += '|'.join(node_attr_list)
|
||||
return node_feature
|
||||
|
||||
@staticmethod
|
||||
def feature_hash(node_feature):
|
||||
return abs(hash(node_feature)) % (10 ** 8)
|
||||
|
||||
def calculate_weight(self, node, node_id, feature_hash):
|
||||
brother_node_count = 0
|
||||
depth = self.dom_tree.depth(node)
|
||||
for brother_node in self.dom_tree.siblings(node_id):
|
||||
brother_node_feature = self.create_feature(brother_node)
|
||||
brother_node_feature_hash = self.feature_hash(brother_node_feature)
|
||||
if brother_node_feature_hash == feature_hash:
|
||||
brother_node_count = brother_node_count + 1
|
||||
if brother_node_count:
|
||||
node_weight = self.initial_weight * self.attenuation_ratio ** depth \
|
||||
* self.attenuation_ratio ** brother_node_count
|
||||
else:
|
||||
node_weight = self.initial_weight * self.attenuation_ratio ** depth
|
||||
return node_weight
|
||||
|
||||
def construct_eigenvector(self, feature_hash, node_weight):
|
||||
feature_hash = feature_hash % self.dimension
|
||||
self.dom_eigenvector[feature_hash] += node_weight
|
||||
|
||||
|
||||
def calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension):
|
||||
a, b = 0, 0
|
||||
for i in range(dimension):
|
||||
a += dom1_eigenvector[i]-dom2_eigenvector[i]
|
||||
if dom1_eigenvector[i] and dom2_eigenvector[i]:
|
||||
b += dom1_eigenvector[i] + dom2_eigenvector[i]
|
||||
pseudodistance = abs(a)/b
|
||||
return pseudodistance
|
||||
|
||||
|
||||
def get_pseudodistance(html_doc1, html_doc2, dimension=5000):
|
||||
"""
|
||||
获取html文档结构相似度
|
||||
|
||||
:param str html_doc1: html文档
|
||||
:param str html_doc2: html文档
|
||||
:param int dimension: 降维后的维数
|
||||
:return 伪距离
|
||||
"""
|
||||
hp1 = HTMLParser(html_doc1)
|
||||
html_doc1_dom_tree = hp1.get_dom_structure_tree()
|
||||
hp2 = HTMLParser(html_doc2)
|
||||
html_doc2_dom_tree = hp2.get_dom_structure_tree()
|
||||
converter = Converter(html_doc1_dom_tree, dimension)
|
||||
dom1_eigenvector = converter.get_eigenvector()
|
||||
converter = Converter(html_doc2_dom_tree, dimension)
|
||||
dom2_eigenvector = converter.get_eigenvector()
|
||||
return calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension)
|
||||
|
||||
|
||||
def is_similar(html_doc1, html_doc2, dimension=5000):
|
||||
"""
|
||||
根据计算出的伪距离来判断是否html页面结构相似
|
||||
|
||||
:param str html_doc1: html文档
|
||||
:param str html_doc2: html文档
|
||||
:param int dimension: 降维后的维数
|
||||
:return 是否相似（伪距离value<0.2时相似，value>0.2时不相似）
|
||||
"""
|
||||
value = get_pseudodistance(html_doc1, html_doc2, dimension)
|
||||
if value > 0.2:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
@@ -741,3 +741,15 @@ def check_random_subdomain(subdomains):
|
||||
if subdomain:
|
||||
logger.log('ALERT', f'Please check whether {subdomain} is correct or not')
|
||||
return True
|
||||
|
||||
|
||||
def get_url_resp(url):
|
||||
timeout = settings.request_timeout
|
||||
verify = settings.request_verify
|
||||
try:
|
||||
resp = requests.get(url, params=None, timeout=timeout, verify=verify)
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', f'Error request {url}')
|
||||
logger.log('DEBUG', e.args)
|
||||
return None
|
||||
return resp
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 43 KiB |
Reference in New Issue
Block a user