diff --git a/Pipfile b/Pipfile index 3f7c4a8..490f42b 100644 --- a/Pipfile +++ b/Pipfile @@ -13,15 +13,13 @@ requests = "*" records = "*" tldextract = "*" exrex = "*" -aiohttp = "*" fire = "*" bs4 = "*" pysocks = "*" cloudscraper = "*" tablib = "*" -psutil = "*" tenacity = "*" -ip2location = "*" +treelib = "*" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index 9671d42..ad71cf1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6556f720e7fb052c3f923f3e7986cc00c1cc230db729ad79685a127c02568c96" + "sha256": "fde2c88929fdf63725faea24a4fc7d82d6792caaca6676c08e96a40227f47e18" }, "pipfile-spec": 6, "requires": { @@ -16,38 +16,6 @@ ] }, "default": { - "aiohttp": { - "hashes": [ - "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e", - "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326", - "sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a", - "sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654", - "sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a", - "sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4", - "sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17", - "sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec", - "sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd", - "sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48", - "sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59", - "sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965" - ], - "index": "pypi", - "version": "==3.6.2" - }, - "async-timeout": { - "hashes": [ - "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", - "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" - ], - "version": "==3.0.1" - }, - "attrs": { - "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" - ], - "version": "==19.3.0" - }, "beautifulsoup4": { "hashes": [ "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7", @@ -79,11 +47,11 @@ }, "cloudscraper": { "hashes": [ - "sha256:1d4e158de3be7a4e651083f13a14a25b75853186e989d9d97db1a08056b7a0b6", - "sha256:c8c255aaa84e80d2cb5bf2c1ad1ffa000ce864595ca4cdb75ff7aab6f97befec" + "sha256:793095bbc37aae84a5c38d9f66b56f3c83a00f0b1ff2d334f75a2b8f88b924af", + "sha256:a7ae28b5dc4e4ef0789ef608c42a4382bf6d3e3dd6b8117c2e3633103131ae43" ], "index": "pypi", - "version": "==1.2.42" + "version": "==1.2.46" }, "colorama": { "hashes": [ @@ -95,11 +63,11 @@ }, "dnspython": { "hashes": [ - "sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01", - "sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d" + "sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7", + "sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d" ], "index": "pypi", - "version": "==1.16.0" + "version": "==2.0.0" }, "docopt": { "hashes": [ @@ -127,22 +95,19 @@ "index": "pypi", "version": "==0.3.1" }, + "future": { + "hashes": [ + "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" + ], + "version": "==0.18.2" + }, "idna": { "hashes": [ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.10" }, - "ip2location": { - "hashes": [ - "sha256:4a83f32098091b6aafaef63b393331bf973bbf54547db82de38f30108b5f079c", - "sha256:bbddbd5e3f79c18a9710afd43be09cb21724d7324557668017c545b5c5e62bc4" - ], - "index": "pypi", - "version": "==8.4.1" - }, "jdcal": { "hashes": [ "sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba", @@ -158,58 +123,17 @@ "index": "pypi", "version": "==0.5.1" }, - "multidict": { - "hashes": [ - "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a", - "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000", - "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2", - "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507", - "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5", - "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7", - "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d", - "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463", - "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19", - "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3", - "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b", - "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c", - "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87", - "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7", - "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430", - "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255", - "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d" - ], - "markers": "python_version >= '3.5'", - "version": "==4.7.6" - }, "openpyxl": { "hashes": [ "sha256:626d38647c063d55803ef4971c4d43226538d4e95cb6260c094e363ee33e10c7" ], "version": "==2.4.11" }, - "psutil": { - "hashes": [ - "sha256:1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058", - "sha256:298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953", - "sha256:60b86f327c198561f101a92be1995f9ae0399736b6eced8f24af41ec64fb88d4", - "sha256:685ec16ca14d079455892f25bd124df26ff9137664af445563c1bd36629b5e0e", - "sha256:73f35ab66c6c7a9ce82ba44b1e9b1050be2a80cd4dcc3352cc108656b115c74f", - "sha256:75e22717d4dbc7ca529ec5063000b2b294fc9a367f9c9ede1f65846c7955fd38", - "sha256:a02f4ac50d4a23253b68233b07e7cdb567bd025b982d5cf0ee78296990c22d9e", - "sha256:d008ddc00c6906ec80040d26dc2d3e3962109e40ad07fd8a12d0284ce5e0e4f8", - "sha256:d84029b190c8a66a946e28b4d3934d2ca1528ec94764b180f7d6ea57b0e75e26", - "sha256:e2d0c5b07c6fe5a87fa27b7855017edb0d52ee73b71e6ee368fae268605cc3f5", - "sha256:f344ca230dd8e8d5eee16827596f1c22ec0876127c28e800d7ae20ed44c4b310" - ], - "index": "pypi", - "version": "==5.7.0" - }, "pyparsing": { "hashes": [ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "pysocks": { @@ -256,7 +180,6 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "soupsieve": { @@ -264,42 +187,45 @@ "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" ], - "markers": "python_version >= '3.5'", "version": "==2.0.1" }, "sqlalchemy": { "hashes": [ - "sha256:0942a3a0df3f6131580eddd26d99071b48cfe5aaf3eab2783076fbc5a1c1882e", - "sha256:0ec575db1b54909750332c2e335c2bb11257883914a03bc5a3306a4488ecc772", - "sha256:109581ccc8915001e8037b73c29590e78ce74be49ca0a3630a23831f9e3ed6c7", - "sha256:16593fd748944726540cd20f7e83afec816c2ac96b082e26ae226e8f7e9688cf", - "sha256:427273b08efc16a85aa2b39892817e78e3ed074fcb89b2a51c4979bae7e7ba98", - "sha256:50c4ee32f0e1581828843267d8de35c3298e86ceecd5e9017dc45788be70a864", - "sha256:512a85c3c8c3995cc91af3e90f38f460da5d3cade8dc3a229c8e0879037547c9", - "sha256:57aa843b783179ab72e863512e14bdcba186641daf69e4e3a5761d705dcc35b1", - "sha256:621f58cd921cd71ba6215c42954ffaa8a918eecd8c535d97befa1a8acad986dd", - "sha256:6ac2558631a81b85e7fb7a44e5035347938b0a73f5fdc27a8566777d0792a6a4", - "sha256:716754d0b5490bdcf68e1e4925edc02ac07209883314ad01a137642ddb2056f1", - "sha256:736d41cfebedecc6f159fc4ac0769dc89528a989471dc1d378ba07d29a60ba1c", - "sha256:8619b86cb68b185a778635be5b3e6018623c0761dde4df2f112896424aa27bd8", - "sha256:87fad64529cde4f1914a5b9c383628e1a8f9e3930304c09cf22c2ae118a1280e", - "sha256:89494df7f93b1836cae210c42864b292f9b31eeabca4810193761990dc689cce", - "sha256:8cac7bb373a5f1423e28de3fd5fc8063b9c8ffe8957dc1b1a59cb90453db6da1", - "sha256:8fd452dc3d49b3cc54483e033de6c006c304432e6f84b74d7b2c68afa2569ae5", - "sha256:adad60eea2c4c2a1875eb6305a0b6e61a83163f8e233586a4d6a55221ef984fe", - "sha256:c26f95e7609b821b5f08a72dab929baa0d685406b953efd7c89423a511d5c413", - "sha256:cbe1324ef52ff26ccde2cb84b8593c8bf930069dfc06c1e616f1bfd4e47f48a3", - "sha256:d05c4adae06bd0c7f696ae3ec8d993ed8ffcc4e11a76b1b35a5af8a099bd2284", - "sha256:d98bc827a1293ae767c8f2f18be3bb5151fd37ddcd7da2a5f9581baeeb7a3fa1", - "sha256:da2fb75f64792c1fc64c82313a00c728a7c301efe6a60b7a9fe35b16b4368ce7", - "sha256:e4624d7edb2576cd72bb83636cd71c8ce544d8e272f308bd80885056972ca299", - "sha256:e89e0d9e106f8a9180a4ca92a6adde60c58b1b0299e1b43bd5e0312f535fbf33", - "sha256:f11c2437fb5f812d020932119ba02d9e2bc29a6eca01a055233a8b449e3e1e7d", - "sha256:f57be5673e12763dd400fea568608700a63ce1c6bd5bdbc3cc3a2c5fdb045274", - "sha256:fc728ece3d5c772c196fd338a99798e7efac7a04f9cb6416299a3638ee9a94cd" + "sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb", + "sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804", + "sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6", + "sha256:26c5ca9d09f0e21b8671a32f7d83caad5be1f6ff45eef5ec2f6fd0db85fc5dc0", + "sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe", + "sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de", + "sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36", + "sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e", + "sha256:465c999ef30b1c7525f81330184121521418a67189053bcf585824d833c05b66", + "sha256:51064ee7938526bab92acd049d41a1dc797422256086b39c08bafeffb9d304c6", + "sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc", + "sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d", + "sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce", + "sha256:6557af9e0d23f46b8cd56f8af08eaac72d2e3c632ac8d5cf4e20215a8dca7cea", + "sha256:73a40d4fcd35fdedce07b5885905753d5d4edf413fbe53544dd871f27d48bd4f", + "sha256:8280f9dae4adb5889ce0bb3ec6a541bf05434db5f9ab7673078c00713d148365", + "sha256:83469ad15262402b0e0974e612546bc0b05f379b5aa9072ebf66d0f8fef16bea", + "sha256:860d0fe234922fd5552b7f807fbb039e3e7ca58c18c8d38aa0d0a95ddf4f6c23", + "sha256:883c9fb62cebd1e7126dd683222b3b919657590c3e2db33bdc50ebbad53e0338", + "sha256:8afcb6f4064d234a43fea108859942d9795c4060ed0fbd9082b0f280181a15c1", + "sha256:96f51489ac187f4bab588cf51f9ff2d40b6d170ac9a4270ffaed535c8404256b", + "sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e", + "sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba", + "sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02", + "sha256:b6ff91356354b7ff3bd208adcf875056d3d886ed7cef90c571aef2ab8a554b12", + "sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86", + "sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d", + "sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7", + "sha256:e49947d583fe4d29af528677e4f0aa21f5e535ca2ae69c48270ebebd0d8843c0", + "sha256:eb1d71643e4154398b02e88a42fc8b29db8c44ce4134cf0f4474bfc5cb5d4dac", + "sha256:f2e8a9c0c8813a468aa659a01af6592f71cd30237ec27c4cc0683f089f90dcfc", + "sha256:fe7fe11019fc3e6600819775a7d55abc5446dda07e9795f5954fdbf8a49e1c37" ], "markers": "python_version >= '3.0'", - "version": "==1.3.18" + "version": "==1.3.19" }, "tablib": { "hashes": [ @@ -325,27 +251,33 @@ }, "tldextract": { "hashes": [ - "sha256:16b2f7e81d89c2a5a914d25bdbddd3932c31a6b510db886c3ce0764a195c0ee7", - "sha256:9aa21a1f7827df4209e242ec4fc2293af5940ec730cde46ea80f66ed97bfc808" + "sha256:ab0e38977a129c72729476d5f8c85a8e1f8e49e9202e1db8dca76e95da7be9a8", + "sha256:c2a8a392edf3ea6fa8be80930f04c3ac29e91fa604cb2139bdf6a37fc1e1ac6d" ], "index": "pypi", - "version": "==2.2.2" + "version": "==2.2.3" }, "tqdm": { "hashes": [ - "sha256:63ef7a6d3eb39f80d6b36e4867566b3d8e5f1fe3d6cb50c5e9ede2b3198ba7b7", - "sha256:7810e627bcf9d983a99d9ff8a0c09674400fd2927eddabeadf153c14a2ec8656" + "sha256:1a336d2b829be50e46b84668691e0a2719f26c97c62846298dd5ae2937e4d5cf", + "sha256:564d632ea2b9cb52979f7956e093e831c28d441c11751682f84c86fc46e4fd21" ], "index": "pypi", - "version": "==4.47.0" + "version": "==4.48.2" + }, + "treelib": { + "hashes": [ + "sha256:1cbfffb2d2b75ccac27d0200cee0507b6fbb0726e0afb9fae017ade5d2ce8788" + ], + "index": "pypi", + "version": "==1.6.1" }, "urllib3": { "hashes": [ - "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", - "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" + "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", + "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.25.9" + "version": "==1.25.10" }, "win32-setctime": { "hashes": [ @@ -354,28 +286,6 @@ ], "markers": "sys_platform == 'win32'", "version": "==1.0.1" - }, - "yarl": { - "hashes": [ - "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce", - "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6", - "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce", - "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae", - "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d", - "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f", - "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b", - "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b", - "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb", - "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462", - "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea", - "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70", - "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1", - "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a", - "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b", - "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080", - "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2" - ], - "version": "==1.4.2" } }, "develop": {} diff --git a/README.md b/README.md index 701be30..59e5ba5 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll) [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability) [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE) -[![python](https://img.shields.io/badge/python-3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/) +[![python](https://img.shields.io/badge/python-3.6|3.7|3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/) [![python](https://img.shields.io/badge/release-v0.3.0-brightgreen)](https://github.com/shmilylty/OneForAll/releases) 👊**OneForAll是一款功能强大的子域收集工具** 📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md) @@ -15,21 +15,6 @@ 📢 请务必花一点时间阅读此文档，有助于你快速熟悉OneForAll！ -
-🐍安装要求 - -OneForAll基于[Python 3.8.0]( https://www.python.org/downloads/release/python-380/ )开发和测试，请使用高于Python 3.8.0的稳定发行版本，其他版本可能会出现一些问题（Windows平台必须使用3.8.0以上版本），安装Python环境可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)。运行以下命令检查Python和pip3版本： -```bash -python -V -pip3 -V -``` -如果你看到类似以下的输出便说明Python环境没有问题： -```bash -Python 3.8.0 -pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python38\site-packages\pip (python 3.8) -``` -
-
✔安装步骤（git 版） diff --git a/brute.py b/brute.py index 76712d8..06a4d4d 100644 --- a/brute.py +++ b/brute.py @@ -10,7 +10,6 @@ OneForAll subdomain brute module import gc import json import time -import random import secrets import exrex @@ -21,67 +20,137 @@ from dns.resolver import NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers import dbexport from common import utils +from common import similarity from config import settings from common.module import Module from config.log import logger -@tenacity.retry(stop=tenacity.stop_after_attempt(3)) -def do_query_a(domain, resolver): +def config_resolver(nameservers): + """ + 配置DNS解析器 + + :param nameservers: 名称解析服务器地址 + """ + resolver = utils.dns_resolver() + resolver.nameservers = nameservers + resolver.rotate = True # 随机使用NS + resolver.cache = None # 不使用DNS缓存 + return resolver + + +def gen_random_subdomains(domain, count): + """ + 生成指定数量的随机子域域名列表 + + :param domain: 主域 + :param count: 数量 + """ + subdomains = set() + if count < 1: + return subdomains + for _ in range(count): + token = secrets.token_hex(4) + subdomains.add(f'{token}.{domain}') + return subdomains + + +def query_a_record(subdomain, resolver): + """ + 查询子域A记录 + + :param subdomain: 子域 + :param resolver: DNS解析器 + """ try: - answer = resolver.query(domain, 'A') - # If resolve random subdomain raise timeout error, try again - except Timeout as e: - logger.log('ALERT', f'DNS resolve timeout, retrying') - logger.log('DEBUG', e.args) - raise tenacity.TryAgain - # If resolve random subdomain raise NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers error - # It means that there is no A record of random subdomain and not use wildcard dns record - except (NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers) as e: - logger.log('DEBUG', e.args) - logger.log('INFOR', f'{domain} seems not use wildcard dns record') - return False + answer = resolver.query(subdomain, 'A') except Exception as e: - logger.log('ALERT', f'Detect {domain} wildcard dns record error') - logger.log('FATAL', e.args) - exit(1) - else: - if answer.rrset is None: - logger.log('ALERT', f'DNS resolve dont have result, retrying') - raise tenacity.TryAgain - ttl = answer.ttl - name = answer.name - ips = {item.address for item in answer} - logger.log('ALERT', f'{domain} use wildcard dns record') - logger.log('ALERT', f'{domain} resolve to: {name} ' - f'IP: {ips} TTL: {ttl}') + logger.log('DEBUG', f'Query {subdomain} wildcard dns record error') + logger.log('DEBUG', e.args) + return False + if answer.rrset is None: + return False + ttl = answer.ttl + name = answer.name + ips = {item.address for item in answer} + logger.log('ALERT', f'{subdomain} resolve to: {name} ' + f'IP: {ips} TTL: {ttl}') + return True + + +def all_resolve_success(subdomains): + """ + 判断是否所有子域都解析成功 + + :param subdomains: 子域列表 + """ + resolver = utils.dns_resolver() + resolver.cache = None # 不使用DNS缓存 + status = set() + for subdomain in subdomains: + status.add(query_a_record(subdomain, resolver)) + return all(status) + + +def all_request_success(subdomains): + """ + 判断是否所有子域都请求成功 + + :param subdomains: 子域列表 + """ + result = list() + for subdomain in subdomains: + url = f'http://{subdomain}' + resp = utils.get_url_resp(url) + if resp: + logger.log('ALERT', f'Request: {url} Status: {resp.status_code} ' + f'Size: {len(resp.content)}') + result.append(resp.text) + else: + result.append(resp) + return all(result), result + + +def any_similar_html(resp_list): + """ + 判断是否有一组HTML页面结构相似 + + :param resp_list: 响应HTML页面 + """ + html_doc1, html_doc2, html_doc3 = resp_list + if similarity.is_similar(html_doc1, html_doc2): return True + if similarity.is_similar(html_doc1, html_doc3): + return True + if similarity.is_similar(html_doc2, html_doc3): + return True + return False -def detect_wildcard(domain, authoritative_ns): +def detect_wildcard(domain): """ Detect use wildcard dns record or not :param str domain: domain - :param list authoritative_ns: authoritative name server :return bool use wildcard dns record or not """ logger.log('INFOR', f'Detecting {domain} use wildcard dns record or not') - token = secrets.token_hex(4) - random_subdomain = f'{token}.{domain}' - resolver = utils.dns_resolver() - resolver.nameservers = authoritative_ns - resolver.rotate = True - resolver.cache = None - try: - wildcard = do_query_a(random_subdomain, resolver) - except Exception as e: - logger.log('DEBUG', e.args) - logger.log('ALERT', f'Multiple detection errors, so temporarily {domain} ' - f'does not use wildcard dns record') + random_subdomains = gen_random_subdomains(domain, 3) + if not all_resolve_success(random_subdomains): return False + is_all_success, all_request_resp = all_request_success(random_subdomains) + if not is_all_success: + return True + return any_similar_html(all_request_resp) + + +def is_enable_wildcard(domain): + is_enable = detect_wildcard(domain) + if is_enable: + logger.log('ALERT', f'The domain {domain} enables wildcard') else: - return wildcard + logger.log('ALERT', f'The domain {domain} disables wildcard') + return is_enable def gen_subdomains(expression, path): @@ -229,9 +298,9 @@ def collect_wildcard_record(domain, authoritative_ns): if not authoritative_ns: return list(), int() resolver = utils.dns_resolver() - resolver.nameservers = authoritative_ns - resolver.rotate = True - resolver.cache = None + resolver.nameservers = authoritative_ns # 使用权威名称服务器 + resolver.rotate = True # 随机使用NS + resolver.cache = None # 不使用DNS缓存 ips = set() ttl = int() ips_stat = dict() @@ -464,14 +533,13 @@ class Brute(Module): Example： brute.py --target domain.com --word True run brute.py --targets ./domains.txt --word True run - brute.py --target domain.com --word True --process 1 run + brute.py --target domain.com --word True --concurrent 2000 run brute.py --target domain.com --word True --wordlist subnames.txt run brute.py --target domain.com --word True --recursive True --depth 2 run brute.py --target d.com --fuzz True --place m.*.d.com --rule '[a-z]' run brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run Note: - --alive True/False Only export alive subdomains or not (default False) --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) --path Result path (default None, automatically generated) @@ -479,7 +547,7 @@ class Brute(Module): :param str target: One domain (target or targets must be provided) :param str targets: File path of one domain per line :param int process: Number of processes (default 1) - :param int concurrent: Number of concurrent (default 10000) + :param int concurrent: Number of concurrent (default 2000) :param bool word: Use word mode generate dictionary (default False) :param str wordlist: Dictionary path used in word mode (default use ./config/default.py) :param bool recursive: Use recursion (default False) @@ -524,8 +592,6 @@ class Brute(Module): self.domain = str() # 当前正在进行爆破的域名 self.ips_times = dict() # IP集合出现次数 self.enable_wildcard = False # 当前域名是否使用泛解析 - self.wildcard_check = settings.brute_wildcard_check - self.wildcard_deal = settings.brute_wildcard_deal self.check_env = True self.quite = False @@ -596,7 +662,7 @@ class Brute(Module): wildcard_ttl = int() # 泛解析TTL整型值 ns_list = query_domain_ns(self.domain) ns_ip_list = query_domain_ns_a(ns_list) # DNS权威名称服务器对应A记录列表 - self.enable_wildcard = detect_wildcard(domain, ns_ip_list) + self.enable_wildcard = is_enable_wildcard(domain) if self.enable_wildcard: wildcard_ips, wildcard_ttl = collect_wildcard_record(domain, diff --git a/common/check.py b/common/check.py new file mode 100644 index 0000000..fe0f6f6 --- /dev/null +++ b/common/check.py @@ -0,0 +1,26 @@ +from common.module import Module + + +class Check(Module): + """ + Check base class + """ + + def __init__(self): + Module.__init__(self) + self.request_status = 1 + + def to_check(self, filenames): + urls = set() + for filename in filenames: + urls.update((f'http://{self.domain}/{filename}', + f'https://{self.domain}/{filename}', + f'http://www.{self.domain}/{filename}', + f'https://www.{self.domain}/{filename}')) + for url in urls: + self.header = self.get_header() + self.proxy = self.get_proxy(self.source) + resp = self.get(url, check=False, ignore=True) + self.subdomains = self.collect_subdomains(resp) + if self.subdomains: + break diff --git a/common/database.py b/common/database.py index a61e22e..5fde260 100644 --- a/common/database.py +++ b/common/database.py @@ -78,8 +78,8 @@ class Database(object): f'cidr text,' f'asn text,' f'org text,' - f'ip2region text,' - f'ip2location text,' + f'addr text,' + f'isp text,' f'resolver text,' f'module text,' f'source text,' @@ -103,13 +103,13 @@ class Database(object): f'insert into "{table_name}" ' f'(id, alive, resolve, request, new, url, subdomain, port, level,' f'cname, content, public, cdn, status, reason, title, banner, header,' - f'history, response, times, ttl, cidr, asn, org, ip2region,' - f'ip2location, resolver, module, source, elapse, find) ' - f'values (:id, :alive, :resolve, :request, :new, :url, ' + f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,' + f'module, source, elapse, find) ' + f'values (:id, :alive, :resolve, :request, :new, :url,' f':subdomain, :port, :level, :cname, :content, :public, :cdn,' f':status, :reason, :title, :banner, :header, :history, :response,' - f':times, :ttl, :cidr, :asn, :org, :ip2region, :ip2location,' - f':resolver, :module, :source, :elapse, :find)', results) + f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,' + f':source, :elapse, :find)', results) except Exception as e: logger.log('ERROR', e) @@ -231,7 +231,7 @@ class Database(object): table_name = table_name.replace('.', '_') query = f'select id, new, alive, request, resolve, url, subdomain, level,' \ f'cname, content, public, cdn, port, status, reason, title, banner,' \ - f'cidr, asn, org, ip2region, ip2location, source from "{table_name}"' + f'cidr, asn, org, addr, isp, source from "{table_name}"' if alive and limit: if limit in ['resolve', 'request']: where = f' where {limit} = 1' diff --git a/common/domain.py b/common/domain.py index 3a00e6e..5867eb1 100644 --- a/common/domain.py +++ b/common/domain.py @@ -9,7 +9,6 @@ class Domain(object): :param str string: input string """ - def __init__(self, string): self.string = str(string) self.regexp = r'\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b' diff --git a/common/ipgeo.py b/common/ipgeo.py deleted file mode 100644 index 73a7ba4..0000000 --- a/common/ipgeo.py +++ /dev/null @@ -1,13 +0,0 @@ -import zipfile -from IP2Location import IP2Location -from config.setting import data_storage_dir - - -class IpGeoInfo(IP2Location): - def __init__(self): - zip_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN.ZIP") - bin_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN") - if not bin_path.exists(): - zf = zipfile.ZipFile(zip_path) - zf.extract('IP2LOCATION-LITE-DB3.BIN', data_storage_dir) - IP2Location.__init__(self, bin_path) diff --git a/common/ipreg.py b/common/ipreg.py index 8f0e802..b65d99e 100644 --- a/common/ipreg.py +++ b/common/ipreg.py @@ -10,6 +10,8 @@ import socket import struct import sys +from config import settings + class IpRegInfo(object): __INDEX_BLOCK_LENGTH = 12 @@ -39,7 +41,8 @@ class IpRegInfo(object): self.__dbBinStr = self.__f.read() # read all the contents in file self.__indexSPtr = self.get_long(self.__dbBinStr, 0) self.__indexLPtr = self.get_long(self.__dbBinStr, 4) - self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) / self.__INDEX_BLOCK_LENGTH) + 1 + self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) / + self.__INDEX_BLOCK_LENGTH) + 1 l, h, data_ptr = (0, self.__indexCount, 0) while l <= h: @@ -67,14 +70,16 @@ class IpRegInfo(object): " binary search method " param: ip """ - if not ip.isdigit(): ip = self.ip2long(ip) + if not ip.isdigit(): + ip = self.ip2long(ip) if self.__indexCount == 0: self.__f.seek(0) super_block = self.__f.read(8) self.__indexSPtr = self.get_long(super_block, 0) self.__indexLPtr = self.get_long(super_block, 4) - self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) / self.__INDEX_BLOCK_LENGTH) + 1 + self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) / + self.__INDEX_BLOCK_LENGTH) + 1 l, h, data_ptr = (0, self.__indexCount, 0) while l <= h: @@ -208,10 +213,9 @@ class IpRegInfo(object): self.__f.seek(data_ptr) data = self.__f.read(data_len) - return { - "city_id": self.get_long(data, 0), - "region": data[4:] - } + info = {"city_id": self.get_long(data, 0), + "region": data[4:].decode('utf-8')} + return info @staticmethod def ip2long(ip): @@ -244,3 +248,27 @@ class IpRegInfo(object): self.__dbBinStr = None self.__headerPtr = None self.__headerSip = None + + +class IpRegData(IpRegInfo): + def __init__(self): + path = settings.data_storage_dir.joinpath('ip2region.db') + IpRegInfo.__init__(self, path) + + def query(self, ip, algorithm='memory'): + algorithms = ['memory', 'binary', 'btree'] + if algorithm not in algorithms: + raise Exception(f"Only three query algorithms are supported: {algorithms}") + if algorithm == 'memory': + result = self.memory_search(ip) + elif algorithm == 'binary': + result = self.binary_search(ip) + else: + result = self.btree_search(ip) + addr_list = result.get('region').split('|') + addr = ''.join(filter(lambda x: x != '0', addr_list[:-1])) + isp = addr_list[-1] + if isp == '0': + isp = '未知' + info = {'addr': addr, 'isp': isp} + return info diff --git a/common/module.py b/common/module.py index a4b18a8..c41735b 100644 --- a/common/module.py +++ b/common/module.py @@ -22,9 +22,9 @@ class Module(object): self.cookie = None self.header = dict() self.proxy = None - self.delay = settings.request_delay # 请求睡眠时延 - self.timeout = settings.request_timeout # 请求超时时间 - self.verify = settings.request_verify # 请求SSL验证 + self.delay = 1 # 请求睡眠时延 + self.timeout = settings.request_timeout_second # 请求超时时间 + self.verify = settings.request_ssl_verify # 请求SSL验证 self.domain = str() # 当前进行子域名收集的主域 self.subdomains = set() # 存放发现的子域 self.infos = dict() # 存放子域有关信息 @@ -73,7 +73,7 @@ class Module(object): :param dict params: request parameters :param bool check: check response :param kwargs: other params - :return: requests's response object + :return: response object """ try: resp = requests.head(url, @@ -93,16 +93,20 @@ class Module(object): return resp return None - def get(self, url, params=None, check=True, **kwargs): + def get(self, url, params=None, check=True, ignore=False, **kwargs): """ Custom get request :param str url: request url :param dict params: request parameters :param bool check: check response + :param bool ignore: ignore error :param kwargs: other params - :return: requests's response object + :return: response object """ + level = 'ERROR' + if ignore: + level = 'DEBUG' try: resp = requests.get(url, params=params, @@ -113,7 +117,7 @@ class Module(object): verify=self.verify, **kwargs) except Exception as e: - logger.log('ERROR', e.args) + logger.log(level, e.args) return None if not check: return resp @@ -129,7 +133,7 @@ class Module(object): :param dict data: request data :param bool check: check response :param kwargs: other params - :return: requests's response object + :return: response object """ try: resp = requests.post(url, @@ -195,7 +199,7 @@ class Module(object): :param str module: module name :return: proxy """ - if not settings.enable_proxy: + if not settings.enable_request_proxy: logger.log('TRACE', f'All modules do not use proxy') return self.proxy if settings.proxy_all_module: @@ -280,8 +284,8 @@ class Module(object): 'cidr': None, 'asn': None, 'org': None, - 'ip2region': None, - 'ip2location': None, + 'addr': None, + 'isp': None, 'resolver': None, 'module': self.module, 'source': self.source, @@ -331,8 +335,8 @@ class Module(object): 'cidr': info.get('cidr'), 'asn': info.get('asn'), 'org': info.get('org'), - 'ip2region': info.get('ip2region'), - 'ip2location': info.get('ip2location'), + 'addr': info.get('addr'), + 'isp': info.get('isp'), 'resolver': info.get('resolver'), 'module': self.module, 'source': self.source, diff --git a/common/query.py b/common/query.py index b0ad133..be1a3a1 100644 --- a/common/query.py +++ b/common/query.py @@ -5,6 +5,5 @@ class Query(Module): """ Query base class """ - def __init__(self): Module.__init__(self) diff --git a/common/request.py b/common/request.py index e6caa5a..1966793 100644 --- a/common/request.py +++ b/common/request.py @@ -1,10 +1,10 @@ import json -import asyncio -import functools +from threading import Thread +from queue import Queue + -import aiohttp import tqdm -from aiohttp import ClientSession +import requests from bs4 import BeautifulSoup from common import utils @@ -12,17 +12,17 @@ from config.log import logger from config import settings -def get_limit_conn(): - count = settings.limit_open_conn +def req_thread_count(): + count = settings.request_thread_count if isinstance(count, int): count = max(16, count) else: - count = utils.get_coroutine_count() - logger.log('DEBUG', f'Request coroutine {count}') + count = utils.get_request_count() + logger.log('DEBUG', f'Number of request threads {count}') return count -def get_ports(port): +def get_port_seq(port): logger.log('DEBUG', 'Getting port range') ports = set() if isinstance(port, (set, list, tuple)): @@ -40,81 +40,33 @@ def get_ports(port): return set(ports) -def gen_req_data(data, ports): +def gen_req_url(domain, port): + if str(port).endswith('443'): + url = f'https://{domain}:{port}' + if port == 443: + url = f'https://{domain}' + return url + url = f'http://{domain}:{port}' + if port == 80: + url = f'http://{domain}' + return url + + +def gen_req_urls(data, ports): logger.log('INFOR', 'Generating request urls') - new_data = [] - for data in data: - resolve = data.get('resolve') + urls = set() + for info in data: + resolve = info.get('resolve') # 解析不成功的子域不进行http请求探测 if resolve != 1: continue - subdomain = data.get('subdomain') + subdomain = info.get('subdomain') for port in ports: - if str(port).endswith('443'): - url = f'https://{subdomain}:{port}' - if port == 443: - url = f'https://{subdomain}' - data['id'] = None - data['url'] = url - data['port'] = port - new_data.append(data) - data = dict(data) # 需要生成一个新的字典对象 - else: - url = f'http://{subdomain}:{port}' - if port == 80: - url = f'http://{subdomain}' - data['id'] = None - data['url'] = url - data['port'] = port - new_data.append(data) - data = dict(data) # 需要生成一个新的字典对象 - return new_data + urls.add(gen_req_url(subdomain, port)) + return urls -async def fetch(session, method, url): - """ - 请求 - - :param session: session对象 - :param method: 请求方法 - :param str url: url地址 - :return: 响应对象和响应文本 - """ - timeout = aiohttp.ClientTimeout(total=None, - connect=None, - sock_read=settings.sockread_timeout, - sock_connect=settings.sockconn_timeout) - try: - if method == 'HEAD': - async with session.head(url, - ssl=settings.verify_ssl, - allow_redirects=settings.allow_redirects, - timeout=timeout, - proxy=settings.aiohttp_proxy) as resp: - text = await resp.text() - else: - async with session.get(url, - ssl=settings.verify_ssl, - allow_redirects=settings.allow_redirects, - timeout=timeout, - proxy=settings.aiohttp_proxy) as resp: - - try: - # 先尝试用utf-8解码 - text = await resp.text(encoding='utf-8', errors='strict') - except UnicodeError: - try: - # 再尝试用gb18030解码 - text = await resp.text(encoding='gb18030', errors='strict') - except UnicodeError: - # 最后尝试自动解码 - text = await resp.text(encoding=None, errors='ignore') - return resp, text - except Exception as e: - return e, None - - -def get_title(markup): +def get_html_title(markup): """ 获取标题 @@ -161,107 +113,107 @@ def get_jump_urls(history): return urls -def request_callback(future, index, datas): - resp, text = future.result() - if isinstance(resp, BaseException): - exception = resp - logger.log('TRACE', exception.args) - name = utils.get_classname(exception) - datas[index]['reason'] = name + ' ' + str(exception) - datas[index]['request'] = 0 - datas[index]['alive'] = 0 - else: - datas[index]['reason'] = resp.reason - datas[index]['status'] = resp.status - datas[index]['request'] = 1 - if resp.status == 400 or resp.status >= 500: - datas[index]['alive'] = 0 - else: - datas[index]['alive'] = 1 - headers = resp.headers - if settings.enable_banner_identify: - datas[index]['banner'] = utils.get_sample_banner(headers) - datas[index]['header'] = json.dumps(dict(headers)) - history = resp.history - datas[index]['history'] = json.dumps(get_jump_urls(history)) - if isinstance(text, str): - title = get_title(text).strip() - datas[index]['title'] = utils.remove_invalid_string(title) - datas[index]['response'] = utils.remove_invalid_string(text) - - -def get_connector(): - count = get_limit_conn() - return aiohttp.TCPConnector(ttl_dns_cache=300, - ssl=settings.verify_ssl, - limit=count, - limit_per_host=settings.limit_per_host) - - -async def async_request(urls): - results = list() - connector = get_connector() - headers = utils.get_random_header() - session = ClientSession(connector=connector, headers=headers) - tasks = [] - for i, url in enumerate(urls): - task = asyncio.ensure_future(fetch(session, 'GET', url)) - tasks.append(task) - if tasks: - futures = asyncio.as_completed(tasks) - for future in tqdm.tqdm(futures, - total=len(tasks), - desc='Request Progress', - ncols=80): - result = await future - results.append(result) - await session.close() - return results - - -async def bulk_request(data, port): - ports = get_ports(port) - no_req_data = utils.get_filtered_data(data) - to_req_data = gen_req_data(data, ports) - method = settings.request_method.upper() - logger.log('INFOR', f'Use {method} method to request') - logger.log('INFOR', 'Async subdomains request in progress') - connector = get_connector() - headers = utils.get_random_header() - session = ClientSession(connector=connector, headers=headers) - tasks = [] - for num, data in enumerate(to_req_data): - url = data.get('url') - task = asyncio.ensure_future(fetch(session, method, url)) - task.add_done_callback(functools.partial(request_callback, - index=num, - datas=to_req_data)) - tasks.append(task) - if tasks: - futures = asyncio.as_completed(tasks) - for future in tqdm.tqdm(futures, - total=len(tasks), - desc='Request Progress', - ncols=80): - await future - await session.close() - return to_req_data + no_req_data - - -def set_loop_policy(): +def get(url, resp_queue, session): + timeout = settings.request_timeout_second + redirect = settings.request_allow_redirect + proxy = utils.get_proxy() try: - import uvloop - except ImportError: - pass + resp = session.get(url, timeout=timeout, allow_redirects=redirect, proxies=proxy) + except Exception as e: + logger.log('DEBUG', e.args) + return + resp_queue.put(resp) + + +def request(urls_queue, resp_queue, session): + while not urls_queue.empty(): + url = urls_queue.get() + get(url, resp_queue, session) + urls_queue.task_done() + + +def progress(urls, resp_queue): + bar = tqdm.tqdm() + bar.total = len(urls) + bar.desc = 'Request Progress' + bar.ncols = 80 + while True: + done = resp_queue.qsize() + bar.n = done + bar.update() + if done == bar.total: + break + + +def get_session(): + header = utils.gen_fake_header() + verify = settings.request_ssl_verify + redirect_limit = settings.request_redirect_limit + session = requests.Session() + session.headers = header + session.verify = verify + session.max_redirects = redirect_limit + return session + + +def bulk_request(urls): + logger.log('INFOR', 'Requesting urls in bulk') + resp_list = list() + urls_queue = Queue() + resp_queue = Queue() + for url in urls: + urls_queue.put(url) + session = get_session() + thread_count = req_thread_count() + + progress_thread = Thread(target=progress, args=(urls, resp_queue)) + progress_thread.start() + + for _ in range(thread_count): + request_thread = Thread(target=request, args=(urls_queue, resp_queue, session)) + request_thread.start() + + urls_queue.join() + + while not resp_queue.empty(): + resp = resp_queue.get() + resp_list.append(resp) + return resp_list + + +def gen_new_info(info, resp): + port = resp.raw._pool.port + info['port'] = port + info['url'] = resp.url + info['reason'] = resp.reason + code = resp.status_code + info['status'] = code + info['request'] = 1 + if code == 400 or code >= 500: + info['alive'] = 0 else: - asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + info['alive'] = 1 + headers = resp.headers + if settings.enable_banner_identify: + info['banner'] = utils.get_sample_banner(headers) + info['header'] = json.dumps(dict(headers)) + history = resp.history + info['history'] = json.dumps(get_jump_urls(history)) + text = utils.decode_resp_text(resp) + title = get_html_title(text).strip() + info['title'] = utils.remove_invalid_string(title) + info['response'] = utils.remove_invalid_string(text) + return info -def set_loop(): - set_loop_policy() - loop = asyncio.get_event_loop() - asyncio.set_event_loop(loop) - return loop +def gen_new_data(data, resp_list): + new_data = list() + for resp in resp_list: + subdomain = resp.raw._pool.host + for info in data: + if info.get('subdomain') == subdomain: + new_data.append(gen_new_info(info, resp)) + return new_data def run_request(domain, data, port): @@ -274,25 +226,18 @@ def run_request(domain, data, port): :return list: result """ logger.log('INFOR', f'Start requesting subdomains of {domain}') - loop = set_loop() data = utils.set_id_none(data) - request_coroutine = bulk_request(data, port) - data = loop.run_until_complete(request_coroutine) - loop.run_until_complete(asyncio.sleep(0.25)) + ports = get_port_seq(port) + filtered_data = utils.get_filtered_data(data) + req_urls = gen_req_urls(data, ports) + resp_list = bulk_request(req_urls) + new_data = gen_new_data(data, resp_list) + data = new_data + filtered_data count = utils.count_alive(data) logger.log('INFOR', f'Found that {domain} has {count} alive subdomains') return data -def urls_request(urls): - logger.log('INFOR', 'Start urls request module') - loop = set_loop() - request_coroutine = async_request(urls) - data = loop.run_until_complete(request_coroutine) - loop.run_until_complete(asyncio.sleep(0.25)) - return data - - def save_db(name, data): """ Save request results to database diff --git a/common/resolve.py b/common/resolve.py index 53baaf7..d5f42b1 100644 --- a/common/resolve.py +++ b/common/resolve.py @@ -5,14 +5,11 @@ from config.log import logger from config import settings from common import utils from common.ipasn import IPAsnInfo -from common.ipgeo import IpGeoInfo -from common.ipreg import IpRegInfo +from common.ipreg import IpRegData ip_asn = IPAsnInfo() -ip_geo = IpGeoInfo() -db_path = settings.data_storage_dir.joinpath('ip2region.db') -ip_reg = IpRegInfo(db_path) +ip_reg = IpRegData() def filter_subdomain(data): @@ -82,12 +79,12 @@ def gen_infos(data, qname, info, infos): cname = list() ips = list() public = list() - ttls = list() - cidrs = list() - asns = list() - orgs = list() - locs = list() - regs = list() + ttl = list() + cidr = list() + asn = list() + org = list() + addr = list() + isp = list() answers = data.get('answers') for answer in answers: if answer.get('type') == 'A': @@ -95,31 +92,26 @@ def gen_infos(data, qname, info, infos): cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号 ip = answer.get('data') ips.append(ip) - ttl = answer.get('ttl') - ttls.append(str(ttl)) - is_public = utils.ip_is_public(ip) - public.append(str(is_public)) + ttl.append(str(answer.get('ttl'))) + public.append(str(utils.ip_is_public(ip))) asn_info = ip_asn.find(ip) - cidrs.append(asn_info.get('cidr')) - asns.append(asn_info.get('asn')) - orgs.append(asn_info.get('org')) - loc = f'{ip_geo.get_country_long(ip)} ' \ - f'{ip_geo.get_region(ip)} ' \ - f'{ip_geo.get_city(ip)}' - locs.append(loc) - reg = ip_reg.memory_search(ip).get('region').decode('utf-8') - regs.append(reg) + cidr.append(asn_info.get('cidr')) + asn.append(asn_info.get('asn')) + org.append(asn_info.get('org')) + ip_info = ip_reg.query(ip) + addr.append(ip_info.get('addr')) + isp.append(ip_info.get('isp')) info['resolve'] = 1 info['reason'] = 'OK' info['cname'] = ','.join(cname) info['content'] = ','.join(ips) info['public'] = ','.join(public) - info['ttl'] = ','.join(ttls) - info['cidr'] = ','.join(cidrs) - info['asn'] = ','.join(asns) - info['org'] = ','.join(orgs) - info['ip2location'] = ','.join(locs) - info['ip2region'] = ','.join(regs) + info['ttl'] = ','.join(ttl) + info['cidr'] = ','.join(cidr) + info['asn'] = ','.join(asn) + info['org'] = ','.join(org) + info['addr'] = ','.join(addr) + info['isp'] = ','.join(isp) infos[qname] = info if not flag: info['alive'] = 0 diff --git a/common/similarity.py b/common/similarity.py new file mode 100644 index 0000000..4070566 --- /dev/null +++ b/common/similarity.py @@ -0,0 +1,138 @@ +""" +根据网页结构判断页面相似性(Determine page similarity based on HTML page structure) +判断方法：根据网页的DOM树确定网页的模板特征向量，对模板特征向量计算网页结构相似性。 +来源地址：https://github.com/SPuerBRead/HTMLSimilarity +计算参考: https://patents.google.com/patent/CN101694668B/zh +""" +from treelib import Tree +from bs4 import BeautifulSoup +import bs4 + + +class DOMTree(object): + def __init__(self, label, attrs): + self.label = label + self.attrs = attrs + + +class HTMLParser(object): + def __init__(self, html): + self.dom_id = 1 + self.dom_tree = Tree() + self.bs_html = BeautifulSoup(html, 'html.parser') + + def get_dom_structure_tree(self): + for content in self.bs_html.contents: + if isinstance(content, bs4.element.Tag): + self.bs_html = content + self.recursive_descendants(self.bs_html, 1) + return self.dom_tree + + def recursive_descendants(self, descendants, parent_id): + if self.dom_id == 1: + self.dom_tree.create_node(descendants.name, self.dom_id, + data=DOMTree(descendants.name, descendants.attrs)) + self.dom_id = self.dom_id + 1 + for child in descendants.contents: + if isinstance(child, bs4.element.Tag): + self.dom_tree.create_node(child.name, self.dom_id, parent_id, + data=DOMTree(child.name, child.attrs)) + self.dom_id = self.dom_id + 1 + self.recursive_descendants(child, self.dom_id - 1) + + +class Converter(object): + def __init__(self, dom_tree, dimension): + self.dom_tree = dom_tree + self.node_info_list = [] + self.dimension = dimension + self.initial_weight = 1 + self.attenuation_ratio = 0.6 + self.dom_eigenvector = {}.fromkeys(range(0, dimension), 0) + + def get_eigenvector(self): + for node_id in range(1, self.dom_tree.size() + 1): + node = self.dom_tree.get_node(node_id) + node_feature = self.create_feature(node) + feature_hash = self.feature_hash(node_feature) + node_weight = self.calculate_weight(node, node_id, feature_hash) + self.construct_eigenvector(feature_hash, node_weight) + return self.dom_eigenvector + + @staticmethod + def create_feature(node): + node_attr_list = [] + node_feature = node.data.label + '|' + for attr in node.data.attrs.keys(): + node_attr_list.append(attr + ':' + str(node.data.attrs[attr])) + node_feature += '|'.join(node_attr_list) + return node_feature + + @staticmethod + def feature_hash(node_feature): + return abs(hash(node_feature)) % (10 ** 8) + + def calculate_weight(self, node, node_id, feature_hash): + brother_node_count = 0 + depth = self.dom_tree.depth(node) + for brother_node in self.dom_tree.siblings(node_id): + brother_node_feature = self.create_feature(brother_node) + brother_node_feature_hash = self.feature_hash(brother_node_feature) + if brother_node_feature_hash == feature_hash: + brother_node_count = brother_node_count + 1 + if brother_node_count: + node_weight = self.initial_weight * self.attenuation_ratio ** depth \ + * self.attenuation_ratio ** brother_node_count + else: + node_weight = self.initial_weight * self.attenuation_ratio ** depth + return node_weight + + def construct_eigenvector(self, feature_hash, node_weight): + feature_hash = feature_hash % self.dimension + self.dom_eigenvector[feature_hash] += node_weight + + +def calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension): + a, b = 0, 0 + for i in range(dimension): + a += dom1_eigenvector[i]-dom2_eigenvector[i] + if dom1_eigenvector[i] and dom2_eigenvector[i]: + b += dom1_eigenvector[i] + dom2_eigenvector[i] + pseudodistance = abs(a)/b + return pseudodistance + + +def get_pseudodistance(html_doc1, html_doc2, dimension=5000): + """ + 获取html文档结构相似度 + + :param str html_doc1: html文档 + :param str html_doc2: html文档 + :param int dimension: 降维后的维数 + :return 伪距离 + """ + hp1 = HTMLParser(html_doc1) + html_doc1_dom_tree = hp1.get_dom_structure_tree() + hp2 = HTMLParser(html_doc2) + html_doc2_dom_tree = hp2.get_dom_structure_tree() + converter = Converter(html_doc1_dom_tree, dimension) + dom1_eigenvector = converter.get_eigenvector() + converter = Converter(html_doc2_dom_tree, dimension) + dom2_eigenvector = converter.get_eigenvector() + return calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension) + + +def is_similar(html_doc1, html_doc2, dimension=5000): + """ + 根据计算出的伪距离来判断是否html页面结构相似 + + :param str html_doc1: html文档 + :param str html_doc2: html文档 + :param int dimension: 降维后的维数 + :return 是否相似（伪距离value<0.2时相似，value>0.2时不相似） + """ + value = get_pseudodistance(html_doc1, html_doc2, dimension) + if value > 0.2: + return False + else: + return True diff --git a/common/utils.py b/common/utils.py index 2b071dc..a8f3f90 100644 --- a/common/utils.py +++ b/common/utils.py @@ -10,7 +10,6 @@ import subprocess from ipaddress import IPv4Address, ip_address from stat import S_IXUSR -import psutil import tenacity import requests from pathlib import Path @@ -49,10 +48,10 @@ def gen_fake_header(): """ Generate fake request headers """ - headers = settings.headers + headers = settings.request_default_headers if not isinstance(headers, dict): headers = dict() - if settings.random_user_agent: + if settings.enable_random_ua: ua = random.choice(user_agents) headers['User-Agent'] = ua headers['Accept-Encoding'] = 'gzip, deflate' @@ -74,7 +73,7 @@ def get_random_proxy(): Get random proxy """ try: - return random.choice(settings.proxy_pool) + return random.choice(settings.request_proxy_pool) except IndexError: return None @@ -83,7 +82,7 @@ def get_proxy(): """ Get proxy """ - if settings.enable_proxy: + if settings.enable_request_proxy: return get_random_proxy() return None @@ -112,26 +111,21 @@ def match_main_domain(domain): def read_target_file(target): - domains = set() + domains = list() with open(target, encoding='utf-8', errors='ignore') as file: for line in file: domain = match_main_domain(line) if not domain: continue - domain = get_main_domain(domain) - if not domain: - continue - domains.add(domain) - return domains + domains.append(domain) + sorted_domains = sorted(set(domains), key=domains.index) + return sorted_domains def get_from_target(target): domains = set() if isinstance(target, str): domain = match_main_domain(target) - if not domain: - return domains - domain = get_main_domain(domain) if not domain: return domains domains.add(domain) @@ -507,27 +501,8 @@ def get_process_num(): return 1 -def get_coroutine_count(): - """ - 根据内存大小获取并发数量 - """ - mem = psutil.virtual_memory() - total_mem = mem.total - g_size = 1024 * 1024 * 1024 - if total_mem < 1 * g_size: - return 16 - elif total_mem < 2 * g_size: - return 32 - elif total_mem < 4 * g_size: - return 64 - elif total_mem < 8 * g_size: - return 128 - elif total_mem < 16 * g_size: - return 256 - elif total_mem < 32 * g_size: - return 512 - else: - return 1024 +def get_request_count(): + return os.cpu_count() * 10 def uniq_dict_list(dict_list): @@ -567,7 +542,6 @@ def check_net(): def check_pre(): logger.log('INFOR', 'Checking dependent environment') - system = platform.system() implementation = platform.python_implementation() version = platform.python_version() if implementation != 'CPython': @@ -576,16 +550,6 @@ def check_pre(): if version < '3.6': logger.log('FATAL', 'OneForAll requires Python 3.6 or higher') exit(1) - if system == 'Windows' and implementation == 'CPython' and version < '3.8': - logger.log('FATAL', 'OneForAll requires Python 3.8 ' - 'or higher when running on Windows') - exit(1) - if system in {"Linux", "Darwin"}: - try: - import uvloop - except ImportError: - logger.log('ALERT', f'Please install the uvloop library manually ' - f'to accelerate subdomain requests') def check_env(): @@ -604,8 +568,8 @@ def check_version(local): api = 'https://api.github.com/repos/shmilylty/OneForAll/releases/latest' header = get_random_header() proxy = get_proxy() - timeout = settings.request_timeout - verify = settings.request_verify + timeout = settings.request_timeout_second + verify = settings.request_ssl_verify try: resp = requests.get(url=api, headers=header, proxies=proxy, timeout=timeout, verify=verify) @@ -613,7 +577,7 @@ def check_version(local): latest = resp_json['tag_name'] except Exception as e: logger.log('ERROR', 'An error occurred while checking the latest version') - logger.log('ERROR', e.args) + logger.log('DEBUG', e.args) return if latest > local: change = resp_json.get("body") @@ -740,3 +704,26 @@ def check_random_subdomain(subdomains): if subdomain: logger.log('ALERT', f'Please check whether {subdomain} is correct or not') return True + + +def get_url_resp(url): + timeout = settings.request_timeout + verify = settings.request_verify + try: + resp = requests.get(url, params=None, timeout=timeout, verify=verify) + except Exception as e: + logger.log('DEBUG', f'Error request {url}') + logger.log('DEBUG', e.args) + return None + return resp + + +def decode_resp_text(resp): + try: + text = resp.text(encoding='utf-8', errors='strict') # 先尝试用utf-8严格解码 + except UnicodeError: + try: + text = resp.text(encoding='gb18030', errors='strict') # 再尝试用gb18030严格解码 + except UnicodeError: + text = resp.text(encoding=None, errors='ignore') # 最后尝试自动解码 + return text diff --git a/config/default.py b/config/default.py index cc4df1e..976f167 100644 --- a/config/default.py +++ b/config/default.py @@ -6,6 +6,9 @@ OneForAll默认配置 import pathlib import warnings +# 禁用所有警告信息 +warnings.filterwarnings("ignore") + # 路径设置 relative_directory = pathlib.Path(__file__).parent.parent # OneForAll代码相对路径 module_dir = relative_directory.joinpath('modules') # OneForAll模块目录 @@ -40,7 +43,7 @@ enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_mo # 只使用ask和baidu搜索引擎收集子域的示例 # enable_partial_module = [('modules.search', 'ask') # ('modules.search', 'baidu')] -module_thread_timeout = 120.0 # 每个收集模块线程超时时间(默认2分钟) +module_thread_timeout = 90.0 # 每个收集模块线程超时时间(默认90秒) # 爆破模块设置 enable_wildcard_check = True # 开启泛解析检测(默认True) @@ -51,7 +54,7 @@ brute_status_format = 'ansi' # 爆破时状态输出格式（默认asni，可 brute_process_num = 1 # 默认1 brute_concurrent_num = 2000 # 并发查询数量(默认2000，最大推荐10000) brute_socket_num = 1 # 爆破时每个进程下的socket数量 -brute_resolve_num = 10 # 解析失败时尝试换名称服务器重查次数 +brute_resolve_num = 15 # 解析失败时尝试换名称服务器重查次数 # 爆破所使用的字典路径 默认data/subdomains.txt brute_wordlist_path = data_storage_dir.joinpath('subnames.txt') # 爆破所使用的字典路径 默认data/cn_nameservers.txt @@ -79,24 +82,39 @@ ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标 banner_process_number = 4 # 识别进程数量(默认4) # 代理设置 -enable_proxy = False # 是否使用代理(全局开关) +enable_request_proxy = False # 是否使用代理(全局开关，默认False) proxy_all_module = False # 代理所有模块 proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch', 'GoogleAPISearch', 'GoogleSearch', 'YahooSearch', 'YandexSearch', 'CrossDomainXml', 'ContentSecurityPolicy'] # 代理自定义的模块 -proxy_pool = [{'http': 'http://127.0.0.1:1080', - 'https': 'https://127.0.0.1:1080'}] # 代理池 -# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808', -# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池 +request_proxy_pool = [{'http': 'http://127.0.0.1:1080', + 'https': 'https://127.0.0.1:1080'}] # 代理池 +# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808', +# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池 -# 网络请求设置 -request_delay = 1 # 请求时延 -request_timeout = 60 # 请求超时 -request_verify = False # 请求SSL验证 -# 禁用所有警告信息 -warnings.filterwarnings("ignore") +# 请求设置 +request_thread_count = None # 请求线程数量(默认None，则根据内存大小设置) +request_timeout_second = 30 # 请求超时秒数(默认30秒) +request_ssl_verify = False # 请求SSL验证(默认False) +request_allow_redirect = True # 请求允许重定向(默认True) +request_redirect_limit = 10 # 请求跳转限制(默认10次) +# 默认请求头 可以在headers里添加自定义请求头 +request_default_headers = { + 'Accept': 'text/html,application/xhtml+xml,' + 'application/xml;q=0.9,*/*;q=0.8', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7', + 'Cache-Control': 'max-age=0', + 'DNT': '1', + 'Referer': 'https://www.google.com/', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36', + 'Upgrade-Insecure-Requests': '1', + 'X-Forwarded-For': '127.0.0.1' +} +enable_random_ua = True # 使用随机UA(默认True，开启可以覆盖request_default_headers的UA) # 搜索模块设置 # 开启全量搜索会尽量去获取搜索引擎搜索的全部结果，不过搜索耗时可能会过长 @@ -112,9 +130,8 @@ resolver_nameservers = [ '8.8.8.8', # Google DNS '1.1.1.1' # CloudFlare DNS ] # 指定查询的DNS域名服务器 -resolver_timeout = 5.0 # 解析超时时间 -resolver_lifetime = 60.0 # 解析存活时间 -limit_resolve_conn = 500 # 限制同一时间解析的数量(默认500) +resolver_timeout = 5.0 # 解析超时时间(默认5.0秒) +resolver_lifetime = 10.0 # 解析存活时间(默认10.0秒) # 请求端口探测设置 # 你可以在端口列表添加自定义端口 @@ -140,42 +157,11 @@ large_ports = [80, 81, 280, 300, 443, 591, 593, 832, 888, 901, 981, 1010, 1080, 18092, 20000, 20720, 24465, 28017, 28080, 30821, 43110, 61600] ports = {'small': small_ports, 'medium': medium_ports, 'large': large_ports} -# aiohttp有关配置 -verify_ssl = False -# aiohttp 支持 HTTP/HTTPS形式的代理 -aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com" -allow_redirects = True # 允许请求跳转 -# 为了保证请求质量 请谨慎更改以下设置 -# request_method只能是HEAD或GET,HEAD请求方法更快，但是不能获取响应体并提取从中title及banner -request_method = 'GET' # 使用请求方法，默认GET -sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒 -sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒 -limit_open_conn = None # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置 -# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数 -limit_per_host = 10 # 0表示不限制,默认10 - common_subnames = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs', 'web', 'www', 'job', 'docs', 'news', 'blog', 'data', 'help', 'live', 'mall', 'blogs', 'files', 'forum', 'store', 'mobile'} -# 请求头设置 -# 可以在headers里添加自定义请求头 -headers = { - 'Accept': 'text/html,application/xhtml+xml,' - 'application/xml;q=0.9,*/*;q=0.8', - 'Accept-Encoding': 'gzip, deflate', - 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7', - 'Cache-Control': 'max-age=0', - 'DNT': '1', - 'Referer': 'https://www.google.com/', - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' - '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36', - 'Upgrade-Insecure-Requests': '1', - 'X-Forwarded-For': '127.0.0.1' -} -random_user_agent = True # 使用随机UA(默认True，开启可以覆盖header的设置) - # 模块API配置 # Censys可以免费注册获取API：https://censys.io/api censys_api_id = '' diff --git a/config/log.py b/config/log.py index 36f026e..ec8bb47 100644 --- a/config/log.py +++ b/config/log.py @@ -35,4 +35,4 @@ logger.level(name='FATAL', no=50, color='', icon='☠️') # 命令终端日志级别默认为INFOR logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True) # 日志文件默认为级别为DEBUG -logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True) +logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True, encoding='utf-8') diff --git a/config/setting.py b/config/setting.py index e6a640b..07fb2d2 100644 --- a/config/setting.py +++ b/config/setting.py @@ -35,11 +35,8 @@ enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_mo # 只使用ask和baidu搜索引擎收集子域的示例 # enable_partial_module = [('modules.search', 'ask') # ('modules.search', 'baidu')] -module_thread_timeout = 120.0 # 每个收集模块线程超时时间(默认2分钟) # 爆破模块设置 -brute_wildcard_check = True # 开启泛解析检测(默认True) -brute_wildcard_deal = True # 开启泛解析处理(默认True) brute_concurrent_num = 2000 # 爆破时并发查询数量(默认2000，最大推荐10000) # 爆破所使用的字典路径 默认data/subdomains.txt brute_wordlist_path = data_storage_dir.joinpath('subnames.txt') @@ -66,37 +63,19 @@ fuzz_rule = None # fuzz域名的正则 示例：'[a-z][0-9]' 表示第一位是 brute_ip_blacklist = {'0.0.0.0', '0.0.0.1'} # IP黑名单 子域解析到IP黑名单则标记为非法子域 ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标记为非法(泛解析)子域 -# banner识别模块设置 -banner_process_number = 4 # 识别进程数量(默认4) - # 代理设置 -enable_proxy = False # 是否使用代理(全局开关) +enable_request_proxy = False # 是否使用代理(全局开关) proxy_all_module = False # 代理所有模块 proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch', 'GoogleAPISearch', 'GoogleSearch', 'YahooSearch', 'YandexSearch', 'CrossDomainXml', 'ContentSecurityPolicy'] # 代理自定义的模块 -proxy_pool = [{'http': 'http://127.0.0.1:1080', - 'https': 'https://127.0.0.1:1080'}] # 代理池 -# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808', -# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池 - - -# 网络请求设置 -request_delay = 1 # 请求时延 -request_timeout = 60 # 请求超时 -request_verify = False # 请求SSL验证 +request_proxy_pool = [{'http': 'http://127.0.0.1:1080', + 'https': 'https://127.0.0.1:1080'}] # 代理池 +# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808', +# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池 # 搜索模块设置 enable_recursive_search = False # 递归搜索子域 search_recursive_times = 2 # 递归搜索层数 -# aiohttp有关配置 -# aiohttp 支持 HTTP/HTTPS形式的代理 -aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com" -# 为了保证请求质量 请谨慎更改以下设置 -sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒 -sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒 -limit_open_conn = None # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置 -# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数 -limit_per_host = 10 # 0表示不限制,默认10 diff --git a/data/IP2LOCATION-LITE-DB3.BIN.ZIP b/data/IP2LOCATION-LITE-DB3.BIN.ZIP deleted file mode 100644 index 9c939a1..0000000 Binary files a/data/IP2LOCATION-LITE-DB3.BIN.ZIP and /dev/null differ diff --git a/dbexport.py b/dbexport.py index 8e05f42..b43bd92 100644 --- a/dbexport.py +++ b/dbexport.py @@ -29,7 +29,6 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f python3 dbexport.py --target table_name --tb True --show False Note: - --alive True/False Only export alive subdomains or not (default False) --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) --path Result directory (default directory is ./results) @@ -82,4 +81,3 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f if __name__ == '__main__': fire.Fire(export) - # export('example.com') diff --git a/docs/en-us/README.md b/docs/en-us/README.md index 8e96ec4..5223e47 100644 --- a/docs/en-us/README.md +++ b/docs/en-us/README.md @@ -4,7 +4,7 @@ [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll) [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability) [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE) -[![python](https://img.shields.io/badge/python-3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/) +[![python](https://img.shields.io/badge/python-3.6|3.7|3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/) [![python](https://img.shields.io/badge/release-v0.3.0-brightgreen)](https://github.com/shmilylty/OneForAll/releases) 👊**OneForAll is a powerful subdomain integration tool** 📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md) @@ -15,23 +15,6 @@ 📢 Please read this document to help you start quickly! -
-🐍Installation requirements - -OneForAll is developed and tested based on [Python 3.8.0](https://www.python.org/downloads/release/python-380/). Recommend use release higher than Python 3.8.0 (Windows platform must use Python 3.8.0 or later). For more information on installing the Python environment, please read [Python 3 installation Guide](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3). - -After installation python, run the following command to check the Python and pip3 versions: -```bash -python -V -pip3 -V -``` -If you see the following output, there is no problem with the Python environment: -```bash -Python 3.8.0 -pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.8) -``` -
-
✔Installation steps (for Git) diff --git a/docs/en-us/usage_help.md b/docs/en-us/usage_help.md index 77a0e91..46b64da 100644 --- a/docs/en-us/usage_help.md +++ b/docs/en-us/usage_help.md @@ -10,141 +10,214 @@ The OneForAll command line interface is based on [Fire](https://github.com/googl 1. **oneforall.py help** - ```bash - python oneforall.py --help - ``` - ```bash - NAME - oneforall.py - OneForAll help summary page - - SYNOPSIS - oneforall.py COMMAND | - - DESCRIPTION - OneForAll is a powerful subdomain integration tool - - Example: - python3 oneforall.py version - python3 oneforall.py check - python3 oneforall.py --target example.com run - python3 oneforall.py --targets ./domains.txt run - python3 oneforall.py --target example.com --alive False run - python3 oneforall.py --target example.com --brute True run - python3 oneforall.py --target example.com --port medium run - python3 oneforall.py --target example.com --format csv run - python3 oneforall.py --target example.com --dns False run - python3 oneforall.py --target example.com --req False run - python3 oneforall.py --target example.com --takeover False run - python3 oneforall.py --target example.com --show True run +```bash +python oneforall.py --help +``` +```bash +NAME + oneforall.py - OneForAll help summary page - - Note: - --alive True/False Only export alive subdomains or not (default False) - --port small/medium/large See details in ./config/setting.py(default small) - --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) - --path Result path (default None, automatically generated) +SYNOPSIS + oneforall.py COMMAND | - FLAGS - --target=TARGET - One domain (target or targets parameters must be provided) - --targets=TARGETS - File path of one domain per line - --brute=BRUTE - Use brute module (default False) - --dns=DNS - Use DNS resolution (default True) - --req=REQ - HTTP request subdomains (default True) - --port=PORT - The port range to request (default small port is 80,443) - --alive=ALIVE - Only export alive subdomains (default False) - --format=FORMAT - Result format (default csv) - --path=PATH - Result path (default None, automatically generated) - --takeover=TAKEOVER - Scan subdomain takeover (default False) +DESCRIPTION + OneForAll is a powerful subdomain integration tool - COMMANDS - COMMAND is one of the following: + Example: + python3 oneforall.py version + python3 oneforall.py check + python3 oneforall.py --target example.com run + python3 oneforall.py --targets ./domains.txt run + python3 oneforall.py --target example.com --alive False run + python3 oneforall.py --target example.com --brute True run + python3 oneforall.py --target example.com --port medium run + python3 oneforall.py --target example.com --format csv run + python3 oneforall.py --target example.com --dns False run + python3 oneforall.py --target example.com --req False run + python3 oneforall.py --target example.com --takeover False run + python3 oneforall.py --target example.com --show True run - check - Check if there is a new version and exit - version - Print version information and exit - ``` + Note: + --port small/medium/large See details in ./config/setting.py(default small) + --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) + --path Result path (default None, automatically generated) + +FLAGS + --target=TARGET + One domain (target or targets parameters must be provided) + --targets=TARGETS + File path of one domain per line + --brute=BRUTE + Use brute module (default False) + --dns=DNS + Use DNS resolution (default True) + --req=REQ + HTTP request subdomains (default True) + --port=PORT + The port range to request (default small port is 80,443) + --alive=ALIVE + Only export alive subdomains (default False) + --format=FORMAT + Result format (default csv) + --path=PATH + Result path (default None, automatically generated) + --takeover=TAKEOVER + Scan subdomain takeover (default False) + +COMMANDS + COMMAND is one of the following: + + check + Check if there is a new version and exit + + version + Print version information and exit +``` 2. **aiobrute.py help** - With regard to the handling of the universal parsing problem, first of all, OneForAll accesses a random subdomain to determine whether universal parsing is used, and if universal parsing is used, it is handled by the following judgment: - - First, it is mainly compared with the pan-parsed IP set and TTL values, see [this article](http://sh3ll.me/archives/201704041222.txt). +With regard to the handling of the universal parsing problem, first of all, OneForAll accesses a random subdomain to determine whether universal parsing is used, and if universal parsing is used, it is handled by the following judgment: +- First, it is mainly compared with the pan-parsed IP set and TTL values, see [this article](http://sh3ll.me/archives/201704041222.txt). - - Second, the number of times to resolve to the same IP collection multiple times (the default is 10, which can be set to size in config.py). - - - Third, considering the blasting efficiency, there is no HTTP response volume similarity comparison and response volume content judgment, this function has not been implemented yet, and will be implemented if necessary. +- Second, the number of times to resolve to the same IP collection multiple times (the default is 10, which can be set to size in config.py). + +- Third, considering the blasting efficiency, there is no HTTP response volume similarity comparison and response volume content judgment, this function has not been implemented yet, and will be implemented if necessary. - ```bash - python aiobrute.py --help - ``` +```bash +python brute.py --help +``` - ```bash - NAME - aiobrute.py - OneForAll multi-process multi-correlation asynchronous subdomain blasting module - - SYNOPSIS - aiobrute.py --target=TARGET - - DESCRIPTION - Example： - python3 aiobrute.py --target example.com run - python3 aiobrute.py --target ./domains.txt run - python3 aiobrute.py --target example.com --process 4 --coroutine 64 run - python3 aiobrute.py --target example.com --wordlist subdomains.txt run - python3 aiobrute.py --target example.com --recursive True --depth 2 run - python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run - - Note: - Parameter valid optional value 1, 0, none indicates that the export is - valid, invalid, and all subdomains, respectively. - - Parameter format have optional values 'txt', 'rst', 'csv', 'tsv', 'json', - 'yaml', 'html', 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'. - If the parameter path is None, the appropriate file is generated in the - project result directory based on the format parameter and the domain - name. - - ARGUMENTS - TARGET - Single domain name or file path for one domain name per line (required) - - FLAGS - --process=PROCESS - Number of processes blasted (default CPU core count) - --coroutine=COROUTINE - Number of coroutines per blasting process (default 1024) - --wordlist=WORDLIST - Specify the dictionary path used for blasting (config.py is used by default) - --recursive=RECURSIVE - Whether to use recursive blasting (default False) - --depth=DEPTH - Depth of recursive blasting (default 2) - --namelist=NAMELIST - Specifies the dictionary path used by recursive blasting (configured by default using config.py) - --fuzz=FUZZ - Whether to use the fuzz mode for blasting (default False, you must specify the fuzz regular rule) - --rule=RULE - Regular rules used by fuzz mode (configured by default using config.py) - --export=EXPORT - Whether to export the blast result (default True) - --valid=VALID - Export validity of subdomains (default None) - --format=FORMAT - Export format (default xls) - --path=PATH - Export path (default None) - --show=SHOW - Terminal display exported data (default False) - ``` \ No newline at end of file +```bash +NAME +brute.py - OneForAll subdomain brute module + +SYNOPSIS + brute.py + +DESCRIPTION + Example： + brute.py --target domain.com --word True run + brute.py --targets ./domains.txt --word True run + brute.py --target domain.com --word True --concurrent 2000 run + brute.py --target domain.com --word True --wordlist subnames.txt run + brute.py --target domain.com --word True --recursive True --depth 2 run + brute.py --target d.com --fuzz True --place m.*.d.com --rule '[a-z]' run + brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run + + Note: + --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) + --path Result path (default None, automatically generated) + + FLAGS + --target=TARGET + One domain (target or targets must be provided) + --targets=TARGETS + File path of one domain per line + --process=PROCESS + Number of processes (default 1) + --concurrent=CONCURRENT + Number of concurrent (default 2000) + --word=WORD + Use word mode generate dictionary (default False) + --wordlist=WORDLIST + Dictionary path used in word mode (default use ./config/default.py) + --recursive=RECURSIVE + Use recursion (default False) + --depth=DEPTH + Recursive depth (default 2) + --nextlist=NEXTLIST + Dictionary file path used by recursive (default use ./config/default.py) + --fuzz=FUZZ + Use fuzz mode generate dictionary (default False) + --place=PLACE + Designated fuzz position (required if use fuzz mode) + --rule=RULE + Specify the regexp rules used in fuzz mode (required if use fuzz mode) + --fuzzlist=FUZZLIST + Dictionary path used in fuzz mode (default use ./config/default.py) + --export=EXPORT + Export the results (default True) + --alive=ALIVE + Only export alive subdomains (default False) + --format=FORMAT + Result format (default csv) + --path=PATH + Result directory (default None) +``` + +3. **takeover.py help** + +```bash +python takeover.py --help +``` + +```bash +NAME +takeover.py - OneForAll subdomain takeover module SYNOPSIS takeover.py DESCRIPTION + +Example: + python3 takeover.py --target www.example.com --format csv run + python3 takeover.py --targets ./subdomains.txt --thread 10 run + +Note: + --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) + --path Result directory (default directory is ./results) + +FLAGS + --target=TARGET + One domain (target or targets must be provided) + --targets=TARGETS + File path of one domain per line + --thread=THREAD + threads number (default 20) + --path=PATH + Result directory (default None) + --format=FORMAT + Result format (default csv) +``` + +4. **dbexport.py help** + +```bash +python dbexport.py --help +``` + +```bash +NAME + dbexport.py - OneForAll export from database module + +SYNOPSIS + dbexport.py TARGET + +DESCRIPTION + Example: + python3 dbexport.py --target name --format csv --dir= ./result.csv + python3 dbexport.py --db result.db --target name --show False + python3 dbexport.py --target table_name --tb True --show False + + Note: + --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) + --path Result directory (default directory is ./results) + +POSITIONAL ARGUMENTS + TARGET + Table to be exported + +FLAGS + --type=TYPE + Type of target + --db=DB + Database path to be exported (default ./results/result.sqlite3) + --alive=ALIVE + Only export the results of alive subdomains (default False) + --limit=LIMIT + Export limit (default None) + --path=PATH + Result directory (default None) + --format=FORMAT + Result format (default csv) + --show=SHOW + Displays the exported data in terminal (default False) +``` \ No newline at end of file diff --git a/docs/field.md b/docs/field.md index c9ce0cb..24b4aa5 100644 --- a/docs/field.md +++ b/docs/field.md @@ -84,19 +84,19 @@ DNS解析返回的TTL值 ### cidr -IP2LOCATION-ASN库查询出的CIDR +ip2location库查询出的CIDR ### asn -IP2LOCATION-ASN库查询出的ASN +ip2location库查询出的ASN -### ip2region +### addr ip2region库查询出的物理地址 -### ip2location +### isp -ip2location库查询出的物理地址 +ip2region库查询出的网络服务提供商 ### resolver diff --git a/docs/usage_help.md b/docs/usage_help.md index 180d995..d1f3dc1 100644 --- a/docs/usage_help.md +++ b/docs/usage_help.md @@ -38,7 +38,6 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实 python3 oneforall.py --target example.com --show True run Note: - --alive True/False 只导出存活子域(默认False) --port small/medium/large 详见./config/setting.py(默认small) --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (结果格式，默认CSV) --path 结果路径(默认None，自动生成) @@ -90,63 +89,56 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实 ```bash NAME - aiobrute.py - OneForAll多进程多协程异步子域爆破模块 + brute.py - OneForAll子域爆破模块 SYNOPSIS - aiobrute.py --target=TARGET + brute.py DESCRIPTION Example： - python3 aiobrute.py --target example.com run - python3 aiobrute.py --target ./domains.txt run - python3 aiobrute.py --target example.com --process 4 --coroutine 64 run - python3 aiobrute.py --target example.com --wordlist subdomains.txt run - python3 aiobrute.py --target example.com --recursive True --depth 2 run - python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run - + brute.py --target domain.com --word True run + brute.py --targets ./domains.txt --word True run + brute.py --target domain.com --word True --coroutine 2000 run + brute.py --target domain.com --word True --wordlist subnames.txt run + brute.py --target domain.com --word True --recursive True --depth 2 run + brute.py --target d.com --fuzz True --place m.*.d.com --rule '[a-z]' run + brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run + Note: - 参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为任务组， - 当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小： - 十万字典建议设置为5000，百万字典设置为50000 - 参数valid可选值1，0，None，分别表示导出有效，无效，全部子域 - 参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx', - 'dbf', 'latex', 'ods' - 参数path为None会根据format参数和域名名称在项目结果目录生成相应文件 - - ARGUMENTS - TARGET - 单个域名或者每行一个域名的文件路径 - - FLAGS - --process=PROCESS - 爆破的进程数(默认CPU核心数) - --coroutine=COROUTINE - 每个爆破进程下的协程数(默认64) - --wordlist=WORDLIST - 指定爆破所使用的字典路径(默认使用config.py配置) - --segment=SEGMENT - 爆破任务分割(默认500) - --recursive=RECURSIVE - 是否使用递归爆破(默认False) - --depth=DEPTH - 递归爆破的深度(默认2) - --namelist=NAMELIST - 指定递归爆破所使用的字典路径(默认使用config.py配置) - --fuzz=FUZZ - 是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则) - --rule=RULE - fuzz模式使用的正则规则(默认使用config.py配置) - --export=EXPORT - 是否导出爆破结果(默认True) - --valid=VALID - 导出子域的有效性(默认None) - --format=FORMAT - 导出格式(默认xlsx) - --path=PATH - 导出路径(默认None) - --show=SHOW - 终端显示导出数据(默认False) - + --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (结果格式，默认CSV) + --path 导出路径(默认None，自动生成) + + FLAGS + --target=TARGET + 单个域名(必须提供target或targets参数) + --targets=TARGETS + 每行一个域名的文件路径 + --process=PROCESS + 爆破的进程数(默认CPU核心数) + --coroutine=COROUTINE + 每个爆破进程下的协程数(默认2000) + --wordlist=WORDLIST + 指定爆破所使用的字典路径(默认使用config.py配置) + --recursive=RECURSIVE + 是否使用递归爆破(默认False) + --depth=DEPTH + 递归爆破的深度(默认2) + --nextlist=NEXTLIST + 指定递归爆破所使用的字典路径(默认使用config.py配置) + --fuzz=FUZZ + 是否使用fuzz模式进行爆破(默认False) + --rule=RULE + fuzz模式使用的正则规则(默认使用config.py配置) + --fuzzlist=FUZZLIST + 指定fuzz模式所使用的字典路径(默认使用config.py配置) + --export=EXPORT + 是否导出爆破结果(默认True) + --alive=ALIVE + 只导出存活子域(默认False) + --format=FORMAT + 导出格式(默认csv) + --path=PATH + 导出路径(默认None) ``` @@ -200,24 +192,26 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实 dbexport.py - OneForAll数据库导出模块 SYNOPSIS - dbexport.py TABLE + dbexport.py TARGET DESCRIPTION Example: - python3 dbexport.py --table name --format csv --path= ./result.csv - python3 dbexport.py --db result.db --table name --show False + python3 dbexport.py --target name --format csv --dir= ./result.csv + python3 dbexport.py --db result.db --target name --show False + python3 dbexport.py --target table_name --tb True --show False Note: - 参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置 - 参数format可选格式有'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx', - 'dbf', 'latex', 'ods' - 参数path为None会根据format参数和域名名称在项目结果目录生成相应文件 + --type target/table 要导出的目标类型(默认target) + --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (结果格式，默认CSV) + --path 结果路径(默认None，自动生成) POSITIONAL ARGUMENTS - TABLE - 要导出的表 + TARGET + 要导出的目标类型 FLAGS + --type=TYPE + 要导出的目标类型(默认target) --db=DB 要导出的数据库路径(默认为results/result.sqlite3) --valid=VALID diff --git a/docs/wildcard_judgment.png b/docs/wildcard_judgment.png new file mode 100644 index 0000000..0f6dc30 Binary files /dev/null and b/docs/wildcard_judgment.png differ diff --git a/modules/check/axfr.py b/modules/check/axfr.py index 55fb43d..5b33d12 100644 --- a/modules/check/axfr.py +++ b/modules/check/axfr.py @@ -11,19 +11,15 @@ import dns.resolver import dns.zone from common import utils -from common.module import Module +from common.check import Check from config.log import logger -class CheckAXFR(Module): - """ - DNS zone transfer vulnerability base class - """ - - def __init__(self, domain: str): - Module.__init__(self) +class AXFR(Check): + def __init__(self, domain): + Check.__init__(self) self.domain = domain - self.module = 'Check' + self.module = 'check' self.source = 'AXFRCheck' self.results = [] @@ -92,7 +88,7 @@ def run(domain): :param str domain: 域名 """ - check = CheckAXFR(domain) + check = AXFR(domain) check.run() diff --git a/modules/check/cdx.py b/modules/check/cdx.py index 3301ae7..ab3ea48 100644 --- a/modules/check/cdx.py +++ b/modules/check/cdx.py @@ -1,33 +1,22 @@ """ 检查crossdomain.xml文件收集子域名 """ - -from common.module import Module +from common.check import Check -class CheckCDX(Module): - """ - 检查crossdomain.xml文件收集子域名 - """ - def __init__(self, domain: str): - Module.__init__(self) +class CrossDomain(Check): + def __init__(self, domain): + Check.__init__(self) self.domain = domain - self.module = 'Check' - self.source = "CrossDomainXml" + self.module = 'check' + self.source = "CrossDomainCheck" def check(self): """ 检查crossdomain.xml收集子域名 """ - urls = [f'http://{self.domain}/crossdomain.xml', - f'https://{self.domain}/crossdomain.xml', - f'http://www.{self.domain}/crossdomain.xml', - f'https://www.{self.domain}/crossdomain.xml'] - for url in urls: - self.header = self.get_header() - self.proxy = self.get_proxy(self.source) - resp = self.get(url, check=False) - self.subdomains = self.collect_subdomains(resp) + filenames = {'crossdomain.xml'} + self.to_check(filenames) def run(self): """ @@ -47,7 +36,7 @@ def run(domain): :param domain: 域名 """ - check = CheckCDX(domain) + check = CrossDomain(domain) check.run() diff --git a/modules/check/cert.py b/modules/check/cert.py index e33c72b..6dc01ec 100644 --- a/modules/check/cert.py +++ b/modules/check/cert.py @@ -1,21 +1,18 @@ -#!/usr/bin/env python3 - """ 检查域名证书收集子域名 """ import socket import ssl -from common.module import Module from config.log import logger +from common.check import Check -class CheckCert(Module): +class CertInfo(Check): def __init__(self, domain): - Module.__init__(self) + Check.__init__(self) self.domain = domain - self.port = 443 # ssl port - self.module = 'Check' + self.module = 'check' self.source = 'CertInfo' def check(self): @@ -24,10 +21,11 @@ class CheckCert(Module): """ try: ctx = ssl.create_default_context() - sock = ctx.wrap_socket(socket.socket(), - server_hostname=self.domain) - sock.connect((self.domain, self.port)) - cert_dict = sock.getpeercert() + sock = socket.socket() + sock.settimeout(10) + wrap_sock = ctx.wrap_socket(sock, server_hostname=self.domain) + wrap_sock.connect((self.domain, 443)) + cert_dict = wrap_sock.getpeercert() except Exception as e: logger.log('DEBUG', e.args) return @@ -52,7 +50,7 @@ def run(domain): :param str domain: 域名 """ - check = CheckCert(domain) + check = CertInfo(domain) check.run() diff --git a/modules/check/csp.py b/modules/check/csp.py index 9da25e1..c61a935 100644 --- a/modules/check/csp.py +++ b/modules/check/csp.py @@ -3,19 +3,19 @@ Collect subdomains from ContentSecurityPolicy """ import requests -from common.module import Module from config.log import logger +from common.check import Check -class CheckCSP(Module): +class CSP(Check): """ Collect subdomains from ContentSecurityPolicy """ def __init__(self, domain, header): - Module.__init__(self) + Check.__init__(self) self.domain = domain - self.module = 'Check' - self.source = 'ContentSecurityPolicy' + self.module = 'check' + self.source = 'CSPCheck' self.csp_header = header def grab_header(self): @@ -34,8 +34,7 @@ class CheckCSP(Module): self.proxy = self.get_proxy(self.source) response = self.get(url, check=False) if response: - csp_header = response.headers - break + return response.headers return csp_header def check(self): @@ -73,7 +72,7 @@ def run(domain, header=None): :param str domain: 域名 :param dict or None header: 响应头 """ - check = CheckCSP(domain, header) + check = CSP(domain, header) check.run() diff --git a/modules/check/nsec.py b/modules/check/nsec.py index 952b648..9c5be4e 100644 --- a/modules/check/nsec.py +++ b/modules/check/nsec.py @@ -1,16 +1,16 @@ # https://www.icann.org/resources/pages/dnssec-what-is-it-why-important-2019-03-20-zh # https://appsecco.com/books/subdomain-enumeration/active_techniques/zone_walking.html -from common.module import Module from common import utils +from common.check import Check -class CheckNSEC(Module): +class NSEC(Check): def __init__(self, domain): - Module.__init__(self) + Check.__init__(self) self.domain = domain self.module = 'check' - self.source = "CheckNSEC" + self.source = "NSECCheck" def walk(self): domain = self.domain @@ -47,7 +47,7 @@ def run(domain): :param str domain: 域名 """ - brute = CheckNSEC(domain) + brute = NSEC(domain) brute.run() diff --git a/modules/check/robots.py b/modules/check/robots.py index 4557d84..1b38c8f 100644 --- a/modules/check/robots.py +++ b/modules/check/robots.py @@ -1,32 +1,22 @@ """ 检查内容安全策略收集子域名收集子域名 """ -from common.module import Module +from common.check import Check -class CheckRobots(Module): - """ - 检查robots.txt收集子域名 - """ +class Robots(Check): def __init__(self, domain): - Module.__init__(self) + Check.__init__(self) self.domain = domain - self.module = 'Check' - self.source = 'Robots' + self.module = 'check' + self.source = 'RobotsCheck' def check(self): """ 正则匹配域名的robots.txt文件中的子域 """ - urls = [f'http://{self.domain}/robots.txt', - f'https://{self.domain}/robots.txt', - f'http://www.{self.domain}/robots.txt', - f'https://www.{self.domain}/robots.txt'] - for url in urls: - self.header = self.get_header() - self.proxy = self.get_proxy(self.source) - resp = self.get(url, check=False, allow_redirects=False) - self.subdomains = self.collect_subdomains(resp) + filenames = {'robots.txt'} + self.to_check(filenames) def run(self): """ @@ -46,7 +36,7 @@ def run(domain): :param str domain: 域名 """ - check = CheckRobots(domain) + check = Robots(domain) check.run() diff --git a/modules/check/sitemap.py b/modules/check/sitemap.py index 26bd26d..1399d27 100644 --- a/modules/check/sitemap.py +++ b/modules/check/sitemap.py @@ -1,37 +1,22 @@ """ 检查内容安全策略收集子域名收集子域名 """ -from common.module import Module +from common.check import Check -class CheckRobots(Module): - """ - 检查sitemap收集子域名 - """ +class Sitemap(Check): def __init__(self, domain): - Module.__init__(self) + Check.__init__(self) self.domain = domain - self.module = 'Check' - self.source = 'Sitemap' + self.module = 'check' + self.source = 'SitemapCheck' def check(self): """ 正则匹配域名的sitemap文件中的子域 """ - urls = [f'http://{self.domain}/sitemap.xml', - f'http://www.{self.domain}/sitemap.xml', - f'http://{self.domain}/sitemap.txt', - f'http://www.{self.domain}/sitemap.txt', - f'http://{self.domain}/sitemap.html', - f'http://www.{self.domain}/sitemap.html', - f'http://{self.domain}/sitemap_index.xml', - f'http://www.{self.domain}/sitemap_index.xml'] - for url in urls: - self.header = self.get_header() - self.proxy = self.get_proxy(self.source) - self.timeout = 10 - resp = self.get(url, check=False) - self.subdomains = self.collect_subdomains(resp) + filenames = {'sitemap.xml', 'sitemap.txt', 'sitemap.html', 'sitemapindex.xml'} + self.to_check(filenames) def run(self): """ @@ -51,7 +36,7 @@ def run(domain): :param str domain: 域名 """ - check = CheckRobots(domain) + check = Sitemap(domain) check.run() diff --git a/modules/collect.py b/modules/collect.py index e012662..ab02a1e 100644 --- a/modules/collect.py +++ b/modules/collect.py @@ -1,38 +1,24 @@ -import time import threading import importlib -import dbexport from config.log import logger from config import settings -from common import utils class Collect(object): - """ - Collect subdomains - """ - - def __init__(self, domain, export=True): - self.domain = utils.get_main_domain(domain) - self.elapse = 0.0 + def __init__(self, domain): + self.domain = domain self.modules = [] self.collect_funcs = [] - self.path = None - self.export = export - self.format = 'csv' def get_mod(self): """ Get modules """ if settings.enable_all_module: - # modules = ['brute', 'certificates', 'crawl', - # 'datasets', 'intelligence', 'search'] # The crawl module has some problems modules = ['certificates', 'check', 'datasets', 'dnsquery', 'intelligence', 'search'] - # modules = ['certificates'] for module in modules: module_path = settings.module_dir.joinpath(module) for path in module_path.rglob('*.py'): @@ -55,7 +41,6 @@ class Collect(object): """ Class entrance """ - start = time.time() logger.log('INFOR', f'Start collecting subdomains of {self.domain}') self.get_mod() self.import_func() @@ -66,8 +51,7 @@ class Collect(object): func_obj, func_name = collect_func thread = threading.Thread(target=func_obj, name=func_name, - args=(self.domain,), - daemon=True) + args=(self.domain,)) threads.append(thread) # Start all threads for thread in threads: @@ -82,15 +66,6 @@ class Collect(object): if thread.is_alive(): logger.log('ALERT', f'{thread.name} module thread timed out') - # Export - if self.export: - if not self.path: - name = f'{self.domain}.{self.format}' - self.path = settings.result_save_dir.joinpath(name) - dbexport.export(self.domain, type='table', path=self.path, format=self.format) - end = time.time() - self.elapse = round(end - start, 1) - if __name__ == '__main__': collect = Collect('example.com') diff --git a/modules/dnsquery/srv.py b/modules/dnsquery/srv.py deleted file mode 100644 index f21bf14..0000000 --- a/modules/dnsquery/srv.py +++ /dev/null @@ -1,95 +0,0 @@ -""" -通过枚举域名常见的SRV记录并做查询来发现子域 -""" - -import json -import queue -import threading - -from common import utils -from common.module import Module -from config.setting import data_storage_dir - - -class BruteSRV(Module): - def __init__(self, domain): - Module.__init__(self) - self.domain = domain - self.module = 'dnsquery' - self.source = "BruteSRV" - self.qtype = 'SRV' # 利用的DNS记录的SRV记录查询子域 - self.thread_num = 10 - self.names_que = queue.Queue() - self.answers_que = queue.Queue() - - def gen_names(self): - path = data_storage_dir.joinpath('srv_prefixes.json') - with open(path, encoding='utf-8', errors='ignore') as file: - prefixes = json.load(file) - names = map(lambda prefix: prefix + self.domain, prefixes) - - for name in names: - self.names_que.put(name) - - def brute(self): - """ - 枚举域名的SRV记录 - """ - self.gen_names() - - for num in range(self.thread_num): - thread = BruteThread(self.names_que, self.answers_que) - thread.name = f'BruteThread-{num}' - thread.daemon = True - thread.start() - - self.names_que.join() - - while not self.answers_que.empty(): - answer = self.answers_que.get() - if answer is None: - continue - for item in answer: - record = str(item) - subdomains = self.match_subdomains(record) - self.subdomains.update(subdomains) - - def run(self): - """ - 类执行入口 - """ - self.begin() - self.brute() - self.finish() - self.save_json() - self.gen_result() - self.save_db() - - -class BruteThread(threading.Thread): - def __init__(self, names_que, answers_que): - threading.Thread.__init__(self) - self.names_que = names_que - self.answers_que = answers_que - - def run(self): - while True: - name = self.names_que.get() - answer = utils.dns_query(name, 'SRV') - self.answers_que.put(answer) - self.names_que.task_done() - - -def run(domain): - """ - 类统一调用入口 - - :param str domain: 域名 - """ - brute = BruteSRV(domain) - brute.run() - - -if __name__ == '__main__': - run('zonetransfer.me') - # run('example.com') diff --git a/modules/finder.py b/modules/finder.py index 12c9867..b21de6b 100644 --- a/modules/finder.py +++ b/modules/finder.py @@ -190,9 +190,10 @@ def find_subdomains(domain, data): subdomains.update(find_in_resp(domain, req_url, rsp_html)) js_urls.update(find_js_urls(domain, req_url, rsp_html)) - resp_data = request.urls_request(js_urls) - for resp, text in resp_data: - if not text: + resp_data = request.bulk_request(js_urls) + for resp in resp_data: + if not resp: continue + text = utils.decode_resp_text(resp) subdomains.update(find_in_resp(domain, resp.url, text)) return subdomains diff --git a/modules/srv.py b/modules/srv.py new file mode 100644 index 0000000..b01dc4c --- /dev/null +++ b/modules/srv.py @@ -0,0 +1,75 @@ +""" +通过枚举域名常见的SRV记录并做查询来发现子域 +""" + +import queue +import threading + +from common import utils +from common.module import Module +from config.setting import data_storage_dir + + +class BruteSRV(Module): + def __init__(self, domain): + Module.__init__(self) + self.domain = domain + self.module = 'BruteSRV' + self.source = "BruteSRV" + self.qtype = 'SRV' + self.thread_num = 20 + self.names_queue = queue.Queue() + self.answers_queue = queue.Queue() + + def fill_queue(self): + path = data_storage_dir.joinpath('srv_prefixes.json') + prefixes = utils.load_json(path) + for prefix in prefixes: + self.names_queue.put(prefix + self.domain) + + def do_brute(self): + for num in range(self.thread_num): + thread = BruteThread(self.names_queue, self.answers_queue) + thread.name = f'BruteThread-{num}' + thread.daemon = True + thread.start() + self.names_queue.join() + + def deal_answers(self): + while not self.answers_queue.empty(): + answer = self.answers_queue.get() + if answer is None: + continue + for item in answer: + record = str(item) + subdomains = self.match_subdomains(record) + self.subdomains.update(subdomains) + + def run(self): + self.begin() + self.fill_queue() + self.do_brute() + self.deal_answers() + self.finish() + self.save_json() + self.gen_result() + self.save_db() + + +class BruteThread(threading.Thread): + def __init__(self, names_queue, answers_queue): + threading.Thread.__init__(self) + self.names_queue = names_queue + self.answers_queue = answers_queue + + def run(self): + while True: + name = self.names_queue.get() + answer = utils.dns_query(name, 'SRV') + self.answers_queue.put(answer) + self.names_queue.task_done() + + +if __name__ == '__main__': + brute = BruteSRV('zonetransfer.me') + brute.run() diff --git a/oneforall.py b/oneforall.py index 184703e..533539a 100644 --- a/oneforall.py +++ b/oneforall.py @@ -17,6 +17,7 @@ from brute import Brute from common import utils, resolve, request from common.database import Database from modules.collect import Collect +from modules.srv import BruteSRV from modules.finder import Finder from modules import iscdn from config import settings @@ -65,7 +66,6 @@ class OneForAll(object): python3 oneforall.py --target example.com --show True run Note: - --alive True/False Only export alive subdomains or not (default False) --port small/medium/large See details in ./config/setting.py(default small) --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) --path Result path (default None, automatically generated) @@ -188,8 +188,12 @@ class OneForAll(object): self.origin_table = self.domain + '_origin_result' self.resolve_table = self.domain + '_resolve_result' - collect = Collect(self.domain, export=False) + collect = Collect(self.domain) collect.run() + + srv = BruteSRV(self.domain) + srv.run() + if self.brute: # Due to there will be a large number of dns resolution requests, # may cause other network tasks to be error @@ -240,7 +244,7 @@ class OneForAll(object): # Scan subdomain takeover if self.takeover: subdomains = utils.get_subdomains(self.data) - takeover = Takeover(subdomains) + takeover = Takeover(targets=subdomains) takeover.run() return self.data diff --git a/requirements.txt b/requirements.txt index f331a15..c9c188e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,24 +1,19 @@ -aiohttp==3.6.2 -async-timeout==3.0.1 -attrs==19.3.0 beautifulsoup4==4.9.1 bs4==0.0.1 certifi==2020.6.20 chardet==3.0.4 -cloudscraper==1.2.42 +cloudscraper==1.2.46 colorama==0.4.3 -dnspython==1.16.0 +dnspython==2.0.0 docopt==0.6.2 et-xmlfile==1.0.1 exrex==0.10.5 fire==0.3.1 +future==0.18.2 idna==2.10 -IP2Location==8.4.1 jdcal==1.4.1 loguru==0.5.1 -multidict==4.7.6 openpyxl==2.4.11 -psutil==5.7.0 pyparsing==2.4.7 PySocks==1.7.1 records==0.5.3 @@ -27,12 +22,12 @@ requests-file==1.5.1 requests-toolbelt==0.9.1 six==1.15.0 soupsieve==2.0.1 -SQLAlchemy==1.3.18 +SQLAlchemy==1.3.19 tablib==2.0.0 tenacity==6.2.0 termcolor==1.1.0 -tldextract==2.2.2 -tqdm==4.47.0 -urllib3==1.25.9 -win32-setctime==1.0.1 -yarl==1.4.2 +tldextract==2.2.3 +tqdm==4.48.2 +treelib==1.6.1 +urllib3==1.25.10 +win32-setctime==1.0.1 \ No newline at end of file diff --git a/takeover.py b/takeover.py index 2a802bd..681be82 100644 --- a/takeover.py +++ b/takeover.py @@ -20,7 +20,6 @@ from config.log import logger from config import settings from common import utils from common.module import Module -from common.domain import Domain def get_fingerprint(): @@ -41,34 +40,32 @@ def get_cname(subdomain): return answer.to_text() # 一个子域只有一个CNAME记录 -def get_maindomain(subdomain): - return Domain(subdomain).registered() - - class Takeover(Module): """ OneForAll subdomain takeover module Example: python3 takeover.py --target www.example.com --format csv run - python3 takeover.py --target ./subdomains.txt --thread 10 run + python3 takeover.py --targets ./subdomains.txt --thread 10 run Note: --format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format) --path Result directory (default directory is ./results) - :param any target: One domain or File path of one domain per line (required) - :param int thread: threads number (default 100) - :param str format: Result format (default csv) - :param str path: Result directory (default None) + :param str target: One domain (target or targets must be provided) + :param str targets: File path of one domain per line + :param int thread: threads number (default 20) + :param str format: Result format (default csv) + :param str path: Result directory (default None) """ - def __init__(self, target, thread=100, path=None, format='csv'): + def __init__(self, target=None, targets=None, thread=20, path=None, format='csv'): Module.__init__(self) self.subdomains = set() self.module = 'Check' self.source = 'Takeover' self.target = target + self.targets = targets self.thread = thread self.path = path self.format = format @@ -101,10 +98,10 @@ class Takeover(Module): cname = get_cname(subdomain) if cname is None: return - maindomain = get_maindomain(cname) + main_domain = utils.get_main_domain(cname) for fingerprint in self.fingerprints: cnames = fingerprint.get('cname') - if maindomain not in cnames: + if main_domain not in cnames: continue responses = fingerprint.get('response') self.compare(subdomain, cname, responses) @@ -116,7 +113,6 @@ class Takeover(Module): self.subdomainq.task_done() def progress(self): - # 设置进度 bar = tqdm() bar.total = len(self.subdomains) bar.desc = 'Check Progress' @@ -127,12 +123,14 @@ class Takeover(Module): bar.update() if done == bar.total: # 完成队列中所有子域的检查退出 break - # bar.close() def run(self): start = time.time() logger.log('INFOR', f'Start running {self.source} module') - self.subdomains = utils.get_domains(self.target) + if isinstance(self.targets, set): + self.subdomains = self.targets + else: + self.subdomains = utils.get_domains(self.target, self.targets) self.format = utils.check_format(self.format, len(self.subdomains)) timestamp = utils.get_timestamp() name = f'takeover_check_result_{timestamp}' diff --git a/test.py b/test.py index 1b62fc9..e59c64f 100644 --- a/test.py +++ b/test.py @@ -8,7 +8,7 @@ Example from oneforall import OneForAll if __name__ == '__main__': - test = OneForAll(target='hackfun.org') + test = OneForAll(target='mi.com') test.brute = True test.req = True test.takeover = True