mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-10-25 17:26:15 +08:00
Merge branch 'master' of https://github.com/shmilylty/OneForAll
This commit is contained in:
Jing Ling
@@ -13,15 +13,13 @@ requests = "*"
|
||||
records = "*"
|
||||
tldextract = "*"
|
||||
exrex = "*"
|
||||
aiohttp = "*"
|
||||
fire = "*"
|
||||
bs4 = "*"
|
||||
pysocks = "*"
|
||||
cloudscraper = "*"
|
||||
tablib = "*"
|
||||
psutil = "*"
|
||||
tenacity = "*"
|
||||
ip2location = "*"
|
||||
treelib = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.8"
|
||||
|
||||
Generated
+62
-152
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "6556f720e7fb052c3f923f3e7986cc00c1cc230db729ad79685a127c02568c96"
|
||||
"sha256": "fde2c88929fdf63725faea24a4fc7d82d6792caaca6676c08e96a40227f47e18"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
@@ -16,38 +16,6 @@
|
||||
]
|
||||
},
|
||||
"default": {
|
||||
"aiohttp": {
|
||||
"hashes": [
|
||||
"sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e",
|
||||
"sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326",
|
||||
"sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a",
|
||||
"sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654",
|
||||
"sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a",
|
||||
"sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4",
|
||||
"sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17",
|
||||
"sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec",
|
||||
"sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd",
|
||||
"sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48",
|
||||
"sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59",
|
||||
"sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.6.2"
|
||||
},
|
||||
"async-timeout": {
|
||||
"hashes": [
|
||||
"sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
|
||||
"sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
|
||||
],
|
||||
"version": "==3.0.1"
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
|
||||
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
|
||||
],
|
||||
"version": "==19.3.0"
|
||||
},
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7",
|
||||
@@ -79,11 +47,11 @@
|
||||
},
|
||||
"cloudscraper": {
|
||||
"hashes": [
|
||||
"sha256:1d4e158de3be7a4e651083f13a14a25b75853186e989d9d97db1a08056b7a0b6",
|
||||
"sha256:c8c255aaa84e80d2cb5bf2c1ad1ffa000ce864595ca4cdb75ff7aab6f97befec"
|
||||
"sha256:793095bbc37aae84a5c38d9f66b56f3c83a00f0b1ff2d334f75a2b8f88b924af",
|
||||
"sha256:a7ae28b5dc4e4ef0789ef608c42a4382bf6d3e3dd6b8117c2e3633103131ae43"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.2.42"
|
||||
"version": "==1.2.46"
|
||||
},
|
||||
"colorama": {
|
||||
"hashes": [
|
||||
@@ -95,11 +63,11 @@
|
||||
},
|
||||
"dnspython": {
|
||||
"hashes": [
|
||||
"sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01",
|
||||
"sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d"
|
||||
"sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7",
|
||||
"sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.16.0"
|
||||
"version": "==2.0.0"
|
||||
},
|
||||
"docopt": {
|
||||
"hashes": [
|
||||
@@ -127,22 +95,19 @@
|
||||
"index": "pypi",
|
||||
"version": "==0.3.1"
|
||||
},
|
||||
"future": {
|
||||
"hashes": [
|
||||
"sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
|
||||
],
|
||||
"version": "==0.18.2"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
|
||||
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==2.10"
|
||||
},
|
||||
"ip2location": {
|
||||
"hashes": [
|
||||
"sha256:4a83f32098091b6aafaef63b393331bf973bbf54547db82de38f30108b5f079c",
|
||||
"sha256:bbddbd5e3f79c18a9710afd43be09cb21724d7324557668017c545b5c5e62bc4"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==8.4.1"
|
||||
},
|
||||
"jdcal": {
|
||||
"hashes": [
|
||||
"sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba",
|
||||
@@ -158,58 +123,17 @@
|
||||
"index": "pypi",
|
||||
"version": "==0.5.1"
|
||||
},
|
||||
"multidict": {
|
||||
"hashes": [
|
||||
"sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a",
|
||||
"sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000",
|
||||
"sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2",
|
||||
"sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507",
|
||||
"sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5",
|
||||
"sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7",
|
||||
"sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d",
|
||||
"sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463",
|
||||
"sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19",
|
||||
"sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3",
|
||||
"sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b",
|
||||
"sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c",
|
||||
"sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87",
|
||||
"sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7",
|
||||
"sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430",
|
||||
"sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255",
|
||||
"sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d"
|
||||
],
|
||||
"markers": "python_version >= '3.5'",
|
||||
"version": "==4.7.6"
|
||||
},
|
||||
"openpyxl": {
|
||||
"hashes": [
|
||||
"sha256:626d38647c063d55803ef4971c4d43226538d4e95cb6260c094e363ee33e10c7"
|
||||
],
|
||||
"version": "==2.4.11"
|
||||
},
|
||||
"psutil": {
|
||||
"hashes": [
|
||||
"sha256:1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058",
|
||||
"sha256:298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953",
|
||||
"sha256:60b86f327c198561f101a92be1995f9ae0399736b6eced8f24af41ec64fb88d4",
|
||||
"sha256:685ec16ca14d079455892f25bd124df26ff9137664af445563c1bd36629b5e0e",
|
||||
"sha256:73f35ab66c6c7a9ce82ba44b1e9b1050be2a80cd4dcc3352cc108656b115c74f",
|
||||
"sha256:75e22717d4dbc7ca529ec5063000b2b294fc9a367f9c9ede1f65846c7955fd38",
|
||||
"sha256:a02f4ac50d4a23253b68233b07e7cdb567bd025b982d5cf0ee78296990c22d9e",
|
||||
"sha256:d008ddc00c6906ec80040d26dc2d3e3962109e40ad07fd8a12d0284ce5e0e4f8",
|
||||
"sha256:d84029b190c8a66a946e28b4d3934d2ca1528ec94764b180f7d6ea57b0e75e26",
|
||||
"sha256:e2d0c5b07c6fe5a87fa27b7855017edb0d52ee73b71e6ee368fae268605cc3f5",
|
||||
"sha256:f344ca230dd8e8d5eee16827596f1c22ec0876127c28e800d7ae20ed44c4b310"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==5.7.0"
|
||||
},
|
||||
"pyparsing": {
|
||||
"hashes": [
|
||||
"sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
|
||||
"sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
|
||||
],
|
||||
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==2.4.7"
|
||||
},
|
||||
"pysocks": {
|
||||
@@ -256,7 +180,6 @@
|
||||
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
|
||||
"sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==1.15.0"
|
||||
},
|
||||
"soupsieve": {
|
||||
@@ -264,42 +187,45 @@
|
||||
"sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
|
||||
"sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
|
||||
],
|
||||
"markers": "python_version >= '3.5'",
|
||||
"version": "==2.0.1"
|
||||
},
|
||||
"sqlalchemy": {
|
||||
"hashes": [
|
||||
"sha256:0942a3a0df3f6131580eddd26d99071b48cfe5aaf3eab2783076fbc5a1c1882e",
|
||||
"sha256:0ec575db1b54909750332c2e335c2bb11257883914a03bc5a3306a4488ecc772",
|
||||
"sha256:109581ccc8915001e8037b73c29590e78ce74be49ca0a3630a23831f9e3ed6c7",
|
||||
"sha256:16593fd748944726540cd20f7e83afec816c2ac96b082e26ae226e8f7e9688cf",
|
||||
"sha256:427273b08efc16a85aa2b39892817e78e3ed074fcb89b2a51c4979bae7e7ba98",
|
||||
"sha256:50c4ee32f0e1581828843267d8de35c3298e86ceecd5e9017dc45788be70a864",
|
||||
"sha256:512a85c3c8c3995cc91af3e90f38f460da5d3cade8dc3a229c8e0879037547c9",
|
||||
"sha256:57aa843b783179ab72e863512e14bdcba186641daf69e4e3a5761d705dcc35b1",
|
||||
"sha256:621f58cd921cd71ba6215c42954ffaa8a918eecd8c535d97befa1a8acad986dd",
|
||||
"sha256:6ac2558631a81b85e7fb7a44e5035347938b0a73f5fdc27a8566777d0792a6a4",
|
||||
"sha256:716754d0b5490bdcf68e1e4925edc02ac07209883314ad01a137642ddb2056f1",
|
||||
"sha256:736d41cfebedecc6f159fc4ac0769dc89528a989471dc1d378ba07d29a60ba1c",
|
||||
"sha256:8619b86cb68b185a778635be5b3e6018623c0761dde4df2f112896424aa27bd8",
|
||||
"sha256:87fad64529cde4f1914a5b9c383628e1a8f9e3930304c09cf22c2ae118a1280e",
|
||||
"sha256:89494df7f93b1836cae210c42864b292f9b31eeabca4810193761990dc689cce",
|
||||
"sha256:8cac7bb373a5f1423e28de3fd5fc8063b9c8ffe8957dc1b1a59cb90453db6da1",
|
||||
"sha256:8fd452dc3d49b3cc54483e033de6c006c304432e6f84b74d7b2c68afa2569ae5",
|
||||
"sha256:adad60eea2c4c2a1875eb6305a0b6e61a83163f8e233586a4d6a55221ef984fe",
|
||||
"sha256:c26f95e7609b821b5f08a72dab929baa0d685406b953efd7c89423a511d5c413",
|
||||
"sha256:cbe1324ef52ff26ccde2cb84b8593c8bf930069dfc06c1e616f1bfd4e47f48a3",
|
||||
"sha256:d05c4adae06bd0c7f696ae3ec8d993ed8ffcc4e11a76b1b35a5af8a099bd2284",
|
||||
"sha256:d98bc827a1293ae767c8f2f18be3bb5151fd37ddcd7da2a5f9581baeeb7a3fa1",
|
||||
"sha256:da2fb75f64792c1fc64c82313a00c728a7c301efe6a60b7a9fe35b16b4368ce7",
|
||||
"sha256:e4624d7edb2576cd72bb83636cd71c8ce544d8e272f308bd80885056972ca299",
|
||||
"sha256:e89e0d9e106f8a9180a4ca92a6adde60c58b1b0299e1b43bd5e0312f535fbf33",
|
||||
"sha256:f11c2437fb5f812d020932119ba02d9e2bc29a6eca01a055233a8b449e3e1e7d",
|
||||
"sha256:f57be5673e12763dd400fea568608700a63ce1c6bd5bdbc3cc3a2c5fdb045274",
|
||||
"sha256:fc728ece3d5c772c196fd338a99798e7efac7a04f9cb6416299a3638ee9a94cd"
|
||||
"sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb",
|
||||
"sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804",
|
||||
"sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6",
|
||||
"sha256:26c5ca9d09f0e21b8671a32f7d83caad5be1f6ff45eef5ec2f6fd0db85fc5dc0",
|
||||
"sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe",
|
||||
"sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de",
|
||||
"sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36",
|
||||
"sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e",
|
||||
"sha256:465c999ef30b1c7525f81330184121521418a67189053bcf585824d833c05b66",
|
||||
"sha256:51064ee7938526bab92acd049d41a1dc797422256086b39c08bafeffb9d304c6",
|
||||
"sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc",
|
||||
"sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d",
|
||||
"sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce",
|
||||
"sha256:6557af9e0d23f46b8cd56f8af08eaac72d2e3c632ac8d5cf4e20215a8dca7cea",
|
||||
"sha256:73a40d4fcd35fdedce07b5885905753d5d4edf413fbe53544dd871f27d48bd4f",
|
||||
"sha256:8280f9dae4adb5889ce0bb3ec6a541bf05434db5f9ab7673078c00713d148365",
|
||||
"sha256:83469ad15262402b0e0974e612546bc0b05f379b5aa9072ebf66d0f8fef16bea",
|
||||
"sha256:860d0fe234922fd5552b7f807fbb039e3e7ca58c18c8d38aa0d0a95ddf4f6c23",
|
||||
"sha256:883c9fb62cebd1e7126dd683222b3b919657590c3e2db33bdc50ebbad53e0338",
|
||||
"sha256:8afcb6f4064d234a43fea108859942d9795c4060ed0fbd9082b0f280181a15c1",
|
||||
"sha256:96f51489ac187f4bab588cf51f9ff2d40b6d170ac9a4270ffaed535c8404256b",
|
||||
"sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e",
|
||||
"sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba",
|
||||
"sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02",
|
||||
"sha256:b6ff91356354b7ff3bd208adcf875056d3d886ed7cef90c571aef2ab8a554b12",
|
||||
"sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86",
|
||||
"sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d",
|
||||
"sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7",
|
||||
"sha256:e49947d583fe4d29af528677e4f0aa21f5e535ca2ae69c48270ebebd0d8843c0",
|
||||
"sha256:eb1d71643e4154398b02e88a42fc8b29db8c44ce4134cf0f4474bfc5cb5d4dac",
|
||||
"sha256:f2e8a9c0c8813a468aa659a01af6592f71cd30237ec27c4cc0683f089f90dcfc",
|
||||
"sha256:fe7fe11019fc3e6600819775a7d55abc5446dda07e9795f5954fdbf8a49e1c37"
|
||||
],
|
||||
"markers": "python_version >= '3.0'",
|
||||
"version": "==1.3.18"
|
||||
"version": "==1.3.19"
|
||||
},
|
||||
"tablib": {
|
||||
"hashes": [
|
||||
@@ -325,27 +251,33 @@
|
||||
},
|
||||
"tldextract": {
|
||||
"hashes": [
|
||||
"sha256:16b2f7e81d89c2a5a914d25bdbddd3932c31a6b510db886c3ce0764a195c0ee7",
|
||||
"sha256:9aa21a1f7827df4209e242ec4fc2293af5940ec730cde46ea80f66ed97bfc808"
|
||||
"sha256:ab0e38977a129c72729476d5f8c85a8e1f8e49e9202e1db8dca76e95da7be9a8",
|
||||
"sha256:c2a8a392edf3ea6fa8be80930f04c3ac29e91fa604cb2139bdf6a37fc1e1ac6d"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.2.2"
|
||||
"version": "==2.2.3"
|
||||
},
|
||||
"tqdm": {
|
||||
"hashes": [
|
||||
"sha256:63ef7a6d3eb39f80d6b36e4867566b3d8e5f1fe3d6cb50c5e9ede2b3198ba7b7",
|
||||
"sha256:7810e627bcf9d983a99d9ff8a0c09674400fd2927eddabeadf153c14a2ec8656"
|
||||
"sha256:1a336d2b829be50e46b84668691e0a2719f26c97c62846298dd5ae2937e4d5cf",
|
||||
"sha256:564d632ea2b9cb52979f7956e093e831c28d441c11751682f84c86fc46e4fd21"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.47.0"
|
||||
"version": "==4.48.2"
|
||||
},
|
||||
"treelib": {
|
||||
"hashes": [
|
||||
"sha256:1cbfffb2d2b75ccac27d0200cee0507b6fbb0726e0afb9fae017ade5d2ce8788"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.6.1"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527",
|
||||
"sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"
|
||||
"sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a",
|
||||
"sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
|
||||
"version": "==1.25.9"
|
||||
"version": "==1.25.10"
|
||||
},
|
||||
"win32-setctime": {
|
||||
"hashes": [
|
||||
@@ -354,28 +286,6 @@
|
||||
],
|
||||
"markers": "sys_platform == 'win32'",
|
||||
"version": "==1.0.1"
|
||||
},
|
||||
"yarl": {
|
||||
"hashes": [
|
||||
"sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce",
|
||||
"sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6",
|
||||
"sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce",
|
||||
"sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae",
|
||||
"sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d",
|
||||
"sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f",
|
||||
"sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b",
|
||||
"sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b",
|
||||
"sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb",
|
||||
"sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462",
|
||||
"sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea",
|
||||
"sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70",
|
||||
"sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1",
|
||||
"sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a",
|
||||
"sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b",
|
||||
"sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080",
|
||||
"sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2"
|
||||
],
|
||||
"version": "==1.4.2"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
[](https://codecov.io/gh/shmilylty/OneForAll)
|
||||
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/releases)
|
||||
|
||||
👊**OneForAll是一款功能强大的子域收集工具** 📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md)
|
||||
@@ -15,21 +15,6 @@
|
||||
|
||||
📢 请务必花一点时间阅读此文档，有助于你快速熟悉OneForAll！
|
||||
|
||||
<details>
|
||||
<summary><b>🐍安装要求</b></summary>
|
||||
|
||||
OneForAll基于[Python 3.8.0]( https://www.python.org/downloads/release/python-380/ )开发和测试，请使用高于Python 3.8.0的稳定发行版本，其他版本可能会出现一些问题（Windows平台必须使用3.8.0以上版本），安装Python环境可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)。运行以下命令检查Python和pip3版本：
|
||||
```bash
|
||||
python -V
|
||||
pip3 -V
|
||||
```
|
||||
如果你看到类似以下的输出便说明Python环境没有问题：
|
||||
```bash
|
||||
Python 3.8.0
|
||||
pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python38\site-packages\pip (python 3.8)
|
||||
```
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>✔安装步骤（git 版）</b></summary>
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ OneForAll subdomain brute module
|
||||
import gc
|
||||
import json
|
||||
import time
|
||||
import random
|
||||
import secrets
|
||||
|
||||
import exrex
|
||||
@@ -21,67 +20,137 @@ from dns.resolver import NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers
|
||||
|
||||
import dbexport
|
||||
from common import utils
|
||||
from common import similarity
|
||||
from config import settings
|
||||
from common.module import Module
|
||||
from config.log import logger
|
||||
|
||||
|
||||
@tenacity.retry(stop=tenacity.stop_after_attempt(3))
|
||||
def do_query_a(domain, resolver):
|
||||
def config_resolver(nameservers):
|
||||
"""
|
||||
配置DNS解析器
|
||||
|
||||
:param nameservers: 名称解析服务器地址
|
||||
"""
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.nameservers = nameservers
|
||||
resolver.rotate = True # 随机使用NS
|
||||
resolver.cache = None # 不使用DNS缓存
|
||||
return resolver
|
||||
|
||||
|
||||
def gen_random_subdomains(domain, count):
|
||||
"""
|
||||
生成指定数量的随机子域域名列表
|
||||
|
||||
:param domain: 主域
|
||||
:param count: 数量
|
||||
"""
|
||||
subdomains = set()
|
||||
if count < 1:
|
||||
return subdomains
|
||||
for _ in range(count):
|
||||
token = secrets.token_hex(4)
|
||||
subdomains.add(f'{token}.{domain}')
|
||||
return subdomains
|
||||
|
||||
|
||||
def query_a_record(subdomain, resolver):
|
||||
"""
|
||||
查询子域A记录
|
||||
|
||||
:param subdomain: 子域
|
||||
:param resolver: DNS解析器
|
||||
"""
|
||||
try:
|
||||
answer = resolver.query(domain, 'A')
|
||||
# If resolve random subdomain raise timeout error, try again
|
||||
except Timeout as e:
|
||||
logger.log('ALERT', f'DNS resolve timeout, retrying')
|
||||
logger.log('DEBUG', e.args)
|
||||
raise tenacity.TryAgain
|
||||
# If resolve random subdomain raise NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers error
|
||||
# It means that there is no A record of random subdomain and not use wildcard dns record
|
||||
except (NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers) as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
logger.log('INFOR', f'{domain} seems not use wildcard dns record')
|
||||
return False
|
||||
answer = resolver.query(subdomain, 'A')
|
||||
except Exception as e:
|
||||
logger.log('ALERT', f'Detect {domain} wildcard dns record error')
|
||||
logger.log('FATAL', e.args)
|
||||
exit(1)
|
||||
else:
|
||||
if answer.rrset is None:
|
||||
logger.log('ALERT', f'DNS resolve dont have result, retrying')
|
||||
raise tenacity.TryAgain
|
||||
ttl = answer.ttl
|
||||
name = answer.name
|
||||
ips = {item.address for item in answer}
|
||||
logger.log('ALERT', f'{domain} use wildcard dns record')
|
||||
logger.log('ALERT', f'{domain} resolve to: {name} '
|
||||
f'IP: {ips} TTL: {ttl}')
|
||||
logger.log('DEBUG', f'Query {subdomain} wildcard dns record error')
|
||||
logger.log('DEBUG', e.args)
|
||||
return False
|
||||
if answer.rrset is None:
|
||||
return False
|
||||
ttl = answer.ttl
|
||||
name = answer.name
|
||||
ips = {item.address for item in answer}
|
||||
logger.log('ALERT', f'{subdomain} resolve to: {name} '
|
||||
f'IP: {ips} TTL: {ttl}')
|
||||
return True
|
||||
|
||||
|
||||
def all_resolve_success(subdomains):
|
||||
"""
|
||||
判断是否所有子域都解析成功
|
||||
|
||||
:param subdomains: 子域列表
|
||||
"""
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.cache = None # 不使用DNS缓存
|
||||
status = set()
|
||||
for subdomain in subdomains:
|
||||
status.add(query_a_record(subdomain, resolver))
|
||||
return all(status)
|
||||
|
||||
|
||||
def all_request_success(subdomains):
|
||||
"""
|
||||
判断是否所有子域都请求成功
|
||||
|
||||
:param subdomains: 子域列表
|
||||
"""
|
||||
result = list()
|
||||
for subdomain in subdomains:
|
||||
url = f'http://{subdomain}'
|
||||
resp = utils.get_url_resp(url)
|
||||
if resp:
|
||||
logger.log('ALERT', f'Request: {url} Status: {resp.status_code} '
|
||||
f'Size: {len(resp.content)}')
|
||||
result.append(resp.text)
|
||||
else:
|
||||
result.append(resp)
|
||||
return all(result), result
|
||||
|
||||
|
||||
def any_similar_html(resp_list):
|
||||
"""
|
||||
判断是否有一组HTML页面结构相似
|
||||
|
||||
:param resp_list: 响应HTML页面
|
||||
"""
|
||||
html_doc1, html_doc2, html_doc3 = resp_list
|
||||
if similarity.is_similar(html_doc1, html_doc2):
|
||||
return True
|
||||
if similarity.is_similar(html_doc1, html_doc3):
|
||||
return True
|
||||
if similarity.is_similar(html_doc2, html_doc3):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def detect_wildcard(domain, authoritative_ns):
|
||||
def detect_wildcard(domain):
|
||||
"""
|
||||
Detect use wildcard dns record or not
|
||||
|
||||
:param str domain: domain
|
||||
:param list authoritative_ns: authoritative name server
|
||||
:return bool use wildcard dns record or not
|
||||
"""
|
||||
logger.log('INFOR', f'Detecting {domain} use wildcard dns record or not')
|
||||
token = secrets.token_hex(4)
|
||||
random_subdomain = f'{token}.{domain}'
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.nameservers = authoritative_ns
|
||||
resolver.rotate = True
|
||||
resolver.cache = None
|
||||
try:
|
||||
wildcard = do_query_a(random_subdomain, resolver)
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
logger.log('ALERT', f'Multiple detection errors, so temporarily {domain} '
|
||||
f'does not use wildcard dns record')
|
||||
random_subdomains = gen_random_subdomains(domain, 3)
|
||||
if not all_resolve_success(random_subdomains):
|
||||
return False
|
||||
is_all_success, all_request_resp = all_request_success(random_subdomains)
|
||||
if not is_all_success:
|
||||
return True
|
||||
return any_similar_html(all_request_resp)
|
||||
|
||||
|
||||
def is_enable_wildcard(domain):
|
||||
is_enable = detect_wildcard(domain)
|
||||
if is_enable:
|
||||
logger.log('ALERT', f'The domain {domain} enables wildcard')
|
||||
else:
|
||||
return wildcard
|
||||
logger.log('ALERT', f'The domain {domain} disables wildcard')
|
||||
return is_enable
|
||||
|
||||
|
||||
def gen_subdomains(expression, path):
|
||||
@@ -229,9 +298,9 @@ def collect_wildcard_record(domain, authoritative_ns):
|
||||
if not authoritative_ns:
|
||||
return list(), int()
|
||||
resolver = utils.dns_resolver()
|
||||
resolver.nameservers = authoritative_ns
|
||||
resolver.rotate = True
|
||||
resolver.cache = None
|
||||
resolver.nameservers = authoritative_ns # 使用权威名称服务器
|
||||
resolver.rotate = True # 随机使用NS
|
||||
resolver.cache = None # 不使用DNS缓存
|
||||
ips = set()
|
||||
ttl = int()
|
||||
ips_stat = dict()
|
||||
@@ -464,14 +533,13 @@ class Brute(Module):
|
||||
Example：
|
||||
brute.py --target domain.com --word True run
|
||||
brute.py --targets ./domains.txt --word True run
|
||||
brute.py --target domain.com --word True --process 1 run
|
||||
brute.py --target domain.com --word True --concurrent 2000 run
|
||||
brute.py --target domain.com --word True --wordlist subnames.txt run
|
||||
brute.py --target domain.com --word True --recursive True --depth 2 run
|
||||
brute.py --target d.com --fuzz True --place m.*.d.com --rule '[a-z]' run
|
||||
brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
|
||||
|
||||
Note:
|
||||
--alive True/False Only export alive subdomains or not (default False)
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--path Result path (default None, automatically generated)
|
||||
|
||||
@@ -479,7 +547,7 @@ class Brute(Module):
|
||||
:param str target: One domain (target or targets must be provided)
|
||||
:param str targets: File path of one domain per line
|
||||
:param int process: Number of processes (default 1)
|
||||
:param int concurrent: Number of concurrent (default 10000)
|
||||
:param int concurrent: Number of concurrent (default 2000)
|
||||
:param bool word: Use word mode generate dictionary (default False)
|
||||
:param str wordlist: Dictionary path used in word mode (default use ./config/default.py)
|
||||
:param bool recursive: Use recursion (default False)
|
||||
@@ -524,8 +592,6 @@ class Brute(Module):
|
||||
self.domain = str() # 当前正在进行爆破的域名
|
||||
self.ips_times = dict() # IP集合出现次数
|
||||
self.enable_wildcard = False # 当前域名是否使用泛解析
|
||||
self.wildcard_check = settings.brute_wildcard_check
|
||||
self.wildcard_deal = settings.brute_wildcard_deal
|
||||
self.check_env = True
|
||||
self.quite = False
|
||||
|
||||
@@ -596,7 +662,7 @@ class Brute(Module):
|
||||
wildcard_ttl = int() # 泛解析TTL整型值
|
||||
ns_list = query_domain_ns(self.domain)
|
||||
ns_ip_list = query_domain_ns_a(ns_list) # DNS权威名称服务器对应A记录列表
|
||||
self.enable_wildcard = detect_wildcard(domain, ns_ip_list)
|
||||
self.enable_wildcard = is_enable_wildcard(domain)
|
||||
|
||||
if self.enable_wildcard:
|
||||
wildcard_ips, wildcard_ttl = collect_wildcard_record(domain,
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
from common.module import Module
|
||||
|
||||
|
||||
class Check(Module):
|
||||
"""
|
||||
Check base class
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
Module.__init__(self)
|
||||
self.request_status = 1
|
||||
|
||||
def to_check(self, filenames):
|
||||
urls = set()
|
||||
for filename in filenames:
|
||||
urls.update((f'http://{self.domain}/{filename}',
|
||||
f'https://{self.domain}/{filename}',
|
||||
f'http://www.{self.domain}/{filename}',
|
||||
f'https://www.{self.domain}/{filename}'))
|
||||
for url in urls:
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(url, check=False, ignore=True)
|
||||
self.subdomains = self.collect_subdomains(resp)
|
||||
if self.subdomains:
|
||||
break
|
||||
+8
-8
@@ -78,8 +78,8 @@ class Database(object):
|
||||
f'cidr text,'
|
||||
f'asn text,'
|
||||
f'org text,'
|
||||
f'ip2region text,'
|
||||
f'ip2location text,'
|
||||
f'addr text,'
|
||||
f'isp text,'
|
||||
f'resolver text,'
|
||||
f'module text,'
|
||||
f'source text,'
|
||||
@@ -103,13 +103,13 @@ class Database(object):
|
||||
f'insert into "{table_name}" '
|
||||
f'(id, alive, resolve, request, new, url, subdomain, port, level,'
|
||||
f'cname, content, public, cdn, status, reason, title, banner, header,'
|
||||
f'history, response, times, ttl, cidr, asn, org, ip2region,'
|
||||
f'ip2location, resolver, module, source, elapse, find) '
|
||||
f'values (:id, :alive, :resolve, :request, :new, :url, '
|
||||
f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,'
|
||||
f'module, source, elapse, find) '
|
||||
f'values (:id, :alive, :resolve, :request, :new, :url,'
|
||||
f':subdomain, :port, :level, :cname, :content, :public, :cdn,'
|
||||
f':status, :reason, :title, :banner, :header, :history, :response,'
|
||||
f':times, :ttl, :cidr, :asn, :org, :ip2region, :ip2location,'
|
||||
f':resolver, :module, :source, :elapse, :find)', results)
|
||||
f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,'
|
||||
f':source, :elapse, :find)', results)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e)
|
||||
|
||||
@@ -231,7 +231,7 @@ class Database(object):
|
||||
table_name = table_name.replace('.', '_')
|
||||
query = f'select id, new, alive, request, resolve, url, subdomain, level,' \
|
||||
f'cname, content, public, cdn, port, status, reason, title, banner,' \
|
||||
f'cidr, asn, org, ip2region, ip2location, source from "{table_name}"'
|
||||
f'cidr, asn, org, addr, isp, source from "{table_name}"'
|
||||
if alive and limit:
|
||||
if limit in ['resolve', 'request']:
|
||||
where = f' where {limit} = 1'
|
||||
|
||||
@@ -9,7 +9,6 @@ class Domain(object):
|
||||
|
||||
:param str string: input string
|
||||
"""
|
||||
|
||||
def __init__(self, string):
|
||||
self.string = str(string)
|
||||
self.regexp = r'\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b'
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
import zipfile
|
||||
from IP2Location import IP2Location
|
||||
from config.setting import data_storage_dir
|
||||
|
||||
|
||||
class IpGeoInfo(IP2Location):
|
||||
def __init__(self):
|
||||
zip_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN.ZIP")
|
||||
bin_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN")
|
||||
if not bin_path.exists():
|
||||
zf = zipfile.ZipFile(zip_path)
|
||||
zf.extract('IP2LOCATION-LITE-DB3.BIN', data_storage_dir)
|
||||
IP2Location.__init__(self, bin_path)
|
||||
+35
-7
@@ -10,6 +10,8 @@ import socket
|
||||
import struct
|
||||
import sys
|
||||
|
||||
from config import settings
|
||||
|
||||
|
||||
class IpRegInfo(object):
|
||||
__INDEX_BLOCK_LENGTH = 12
|
||||
@@ -39,7 +41,8 @@ class IpRegInfo(object):
|
||||
self.__dbBinStr = self.__f.read() # read all the contents in file
|
||||
self.__indexSPtr = self.get_long(self.__dbBinStr, 0)
|
||||
self.__indexLPtr = self.get_long(self.__dbBinStr, 4)
|
||||
self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) / self.__INDEX_BLOCK_LENGTH) + 1
|
||||
self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) /
|
||||
self.__INDEX_BLOCK_LENGTH) + 1
|
||||
|
||||
l, h, data_ptr = (0, self.__indexCount, 0)
|
||||
while l <= h:
|
||||
@@ -67,14 +70,16 @@ class IpRegInfo(object):
|
||||
" binary search method
|
||||
" param: ip
|
||||
"""
|
||||
if not ip.isdigit(): ip = self.ip2long(ip)
|
||||
if not ip.isdigit():
|
||||
ip = self.ip2long(ip)
|
||||
|
||||
if self.__indexCount == 0:
|
||||
self.__f.seek(0)
|
||||
super_block = self.__f.read(8)
|
||||
self.__indexSPtr = self.get_long(super_block, 0)
|
||||
self.__indexLPtr = self.get_long(super_block, 4)
|
||||
self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) / self.__INDEX_BLOCK_LENGTH) + 1
|
||||
self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) /
|
||||
self.__INDEX_BLOCK_LENGTH) + 1
|
||||
|
||||
l, h, data_ptr = (0, self.__indexCount, 0)
|
||||
while l <= h:
|
||||
@@ -208,10 +213,9 @@ class IpRegInfo(object):
|
||||
self.__f.seek(data_ptr)
|
||||
data = self.__f.read(data_len)
|
||||
|
||||
return {
|
||||
"city_id": self.get_long(data, 0),
|
||||
"region": data[4:]
|
||||
}
|
||||
info = {"city_id": self.get_long(data, 0),
|
||||
"region": data[4:].decode('utf-8')}
|
||||
return info
|
||||
|
||||
@staticmethod
|
||||
def ip2long(ip):
|
||||
@@ -244,3 +248,27 @@ class IpRegInfo(object):
|
||||
self.__dbBinStr = None
|
||||
self.__headerPtr = None
|
||||
self.__headerSip = None
|
||||
|
||||
|
||||
class IpRegData(IpRegInfo):
|
||||
def __init__(self):
|
||||
path = settings.data_storage_dir.joinpath('ip2region.db')
|
||||
IpRegInfo.__init__(self, path)
|
||||
|
||||
def query(self, ip, algorithm='memory'):
|
||||
algorithms = ['memory', 'binary', 'btree']
|
||||
if algorithm not in algorithms:
|
||||
raise Exception(f"Only three query algorithms are supported: {algorithms}")
|
||||
if algorithm == 'memory':
|
||||
result = self.memory_search(ip)
|
||||
elif algorithm == 'binary':
|
||||
result = self.binary_search(ip)
|
||||
else:
|
||||
result = self.btree_search(ip)
|
||||
addr_list = result.get('region').split('|')
|
||||
addr = ''.join(filter(lambda x: x != '0', addr_list[:-1]))
|
||||
isp = addr_list[-1]
|
||||
if isp == '0':
|
||||
isp = '未知'
|
||||
info = {'addr': addr, 'isp': isp}
|
||||
return info
|
||||
|
||||
+17
-13
@@ -22,9 +22,9 @@ class Module(object):
|
||||
self.cookie = None
|
||||
self.header = dict()
|
||||
self.proxy = None
|
||||
self.delay = settings.request_delay # 请求睡眠时延
|
||||
self.timeout = settings.request_timeout # 请求超时时间
|
||||
self.verify = settings.request_verify # 请求SSL验证
|
||||
self.delay = 1 # 请求睡眠时延
|
||||
self.timeout = settings.request_timeout_second # 请求超时时间
|
||||
self.verify = settings.request_ssl_verify # 请求SSL验证
|
||||
self.domain = str() # 当前进行子域名收集的主域
|
||||
self.subdomains = set() # 存放发现的子域
|
||||
self.infos = dict() # 存放子域有关信息
|
||||
@@ -73,7 +73,7 @@ class Module(object):
|
||||
:param dict params: request parameters
|
||||
:param bool check: check response
|
||||
:param kwargs: other params
|
||||
:return: requests's response object
|
||||
:return: response object
|
||||
"""
|
||||
try:
|
||||
resp = requests.head(url,
|
||||
@@ -93,16 +93,20 @@ class Module(object):
|
||||
return resp
|
||||
return None
|
||||
|
||||
def get(self, url, params=None, check=True, **kwargs):
|
||||
def get(self, url, params=None, check=True, ignore=False, **kwargs):
|
||||
"""
|
||||
Custom get request
|
||||
|
||||
:param str url: request url
|
||||
:param dict params: request parameters
|
||||
:param bool check: check response
|
||||
:param bool ignore: ignore error
|
||||
:param kwargs: other params
|
||||
:return: requests's response object
|
||||
:return: response object
|
||||
"""
|
||||
level = 'ERROR'
|
||||
if ignore:
|
||||
level = 'DEBUG'
|
||||
try:
|
||||
resp = requests.get(url,
|
||||
params=params,
|
||||
@@ -113,7 +117,7 @@ class Module(object):
|
||||
verify=self.verify,
|
||||
**kwargs)
|
||||
except Exception as e:
|
||||
logger.log('ERROR', e.args)
|
||||
logger.log(level, e.args)
|
||||
return None
|
||||
if not check:
|
||||
return resp
|
||||
@@ -129,7 +133,7 @@ class Module(object):
|
||||
:param dict data: request data
|
||||
:param bool check: check response
|
||||
:param kwargs: other params
|
||||
:return: requests's response object
|
||||
:return: response object
|
||||
"""
|
||||
try:
|
||||
resp = requests.post(url,
|
||||
@@ -195,7 +199,7 @@ class Module(object):
|
||||
:param str module: module name
|
||||
:return: proxy
|
||||
"""
|
||||
if not settings.enable_proxy:
|
||||
if not settings.enable_request_proxy:
|
||||
logger.log('TRACE', f'All modules do not use proxy')
|
||||
return self.proxy
|
||||
if settings.proxy_all_module:
|
||||
@@ -280,8 +284,8 @@ class Module(object):
|
||||
'cidr': None,
|
||||
'asn': None,
|
||||
'org': None,
|
||||
'ip2region': None,
|
||||
'ip2location': None,
|
||||
'addr': None,
|
||||
'isp': None,
|
||||
'resolver': None,
|
||||
'module': self.module,
|
||||
'source': self.source,
|
||||
@@ -331,8 +335,8 @@ class Module(object):
|
||||
'cidr': info.get('cidr'),
|
||||
'asn': info.get('asn'),
|
||||
'org': info.get('org'),
|
||||
'ip2region': info.get('ip2region'),
|
||||
'ip2location': info.get('ip2location'),
|
||||
'addr': info.get('addr'),
|
||||
'isp': info.get('isp'),
|
||||
'resolver': info.get('resolver'),
|
||||
'module': self.module,
|
||||
'source': self.source,
|
||||
|
||||
@@ -5,6 +5,5 @@ class Query(Module):
|
||||
"""
|
||||
Query base class
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
Module.__init__(self)
|
||||
|
||||
+132
-187
@@ -1,10 +1,10 @@
|
||||
import json
|
||||
import asyncio
|
||||
import functools
|
||||
from threading import Thread
|
||||
from queue import Queue
|
||||
|
||||
|
||||
import aiohttp
|
||||
import tqdm
|
||||
from aiohttp import ClientSession
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from common import utils
|
||||
@@ -12,17 +12,17 @@ from config.log import logger
|
||||
from config import settings
|
||||
|
||||
|
||||
def get_limit_conn():
|
||||
count = settings.limit_open_conn
|
||||
def req_thread_count():
|
||||
count = settings.request_thread_count
|
||||
if isinstance(count, int):
|
||||
count = max(16, count)
|
||||
else:
|
||||
count = utils.get_coroutine_count()
|
||||
logger.log('DEBUG', f'Request coroutine {count}')
|
||||
count = utils.get_request_count()
|
||||
logger.log('DEBUG', f'Number of request threads {count}')
|
||||
return count
|
||||
|
||||
|
||||
def get_ports(port):
|
||||
def get_port_seq(port):
|
||||
logger.log('DEBUG', 'Getting port range')
|
||||
ports = set()
|
||||
if isinstance(port, (set, list, tuple)):
|
||||
@@ -40,81 +40,33 @@ def get_ports(port):
|
||||
return set(ports)
|
||||
|
||||
|
||||
def gen_req_data(data, ports):
|
||||
def gen_req_url(domain, port):
|
||||
if str(port).endswith('443'):
|
||||
url = f'https://{domain}:{port}'
|
||||
if port == 443:
|
||||
url = f'https://{domain}'
|
||||
return url
|
||||
url = f'http://{domain}:{port}'
|
||||
if port == 80:
|
||||
url = f'http://{domain}'
|
||||
return url
|
||||
|
||||
|
||||
def gen_req_urls(data, ports):
|
||||
logger.log('INFOR', 'Generating request urls')
|
||||
new_data = []
|
||||
for data in data:
|
||||
resolve = data.get('resolve')
|
||||
urls = set()
|
||||
for info in data:
|
||||
resolve = info.get('resolve')
|
||||
# 解析不成功的子域不进行http请求探测
|
||||
if resolve != 1:
|
||||
continue
|
||||
subdomain = data.get('subdomain')
|
||||
subdomain = info.get('subdomain')
|
||||
for port in ports:
|
||||
if str(port).endswith('443'):
|
||||
url = f'https://{subdomain}:{port}'
|
||||
if port == 443:
|
||||
url = f'https://{subdomain}'
|
||||
data['id'] = None
|
||||
data['url'] = url
|
||||
data['port'] = port
|
||||
new_data.append(data)
|
||||
data = dict(data) # 需要生成一个新的字典对象
|
||||
else:
|
||||
url = f'http://{subdomain}:{port}'
|
||||
if port == 80:
|
||||
url = f'http://{subdomain}'
|
||||
data['id'] = None
|
||||
data['url'] = url
|
||||
data['port'] = port
|
||||
new_data.append(data)
|
||||
data = dict(data) # 需要生成一个新的字典对象
|
||||
return new_data
|
||||
urls.add(gen_req_url(subdomain, port))
|
||||
return urls
|
||||
|
||||
|
||||
async def fetch(session, method, url):
|
||||
"""
|
||||
请求
|
||||
|
||||
:param session: session对象
|
||||
:param method: 请求方法
|
||||
:param str url: url地址
|
||||
:return: 响应对象和响应文本
|
||||
"""
|
||||
timeout = aiohttp.ClientTimeout(total=None,
|
||||
connect=None,
|
||||
sock_read=settings.sockread_timeout,
|
||||
sock_connect=settings.sockconn_timeout)
|
||||
try:
|
||||
if method == 'HEAD':
|
||||
async with session.head(url,
|
||||
ssl=settings.verify_ssl,
|
||||
allow_redirects=settings.allow_redirects,
|
||||
timeout=timeout,
|
||||
proxy=settings.aiohttp_proxy) as resp:
|
||||
text = await resp.text()
|
||||
else:
|
||||
async with session.get(url,
|
||||
ssl=settings.verify_ssl,
|
||||
allow_redirects=settings.allow_redirects,
|
||||
timeout=timeout,
|
||||
proxy=settings.aiohttp_proxy) as resp:
|
||||
|
||||
try:
|
||||
# 先尝试用utf-8解码
|
||||
text = await resp.text(encoding='utf-8', errors='strict')
|
||||
except UnicodeError:
|
||||
try:
|
||||
# 再尝试用gb18030解码
|
||||
text = await resp.text(encoding='gb18030', errors='strict')
|
||||
except UnicodeError:
|
||||
# 最后尝试自动解码
|
||||
text = await resp.text(encoding=None, errors='ignore')
|
||||
return resp, text
|
||||
except Exception as e:
|
||||
return e, None
|
||||
|
||||
|
||||
def get_title(markup):
|
||||
def get_html_title(markup):
|
||||
"""
|
||||
获取标题
|
||||
|
||||
@@ -161,107 +113,107 @@ def get_jump_urls(history):
|
||||
return urls
|
||||
|
||||
|
||||
def request_callback(future, index, datas):
|
||||
resp, text = future.result()
|
||||
if isinstance(resp, BaseException):
|
||||
exception = resp
|
||||
logger.log('TRACE', exception.args)
|
||||
name = utils.get_classname(exception)
|
||||
datas[index]['reason'] = name + ' ' + str(exception)
|
||||
datas[index]['request'] = 0
|
||||
datas[index]['alive'] = 0
|
||||
else:
|
||||
datas[index]['reason'] = resp.reason
|
||||
datas[index]['status'] = resp.status
|
||||
datas[index]['request'] = 1
|
||||
if resp.status == 400 or resp.status >= 500:
|
||||
datas[index]['alive'] = 0
|
||||
else:
|
||||
datas[index]['alive'] = 1
|
||||
headers = resp.headers
|
||||
if settings.enable_banner_identify:
|
||||
datas[index]['banner'] = utils.get_sample_banner(headers)
|
||||
datas[index]['header'] = json.dumps(dict(headers))
|
||||
history = resp.history
|
||||
datas[index]['history'] = json.dumps(get_jump_urls(history))
|
||||
if isinstance(text, str):
|
||||
title = get_title(text).strip()
|
||||
datas[index]['title'] = utils.remove_invalid_string(title)
|
||||
datas[index]['response'] = utils.remove_invalid_string(text)
|
||||
|
||||
|
||||
def get_connector():
|
||||
count = get_limit_conn()
|
||||
return aiohttp.TCPConnector(ttl_dns_cache=300,
|
||||
ssl=settings.verify_ssl,
|
||||
limit=count,
|
||||
limit_per_host=settings.limit_per_host)
|
||||
|
||||
|
||||
async def async_request(urls):
|
||||
results = list()
|
||||
connector = get_connector()
|
||||
headers = utils.get_random_header()
|
||||
session = ClientSession(connector=connector, headers=headers)
|
||||
tasks = []
|
||||
for i, url in enumerate(urls):
|
||||
task = asyncio.ensure_future(fetch(session, 'GET', url))
|
||||
tasks.append(task)
|
||||
if tasks:
|
||||
futures = asyncio.as_completed(tasks)
|
||||
for future in tqdm.tqdm(futures,
|
||||
total=len(tasks),
|
||||
desc='Request Progress',
|
||||
ncols=80):
|
||||
result = await future
|
||||
results.append(result)
|
||||
await session.close()
|
||||
return results
|
||||
|
||||
|
||||
async def bulk_request(data, port):
|
||||
ports = get_ports(port)
|
||||
no_req_data = utils.get_filtered_data(data)
|
||||
to_req_data = gen_req_data(data, ports)
|
||||
method = settings.request_method.upper()
|
||||
logger.log('INFOR', f'Use {method} method to request')
|
||||
logger.log('INFOR', 'Async subdomains request in progress')
|
||||
connector = get_connector()
|
||||
headers = utils.get_random_header()
|
||||
session = ClientSession(connector=connector, headers=headers)
|
||||
tasks = []
|
||||
for num, data in enumerate(to_req_data):
|
||||
url = data.get('url')
|
||||
task = asyncio.ensure_future(fetch(session, method, url))
|
||||
task.add_done_callback(functools.partial(request_callback,
|
||||
index=num,
|
||||
datas=to_req_data))
|
||||
tasks.append(task)
|
||||
if tasks:
|
||||
futures = asyncio.as_completed(tasks)
|
||||
for future in tqdm.tqdm(futures,
|
||||
total=len(tasks),
|
||||
desc='Request Progress',
|
||||
ncols=80):
|
||||
await future
|
||||
await session.close()
|
||||
return to_req_data + no_req_data
|
||||
|
||||
|
||||
def set_loop_policy():
|
||||
def get(url, resp_queue, session):
|
||||
timeout = settings.request_timeout_second
|
||||
redirect = settings.request_allow_redirect
|
||||
proxy = utils.get_proxy()
|
||||
try:
|
||||
import uvloop
|
||||
except ImportError:
|
||||
pass
|
||||
resp = session.get(url, timeout=timeout, allow_redirects=redirect, proxies=proxy)
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
return
|
||||
resp_queue.put(resp)
|
||||
|
||||
|
||||
def request(urls_queue, resp_queue, session):
|
||||
while not urls_queue.empty():
|
||||
url = urls_queue.get()
|
||||
get(url, resp_queue, session)
|
||||
urls_queue.task_done()
|
||||
|
||||
|
||||
def progress(urls, resp_queue):
|
||||
bar = tqdm.tqdm()
|
||||
bar.total = len(urls)
|
||||
bar.desc = 'Request Progress'
|
||||
bar.ncols = 80
|
||||
while True:
|
||||
done = resp_queue.qsize()
|
||||
bar.n = done
|
||||
bar.update()
|
||||
if done == bar.total:
|
||||
break
|
||||
|
||||
|
||||
def get_session():
|
||||
header = utils.gen_fake_header()
|
||||
verify = settings.request_ssl_verify
|
||||
redirect_limit = settings.request_redirect_limit
|
||||
session = requests.Session()
|
||||
session.headers = header
|
||||
session.verify = verify
|
||||
session.max_redirects = redirect_limit
|
||||
return session
|
||||
|
||||
|
||||
def bulk_request(urls):
|
||||
logger.log('INFOR', 'Requesting urls in bulk')
|
||||
resp_list = list()
|
||||
urls_queue = Queue()
|
||||
resp_queue = Queue()
|
||||
for url in urls:
|
||||
urls_queue.put(url)
|
||||
session = get_session()
|
||||
thread_count = req_thread_count()
|
||||
|
||||
progress_thread = Thread(target=progress, args=(urls, resp_queue))
|
||||
progress_thread.start()
|
||||
|
||||
for _ in range(thread_count):
|
||||
request_thread = Thread(target=request, args=(urls_queue, resp_queue, session))
|
||||
request_thread.start()
|
||||
|
||||
urls_queue.join()
|
||||
|
||||
while not resp_queue.empty():
|
||||
resp = resp_queue.get()
|
||||
resp_list.append(resp)
|
||||
return resp_list
|
||||
|
||||
|
||||
def gen_new_info(info, resp):
|
||||
port = resp.raw._pool.port
|
||||
info['port'] = port
|
||||
info['url'] = resp.url
|
||||
info['reason'] = resp.reason
|
||||
code = resp.status_code
|
||||
info['status'] = code
|
||||
info['request'] = 1
|
||||
if code == 400 or code >= 500:
|
||||
info['alive'] = 0
|
||||
else:
|
||||
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
|
||||
info['alive'] = 1
|
||||
headers = resp.headers
|
||||
if settings.enable_banner_identify:
|
||||
info['banner'] = utils.get_sample_banner(headers)
|
||||
info['header'] = json.dumps(dict(headers))
|
||||
history = resp.history
|
||||
info['history'] = json.dumps(get_jump_urls(history))
|
||||
text = utils.decode_resp_text(resp)
|
||||
title = get_html_title(text).strip()
|
||||
info['title'] = utils.remove_invalid_string(title)
|
||||
info['response'] = utils.remove_invalid_string(text)
|
||||
return info
|
||||
|
||||
|
||||
def set_loop():
|
||||
set_loop_policy()
|
||||
loop = asyncio.get_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
return loop
|
||||
def gen_new_data(data, resp_list):
|
||||
new_data = list()
|
||||
for resp in resp_list:
|
||||
subdomain = resp.raw._pool.host
|
||||
for info in data:
|
||||
if info.get('subdomain') == subdomain:
|
||||
new_data.append(gen_new_info(info, resp))
|
||||
return new_data
|
||||
|
||||
|
||||
def run_request(domain, data, port):
|
||||
@@ -274,25 +226,18 @@ def run_request(domain, data, port):
|
||||
:return list: result
|
||||
"""
|
||||
logger.log('INFOR', f'Start requesting subdomains of {domain}')
|
||||
loop = set_loop()
|
||||
data = utils.set_id_none(data)
|
||||
request_coroutine = bulk_request(data, port)
|
||||
data = loop.run_until_complete(request_coroutine)
|
||||
loop.run_until_complete(asyncio.sleep(0.25))
|
||||
ports = get_port_seq(port)
|
||||
filtered_data = utils.get_filtered_data(data)
|
||||
req_urls = gen_req_urls(data, ports)
|
||||
resp_list = bulk_request(req_urls)
|
||||
new_data = gen_new_data(data, resp_list)
|
||||
data = new_data + filtered_data
|
||||
count = utils.count_alive(data)
|
||||
logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
|
||||
return data
|
||||
|
||||
|
||||
def urls_request(urls):
|
||||
logger.log('INFOR', 'Start urls request module')
|
||||
loop = set_loop()
|
||||
request_coroutine = async_request(urls)
|
||||
data = loop.run_until_complete(request_coroutine)
|
||||
loop.run_until_complete(asyncio.sleep(0.25))
|
||||
return data
|
||||
|
||||
|
||||
def save_db(name, data):
|
||||
"""
|
||||
Save request results to database
|
||||
|
||||
+22
-30
@@ -5,14 +5,11 @@ from config.log import logger
|
||||
from config import settings
|
||||
from common import utils
|
||||
from common.ipasn import IPAsnInfo
|
||||
from common.ipgeo import IpGeoInfo
|
||||
from common.ipreg import IpRegInfo
|
||||
from common.ipreg import IpRegData
|
||||
|
||||
|
||||
ip_asn = IPAsnInfo()
|
||||
ip_geo = IpGeoInfo()
|
||||
db_path = settings.data_storage_dir.joinpath('ip2region.db')
|
||||
ip_reg = IpRegInfo(db_path)
|
||||
ip_reg = IpRegData()
|
||||
|
||||
|
||||
def filter_subdomain(data):
|
||||
@@ -82,12 +79,12 @@ def gen_infos(data, qname, info, infos):
|
||||
cname = list()
|
||||
ips = list()
|
||||
public = list()
|
||||
ttls = list()
|
||||
cidrs = list()
|
||||
asns = list()
|
||||
orgs = list()
|
||||
locs = list()
|
||||
regs = list()
|
||||
ttl = list()
|
||||
cidr = list()
|
||||
asn = list()
|
||||
org = list()
|
||||
addr = list()
|
||||
isp = list()
|
||||
answers = data.get('answers')
|
||||
for answer in answers:
|
||||
if answer.get('type') == 'A':
|
||||
@@ -95,31 +92,26 @@ def gen_infos(data, qname, info, infos):
|
||||
cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号
|
||||
ip = answer.get('data')
|
||||
ips.append(ip)
|
||||
ttl = answer.get('ttl')
|
||||
ttls.append(str(ttl))
|
||||
is_public = utils.ip_is_public(ip)
|
||||
public.append(str(is_public))
|
||||
ttl.append(str(answer.get('ttl')))
|
||||
public.append(str(utils.ip_is_public(ip)))
|
||||
asn_info = ip_asn.find(ip)
|
||||
cidrs.append(asn_info.get('cidr'))
|
||||
asns.append(asn_info.get('asn'))
|
||||
orgs.append(asn_info.get('org'))
|
||||
loc = f'{ip_geo.get_country_long(ip)} ' \
|
||||
f'{ip_geo.get_region(ip)} ' \
|
||||
f'{ip_geo.get_city(ip)}'
|
||||
locs.append(loc)
|
||||
reg = ip_reg.memory_search(ip).get('region').decode('utf-8')
|
||||
regs.append(reg)
|
||||
cidr.append(asn_info.get('cidr'))
|
||||
asn.append(asn_info.get('asn'))
|
||||
org.append(asn_info.get('org'))
|
||||
ip_info = ip_reg.query(ip)
|
||||
addr.append(ip_info.get('addr'))
|
||||
isp.append(ip_info.get('isp'))
|
||||
info['resolve'] = 1
|
||||
info['reason'] = 'OK'
|
||||
info['cname'] = ','.join(cname)
|
||||
info['content'] = ','.join(ips)
|
||||
info['public'] = ','.join(public)
|
||||
info['ttl'] = ','.join(ttls)
|
||||
info['cidr'] = ','.join(cidrs)
|
||||
info['asn'] = ','.join(asns)
|
||||
info['org'] = ','.join(orgs)
|
||||
info['ip2location'] = ','.join(locs)
|
||||
info['ip2region'] = ','.join(regs)
|
||||
info['ttl'] = ','.join(ttl)
|
||||
info['cidr'] = ','.join(cidr)
|
||||
info['asn'] = ','.join(asn)
|
||||
info['org'] = ','.join(org)
|
||||
info['addr'] = ','.join(addr)
|
||||
info['isp'] = ','.join(isp)
|
||||
infos[qname] = info
|
||||
if not flag:
|
||||
info['alive'] = 0
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"""
|
||||
根据网页结构判断页面相似性(Determine page similarity based on HTML page structure)
|
||||
判断方法：根据网页的DOM树确定网页的模板特征向量，对模板特征向量计算网页结构相似性。
|
||||
来源地址：https://github.com/SPuerBRead/HTMLSimilarity
|
||||
计算参考: https://patents.google.com/patent/CN101694668B/zh
|
||||
"""
|
||||
from treelib import Tree
|
||||
from bs4 import BeautifulSoup
|
||||
import bs4
|
||||
|
||||
|
||||
class DOMTree(object):
|
||||
def __init__(self, label, attrs):
|
||||
self.label = label
|
||||
self.attrs = attrs
|
||||
|
||||
|
||||
class HTMLParser(object):
|
||||
def __init__(self, html):
|
||||
self.dom_id = 1
|
||||
self.dom_tree = Tree()
|
||||
self.bs_html = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
def get_dom_structure_tree(self):
|
||||
for content in self.bs_html.contents:
|
||||
if isinstance(content, bs4.element.Tag):
|
||||
self.bs_html = content
|
||||
self.recursive_descendants(self.bs_html, 1)
|
||||
return self.dom_tree
|
||||
|
||||
def recursive_descendants(self, descendants, parent_id):
|
||||
if self.dom_id == 1:
|
||||
self.dom_tree.create_node(descendants.name, self.dom_id,
|
||||
data=DOMTree(descendants.name, descendants.attrs))
|
||||
self.dom_id = self.dom_id + 1
|
||||
for child in descendants.contents:
|
||||
if isinstance(child, bs4.element.Tag):
|
||||
self.dom_tree.create_node(child.name, self.dom_id, parent_id,
|
||||
data=DOMTree(child.name, child.attrs))
|
||||
self.dom_id = self.dom_id + 1
|
||||
self.recursive_descendants(child, self.dom_id - 1)
|
||||
|
||||
|
||||
class Converter(object):
|
||||
def __init__(self, dom_tree, dimension):
|
||||
self.dom_tree = dom_tree
|
||||
self.node_info_list = []
|
||||
self.dimension = dimension
|
||||
self.initial_weight = 1
|
||||
self.attenuation_ratio = 0.6
|
||||
self.dom_eigenvector = {}.fromkeys(range(0, dimension), 0)
|
||||
|
||||
def get_eigenvector(self):
|
||||
for node_id in range(1, self.dom_tree.size() + 1):
|
||||
node = self.dom_tree.get_node(node_id)
|
||||
node_feature = self.create_feature(node)
|
||||
feature_hash = self.feature_hash(node_feature)
|
||||
node_weight = self.calculate_weight(node, node_id, feature_hash)
|
||||
self.construct_eigenvector(feature_hash, node_weight)
|
||||
return self.dom_eigenvector
|
||||
|
||||
@staticmethod
|
||||
def create_feature(node):
|
||||
node_attr_list = []
|
||||
node_feature = node.data.label + '|'
|
||||
for attr in node.data.attrs.keys():
|
||||
node_attr_list.append(attr + ':' + str(node.data.attrs[attr]))
|
||||
node_feature += '|'.join(node_attr_list)
|
||||
return node_feature
|
||||
|
||||
@staticmethod
|
||||
def feature_hash(node_feature):
|
||||
return abs(hash(node_feature)) % (10 ** 8)
|
||||
|
||||
def calculate_weight(self, node, node_id, feature_hash):
|
||||
brother_node_count = 0
|
||||
depth = self.dom_tree.depth(node)
|
||||
for brother_node in self.dom_tree.siblings(node_id):
|
||||
brother_node_feature = self.create_feature(brother_node)
|
||||
brother_node_feature_hash = self.feature_hash(brother_node_feature)
|
||||
if brother_node_feature_hash == feature_hash:
|
||||
brother_node_count = brother_node_count + 1
|
||||
if brother_node_count:
|
||||
node_weight = self.initial_weight * self.attenuation_ratio ** depth \
|
||||
* self.attenuation_ratio ** brother_node_count
|
||||
else:
|
||||
node_weight = self.initial_weight * self.attenuation_ratio ** depth
|
||||
return node_weight
|
||||
|
||||
def construct_eigenvector(self, feature_hash, node_weight):
|
||||
feature_hash = feature_hash % self.dimension
|
||||
self.dom_eigenvector[feature_hash] += node_weight
|
||||
|
||||
|
||||
def calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension):
|
||||
a, b = 0, 0
|
||||
for i in range(dimension):
|
||||
a += dom1_eigenvector[i]-dom2_eigenvector[i]
|
||||
if dom1_eigenvector[i] and dom2_eigenvector[i]:
|
||||
b += dom1_eigenvector[i] + dom2_eigenvector[i]
|
||||
pseudodistance = abs(a)/b
|
||||
return pseudodistance
|
||||
|
||||
|
||||
def get_pseudodistance(html_doc1, html_doc2, dimension=5000):
|
||||
"""
|
||||
获取html文档结构相似度
|
||||
|
||||
:param str html_doc1: html文档
|
||||
:param str html_doc2: html文档
|
||||
:param int dimension: 降维后的维数
|
||||
:return 伪距离
|
||||
"""
|
||||
hp1 = HTMLParser(html_doc1)
|
||||
html_doc1_dom_tree = hp1.get_dom_structure_tree()
|
||||
hp2 = HTMLParser(html_doc2)
|
||||
html_doc2_dom_tree = hp2.get_dom_structure_tree()
|
||||
converter = Converter(html_doc1_dom_tree, dimension)
|
||||
dom1_eigenvector = converter.get_eigenvector()
|
||||
converter = Converter(html_doc2_dom_tree, dimension)
|
||||
dom2_eigenvector = converter.get_eigenvector()
|
||||
return calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension)
|
||||
|
||||
|
||||
def is_similar(html_doc1, html_doc2, dimension=5000):
|
||||
"""
|
||||
根据计算出的伪距离来判断是否html页面结构相似
|
||||
|
||||
:param str html_doc1: html文档
|
||||
:param str html_doc2: html文档
|
||||
:param int dimension: 降维后的维数
|
||||
:return 是否相似（伪距离value<0.2时相似，value>0.2时不相似）
|
||||
"""
|
||||
value = get_pseudodistance(html_doc1, html_doc2, dimension)
|
||||
if value > 0.2:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
+36
-49
@@ -10,7 +10,6 @@ import subprocess
|
||||
from ipaddress import IPv4Address, ip_address
|
||||
from stat import S_IXUSR
|
||||
|
||||
import psutil
|
||||
import tenacity
|
||||
import requests
|
||||
from pathlib import Path
|
||||
@@ -49,10 +48,10 @@ def gen_fake_header():
|
||||
"""
|
||||
Generate fake request headers
|
||||
"""
|
||||
headers = settings.headers
|
||||
headers = settings.request_default_headers
|
||||
if not isinstance(headers, dict):
|
||||
headers = dict()
|
||||
if settings.random_user_agent:
|
||||
if settings.enable_random_ua:
|
||||
ua = random.choice(user_agents)
|
||||
headers['User-Agent'] = ua
|
||||
headers['Accept-Encoding'] = 'gzip, deflate'
|
||||
@@ -74,7 +73,7 @@ def get_random_proxy():
|
||||
Get random proxy
|
||||
"""
|
||||
try:
|
||||
return random.choice(settings.proxy_pool)
|
||||
return random.choice(settings.request_proxy_pool)
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
@@ -83,7 +82,7 @@ def get_proxy():
|
||||
"""
|
||||
Get proxy
|
||||
"""
|
||||
if settings.enable_proxy:
|
||||
if settings.enable_request_proxy:
|
||||
return get_random_proxy()
|
||||
return None
|
||||
|
||||
@@ -112,26 +111,21 @@ def match_main_domain(domain):
|
||||
|
||||
|
||||
def read_target_file(target):
|
||||
domains = set()
|
||||
domains = list()
|
||||
with open(target, encoding='utf-8', errors='ignore') as file:
|
||||
for line in file:
|
||||
domain = match_main_domain(line)
|
||||
if not domain:
|
||||
continue
|
||||
domain = get_main_domain(domain)
|
||||
if not domain:
|
||||
continue
|
||||
domains.add(domain)
|
||||
return domains
|
||||
domains.append(domain)
|
||||
sorted_domains = sorted(set(domains), key=domains.index)
|
||||
return sorted_domains
|
||||
|
||||
|
||||
def get_from_target(target):
|
||||
domains = set()
|
||||
if isinstance(target, str):
|
||||
domain = match_main_domain(target)
|
||||
if not domain:
|
||||
return domains
|
||||
domain = get_main_domain(domain)
|
||||
if not domain:
|
||||
return domains
|
||||
domains.add(domain)
|
||||
@@ -507,27 +501,8 @@ def get_process_num():
|
||||
return 1
|
||||
|
||||
|
||||
def get_coroutine_count():
|
||||
"""
|
||||
根据内存大小获取并发数量
|
||||
"""
|
||||
mem = psutil.virtual_memory()
|
||||
total_mem = mem.total
|
||||
g_size = 1024 * 1024 * 1024
|
||||
if total_mem < 1 * g_size:
|
||||
return 16
|
||||
elif total_mem < 2 * g_size:
|
||||
return 32
|
||||
elif total_mem < 4 * g_size:
|
||||
return 64
|
||||
elif total_mem < 8 * g_size:
|
||||
return 128
|
||||
elif total_mem < 16 * g_size:
|
||||
return 256
|
||||
elif total_mem < 32 * g_size:
|
||||
return 512
|
||||
else:
|
||||
return 1024
|
||||
def get_request_count():
|
||||
return os.cpu_count() * 10
|
||||
|
||||
|
||||
def uniq_dict_list(dict_list):
|
||||
@@ -567,7 +542,6 @@ def check_net():
|
||||
|
||||
def check_pre():
|
||||
logger.log('INFOR', 'Checking dependent environment')
|
||||
system = platform.system()
|
||||
implementation = platform.python_implementation()
|
||||
version = platform.python_version()
|
||||
if implementation != 'CPython':
|
||||
@@ -576,16 +550,6 @@ def check_pre():
|
||||
if version < '3.6':
|
||||
logger.log('FATAL', 'OneForAll requires Python 3.6 or higher')
|
||||
exit(1)
|
||||
if system == 'Windows' and implementation == 'CPython' and version < '3.8':
|
||||
logger.log('FATAL', 'OneForAll requires Python 3.8 '
|
||||
'or higher when running on Windows')
|
||||
exit(1)
|
||||
if system in {"Linux", "Darwin"}:
|
||||
try:
|
||||
import uvloop
|
||||
except ImportError:
|
||||
logger.log('ALERT', f'Please install the uvloop library manually '
|
||||
f'to accelerate subdomain requests')
|
||||
|
||||
|
||||
def check_env():
|
||||
@@ -604,8 +568,8 @@ def check_version(local):
|
||||
api = 'https://api.github.com/repos/shmilylty/OneForAll/releases/latest'
|
||||
header = get_random_header()
|
||||
proxy = get_proxy()
|
||||
timeout = settings.request_timeout
|
||||
verify = settings.request_verify
|
||||
timeout = settings.request_timeout_second
|
||||
verify = settings.request_ssl_verify
|
||||
try:
|
||||
resp = requests.get(url=api, headers=header, proxies=proxy,
|
||||
timeout=timeout, verify=verify)
|
||||
@@ -613,7 +577,7 @@ def check_version(local):
|
||||
latest = resp_json['tag_name']
|
||||
except Exception as e:
|
||||
logger.log('ERROR', 'An error occurred while checking the latest version')
|
||||
logger.log('ERROR', e.args)
|
||||
logger.log('DEBUG', e.args)
|
||||
return
|
||||
if latest > local:
|
||||
change = resp_json.get("body")
|
||||
@@ -740,3 +704,26 @@ def check_random_subdomain(subdomains):
|
||||
if subdomain:
|
||||
logger.log('ALERT', f'Please check whether {subdomain} is correct or not')
|
||||
return True
|
||||
|
||||
|
||||
def get_url_resp(url):
|
||||
timeout = settings.request_timeout
|
||||
verify = settings.request_verify
|
||||
try:
|
||||
resp = requests.get(url, params=None, timeout=timeout, verify=verify)
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', f'Error request {url}')
|
||||
logger.log('DEBUG', e.args)
|
||||
return None
|
||||
return resp
|
||||
|
||||
|
||||
def decode_resp_text(resp):
|
||||
try:
|
||||
text = resp.text(encoding='utf-8', errors='strict') # 先尝试用utf-8严格解码
|
||||
except UnicodeError:
|
||||
try:
|
||||
text = resp.text(encoding='gb18030', errors='strict') # 再尝试用gb18030严格解码
|
||||
except UnicodeError:
|
||||
text = resp.text(encoding=None, errors='ignore') # 最后尝试自动解码
|
||||
return text
|
||||
|
||||
+33
-47
@@ -6,6 +6,9 @@ OneForAll默认配置
|
||||
import pathlib
|
||||
import warnings
|
||||
|
||||
# 禁用所有警告信息
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
# 路径设置
|
||||
relative_directory = pathlib.Path(__file__).parent.parent # OneForAll代码相对路径
|
||||
module_dir = relative_directory.joinpath('modules') # OneForAll模块目录
|
||||
@@ -40,7 +43,7 @@ enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_mo
|
||||
# 只使用ask和baidu搜索引擎收集子域的示例
|
||||
# enable_partial_module = [('modules.search', 'ask')
|
||||
# ('modules.search', 'baidu')]
|
||||
module_thread_timeout = 120.0 # 每个收集模块线程超时时间(默认2分钟)
|
||||
module_thread_timeout = 90.0 # 每个收集模块线程超时时间(默认90秒)
|
||||
|
||||
# 爆破模块设置
|
||||
enable_wildcard_check = True # 开启泛解析检测(默认True)
|
||||
@@ -51,7 +54,7 @@ brute_status_format = 'ansi' # 爆破时状态输出格式（默认asni，可
|
||||
brute_process_num = 1 # 默认1
|
||||
brute_concurrent_num = 2000 # 并发查询数量(默认2000，最大推荐10000)
|
||||
brute_socket_num = 1 # 爆破时每个进程下的socket数量
|
||||
brute_resolve_num = 10 # 解析失败时尝试换名称服务器重查次数
|
||||
brute_resolve_num = 15 # 解析失败时尝试换名称服务器重查次数
|
||||
# 爆破所使用的字典路径 默认data/subdomains.txt
|
||||
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
|
||||
# 爆破所使用的字典路径 默认data/cn_nameservers.txt
|
||||
@@ -79,24 +82,39 @@ ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标
|
||||
banner_process_number = 4 # 识别进程数量(默认4)
|
||||
|
||||
# 代理设置
|
||||
enable_proxy = False # 是否使用代理(全局开关)
|
||||
enable_request_proxy = False # 是否使用代理(全局开关，默认False)
|
||||
proxy_all_module = False # 代理所有模块
|
||||
proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
|
||||
'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
|
||||
'YandexSearch', 'CrossDomainXml',
|
||||
'ContentSecurityPolicy'] # 代理自定义的模块
|
||||
proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
||||
'https': 'https://127.0.0.1:1080'}] # 代理池
|
||||
# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
||||
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
||||
request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
||||
'https': 'https://127.0.0.1:1080'}] # 代理池
|
||||
# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
||||
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
||||
|
||||
|
||||
# 网络请求设置
|
||||
request_delay = 1 # 请求时延
|
||||
request_timeout = 60 # 请求超时
|
||||
request_verify = False # 请求SSL验证
|
||||
# 禁用所有警告信息
|
||||
warnings.filterwarnings("ignore")
|
||||
# 请求设置
|
||||
request_thread_count = None # 请求线程数量(默认None，则根据内存大小设置)
|
||||
request_timeout_second = 30 # 请求超时秒数(默认30秒)
|
||||
request_ssl_verify = False # 请求SSL验证(默认False)
|
||||
request_allow_redirect = True # 请求允许重定向(默认True)
|
||||
request_redirect_limit = 10 # 请求跳转限制(默认10次)
|
||||
# 默认请求头 可以在headers里添加自定义请求头
|
||||
request_default_headers = {
|
||||
'Accept': 'text/html,application/xhtml+xml,'
|
||||
'application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
|
||||
'Cache-Control': 'max-age=0',
|
||||
'DNT': '1',
|
||||
'Referer': 'https://www.google.com/',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||||
'(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'X-Forwarded-For': '127.0.0.1'
|
||||
}
|
||||
enable_random_ua = True # 使用随机UA(默认True，开启可以覆盖request_default_headers的UA)
|
||||
|
||||
# 搜索模块设置
|
||||
# 开启全量搜索会尽量去获取搜索引擎搜索的全部结果，不过搜索耗时可能会过长
|
||||
@@ -112,9 +130,8 @@ resolver_nameservers = [
|
||||
'8.8.8.8', # Google DNS
|
||||
'1.1.1.1' # CloudFlare DNS
|
||||
] # 指定查询的DNS域名服务器
|
||||
resolver_timeout = 5.0 # 解析超时时间
|
||||
resolver_lifetime = 60.0 # 解析存活时间
|
||||
limit_resolve_conn = 500 # 限制同一时间解析的数量(默认500)
|
||||
resolver_timeout = 5.0 # 解析超时时间(默认5.0秒)
|
||||
resolver_lifetime = 10.0 # 解析存活时间(默认10.0秒)
|
||||
|
||||
# 请求端口探测设置
|
||||
# 你可以在端口列表添加自定义端口
|
||||
@@ -140,42 +157,11 @@ large_ports = [80, 81, 280, 300, 443, 591, 593, 832, 888, 901, 981, 1010, 1080,
|
||||
18092, 20000, 20720, 24465, 28017, 28080, 30821, 43110, 61600]
|
||||
ports = {'small': small_ports, 'medium': medium_ports, 'large': large_ports}
|
||||
|
||||
# aiohttp有关配置
|
||||
verify_ssl = False
|
||||
# aiohttp 支持 HTTP/HTTPS形式的代理
|
||||
aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com"
|
||||
allow_redirects = True # 允许请求跳转
|
||||
# 为了保证请求质量 请谨慎更改以下设置
|
||||
# request_method只能是HEAD或GET,HEAD请求方法更快，但是不能获取响应体并提取从中title及banner
|
||||
request_method = 'GET' # 使用请求方法，默认GET
|
||||
sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒
|
||||
sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒
|
||||
limit_open_conn = None # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置
|
||||
# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
|
||||
limit_per_host = 10 # 0表示不限制,默认10
|
||||
|
||||
common_subnames = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
|
||||
'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
|
||||
'help', 'live', 'mall', 'blogs', 'files', 'forum',
|
||||
'store', 'mobile'}
|
||||
|
||||
# 请求头设置
|
||||
# 可以在headers里添加自定义请求头
|
||||
headers = {
|
||||
'Accept': 'text/html,application/xhtml+xml,'
|
||||
'application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
|
||||
'Cache-Control': 'max-age=0',
|
||||
'DNT': '1',
|
||||
'Referer': 'https://www.google.com/',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||||
'(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'X-Forwarded-For': '127.0.0.1'
|
||||
}
|
||||
random_user_agent = True # 使用随机UA(默认True，开启可以覆盖header的设置)
|
||||
|
||||
# 模块API配置
|
||||
# Censys可以免费注册获取API：https://censys.io/api
|
||||
censys_api_id = ''
|
||||
|
||||
+1
-1
@@ -35,4 +35,4 @@ logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')
|
||||
# 命令终端日志级别默认为INFOR
|
||||
logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True)
|
||||
# 日志文件默认为级别为DEBUG
|
||||
logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True)
|
||||
logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True, encoding='utf-8')
|
||||
|
||||
+5
-26
@@ -35,11 +35,8 @@ enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_mo
|
||||
# 只使用ask和baidu搜索引擎收集子域的示例
|
||||
# enable_partial_module = [('modules.search', 'ask')
|
||||
# ('modules.search', 'baidu')]
|
||||
module_thread_timeout = 120.0 # 每个收集模块线程超时时间(默认2分钟)
|
||||
|
||||
# 爆破模块设置
|
||||
brute_wildcard_check = True # 开启泛解析检测(默认True)
|
||||
brute_wildcard_deal = True # 开启泛解析处理(默认True)
|
||||
brute_concurrent_num = 2000 # 爆破时并发查询数量(默认2000，最大推荐10000)
|
||||
# 爆破所使用的字典路径 默认data/subdomains.txt
|
||||
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
|
||||
@@ -66,37 +63,19 @@ fuzz_rule = None # fuzz域名的正则 示例：'[a-z][0-9]' 表示第一位是
|
||||
brute_ip_blacklist = {'0.0.0.0', '0.0.0.1'} # IP黑名单 子域解析到IP黑名单则标记为非法子域
|
||||
ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标记为非法(泛解析)子域
|
||||
|
||||
# banner识别模块设置
|
||||
banner_process_number = 4 # 识别进程数量(默认4)
|
||||
|
||||
# 代理设置
|
||||
enable_proxy = False # 是否使用代理(全局开关)
|
||||
enable_request_proxy = False # 是否使用代理(全局开关)
|
||||
proxy_all_module = False # 代理所有模块
|
||||
proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
|
||||
'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
|
||||
'YandexSearch', 'CrossDomainXml',
|
||||
'ContentSecurityPolicy'] # 代理自定义的模块
|
||||
proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
||||
'https': 'https://127.0.0.1:1080'}] # 代理池
|
||||
# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
||||
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
||||
|
||||
|
||||
# 网络请求设置
|
||||
request_delay = 1 # 请求时延
|
||||
request_timeout = 60 # 请求超时
|
||||
request_verify = False # 请求SSL验证
|
||||
request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
||||
'https': 'https://127.0.0.1:1080'}] # 代理池
|
||||
# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
||||
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
||||
|
||||
# 搜索模块设置
|
||||
enable_recursive_search = False # 递归搜索子域
|
||||
search_recursive_times = 2 # 递归搜索层数
|
||||
|
||||
# aiohttp有关配置
|
||||
# aiohttp 支持 HTTP/HTTPS形式的代理
|
||||
aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com"
|
||||
# 为了保证请求质量 请谨慎更改以下设置
|
||||
sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒
|
||||
sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒
|
||||
limit_open_conn = None # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置
|
||||
# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
|
||||
limit_per_host = 10 # 0表示不限制,默认10
|
||||
|
||||
Binary file not shown.
@@ -29,7 +29,6 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f
|
||||
python3 dbexport.py --target table_name --tb True --show False
|
||||
|
||||
Note:
|
||||
--alive True/False Only export alive subdomains or not (default False)
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--path Result directory (default directory is ./results)
|
||||
|
||||
@@ -82,4 +81,3 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f
|
||||
|
||||
if __name__ == '__main__':
|
||||
fire.Fire(export)
|
||||
# export('example.com')
|
||||
|
||||
+1
-18
@@ -4,7 +4,7 @@
|
||||
[](https://codecov.io/gh/shmilylty/OneForAll)
|
||||
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/releases)
|
||||
|
||||
👊**OneForAll is a powerful subdomain integration tool** 📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
|
||||
@@ -15,23 +15,6 @@
|
||||
|
||||
📢 Please read this document to help you start quickly!
|
||||
|
||||
<details>
|
||||
<summary><b>🐍Installation requirements</b></summary>
|
||||
|
||||
OneForAll is developed and tested based on [Python 3.8.0](https://www.python.org/downloads/release/python-380/). Recommend use release higher than Python 3.8.0 (Windows platform must use Python 3.8.0 or later). For more information on installing the Python environment, please read [Python 3 installation Guide](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3).
|
||||
|
||||
After installation python, run the following command to check the Python and pip3 versions:
|
||||
```bash
|
||||
python -V
|
||||
pip3 -V
|
||||
```
|
||||
If you see the following output, there is no problem with the Python environment:
|
||||
```bash
|
||||
Python 3.8.0
|
||||
pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.8)
|
||||
```
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>✔Installation steps (for Git)</b></summary>
|
||||
|
||||
|
||||
+199
-126
@@ -10,141 +10,214 @@ The OneForAll command line interface is based on [Fire](https://github.com/googl
|
||||
|
||||
1. **oneforall.py help**
|
||||
|
||||
```bash
|
||||
python oneforall.py --help
|
||||
```
|
||||
```bash
|
||||
NAME
|
||||
oneforall.py - OneForAll help summary page
|
||||
|
||||
SYNOPSIS
|
||||
oneforall.py COMMAND | <flags>
|
||||
|
||||
DESCRIPTION
|
||||
OneForAll is a powerful subdomain integration tool
|
||||
|
||||
Example:
|
||||
python3 oneforall.py version
|
||||
python3 oneforall.py check
|
||||
python3 oneforall.py --target example.com run
|
||||
python3 oneforall.py --targets ./domains.txt run
|
||||
python3 oneforall.py --target example.com --alive False run
|
||||
python3 oneforall.py --target example.com --brute True run
|
||||
python3 oneforall.py --target example.com --port medium run
|
||||
python3 oneforall.py --target example.com --format csv run
|
||||
python3 oneforall.py --target example.com --dns False run
|
||||
python3 oneforall.py --target example.com --req False run
|
||||
python3 oneforall.py --target example.com --takeover False run
|
||||
python3 oneforall.py --target example.com --show True run
|
||||
```bash
|
||||
python oneforall.py --help
|
||||
```
|
||||
```bash
|
||||
NAME
|
||||
oneforall.py - OneForAll help summary page
|
||||
|
||||
|
||||
Note:
|
||||
--alive True/False Only export alive subdomains or not (default False)
|
||||
--port small/medium/large See details in ./config/setting.py(default small)
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--path Result path (default None, automatically generated)
|
||||
SYNOPSIS
|
||||
oneforall.py COMMAND | <flags>
|
||||
|
||||
FLAGS
|
||||
--target=TARGET
|
||||
One domain (target or targets parameters must be provided)
|
||||
--targets=TARGETS
|
||||
File path of one domain per line
|
||||
--brute=BRUTE
|
||||
Use brute module (default False)
|
||||
--dns=DNS
|
||||
Use DNS resolution (default True)
|
||||
--req=REQ
|
||||
HTTP request subdomains (default True)
|
||||
--port=PORT
|
||||
The port range to request (default small port is 80,443)
|
||||
--alive=ALIVE
|
||||
Only export alive subdomains (default False)
|
||||
--format=FORMAT
|
||||
Result format (default csv)
|
||||
--path=PATH
|
||||
Result path (default None, automatically generated)
|
||||
--takeover=TAKEOVER
|
||||
Scan subdomain takeover (default False)
|
||||
DESCRIPTION
|
||||
OneForAll is a powerful subdomain integration tool
|
||||
|
||||
COMMANDS
|
||||
COMMAND is one of the following:
|
||||
Example:
|
||||
python3 oneforall.py version
|
||||
python3 oneforall.py check
|
||||
python3 oneforall.py --target example.com run
|
||||
python3 oneforall.py --targets ./domains.txt run
|
||||
python3 oneforall.py --target example.com --alive False run
|
||||
python3 oneforall.py --target example.com --brute True run
|
||||
python3 oneforall.py --target example.com --port medium run
|
||||
python3 oneforall.py --target example.com --format csv run
|
||||
python3 oneforall.py --target example.com --dns False run
|
||||
python3 oneforall.py --target example.com --req False run
|
||||
python3 oneforall.py --target example.com --takeover False run
|
||||
python3 oneforall.py --target example.com --show True run
|
||||
|
||||
check
|
||||
Check if there is a new version and exit
|
||||
|
||||
version
|
||||
Print version information and exit
|
||||
```
|
||||
Note:
|
||||
--port small/medium/large See details in ./config/setting.py(default small)
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--path Result path (default None, automatically generated)
|
||||
|
||||
FLAGS
|
||||
--target=TARGET
|
||||
One domain (target or targets parameters must be provided)
|
||||
--targets=TARGETS
|
||||
File path of one domain per line
|
||||
--brute=BRUTE
|
||||
Use brute module (default False)
|
||||
--dns=DNS
|
||||
Use DNS resolution (default True)
|
||||
--req=REQ
|
||||
HTTP request subdomains (default True)
|
||||
--port=PORT
|
||||
The port range to request (default small port is 80,443)
|
||||
--alive=ALIVE
|
||||
Only export alive subdomains (default False)
|
||||
--format=FORMAT
|
||||
Result format (default csv)
|
||||
--path=PATH
|
||||
Result path (default None, automatically generated)
|
||||
--takeover=TAKEOVER
|
||||
Scan subdomain takeover (default False)
|
||||
|
||||
COMMANDS
|
||||
COMMAND is one of the following:
|
||||
|
||||
check
|
||||
Check if there is a new version and exit
|
||||
|
||||
version
|
||||
Print version information and exit
|
||||
```
|
||||
|
||||
2. **aiobrute.py help**
|
||||
|
||||
With regard to the handling of the universal parsing problem, first of all, OneForAll accesses a random subdomain to determine whether universal parsing is used, and if universal parsing is used, it is handled by the following judgment:
|
||||
- First, it is mainly compared with the pan-parsed IP set and TTL values, see [this article](http://sh3ll.me/archives/201704041222.txt).
|
||||
With regard to the handling of the universal parsing problem, first of all, OneForAll accesses a random subdomain to determine whether universal parsing is used, and if universal parsing is used, it is handled by the following judgment:
|
||||
- First, it is mainly compared with the pan-parsed IP set and TTL values, see [this article](http://sh3ll.me/archives/201704041222.txt).
|
||||
|
||||
- Second, the number of times to resolve to the same IP collection multiple times (the default is 10, which can be set to size in config.py).
|
||||
|
||||
- Third, considering the blasting efficiency, there is no HTTP response volume similarity comparison and response volume content judgment, this function has not been implemented yet, and will be implemented if necessary.
|
||||
- Second, the number of times to resolve to the same IP collection multiple times (the default is 10, which can be set to size in config.py).
|
||||
|
||||
- Third, considering the blasting efficiency, there is no HTTP response volume similarity comparison and response volume content judgment, this function has not been implemented yet, and will be implemented if necessary.
|
||||
|
||||
|
||||
```bash
|
||||
python aiobrute.py --help
|
||||
```
|
||||
```bash
|
||||
python brute.py --help
|
||||
```
|
||||
|
||||
```bash
|
||||
NAME
|
||||
aiobrute.py - OneForAll multi-process multi-correlation asynchronous subdomain blasting module
|
||||
|
||||
SYNOPSIS
|
||||
aiobrute.py --target=TARGET <flags>
|
||||
|
||||
DESCRIPTION
|
||||
Example：
|
||||
python3 aiobrute.py --target example.com run
|
||||
python3 aiobrute.py --target ./domains.txt run
|
||||
python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
|
||||
python3 aiobrute.py --target example.com --wordlist subdomains.txt run
|
||||
python3 aiobrute.py --target example.com --recursive True --depth 2 run
|
||||
python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
|
||||
|
||||
Note:
|
||||
Parameter valid optional value 1, 0, none indicates that the export is
|
||||
valid, invalid, and all subdomains, respectively.
|
||||
|
||||
Parameter format have optional values 'txt', 'rst', 'csv', 'tsv', 'json',
|
||||
'yaml', 'html', 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'.
|
||||
If the parameter path is None, the appropriate file is generated in the
|
||||
project result directory based on the format parameter and the domain
|
||||
name.
|
||||
|
||||
ARGUMENTS
|
||||
TARGET
|
||||
Single domain name or file path for one domain name per line (required)
|
||||
|
||||
FLAGS
|
||||
--process=PROCESS
|
||||
Number of processes blasted (default CPU core count)
|
||||
--coroutine=COROUTINE
|
||||
Number of coroutines per blasting process (default 1024)
|
||||
--wordlist=WORDLIST
|
||||
Specify the dictionary path used for blasting (config.py is used by default)
|
||||
--recursive=RECURSIVE
|
||||
Whether to use recursive blasting (default False)
|
||||
--depth=DEPTH
|
||||
Depth of recursive blasting (default 2)
|
||||
--namelist=NAMELIST
|
||||
Specifies the dictionary path used by recursive blasting (configured by default using config.py)
|
||||
--fuzz=FUZZ
|
||||
Whether to use the fuzz mode for blasting (default False, you must specify the fuzz regular rule)
|
||||
--rule=RULE
|
||||
Regular rules used by fuzz mode (configured by default using config.py)
|
||||
--export=EXPORT
|
||||
Whether to export the blast result (default True)
|
||||
--valid=VALID
|
||||
Export validity of subdomains (default None)
|
||||
--format=FORMAT
|
||||
Export format (default xls)
|
||||
--path=PATH
|
||||
Export path (default None)
|
||||
--show=SHOW
|
||||
Terminal display exported data (default False)
|
||||
```
|
||||
```bash
|
||||
NAME
|
||||
brute.py - OneForAll subdomain brute module
|
||||
|
||||
SYNOPSIS
|
||||
brute.py <flags>
|
||||
|
||||
DESCRIPTION
|
||||
Example：
|
||||
brute.py --target domain.com --word True run
|
||||
brute.py --targets ./domains.txt --word True run
|
||||
brute.py --target domain.com --word True --concurrent 2000 run
|
||||
brute.py --target domain.com --word True --wordlist subnames.txt run
|
||||
brute.py --target domain.com --word True --recursive True --depth 2 run
|
||||
brute.py --target d.com --fuzz True --place m.*.d.com --rule '[a-z]' run
|
||||
brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
|
||||
|
||||
Note:
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--path Result path (default None, automatically generated)
|
||||
|
||||
FLAGS
|
||||
--target=TARGET
|
||||
One domain (target or targets must be provided)
|
||||
--targets=TARGETS
|
||||
File path of one domain per line
|
||||
--process=PROCESS
|
||||
Number of processes (default 1)
|
||||
--concurrent=CONCURRENT
|
||||
Number of concurrent (default 2000)
|
||||
--word=WORD
|
||||
Use word mode generate dictionary (default False)
|
||||
--wordlist=WORDLIST
|
||||
Dictionary path used in word mode (default use ./config/default.py)
|
||||
--recursive=RECURSIVE
|
||||
Use recursion (default False)
|
||||
--depth=DEPTH
|
||||
Recursive depth (default 2)
|
||||
--nextlist=NEXTLIST
|
||||
Dictionary file path used by recursive (default use ./config/default.py)
|
||||
--fuzz=FUZZ
|
||||
Use fuzz mode generate dictionary (default False)
|
||||
--place=PLACE
|
||||
Designated fuzz position (required if use fuzz mode)
|
||||
--rule=RULE
|
||||
Specify the regexp rules used in fuzz mode (required if use fuzz mode)
|
||||
--fuzzlist=FUZZLIST
|
||||
Dictionary path used in fuzz mode (default use ./config/default.py)
|
||||
--export=EXPORT
|
||||
Export the results (default True)
|
||||
--alive=ALIVE
|
||||
Only export alive subdomains (default False)
|
||||
--format=FORMAT
|
||||
Result format (default csv)
|
||||
--path=PATH
|
||||
Result directory (default None)
|
||||
```
|
||||
|
||||
3. **takeover.py help**
|
||||
|
||||
```bash
|
||||
python takeover.py --help
|
||||
```
|
||||
|
||||
```bash
|
||||
NAME
|
||||
takeover.py - OneForAll subdomain takeover module SYNOPSIS takeover.py <flags> DESCRIPTION
|
||||
|
||||
Example:
|
||||
python3 takeover.py --target www.example.com --format csv run
|
||||
python3 takeover.py --targets ./subdomains.txt --thread 10 run
|
||||
|
||||
Note:
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--path Result directory (default directory is ./results)
|
||||
|
||||
FLAGS
|
||||
--target=TARGET
|
||||
One domain (target or targets must be provided)
|
||||
--targets=TARGETS
|
||||
File path of one domain per line
|
||||
--thread=THREAD
|
||||
threads number (default 20)
|
||||
--path=PATH
|
||||
Result directory (default None)
|
||||
--format=FORMAT
|
||||
Result format (default csv)
|
||||
```
|
||||
|
||||
4. **dbexport.py help**
|
||||
|
||||
```bash
|
||||
python dbexport.py --help
|
||||
```
|
||||
|
||||
```bash
|
||||
NAME
|
||||
dbexport.py - OneForAll export from database module
|
||||
|
||||
SYNOPSIS
|
||||
dbexport.py TARGET <flags>
|
||||
|
||||
DESCRIPTION
|
||||
Example:
|
||||
python3 dbexport.py --target name --format csv --dir= ./result.csv
|
||||
python3 dbexport.py --db result.db --target name --show False
|
||||
python3 dbexport.py --target table_name --tb True --show False
|
||||
|
||||
Note:
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--path Result directory (default directory is ./results)
|
||||
|
||||
POSITIONAL ARGUMENTS
|
||||
TARGET
|
||||
Table to be exported
|
||||
|
||||
FLAGS
|
||||
--type=TYPE
|
||||
Type of target
|
||||
--db=DB
|
||||
Database path to be exported (default ./results/result.sqlite3)
|
||||
--alive=ALIVE
|
||||
Only export the results of alive subdomains (default False)
|
||||
--limit=LIMIT
|
||||
Export limit (default None)
|
||||
--path=PATH
|
||||
Result directory (default None)
|
||||
--format=FORMAT
|
||||
Result format (default csv)
|
||||
--show=SHOW
|
||||
Displays the exported data in terminal (default False)
|
||||
```
|
||||
+5
-5
@@ -84,19 +84,19 @@ DNS解析返回的TTL值
|
||||
|
||||
### cidr
|
||||
|
||||
IP2LOCATION-ASN库查询出的CIDR
|
||||
ip2location库查询出的CIDR
|
||||
|
||||
### asn
|
||||
|
||||
IP2LOCATION-ASN库查询出的ASN
|
||||
ip2location库查询出的ASN
|
||||
|
||||
### ip2region
|
||||
### addr
|
||||
|
||||
ip2region库查询出的物理地址
|
||||
|
||||
### ip2location
|
||||
### isp
|
||||
|
||||
ip2location库查询出的物理地址
|
||||
ip2region库查询出的网络服务提供商
|
||||
|
||||
### resolver
|
||||
|
||||
|
||||
+55
-61
@@ -38,7 +38,6 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
|
||||
python3 oneforall.py --target example.com --show True run
|
||||
|
||||
Note:
|
||||
--alive True/False 只导出存活子域(默认False)
|
||||
--port small/medium/large 详见./config/setting.py(默认small)
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (结果格式，默认CSV)
|
||||
--path 结果路径(默认None，自动生成)
|
||||
@@ -90,63 +89,56 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
|
||||
|
||||
```bash
|
||||
NAME
|
||||
aiobrute.py - OneForAll多进程多协程异步子域爆破模块
|
||||
brute.py - OneForAll子域爆破模块
|
||||
|
||||
SYNOPSIS
|
||||
aiobrute.py --target=TARGET <flags>
|
||||
brute.py <flags>
|
||||
|
||||
DESCRIPTION
|
||||
Example：
|
||||
python3 aiobrute.py --target example.com run
|
||||
python3 aiobrute.py --target ./domains.txt run
|
||||
python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
|
||||
python3 aiobrute.py --target example.com --wordlist subdomains.txt run
|
||||
python3 aiobrute.py --target example.com --recursive True --depth 2 run
|
||||
python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
|
||||
|
||||
brute.py --target domain.com --word True run
|
||||
brute.py --targets ./domains.txt --word True run
|
||||
brute.py --target domain.com --word True --coroutine 2000 run
|
||||
brute.py --target domain.com --word True --wordlist subnames.txt run
|
||||
brute.py --target domain.com --word True --recursive True --depth 2 run
|
||||
brute.py --target d.com --fuzz True --place m.*.d.com --rule '[a-z]' run
|
||||
brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
|
||||
|
||||
Note:
|
||||
参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为任务组，
|
||||
当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
|
||||
十万字典建议设置为5000，百万字典设置为50000
|
||||
参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
|
||||
参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
|
||||
'dbf', 'latex', 'ods'
|
||||
参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
|
||||
|
||||
ARGUMENTS
|
||||
TARGET
|
||||
单个域名或者每行一个域名的文件路径
|
||||
|
||||
FLAGS
|
||||
--process=PROCESS
|
||||
爆破的进程数(默认CPU核心数)
|
||||
--coroutine=COROUTINE
|
||||
每个爆破进程下的协程数(默认64)
|
||||
--wordlist=WORDLIST
|
||||
指定爆破所使用的字典路径(默认使用config.py配置)
|
||||
--segment=SEGMENT
|
||||
爆破任务分割(默认500)
|
||||
--recursive=RECURSIVE
|
||||
是否使用递归爆破(默认False)
|
||||
--depth=DEPTH
|
||||
递归爆破的深度(默认2)
|
||||
--namelist=NAMELIST
|
||||
指定递归爆破所使用的字典路径(默认使用config.py配置)
|
||||
--fuzz=FUZZ
|
||||
是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
|
||||
--rule=RULE
|
||||
fuzz模式使用的正则规则(默认使用config.py配置)
|
||||
--export=EXPORT
|
||||
是否导出爆破结果(默认True)
|
||||
--valid=VALID
|
||||
导出子域的有效性(默认None)
|
||||
--format=FORMAT
|
||||
导出格式(默认xlsx)
|
||||
--path=PATH
|
||||
导出路径(默认None)
|
||||
--show=SHOW
|
||||
终端显示导出数据(默认False)
|
||||
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (结果格式，默认CSV)
|
||||
--path 导出路径(默认None，自动生成)
|
||||
|
||||
FLAGS
|
||||
--target=TARGET
|
||||
单个域名(必须提供target或targets参数)
|
||||
--targets=TARGETS
|
||||
每行一个域名的文件路径
|
||||
--process=PROCESS
|
||||
爆破的进程数(默认CPU核心数)
|
||||
--coroutine=COROUTINE
|
||||
每个爆破进程下的协程数(默认2000)
|
||||
--wordlist=WORDLIST
|
||||
指定爆破所使用的字典路径(默认使用config.py配置)
|
||||
--recursive=RECURSIVE
|
||||
是否使用递归爆破(默认False)
|
||||
--depth=DEPTH
|
||||
递归爆破的深度(默认2)
|
||||
--nextlist=NEXTLIST
|
||||
指定递归爆破所使用的字典路径(默认使用config.py配置)
|
||||
--fuzz=FUZZ
|
||||
是否使用fuzz模式进行爆破(默认False)
|
||||
--rule=RULE
|
||||
fuzz模式使用的正则规则(默认使用config.py配置)
|
||||
--fuzzlist=FUZZLIST
|
||||
指定fuzz模式所使用的字典路径(默认使用config.py配置)
|
||||
--export=EXPORT
|
||||
是否导出爆破结果(默认True)
|
||||
--alive=ALIVE
|
||||
只导出存活子域(默认False)
|
||||
--format=FORMAT
|
||||
导出格式(默认csv)
|
||||
--path=PATH
|
||||
导出路径(默认None)
|
||||
```
|
||||
|
||||
|
||||
@@ -200,24 +192,26 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
|
||||
dbexport.py - OneForAll数据库导出模块
|
||||
|
||||
SYNOPSIS
|
||||
dbexport.py TABLE <flags>
|
||||
dbexport.py TARGET <flags>
|
||||
|
||||
DESCRIPTION
|
||||
Example:
|
||||
python3 dbexport.py --table name --format csv --path= ./result.csv
|
||||
python3 dbexport.py --db result.db --table name --show False
|
||||
python3 dbexport.py --target name --format csv --dir= ./result.csv
|
||||
python3 dbexport.py --db result.db --target name --show False
|
||||
python3 dbexport.py --target table_name --tb True --show False
|
||||
|
||||
Note:
|
||||
参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
|
||||
参数format可选格式有'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
|
||||
'dbf', 'latex', 'ods'
|
||||
参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
|
||||
--type target/table 要导出的目标类型(默认target)
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (结果格式，默认CSV)
|
||||
--path 结果路径(默认None，自动生成)
|
||||
|
||||
POSITIONAL ARGUMENTS
|
||||
TABLE
|
||||
要导出的表
|
||||
TARGET
|
||||
要导出的目标类型
|
||||
|
||||
FLAGS
|
||||
--type=TYPE
|
||||
要导出的目标类型(默认target)
|
||||
--db=DB
|
||||
要导出的数据库路径(默认为results/result.sqlite3)
|
||||
--valid=VALID
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 43 KiB |
+6
-10
@@ -11,19 +11,15 @@ import dns.resolver
|
||||
import dns.zone
|
||||
|
||||
from common import utils
|
||||
from common.module import Module
|
||||
from common.check import Check
|
||||
from config.log import logger
|
||||
|
||||
|
||||
class CheckAXFR(Module):
|
||||
"""
|
||||
DNS zone transfer vulnerability base class
|
||||
"""
|
||||
|
||||
def __init__(self, domain: str):
|
||||
Module.__init__(self)
|
||||
class AXFR(Check):
|
||||
def __init__(self, domain):
|
||||
Check.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Check'
|
||||
self.module = 'check'
|
||||
self.source = 'AXFRCheck'
|
||||
self.results = []
|
||||
|
||||
@@ -92,7 +88,7 @@ def run(domain):
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
check = CheckAXFR(domain)
|
||||
check = AXFR(domain)
|
||||
check.run()
|
||||
|
||||
|
||||
|
||||
+9
-20
@@ -1,33 +1,22 @@
|
||||
"""
|
||||
检查crossdomain.xml文件收集子域名
|
||||
"""
|
||||
|
||||
from common.module import Module
|
||||
from common.check import Check
|
||||
|
||||
|
||||
class CheckCDX(Module):
|
||||
"""
|
||||
检查crossdomain.xml文件收集子域名
|
||||
"""
|
||||
def __init__(self, domain: str):
|
||||
Module.__init__(self)
|
||||
class CrossDomain(Check):
|
||||
def __init__(self, domain):
|
||||
Check.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Check'
|
||||
self.source = "CrossDomainXml"
|
||||
self.module = 'check'
|
||||
self.source = "CrossDomainCheck"
|
||||
|
||||
def check(self):
|
||||
"""
|
||||
检查crossdomain.xml收集子域名
|
||||
"""
|
||||
urls = [f'http://{self.domain}/crossdomain.xml',
|
||||
f'https://{self.domain}/crossdomain.xml',
|
||||
f'http://www.{self.domain}/crossdomain.xml',
|
||||
f'https://www.{self.domain}/crossdomain.xml']
|
||||
for url in urls:
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(url, check=False)
|
||||
self.subdomains = self.collect_subdomains(resp)
|
||||
filenames = {'crossdomain.xml'}
|
||||
self.to_check(filenames)
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
@@ -47,7 +36,7 @@ def run(domain):
|
||||
|
||||
:param domain: 域名
|
||||
"""
|
||||
check = CheckCDX(domain)
|
||||
check = CrossDomain(domain)
|
||||
check.run()
|
||||
|
||||
|
||||
|
||||
+10
-12
@@ -1,21 +1,18 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
检查域名证书收集子域名
|
||||
"""
|
||||
import socket
|
||||
import ssl
|
||||
|
||||
from common.module import Module
|
||||
from config.log import logger
|
||||
from common.check import Check
|
||||
|
||||
|
||||
class CheckCert(Module):
|
||||
class CertInfo(Check):
|
||||
def __init__(self, domain):
|
||||
Module.__init__(self)
|
||||
Check.__init__(self)
|
||||
self.domain = domain
|
||||
self.port = 443 # ssl port
|
||||
self.module = 'Check'
|
||||
self.module = 'check'
|
||||
self.source = 'CertInfo'
|
||||
|
||||
def check(self):
|
||||
@@ -24,10 +21,11 @@ class CheckCert(Module):
|
||||
"""
|
||||
try:
|
||||
ctx = ssl.create_default_context()
|
||||
sock = ctx.wrap_socket(socket.socket(),
|
||||
server_hostname=self.domain)
|
||||
sock.connect((self.domain, self.port))
|
||||
cert_dict = sock.getpeercert()
|
||||
sock = socket.socket()
|
||||
sock.settimeout(10)
|
||||
wrap_sock = ctx.wrap_socket(sock, server_hostname=self.domain)
|
||||
wrap_sock.connect((self.domain, 443))
|
||||
cert_dict = wrap_sock.getpeercert()
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
return
|
||||
@@ -52,7 +50,7 @@ def run(domain):
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
check = CheckCert(domain)
|
||||
check = CertInfo(domain)
|
||||
check.run()
|
||||
|
||||
|
||||
|
||||
@@ -3,19 +3,19 @@ Collect subdomains from ContentSecurityPolicy
|
||||
"""
|
||||
import requests
|
||||
|
||||
from common.module import Module
|
||||
from config.log import logger
|
||||
from common.check import Check
|
||||
|
||||
|
||||
class CheckCSP(Module):
|
||||
class CSP(Check):
|
||||
"""
|
||||
Collect subdomains from ContentSecurityPolicy
|
||||
"""
|
||||
def __init__(self, domain, header):
|
||||
Module.__init__(self)
|
||||
Check.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Check'
|
||||
self.source = 'ContentSecurityPolicy'
|
||||
self.module = 'check'
|
||||
self.source = 'CSPCheck'
|
||||
self.csp_header = header
|
||||
|
||||
def grab_header(self):
|
||||
@@ -34,8 +34,7 @@ class CheckCSP(Module):
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
response = self.get(url, check=False)
|
||||
if response:
|
||||
csp_header = response.headers
|
||||
break
|
||||
return response.headers
|
||||
return csp_header
|
||||
|
||||
def check(self):
|
||||
@@ -73,7 +72,7 @@ def run(domain, header=None):
|
||||
:param str domain: 域名
|
||||
:param dict or None header: 响应头
|
||||
"""
|
||||
check = CheckCSP(domain, header)
|
||||
check = CSP(domain, header)
|
||||
check.run()
|
||||
|
||||
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
# https://www.icann.org/resources/pages/dnssec-what-is-it-why-important-2019-03-20-zh
|
||||
# https://appsecco.com/books/subdomain-enumeration/active_techniques/zone_walking.html
|
||||
|
||||
from common.module import Module
|
||||
from common import utils
|
||||
from common.check import Check
|
||||
|
||||
|
||||
class CheckNSEC(Module):
|
||||
class NSEC(Check):
|
||||
def __init__(self, domain):
|
||||
Module.__init__(self)
|
||||
Check.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'check'
|
||||
self.source = "CheckNSEC"
|
||||
self.source = "NSECCheck"
|
||||
|
||||
def walk(self):
|
||||
domain = self.domain
|
||||
@@ -47,7 +47,7 @@ def run(domain):
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
brute = CheckNSEC(domain)
|
||||
brute = NSEC(domain)
|
||||
brute.run()
|
||||
|
||||
|
||||
|
||||
+8
-18
@@ -1,32 +1,22 @@
|
||||
"""
|
||||
检查内容安全策略收集子域名收集子域名
|
||||
"""
|
||||
from common.module import Module
|
||||
from common.check import Check
|
||||
|
||||
|
||||
class CheckRobots(Module):
|
||||
"""
|
||||
检查robots.txt收集子域名
|
||||
"""
|
||||
class Robots(Check):
|
||||
def __init__(self, domain):
|
||||
Module.__init__(self)
|
||||
Check.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Check'
|
||||
self.source = 'Robots'
|
||||
self.module = 'check'
|
||||
self.source = 'RobotsCheck'
|
||||
|
||||
def check(self):
|
||||
"""
|
||||
正则匹配域名的robots.txt文件中的子域
|
||||
"""
|
||||
urls = [f'http://{self.domain}/robots.txt',
|
||||
f'https://{self.domain}/robots.txt',
|
||||
f'http://www.{self.domain}/robots.txt',
|
||||
f'https://www.{self.domain}/robots.txt']
|
||||
for url in urls:
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
resp = self.get(url, check=False, allow_redirects=False)
|
||||
self.subdomains = self.collect_subdomains(resp)
|
||||
filenames = {'robots.txt'}
|
||||
self.to_check(filenames)
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
@@ -46,7 +36,7 @@ def run(domain):
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
check = CheckRobots(domain)
|
||||
check = Robots(domain)
|
||||
check.run()
|
||||
|
||||
|
||||
|
||||
@@ -1,37 +1,22 @@
|
||||
"""
|
||||
检查内容安全策略收集子域名收集子域名
|
||||
"""
|
||||
from common.module import Module
|
||||
from common.check import Check
|
||||
|
||||
|
||||
class CheckRobots(Module):
|
||||
"""
|
||||
检查sitemap收集子域名
|
||||
"""
|
||||
class Sitemap(Check):
|
||||
def __init__(self, domain):
|
||||
Module.__init__(self)
|
||||
Check.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'Check'
|
||||
self.source = 'Sitemap'
|
||||
self.module = 'check'
|
||||
self.source = 'SitemapCheck'
|
||||
|
||||
def check(self):
|
||||
"""
|
||||
正则匹配域名的sitemap文件中的子域
|
||||
"""
|
||||
urls = [f'http://{self.domain}/sitemap.xml',
|
||||
f'http://www.{self.domain}/sitemap.xml',
|
||||
f'http://{self.domain}/sitemap.txt',
|
||||
f'http://www.{self.domain}/sitemap.txt',
|
||||
f'http://{self.domain}/sitemap.html',
|
||||
f'http://www.{self.domain}/sitemap.html',
|
||||
f'http://{self.domain}/sitemap_index.xml',
|
||||
f'http://www.{self.domain}/sitemap_index.xml']
|
||||
for url in urls:
|
||||
self.header = self.get_header()
|
||||
self.proxy = self.get_proxy(self.source)
|
||||
self.timeout = 10
|
||||
resp = self.get(url, check=False)
|
||||
self.subdomains = self.collect_subdomains(resp)
|
||||
filenames = {'sitemap.xml', 'sitemap.txt', 'sitemap.html', 'sitemapindex.xml'}
|
||||
self.to_check(filenames)
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
@@ -51,7 +36,7 @@ def run(domain):
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
check = CheckRobots(domain)
|
||||
check = Sitemap(domain)
|
||||
check.run()
|
||||
|
||||
|
||||
|
||||
+3
-28
@@ -1,38 +1,24 @@
|
||||
import time
|
||||
import threading
|
||||
import importlib
|
||||
|
||||
import dbexport
|
||||
from config.log import logger
|
||||
from config import settings
|
||||
from common import utils
|
||||
|
||||
|
||||
class Collect(object):
|
||||
"""
|
||||
Collect subdomains
|
||||
"""
|
||||
|
||||
def __init__(self, domain, export=True):
|
||||
self.domain = utils.get_main_domain(domain)
|
||||
self.elapse = 0.0
|
||||
def __init__(self, domain):
|
||||
self.domain = domain
|
||||
self.modules = []
|
||||
self.collect_funcs = []
|
||||
self.path = None
|
||||
self.export = export
|
||||
self.format = 'csv'
|
||||
|
||||
def get_mod(self):
|
||||
"""
|
||||
Get modules
|
||||
"""
|
||||
if settings.enable_all_module:
|
||||
# modules = ['brute', 'certificates', 'crawl',
|
||||
# 'datasets', 'intelligence', 'search']
|
||||
# The crawl module has some problems
|
||||
modules = ['certificates', 'check', 'datasets',
|
||||
'dnsquery', 'intelligence', 'search']
|
||||
# modules = ['certificates']
|
||||
for module in modules:
|
||||
module_path = settings.module_dir.joinpath(module)
|
||||
for path in module_path.rglob('*.py'):
|
||||
@@ -55,7 +41,6 @@ class Collect(object):
|
||||
"""
|
||||
Class entrance
|
||||
"""
|
||||
start = time.time()
|
||||
logger.log('INFOR', f'Start collecting subdomains of {self.domain}')
|
||||
self.get_mod()
|
||||
self.import_func()
|
||||
@@ -66,8 +51,7 @@ class Collect(object):
|
||||
func_obj, func_name = collect_func
|
||||
thread = threading.Thread(target=func_obj,
|
||||
name=func_name,
|
||||
args=(self.domain,),
|
||||
daemon=True)
|
||||
args=(self.domain,))
|
||||
threads.append(thread)
|
||||
# Start all threads
|
||||
for thread in threads:
|
||||
@@ -82,15 +66,6 @@ class Collect(object):
|
||||
if thread.is_alive():
|
||||
logger.log('ALERT', f'{thread.name} module thread timed out')
|
||||
|
||||
# Export
|
||||
if self.export:
|
||||
if not self.path:
|
||||
name = f'{self.domain}.{self.format}'
|
||||
self.path = settings.result_save_dir.joinpath(name)
|
||||
dbexport.export(self.domain, type='table', path=self.path, format=self.format)
|
||||
end = time.time()
|
||||
self.elapse = round(end - start, 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
collect = Collect('example.com')
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
"""
|
||||
通过枚举域名常见的SRV记录并做查询来发现子域
|
||||
"""
|
||||
|
||||
import json
|
||||
import queue
|
||||
import threading
|
||||
|
||||
from common import utils
|
||||
from common.module import Module
|
||||
from config.setting import data_storage_dir
|
||||
|
||||
|
||||
class BruteSRV(Module):
|
||||
def __init__(self, domain):
|
||||
Module.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'dnsquery'
|
||||
self.source = "BruteSRV"
|
||||
self.qtype = 'SRV' # 利用的DNS记录的SRV记录查询子域
|
||||
self.thread_num = 10
|
||||
self.names_que = queue.Queue()
|
||||
self.answers_que = queue.Queue()
|
||||
|
||||
def gen_names(self):
|
||||
path = data_storage_dir.joinpath('srv_prefixes.json')
|
||||
with open(path, encoding='utf-8', errors='ignore') as file:
|
||||
prefixes = json.load(file)
|
||||
names = map(lambda prefix: prefix + self.domain, prefixes)
|
||||
|
||||
for name in names:
|
||||
self.names_que.put(name)
|
||||
|
||||
def brute(self):
|
||||
"""
|
||||
枚举域名的SRV记录
|
||||
"""
|
||||
self.gen_names()
|
||||
|
||||
for num in range(self.thread_num):
|
||||
thread = BruteThread(self.names_que, self.answers_que)
|
||||
thread.name = f'BruteThread-{num}'
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
self.names_que.join()
|
||||
|
||||
while not self.answers_que.empty():
|
||||
answer = self.answers_que.get()
|
||||
if answer is None:
|
||||
continue
|
||||
for item in answer:
|
||||
record = str(item)
|
||||
subdomains = self.match_subdomains(record)
|
||||
self.subdomains.update(subdomains)
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
类执行入口
|
||||
"""
|
||||
self.begin()
|
||||
self.brute()
|
||||
self.finish()
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
|
||||
|
||||
class BruteThread(threading.Thread):
|
||||
def __init__(self, names_que, answers_que):
|
||||
threading.Thread.__init__(self)
|
||||
self.names_que = names_que
|
||||
self.answers_que = answers_que
|
||||
|
||||
def run(self):
|
||||
while True:
|
||||
name = self.names_que.get()
|
||||
answer = utils.dns_query(name, 'SRV')
|
||||
self.answers_que.put(answer)
|
||||
self.names_que.task_done()
|
||||
|
||||
|
||||
def run(domain):
|
||||
"""
|
||||
类统一调用入口
|
||||
|
||||
:param str domain: 域名
|
||||
"""
|
||||
brute = BruteSRV(domain)
|
||||
brute.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run('zonetransfer.me')
|
||||
# run('example.com')
|
||||
+4
-3
@@ -190,9 +190,10 @@ def find_subdomains(domain, data):
|
||||
subdomains.update(find_in_resp(domain, req_url, rsp_html))
|
||||
js_urls.update(find_js_urls(domain, req_url, rsp_html))
|
||||
|
||||
resp_data = request.urls_request(js_urls)
|
||||
for resp, text in resp_data:
|
||||
if not text:
|
||||
resp_data = request.bulk_request(js_urls)
|
||||
for resp in resp_data:
|
||||
if not resp:
|
||||
continue
|
||||
text = utils.decode_resp_text(resp)
|
||||
subdomains.update(find_in_resp(domain, resp.url, text))
|
||||
return subdomains
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""
|
||||
通过枚举域名常见的SRV记录并做查询来发现子域
|
||||
"""
|
||||
|
||||
import queue
|
||||
import threading
|
||||
|
||||
from common import utils
|
||||
from common.module import Module
|
||||
from config.setting import data_storage_dir
|
||||
|
||||
|
||||
class BruteSRV(Module):
|
||||
def __init__(self, domain):
|
||||
Module.__init__(self)
|
||||
self.domain = domain
|
||||
self.module = 'BruteSRV'
|
||||
self.source = "BruteSRV"
|
||||
self.qtype = 'SRV'
|
||||
self.thread_num = 20
|
||||
self.names_queue = queue.Queue()
|
||||
self.answers_queue = queue.Queue()
|
||||
|
||||
def fill_queue(self):
|
||||
path = data_storage_dir.joinpath('srv_prefixes.json')
|
||||
prefixes = utils.load_json(path)
|
||||
for prefix in prefixes:
|
||||
self.names_queue.put(prefix + self.domain)
|
||||
|
||||
def do_brute(self):
|
||||
for num in range(self.thread_num):
|
||||
thread = BruteThread(self.names_queue, self.answers_queue)
|
||||
thread.name = f'BruteThread-{num}'
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
self.names_queue.join()
|
||||
|
||||
def deal_answers(self):
|
||||
while not self.answers_queue.empty():
|
||||
answer = self.answers_queue.get()
|
||||
if answer is None:
|
||||
continue
|
||||
for item in answer:
|
||||
record = str(item)
|
||||
subdomains = self.match_subdomains(record)
|
||||
self.subdomains.update(subdomains)
|
||||
|
||||
def run(self):
|
||||
self.begin()
|
||||
self.fill_queue()
|
||||
self.do_brute()
|
||||
self.deal_answers()
|
||||
self.finish()
|
||||
self.save_json()
|
||||
self.gen_result()
|
||||
self.save_db()
|
||||
|
||||
|
||||
class BruteThread(threading.Thread):
|
||||
def __init__(self, names_queue, answers_queue):
|
||||
threading.Thread.__init__(self)
|
||||
self.names_queue = names_queue
|
||||
self.answers_queue = answers_queue
|
||||
|
||||
def run(self):
|
||||
while True:
|
||||
name = self.names_queue.get()
|
||||
answer = utils.dns_query(name, 'SRV')
|
||||
self.answers_queue.put(answer)
|
||||
self.names_queue.task_done()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
brute = BruteSRV('zonetransfer.me')
|
||||
brute.run()
|
||||
+7
-3
@@ -17,6 +17,7 @@ from brute import Brute
|
||||
from common import utils, resolve, request
|
||||
from common.database import Database
|
||||
from modules.collect import Collect
|
||||
from modules.srv import BruteSRV
|
||||
from modules.finder import Finder
|
||||
from modules import iscdn
|
||||
from config import settings
|
||||
@@ -65,7 +66,6 @@ class OneForAll(object):
|
||||
python3 oneforall.py --target example.com --show True run
|
||||
|
||||
Note:
|
||||
--alive True/False Only export alive subdomains or not (default False)
|
||||
--port small/medium/large See details in ./config/setting.py(default small)
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--path Result path (default None, automatically generated)
|
||||
@@ -188,8 +188,12 @@ class OneForAll(object):
|
||||
self.origin_table = self.domain + '_origin_result'
|
||||
self.resolve_table = self.domain + '_resolve_result'
|
||||
|
||||
collect = Collect(self.domain, export=False)
|
||||
collect = Collect(self.domain)
|
||||
collect.run()
|
||||
|
||||
srv = BruteSRV(self.domain)
|
||||
srv.run()
|
||||
|
||||
if self.brute:
|
||||
# Due to there will be a large number of dns resolution requests,
|
||||
# may cause other network tasks to be error
|
||||
@@ -240,7 +244,7 @@ class OneForAll(object):
|
||||
# Scan subdomain takeover
|
||||
if self.takeover:
|
||||
subdomains = utils.get_subdomains(self.data)
|
||||
takeover = Takeover(subdomains)
|
||||
takeover = Takeover(targets=subdomains)
|
||||
takeover.run()
|
||||
return self.data
|
||||
|
||||
|
||||
+9
-14
@@ -1,24 +1,19 @@
|
||||
aiohttp==3.6.2
|
||||
async-timeout==3.0.1
|
||||
attrs==19.3.0
|
||||
beautifulsoup4==4.9.1
|
||||
bs4==0.0.1
|
||||
certifi==2020.6.20
|
||||
chardet==3.0.4
|
||||
cloudscraper==1.2.42
|
||||
cloudscraper==1.2.46
|
||||
colorama==0.4.3
|
||||
dnspython==1.16.0
|
||||
dnspython==2.0.0
|
||||
docopt==0.6.2
|
||||
et-xmlfile==1.0.1
|
||||
exrex==0.10.5
|
||||
fire==0.3.1
|
||||
future==0.18.2
|
||||
idna==2.10
|
||||
IP2Location==8.4.1
|
||||
jdcal==1.4.1
|
||||
loguru==0.5.1
|
||||
multidict==4.7.6
|
||||
openpyxl==2.4.11
|
||||
psutil==5.7.0
|
||||
pyparsing==2.4.7
|
||||
PySocks==1.7.1
|
||||
records==0.5.3
|
||||
@@ -27,12 +22,12 @@ requests-file==1.5.1
|
||||
requests-toolbelt==0.9.1
|
||||
six==1.15.0
|
||||
soupsieve==2.0.1
|
||||
SQLAlchemy==1.3.18
|
||||
SQLAlchemy==1.3.19
|
||||
tablib==2.0.0
|
||||
tenacity==6.2.0
|
||||
termcolor==1.1.0
|
||||
tldextract==2.2.2
|
||||
tqdm==4.47.0
|
||||
urllib3==1.25.9
|
||||
win32-setctime==1.0.1
|
||||
yarl==1.4.2
|
||||
tldextract==2.2.3
|
||||
tqdm==4.48.2
|
||||
treelib==1.6.1
|
||||
urllib3==1.25.10
|
||||
win32-setctime==1.0.1
|
||||
+14
-16
@@ -20,7 +20,6 @@ from config.log import logger
|
||||
from config import settings
|
||||
from common import utils
|
||||
from common.module import Module
|
||||
from common.domain import Domain
|
||||
|
||||
|
||||
def get_fingerprint():
|
||||
@@ -41,34 +40,32 @@ def get_cname(subdomain):
|
||||
return answer.to_text() # 一个子域只有一个CNAME记录
|
||||
|
||||
|
||||
def get_maindomain(subdomain):
|
||||
return Domain(subdomain).registered()
|
||||
|
||||
|
||||
class Takeover(Module):
|
||||
"""
|
||||
OneForAll subdomain takeover module
|
||||
|
||||
Example:
|
||||
python3 takeover.py --target www.example.com --format csv run
|
||||
python3 takeover.py --target ./subdomains.txt --thread 10 run
|
||||
python3 takeover.py --targets ./subdomains.txt --thread 10 run
|
||||
|
||||
Note:
|
||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||
--path Result directory (default directory is ./results)
|
||||
|
||||
:param any target: One domain or File path of one domain per line (required)
|
||||
:param int thread: threads number (default 100)
|
||||
:param str format: Result format (default csv)
|
||||
:param str path: Result directory (default None)
|
||||
:param str target: One domain (target or targets must be provided)
|
||||
:param str targets: File path of one domain per line
|
||||
:param int thread: threads number (default 20)
|
||||
:param str format: Result format (default csv)
|
||||
:param str path: Result directory (default None)
|
||||
"""
|
||||
|
||||
def __init__(self, target, thread=100, path=None, format='csv'):
|
||||
def __init__(self, target=None, targets=None, thread=20, path=None, format='csv'):
|
||||
Module.__init__(self)
|
||||
self.subdomains = set()
|
||||
self.module = 'Check'
|
||||
self.source = 'Takeover'
|
||||
self.target = target
|
||||
self.targets = targets
|
||||
self.thread = thread
|
||||
self.path = path
|
||||
self.format = format
|
||||
@@ -101,10 +98,10 @@ class Takeover(Module):
|
||||
cname = get_cname(subdomain)
|
||||
if cname is None:
|
||||
return
|
||||
maindomain = get_maindomain(cname)
|
||||
main_domain = utils.get_main_domain(cname)
|
||||
for fingerprint in self.fingerprints:
|
||||
cnames = fingerprint.get('cname')
|
||||
if maindomain not in cnames:
|
||||
if main_domain not in cnames:
|
||||
continue
|
||||
responses = fingerprint.get('response')
|
||||
self.compare(subdomain, cname, responses)
|
||||
@@ -116,7 +113,6 @@ class Takeover(Module):
|
||||
self.subdomainq.task_done()
|
||||
|
||||
def progress(self):
|
||||
# 设置进度
|
||||
bar = tqdm()
|
||||
bar.total = len(self.subdomains)
|
||||
bar.desc = 'Check Progress'
|
||||
@@ -127,12 +123,14 @@ class Takeover(Module):
|
||||
bar.update()
|
||||
if done == bar.total: # 完成队列中所有子域的检查退出
|
||||
break
|
||||
# bar.close()
|
||||
|
||||
def run(self):
|
||||
start = time.time()
|
||||
logger.log('INFOR', f'Start running {self.source} module')
|
||||
self.subdomains = utils.get_domains(self.target)
|
||||
if isinstance(self.targets, set):
|
||||
self.subdomains = self.targets
|
||||
else:
|
||||
self.subdomains = utils.get_domains(self.target, self.targets)
|
||||
self.format = utils.check_format(self.format, len(self.subdomains))
|
||||
timestamp = utils.get_timestamp()
|
||||
name = f'takeover_check_result_{timestamp}'
|
||||
|
||||
Reference in New Issue
Block a user