mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-10-27 19:25:52 +08:00
Merge branch 'master' of https://github.com/shmilylty/OneForAll
This commit is contained in:
Jing Ling
@@ -13,15 +13,13 @@ requests = "*"
|
|||||||
records = "*"
|
records = "*"
|
||||||
tldextract = "*"
|
tldextract = "*"
|
||||||
exrex = "*"
|
exrex = "*"
|
||||||
aiohttp = "*"
|
|
||||||
fire = "*"
|
fire = "*"
|
||||||
bs4 = "*"
|
bs4 = "*"
|
||||||
pysocks = "*"
|
pysocks = "*"
|
||||||
cloudscraper = "*"
|
cloudscraper = "*"
|
||||||
tablib = "*"
|
tablib = "*"
|
||||||
psutil = "*"
|
|
||||||
tenacity = "*"
|
tenacity = "*"
|
||||||
ip2location = "*"
|
treelib = "*"
|
||||||
|
|
||||||
[requires]
|
[requires]
|
||||||
python_version = "3.8"
|
python_version = "3.8"
|
||||||
|
|||||||
Generated
+62
-152
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"_meta": {
|
"_meta": {
|
||||||
"hash": {
|
"hash": {
|
||||||
"sha256": "6556f720e7fb052c3f923f3e7986cc00c1cc230db729ad79685a127c02568c96"
|
"sha256": "fde2c88929fdf63725faea24a4fc7d82d6792caaca6676c08e96a40227f47e18"
|
||||||
},
|
},
|
||||||
"pipfile-spec": 6,
|
"pipfile-spec": 6,
|
||||||
"requires": {
|
"requires": {
|
||||||
@@ -16,38 +16,6 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"default": {
|
"default": {
|
||||||
"aiohttp": {
|
|
||||||
"hashes": [
|
|
||||||
"sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e",
|
|
||||||
"sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326",
|
|
||||||
"sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a",
|
|
||||||
"sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654",
|
|
||||||
"sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a",
|
|
||||||
"sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4",
|
|
||||||
"sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17",
|
|
||||||
"sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec",
|
|
||||||
"sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd",
|
|
||||||
"sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48",
|
|
||||||
"sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59",
|
|
||||||
"sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965"
|
|
||||||
],
|
|
||||||
"index": "pypi",
|
|
||||||
"version": "==3.6.2"
|
|
||||||
},
|
|
||||||
"async-timeout": {
|
|
||||||
"hashes": [
|
|
||||||
"sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
|
|
||||||
"sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
|
|
||||||
],
|
|
||||||
"version": "==3.0.1"
|
|
||||||
},
|
|
||||||
"attrs": {
|
|
||||||
"hashes": [
|
|
||||||
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
|
|
||||||
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
|
|
||||||
],
|
|
||||||
"version": "==19.3.0"
|
|
||||||
},
|
|
||||||
"beautifulsoup4": {
|
"beautifulsoup4": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7",
|
"sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7",
|
||||||
@@ -79,11 +47,11 @@
|
|||||||
},
|
},
|
||||||
"cloudscraper": {
|
"cloudscraper": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:1d4e158de3be7a4e651083f13a14a25b75853186e989d9d97db1a08056b7a0b6",
|
"sha256:793095bbc37aae84a5c38d9f66b56f3c83a00f0b1ff2d334f75a2b8f88b924af",
|
||||||
"sha256:c8c255aaa84e80d2cb5bf2c1ad1ffa000ce864595ca4cdb75ff7aab6f97befec"
|
"sha256:a7ae28b5dc4e4ef0789ef608c42a4382bf6d3e3dd6b8117c2e3633103131ae43"
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==1.2.42"
|
"version": "==1.2.46"
|
||||||
},
|
},
|
||||||
"colorama": {
|
"colorama": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
@@ -95,11 +63,11 @@
|
|||||||
},
|
},
|
||||||
"dnspython": {
|
"dnspython": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01",
|
"sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7",
|
||||||
"sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d"
|
"sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d"
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==1.16.0"
|
"version": "==2.0.0"
|
||||||
},
|
},
|
||||||
"docopt": {
|
"docopt": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
@@ -127,22 +95,19 @@
|
|||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==0.3.1"
|
"version": "==0.3.1"
|
||||||
},
|
},
|
||||||
|
"future": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
|
||||||
|
],
|
||||||
|
"version": "==0.18.2"
|
||||||
|
},
|
||||||
"idna": {
|
"idna": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
|
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
|
||||||
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
|
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
|
||||||
],
|
],
|
||||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
|
||||||
"version": "==2.10"
|
"version": "==2.10"
|
||||||
},
|
},
|
||||||
"ip2location": {
|
|
||||||
"hashes": [
|
|
||||||
"sha256:4a83f32098091b6aafaef63b393331bf973bbf54547db82de38f30108b5f079c",
|
|
||||||
"sha256:bbddbd5e3f79c18a9710afd43be09cb21724d7324557668017c545b5c5e62bc4"
|
|
||||||
],
|
|
||||||
"index": "pypi",
|
|
||||||
"version": "==8.4.1"
|
|
||||||
},
|
|
||||||
"jdcal": {
|
"jdcal": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba",
|
"sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba",
|
||||||
@@ -158,58 +123,17 @@
|
|||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==0.5.1"
|
"version": "==0.5.1"
|
||||||
},
|
},
|
||||||
"multidict": {
|
|
||||||
"hashes": [
|
|
||||||
"sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a",
|
|
||||||
"sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000",
|
|
||||||
"sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2",
|
|
||||||
"sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507",
|
|
||||||
"sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5",
|
|
||||||
"sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7",
|
|
||||||
"sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d",
|
|
||||||
"sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463",
|
|
||||||
"sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19",
|
|
||||||
"sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3",
|
|
||||||
"sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b",
|
|
||||||
"sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c",
|
|
||||||
"sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87",
|
|
||||||
"sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7",
|
|
||||||
"sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430",
|
|
||||||
"sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255",
|
|
||||||
"sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d"
|
|
||||||
],
|
|
||||||
"markers": "python_version >= '3.5'",
|
|
||||||
"version": "==4.7.6"
|
|
||||||
},
|
|
||||||
"openpyxl": {
|
"openpyxl": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:626d38647c063d55803ef4971c4d43226538d4e95cb6260c094e363ee33e10c7"
|
"sha256:626d38647c063d55803ef4971c4d43226538d4e95cb6260c094e363ee33e10c7"
|
||||||
],
|
],
|
||||||
"version": "==2.4.11"
|
"version": "==2.4.11"
|
||||||
},
|
},
|
||||||
"psutil": {
|
|
||||||
"hashes": [
|
|
||||||
"sha256:1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058",
|
|
||||||
"sha256:298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953",
|
|
||||||
"sha256:60b86f327c198561f101a92be1995f9ae0399736b6eced8f24af41ec64fb88d4",
|
|
||||||
"sha256:685ec16ca14d079455892f25bd124df26ff9137664af445563c1bd36629b5e0e",
|
|
||||||
"sha256:73f35ab66c6c7a9ce82ba44b1e9b1050be2a80cd4dcc3352cc108656b115c74f",
|
|
||||||
"sha256:75e22717d4dbc7ca529ec5063000b2b294fc9a367f9c9ede1f65846c7955fd38",
|
|
||||||
"sha256:a02f4ac50d4a23253b68233b07e7cdb567bd025b982d5cf0ee78296990c22d9e",
|
|
||||||
"sha256:d008ddc00c6906ec80040d26dc2d3e3962109e40ad07fd8a12d0284ce5e0e4f8",
|
|
||||||
"sha256:d84029b190c8a66a946e28b4d3934d2ca1528ec94764b180f7d6ea57b0e75e26",
|
|
||||||
"sha256:e2d0c5b07c6fe5a87fa27b7855017edb0d52ee73b71e6ee368fae268605cc3f5",
|
|
||||||
"sha256:f344ca230dd8e8d5eee16827596f1c22ec0876127c28e800d7ae20ed44c4b310"
|
|
||||||
],
|
|
||||||
"index": "pypi",
|
|
||||||
"version": "==5.7.0"
|
|
||||||
},
|
|
||||||
"pyparsing": {
|
"pyparsing": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
|
"sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
|
||||||
"sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
|
"sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
|
||||||
],
|
],
|
||||||
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
|
||||||
"version": "==2.4.7"
|
"version": "==2.4.7"
|
||||||
},
|
},
|
||||||
"pysocks": {
|
"pysocks": {
|
||||||
@@ -256,7 +180,6 @@
|
|||||||
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
|
"sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
|
||||||
"sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
|
"sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
|
||||||
],
|
],
|
||||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
|
||||||
"version": "==1.15.0"
|
"version": "==1.15.0"
|
||||||
},
|
},
|
||||||
"soupsieve": {
|
"soupsieve": {
|
||||||
@@ -264,42 +187,45 @@
|
|||||||
"sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
|
"sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
|
||||||
"sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
|
"sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
|
||||||
],
|
],
|
||||||
"markers": "python_version >= '3.5'",
|
|
||||||
"version": "==2.0.1"
|
"version": "==2.0.1"
|
||||||
},
|
},
|
||||||
"sqlalchemy": {
|
"sqlalchemy": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:0942a3a0df3f6131580eddd26d99071b48cfe5aaf3eab2783076fbc5a1c1882e",
|
"sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb",
|
||||||
"sha256:0ec575db1b54909750332c2e335c2bb11257883914a03bc5a3306a4488ecc772",
|
"sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804",
|
||||||
"sha256:109581ccc8915001e8037b73c29590e78ce74be49ca0a3630a23831f9e3ed6c7",
|
"sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6",
|
||||||
"sha256:16593fd748944726540cd20f7e83afec816c2ac96b082e26ae226e8f7e9688cf",
|
"sha256:26c5ca9d09f0e21b8671a32f7d83caad5be1f6ff45eef5ec2f6fd0db85fc5dc0",
|
||||||
"sha256:427273b08efc16a85aa2b39892817e78e3ed074fcb89b2a51c4979bae7e7ba98",
|
"sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe",
|
||||||
"sha256:50c4ee32f0e1581828843267d8de35c3298e86ceecd5e9017dc45788be70a864",
|
"sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de",
|
||||||
"sha256:512a85c3c8c3995cc91af3e90f38f460da5d3cade8dc3a229c8e0879037547c9",
|
"sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36",
|
||||||
"sha256:57aa843b783179ab72e863512e14bdcba186641daf69e4e3a5761d705dcc35b1",
|
"sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e",
|
||||||
"sha256:621f58cd921cd71ba6215c42954ffaa8a918eecd8c535d97befa1a8acad986dd",
|
"sha256:465c999ef30b1c7525f81330184121521418a67189053bcf585824d833c05b66",
|
||||||
"sha256:6ac2558631a81b85e7fb7a44e5035347938b0a73f5fdc27a8566777d0792a6a4",
|
"sha256:51064ee7938526bab92acd049d41a1dc797422256086b39c08bafeffb9d304c6",
|
||||||
"sha256:716754d0b5490bdcf68e1e4925edc02ac07209883314ad01a137642ddb2056f1",
|
"sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc",
|
||||||
"sha256:736d41cfebedecc6f159fc4ac0769dc89528a989471dc1d378ba07d29a60ba1c",
|
"sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d",
|
||||||
"sha256:8619b86cb68b185a778635be5b3e6018623c0761dde4df2f112896424aa27bd8",
|
"sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce",
|
||||||
"sha256:87fad64529cde4f1914a5b9c383628e1a8f9e3930304c09cf22c2ae118a1280e",
|
"sha256:6557af9e0d23f46b8cd56f8af08eaac72d2e3c632ac8d5cf4e20215a8dca7cea",
|
||||||
"sha256:89494df7f93b1836cae210c42864b292f9b31eeabca4810193761990dc689cce",
|
"sha256:73a40d4fcd35fdedce07b5885905753d5d4edf413fbe53544dd871f27d48bd4f",
|
||||||
"sha256:8cac7bb373a5f1423e28de3fd5fc8063b9c8ffe8957dc1b1a59cb90453db6da1",
|
"sha256:8280f9dae4adb5889ce0bb3ec6a541bf05434db5f9ab7673078c00713d148365",
|
||||||
"sha256:8fd452dc3d49b3cc54483e033de6c006c304432e6f84b74d7b2c68afa2569ae5",
|
"sha256:83469ad15262402b0e0974e612546bc0b05f379b5aa9072ebf66d0f8fef16bea",
|
||||||
"sha256:adad60eea2c4c2a1875eb6305a0b6e61a83163f8e233586a4d6a55221ef984fe",
|
"sha256:860d0fe234922fd5552b7f807fbb039e3e7ca58c18c8d38aa0d0a95ddf4f6c23",
|
||||||
"sha256:c26f95e7609b821b5f08a72dab929baa0d685406b953efd7c89423a511d5c413",
|
"sha256:883c9fb62cebd1e7126dd683222b3b919657590c3e2db33bdc50ebbad53e0338",
|
||||||
"sha256:cbe1324ef52ff26ccde2cb84b8593c8bf930069dfc06c1e616f1bfd4e47f48a3",
|
"sha256:8afcb6f4064d234a43fea108859942d9795c4060ed0fbd9082b0f280181a15c1",
|
||||||
"sha256:d05c4adae06bd0c7f696ae3ec8d993ed8ffcc4e11a76b1b35a5af8a099bd2284",
|
"sha256:96f51489ac187f4bab588cf51f9ff2d40b6d170ac9a4270ffaed535c8404256b",
|
||||||
"sha256:d98bc827a1293ae767c8f2f18be3bb5151fd37ddcd7da2a5f9581baeeb7a3fa1",
|
"sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e",
|
||||||
"sha256:da2fb75f64792c1fc64c82313a00c728a7c301efe6a60b7a9fe35b16b4368ce7",
|
"sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba",
|
||||||
"sha256:e4624d7edb2576cd72bb83636cd71c8ce544d8e272f308bd80885056972ca299",
|
"sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02",
|
||||||
"sha256:e89e0d9e106f8a9180a4ca92a6adde60c58b1b0299e1b43bd5e0312f535fbf33",
|
"sha256:b6ff91356354b7ff3bd208adcf875056d3d886ed7cef90c571aef2ab8a554b12",
|
||||||
"sha256:f11c2437fb5f812d020932119ba02d9e2bc29a6eca01a055233a8b449e3e1e7d",
|
"sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86",
|
||||||
"sha256:f57be5673e12763dd400fea568608700a63ce1c6bd5bdbc3cc3a2c5fdb045274",
|
"sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d",
|
||||||
"sha256:fc728ece3d5c772c196fd338a99798e7efac7a04f9cb6416299a3638ee9a94cd"
|
"sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7",
|
||||||
|
"sha256:e49947d583fe4d29af528677e4f0aa21f5e535ca2ae69c48270ebebd0d8843c0",
|
||||||
|
"sha256:eb1d71643e4154398b02e88a42fc8b29db8c44ce4134cf0f4474bfc5cb5d4dac",
|
||||||
|
"sha256:f2e8a9c0c8813a468aa659a01af6592f71cd30237ec27c4cc0683f089f90dcfc",
|
||||||
|
"sha256:fe7fe11019fc3e6600819775a7d55abc5446dda07e9795f5954fdbf8a49e1c37"
|
||||||
],
|
],
|
||||||
"markers": "python_version >= '3.0'",
|
"markers": "python_version >= '3.0'",
|
||||||
"version": "==1.3.18"
|
"version": "==1.3.19"
|
||||||
},
|
},
|
||||||
"tablib": {
|
"tablib": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
@@ -325,27 +251,33 @@
|
|||||||
},
|
},
|
||||||
"tldextract": {
|
"tldextract": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:16b2f7e81d89c2a5a914d25bdbddd3932c31a6b510db886c3ce0764a195c0ee7",
|
"sha256:ab0e38977a129c72729476d5f8c85a8e1f8e49e9202e1db8dca76e95da7be9a8",
|
||||||
"sha256:9aa21a1f7827df4209e242ec4fc2293af5940ec730cde46ea80f66ed97bfc808"
|
"sha256:c2a8a392edf3ea6fa8be80930f04c3ac29e91fa604cb2139bdf6a37fc1e1ac6d"
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==2.2.2"
|
"version": "==2.2.3"
|
||||||
},
|
},
|
||||||
"tqdm": {
|
"tqdm": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:63ef7a6d3eb39f80d6b36e4867566b3d8e5f1fe3d6cb50c5e9ede2b3198ba7b7",
|
"sha256:1a336d2b829be50e46b84668691e0a2719f26c97c62846298dd5ae2937e4d5cf",
|
||||||
"sha256:7810e627bcf9d983a99d9ff8a0c09674400fd2927eddabeadf153c14a2ec8656"
|
"sha256:564d632ea2b9cb52979f7956e093e831c28d441c11751682f84c86fc46e4fd21"
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==4.47.0"
|
"version": "==4.48.2"
|
||||||
|
},
|
||||||
|
"treelib": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:1cbfffb2d2b75ccac27d0200cee0507b6fbb0726e0afb9fae017ade5d2ce8788"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==1.6.1"
|
||||||
},
|
},
|
||||||
"urllib3": {
|
"urllib3": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527",
|
"sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a",
|
||||||
"sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"
|
"sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"
|
||||||
],
|
],
|
||||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
|
"version": "==1.25.10"
|
||||||
"version": "==1.25.9"
|
|
||||||
},
|
},
|
||||||
"win32-setctime": {
|
"win32-setctime": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
@@ -354,28 +286,6 @@
|
|||||||
],
|
],
|
||||||
"markers": "sys_platform == 'win32'",
|
"markers": "sys_platform == 'win32'",
|
||||||
"version": "==1.0.1"
|
"version": "==1.0.1"
|
||||||
},
|
|
||||||
"yarl": {
|
|
||||||
"hashes": [
|
|
||||||
"sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce",
|
|
||||||
"sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6",
|
|
||||||
"sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce",
|
|
||||||
"sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae",
|
|
||||||
"sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d",
|
|
||||||
"sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f",
|
|
||||||
"sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b",
|
|
||||||
"sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b",
|
|
||||||
"sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb",
|
|
||||||
"sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462",
|
|
||||||
"sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea",
|
|
||||||
"sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70",
|
|
||||||
"sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1",
|
|
||||||
"sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a",
|
|
||||||
"sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b",
|
|
||||||
"sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080",
|
|
||||||
"sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2"
|
|
||||||
],
|
|
||||||
"version": "==1.4.2"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"develop": {}
|
"develop": {}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
[](https://codecov.io/gh/shmilylty/OneForAll)
|
[](https://codecov.io/gh/shmilylty/OneForAll)
|
||||||
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
|
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
|
||||||
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
|
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
|
||||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||||
[](https://github.com/shmilylty/OneForAll/releases)
|
[](https://github.com/shmilylty/OneForAll/releases)
|
||||||
|
|
||||||
👊**OneForAll是一款功能强大的子域收集工具** 📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md)
|
👊**OneForAll是一款功能强大的子域收集工具** 📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md)
|
||||||
@@ -15,21 +15,6 @@
|
|||||||
|
|
||||||
📢 请务必花一点时间阅读此文档，有助于你快速熟悉OneForAll！
|
📢 请务必花一点时间阅读此文档，有助于你快速熟悉OneForAll！
|
||||||
|
|
||||||
<details>
|
|
||||||
<summary><b>🐍安装要求</b></summary>
|
|
||||||
|
|
||||||
OneForAll基于[Python 3.8.0]( https://www.python.org/downloads/release/python-380/ )开发和测试，请使用高于Python 3.8.0的稳定发行版本，其他版本可能会出现一些问题（Windows平台必须使用3.8.0以上版本），安装Python环境可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)。运行以下命令检查Python和pip3版本：
|
|
||||||
```bash
|
|
||||||
python -V
|
|
||||||
pip3 -V
|
|
||||||
```
|
|
||||||
如果你看到类似以下的输出便说明Python环境没有问题：
|
|
||||||
```bash
|
|
||||||
Python 3.8.0
|
|
||||||
pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python38\site-packages\pip (python 3.8)
|
|
||||||
```
|
|
||||||
</details>
|
|
||||||
|
|
||||||
<details>
|
<details>
|
||||||
<summary><b>✔安装步骤（git 版）</b></summary>
|
<summary><b>✔安装步骤（git 版）</b></summary>
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ OneForAll subdomain brute module
|
|||||||
import gc
|
import gc
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import random
|
|
||||||
import secrets
|
import secrets
|
||||||
|
|
||||||
import exrex
|
import exrex
|
||||||
@@ -21,67 +20,137 @@ from dns.resolver import NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers
|
|||||||
|
|
||||||
import dbexport
|
import dbexport
|
||||||
from common import utils
|
from common import utils
|
||||||
|
from common import similarity
|
||||||
from config import settings
|
from config import settings
|
||||||
from common.module import Module
|
from common.module import Module
|
||||||
from config.log import logger
|
from config.log import logger
|
||||||
|
|
||||||
|
|
||||||
@tenacity.retry(stop=tenacity.stop_after_attempt(3))
|
def config_resolver(nameservers):
|
||||||
def do_query_a(domain, resolver):
|
"""
|
||||||
|
配置DNS解析器
|
||||||
|
|
||||||
|
:param nameservers: 名称解析服务器地址
|
||||||
|
"""
|
||||||
|
resolver = utils.dns_resolver()
|
||||||
|
resolver.nameservers = nameservers
|
||||||
|
resolver.rotate = True # 随机使用NS
|
||||||
|
resolver.cache = None # 不使用DNS缓存
|
||||||
|
return resolver
|
||||||
|
|
||||||
|
|
||||||
|
def gen_random_subdomains(domain, count):
|
||||||
|
"""
|
||||||
|
生成指定数量的随机子域域名列表
|
||||||
|
|
||||||
|
:param domain: 主域
|
||||||
|
:param count: 数量
|
||||||
|
"""
|
||||||
|
subdomains = set()
|
||||||
|
if count < 1:
|
||||||
|
return subdomains
|
||||||
|
for _ in range(count):
|
||||||
|
token = secrets.token_hex(4)
|
||||||
|
subdomains.add(f'{token}.{domain}')
|
||||||
|
return subdomains
|
||||||
|
|
||||||
|
|
||||||
|
def query_a_record(subdomain, resolver):
|
||||||
|
"""
|
||||||
|
查询子域A记录
|
||||||
|
|
||||||
|
:param subdomain: 子域
|
||||||
|
:param resolver: DNS解析器
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
answer = resolver.query(domain, 'A')
|
answer = resolver.query(subdomain, 'A')
|
||||||
# If resolve random subdomain raise timeout error, try again
|
|
||||||
except Timeout as e:
|
|
||||||
logger.log('ALERT', f'DNS resolve timeout, retrying')
|
|
||||||
logger.log('DEBUG', e.args)
|
|
||||||
raise tenacity.TryAgain
|
|
||||||
# If resolve random subdomain raise NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers error
|
|
||||||
# It means that there is no A record of random subdomain and not use wildcard dns record
|
|
||||||
except (NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers) as e:
|
|
||||||
logger.log('DEBUG', e.args)
|
|
||||||
logger.log('INFOR', f'{domain} seems not use wildcard dns record')
|
|
||||||
return False
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.log('ALERT', f'Detect {domain} wildcard dns record error')
|
logger.log('DEBUG', f'Query {subdomain} wildcard dns record error')
|
||||||
logger.log('FATAL', e.args)
|
logger.log('DEBUG', e.args)
|
||||||
exit(1)
|
return False
|
||||||
else:
|
|
||||||
if answer.rrset is None:
|
if answer.rrset is None:
|
||||||
logger.log('ALERT', f'DNS resolve dont have result, retrying')
|
return False
|
||||||
raise tenacity.TryAgain
|
|
||||||
ttl = answer.ttl
|
ttl = answer.ttl
|
||||||
name = answer.name
|
name = answer.name
|
||||||
ips = {item.address for item in answer}
|
ips = {item.address for item in answer}
|
||||||
logger.log('ALERT', f'{domain} use wildcard dns record')
|
logger.log('ALERT', f'{subdomain} resolve to: {name} '
|
||||||
logger.log('ALERT', f'{domain} resolve to: {name} '
|
|
||||||
f'IP: {ips} TTL: {ttl}')
|
f'IP: {ips} TTL: {ttl}')
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def detect_wildcard(domain, authoritative_ns):
|
def all_resolve_success(subdomains):
|
||||||
|
"""
|
||||||
|
判断是否所有子域都解析成功
|
||||||
|
|
||||||
|
:param subdomains: 子域列表
|
||||||
|
"""
|
||||||
|
resolver = utils.dns_resolver()
|
||||||
|
resolver.cache = None # 不使用DNS缓存
|
||||||
|
status = set()
|
||||||
|
for subdomain in subdomains:
|
||||||
|
status.add(query_a_record(subdomain, resolver))
|
||||||
|
return all(status)
|
||||||
|
|
||||||
|
|
||||||
|
def all_request_success(subdomains):
|
||||||
|
"""
|
||||||
|
判断是否所有子域都请求成功
|
||||||
|
|
||||||
|
:param subdomains: 子域列表
|
||||||
|
"""
|
||||||
|
result = list()
|
||||||
|
for subdomain in subdomains:
|
||||||
|
url = f'http://{subdomain}'
|
||||||
|
resp = utils.get_url_resp(url)
|
||||||
|
if resp:
|
||||||
|
logger.log('ALERT', f'Request: {url} Status: {resp.status_code} '
|
||||||
|
f'Size: {len(resp.content)}')
|
||||||
|
result.append(resp.text)
|
||||||
|
else:
|
||||||
|
result.append(resp)
|
||||||
|
return all(result), result
|
||||||
|
|
||||||
|
|
||||||
|
def any_similar_html(resp_list):
|
||||||
|
"""
|
||||||
|
判断是否有一组HTML页面结构相似
|
||||||
|
|
||||||
|
:param resp_list: 响应HTML页面
|
||||||
|
"""
|
||||||
|
html_doc1, html_doc2, html_doc3 = resp_list
|
||||||
|
if similarity.is_similar(html_doc1, html_doc2):
|
||||||
|
return True
|
||||||
|
if similarity.is_similar(html_doc1, html_doc3):
|
||||||
|
return True
|
||||||
|
if similarity.is_similar(html_doc2, html_doc3):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def detect_wildcard(domain):
|
||||||
"""
|
"""
|
||||||
Detect use wildcard dns record or not
|
Detect use wildcard dns record or not
|
||||||
|
|
||||||
:param str domain: domain
|
:param str domain: domain
|
||||||
:param list authoritative_ns: authoritative name server
|
|
||||||
:return bool use wildcard dns record or not
|
:return bool use wildcard dns record or not
|
||||||
"""
|
"""
|
||||||
logger.log('INFOR', f'Detecting {domain} use wildcard dns record or not')
|
logger.log('INFOR', f'Detecting {domain} use wildcard dns record or not')
|
||||||
token = secrets.token_hex(4)
|
random_subdomains = gen_random_subdomains(domain, 3)
|
||||||
random_subdomain = f'{token}.{domain}'
|
if not all_resolve_success(random_subdomains):
|
||||||
resolver = utils.dns_resolver()
|
|
||||||
resolver.nameservers = authoritative_ns
|
|
||||||
resolver.rotate = True
|
|
||||||
resolver.cache = None
|
|
||||||
try:
|
|
||||||
wildcard = do_query_a(random_subdomain, resolver)
|
|
||||||
except Exception as e:
|
|
||||||
logger.log('DEBUG', e.args)
|
|
||||||
logger.log('ALERT', f'Multiple detection errors, so temporarily {domain} '
|
|
||||||
f'does not use wildcard dns record')
|
|
||||||
return False
|
return False
|
||||||
|
is_all_success, all_request_resp = all_request_success(random_subdomains)
|
||||||
|
if not is_all_success:
|
||||||
|
return True
|
||||||
|
return any_similar_html(all_request_resp)
|
||||||
|
|
||||||
|
|
||||||
|
def is_enable_wildcard(domain):
|
||||||
|
is_enable = detect_wildcard(domain)
|
||||||
|
if is_enable:
|
||||||
|
logger.log('ALERT', f'The domain {domain} enables wildcard')
|
||||||
else:
|
else:
|
||||||
return wildcard
|
logger.log('ALERT', f'The domain {domain} disables wildcard')
|
||||||
|
return is_enable
|
||||||
|
|
||||||
|
|
||||||
def gen_subdomains(expression, path):
|
def gen_subdomains(expression, path):
|
||||||
@@ -229,9 +298,9 @@ def collect_wildcard_record(domain, authoritative_ns):
|
|||||||
if not authoritative_ns:
|
if not authoritative_ns:
|
||||||
return list(), int()
|
return list(), int()
|
||||||
resolver = utils.dns_resolver()
|
resolver = utils.dns_resolver()
|
||||||
resolver.nameservers = authoritative_ns
|
resolver.nameservers = authoritative_ns # 使用权威名称服务器
|
||||||
resolver.rotate = True
|
resolver.rotate = True # 随机使用NS
|
||||||
resolver.cache = None
|
resolver.cache = None # 不使用DNS缓存
|
||||||
ips = set()
|
ips = set()
|
||||||
ttl = int()
|
ttl = int()
|
||||||
ips_stat = dict()
|
ips_stat = dict()
|
||||||
@@ -464,14 +533,13 @@ class Brute(Module):
|
|||||||
Example：
|
Example：
|
||||||
brute.py --target domain.com --word True run
|
brute.py --target domain.com --word True run
|
||||||
brute.py --targets ./domains.txt --word True run
|
brute.py --targets ./domains.txt --word True run
|
||||||
brute.py --target domain.com --word True --process 1 run
|
brute.py --target domain.com --word True --concurrent 2000 run
|
||||||
brute.py --target domain.com --word True --wordlist subnames.txt run
|
brute.py --target domain.com --word True --wordlist subnames.txt run
|
||||||
brute.py --target domain.com --word True --recursive True --depth 2 run
|
brute.py --target domain.com --word True --recursive True --depth 2 run
|
||||||
brute.py --target d.com --fuzz True --place m.*.d.com --rule '[a-z]' run
|
brute.py --target d.com --fuzz True --place m.*.d.com --rule '[a-z]' run
|
||||||
brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
|
brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
--alive True/False Only export alive subdomains or not (default False)
|
|
||||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||||
--path Result path (default None, automatically generated)
|
--path Result path (default None, automatically generated)
|
||||||
|
|
||||||
@@ -479,7 +547,7 @@ class Brute(Module):
|
|||||||
:param str target: One domain (target or targets must be provided)
|
:param str target: One domain (target or targets must be provided)
|
||||||
:param str targets: File path of one domain per line
|
:param str targets: File path of one domain per line
|
||||||
:param int process: Number of processes (default 1)
|
:param int process: Number of processes (default 1)
|
||||||
:param int concurrent: Number of concurrent (default 10000)
|
:param int concurrent: Number of concurrent (default 2000)
|
||||||
:param bool word: Use word mode generate dictionary (default False)
|
:param bool word: Use word mode generate dictionary (default False)
|
||||||
:param str wordlist: Dictionary path used in word mode (default use ./config/default.py)
|
:param str wordlist: Dictionary path used in word mode (default use ./config/default.py)
|
||||||
:param bool recursive: Use recursion (default False)
|
:param bool recursive: Use recursion (default False)
|
||||||
@@ -524,8 +592,6 @@ class Brute(Module):
|
|||||||
self.domain = str() # 当前正在进行爆破的域名
|
self.domain = str() # 当前正在进行爆破的域名
|
||||||
self.ips_times = dict() # IP集合出现次数
|
self.ips_times = dict() # IP集合出现次数
|
||||||
self.enable_wildcard = False # 当前域名是否使用泛解析
|
self.enable_wildcard = False # 当前域名是否使用泛解析
|
||||||
self.wildcard_check = settings.brute_wildcard_check
|
|
||||||
self.wildcard_deal = settings.brute_wildcard_deal
|
|
||||||
self.check_env = True
|
self.check_env = True
|
||||||
self.quite = False
|
self.quite = False
|
||||||
|
|
||||||
@@ -596,7 +662,7 @@ class Brute(Module):
|
|||||||
wildcard_ttl = int() # 泛解析TTL整型值
|
wildcard_ttl = int() # 泛解析TTL整型值
|
||||||
ns_list = query_domain_ns(self.domain)
|
ns_list = query_domain_ns(self.domain)
|
||||||
ns_ip_list = query_domain_ns_a(ns_list) # DNS权威名称服务器对应A记录列表
|
ns_ip_list = query_domain_ns_a(ns_list) # DNS权威名称服务器对应A记录列表
|
||||||
self.enable_wildcard = detect_wildcard(domain, ns_ip_list)
|
self.enable_wildcard = is_enable_wildcard(domain)
|
||||||
|
|
||||||
if self.enable_wildcard:
|
if self.enable_wildcard:
|
||||||
wildcard_ips, wildcard_ttl = collect_wildcard_record(domain,
|
wildcard_ips, wildcard_ttl = collect_wildcard_record(domain,
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
from common.module import Module
|
||||||
|
|
||||||
|
|
||||||
|
class Check(Module):
|
||||||
|
"""
|
||||||
|
Check base class
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
Module.__init__(self)
|
||||||
|
self.request_status = 1
|
||||||
|
|
||||||
|
def to_check(self, filenames):
|
||||||
|
urls = set()
|
||||||
|
for filename in filenames:
|
||||||
|
urls.update((f'http://{self.domain}/{filename}',
|
||||||
|
f'https://{self.domain}/{filename}',
|
||||||
|
f'http://www.{self.domain}/{filename}',
|
||||||
|
f'https://www.{self.domain}/{filename}'))
|
||||||
|
for url in urls:
|
||||||
|
self.header = self.get_header()
|
||||||
|
self.proxy = self.get_proxy(self.source)
|
||||||
|
resp = self.get(url, check=False, ignore=True)
|
||||||
|
self.subdomains = self.collect_subdomains(resp)
|
||||||
|
if self.subdomains:
|
||||||
|
break
|
||||||
+7
-7
@@ -78,8 +78,8 @@ class Database(object):
|
|||||||
f'cidr text,'
|
f'cidr text,'
|
||||||
f'asn text,'
|
f'asn text,'
|
||||||
f'org text,'
|
f'org text,'
|
||||||
f'ip2region text,'
|
f'addr text,'
|
||||||
f'ip2location text,'
|
f'isp text,'
|
||||||
f'resolver text,'
|
f'resolver text,'
|
||||||
f'module text,'
|
f'module text,'
|
||||||
f'source text,'
|
f'source text,'
|
||||||
@@ -103,13 +103,13 @@ class Database(object):
|
|||||||
f'insert into "{table_name}" '
|
f'insert into "{table_name}" '
|
||||||
f'(id, alive, resolve, request, new, url, subdomain, port, level,'
|
f'(id, alive, resolve, request, new, url, subdomain, port, level,'
|
||||||
f'cname, content, public, cdn, status, reason, title, banner, header,'
|
f'cname, content, public, cdn, status, reason, title, banner, header,'
|
||||||
f'history, response, times, ttl, cidr, asn, org, ip2region,'
|
f'history, response, times, ttl, cidr, asn, org, addr, isp, resolver,'
|
||||||
f'ip2location, resolver, module, source, elapse, find) '
|
f'module, source, elapse, find) '
|
||||||
f'values (:id, :alive, :resolve, :request, :new, :url,'
|
f'values (:id, :alive, :resolve, :request, :new, :url,'
|
||||||
f':subdomain, :port, :level, :cname, :content, :public, :cdn,'
|
f':subdomain, :port, :level, :cname, :content, :public, :cdn,'
|
||||||
f':status, :reason, :title, :banner, :header, :history, :response,'
|
f':status, :reason, :title, :banner, :header, :history, :response,'
|
||||||
f':times, :ttl, :cidr, :asn, :org, :ip2region, :ip2location,'
|
f':times, :ttl, :cidr, :asn, :org, :addr, :isp, :resolver, :module,'
|
||||||
f':resolver, :module, :source, :elapse, :find)', results)
|
f':source, :elapse, :find)', results)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.log('ERROR', e)
|
logger.log('ERROR', e)
|
||||||
|
|
||||||
@@ -231,7 +231,7 @@ class Database(object):
|
|||||||
table_name = table_name.replace('.', '_')
|
table_name = table_name.replace('.', '_')
|
||||||
query = f'select id, new, alive, request, resolve, url, subdomain, level,' \
|
query = f'select id, new, alive, request, resolve, url, subdomain, level,' \
|
||||||
f'cname, content, public, cdn, port, status, reason, title, banner,' \
|
f'cname, content, public, cdn, port, status, reason, title, banner,' \
|
||||||
f'cidr, asn, org, ip2region, ip2location, source from "{table_name}"'
|
f'cidr, asn, org, addr, isp, source from "{table_name}"'
|
||||||
if alive and limit:
|
if alive and limit:
|
||||||
if limit in ['resolve', 'request']:
|
if limit in ['resolve', 'request']:
|
||||||
where = f' where {limit} = 1'
|
where = f' where {limit} = 1'
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ class Domain(object):
|
|||||||
|
|
||||||
:param str string: input string
|
:param str string: input string
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, string):
|
def __init__(self, string):
|
||||||
self.string = str(string)
|
self.string = str(string)
|
||||||
self.regexp = r'\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b'
|
self.regexp = r'\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b'
|
||||||
|
|||||||
@@ -1,13 +0,0 @@
|
|||||||
import zipfile
|
|
||||||
from IP2Location import IP2Location
|
|
||||||
from config.setting import data_storage_dir
|
|
||||||
|
|
||||||
|
|
||||||
class IpGeoInfo(IP2Location):
|
|
||||||
def __init__(self):
|
|
||||||
zip_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN.ZIP")
|
|
||||||
bin_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN")
|
|
||||||
if not bin_path.exists():
|
|
||||||
zf = zipfile.ZipFile(zip_path)
|
|
||||||
zf.extract('IP2LOCATION-LITE-DB3.BIN', data_storage_dir)
|
|
||||||
IP2Location.__init__(self, bin_path)
|
|
||||||
+35
-7
@@ -10,6 +10,8 @@ import socket
|
|||||||
import struct
|
import struct
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
from config import settings
|
||||||
|
|
||||||
|
|
||||||
class IpRegInfo(object):
|
class IpRegInfo(object):
|
||||||
__INDEX_BLOCK_LENGTH = 12
|
__INDEX_BLOCK_LENGTH = 12
|
||||||
@@ -39,7 +41,8 @@ class IpRegInfo(object):
|
|||||||
self.__dbBinStr = self.__f.read() # read all the contents in file
|
self.__dbBinStr = self.__f.read() # read all the contents in file
|
||||||
self.__indexSPtr = self.get_long(self.__dbBinStr, 0)
|
self.__indexSPtr = self.get_long(self.__dbBinStr, 0)
|
||||||
self.__indexLPtr = self.get_long(self.__dbBinStr, 4)
|
self.__indexLPtr = self.get_long(self.__dbBinStr, 4)
|
||||||
self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) / self.__INDEX_BLOCK_LENGTH) + 1
|
self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) /
|
||||||
|
self.__INDEX_BLOCK_LENGTH) + 1
|
||||||
|
|
||||||
l, h, data_ptr = (0, self.__indexCount, 0)
|
l, h, data_ptr = (0, self.__indexCount, 0)
|
||||||
while l <= h:
|
while l <= h:
|
||||||
@@ -67,14 +70,16 @@ class IpRegInfo(object):
|
|||||||
" binary search method
|
" binary search method
|
||||||
" param: ip
|
" param: ip
|
||||||
"""
|
"""
|
||||||
if not ip.isdigit(): ip = self.ip2long(ip)
|
if not ip.isdigit():
|
||||||
|
ip = self.ip2long(ip)
|
||||||
|
|
||||||
if self.__indexCount == 0:
|
if self.__indexCount == 0:
|
||||||
self.__f.seek(0)
|
self.__f.seek(0)
|
||||||
super_block = self.__f.read(8)
|
super_block = self.__f.read(8)
|
||||||
self.__indexSPtr = self.get_long(super_block, 0)
|
self.__indexSPtr = self.get_long(super_block, 0)
|
||||||
self.__indexLPtr = self.get_long(super_block, 4)
|
self.__indexLPtr = self.get_long(super_block, 4)
|
||||||
self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) / self.__INDEX_BLOCK_LENGTH) + 1
|
self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) /
|
||||||
|
self.__INDEX_BLOCK_LENGTH) + 1
|
||||||
|
|
||||||
l, h, data_ptr = (0, self.__indexCount, 0)
|
l, h, data_ptr = (0, self.__indexCount, 0)
|
||||||
while l <= h:
|
while l <= h:
|
||||||
@@ -208,10 +213,9 @@ class IpRegInfo(object):
|
|||||||
self.__f.seek(data_ptr)
|
self.__f.seek(data_ptr)
|
||||||
data = self.__f.read(data_len)
|
data = self.__f.read(data_len)
|
||||||
|
|
||||||
return {
|
info = {"city_id": self.get_long(data, 0),
|
||||||
"city_id": self.get_long(data, 0),
|
"region": data[4:].decode('utf-8')}
|
||||||
"region": data[4:]
|
return info
|
||||||
}
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def ip2long(ip):
|
def ip2long(ip):
|
||||||
@@ -244,3 +248,27 @@ class IpRegInfo(object):
|
|||||||
self.__dbBinStr = None
|
self.__dbBinStr = None
|
||||||
self.__headerPtr = None
|
self.__headerPtr = None
|
||||||
self.__headerSip = None
|
self.__headerSip = None
|
||||||
|
|
||||||
|
|
||||||
|
class IpRegData(IpRegInfo):
|
||||||
|
def __init__(self):
|
||||||
|
path = settings.data_storage_dir.joinpath('ip2region.db')
|
||||||
|
IpRegInfo.__init__(self, path)
|
||||||
|
|
||||||
|
def query(self, ip, algorithm='memory'):
|
||||||
|
algorithms = ['memory', 'binary', 'btree']
|
||||||
|
if algorithm not in algorithms:
|
||||||
|
raise Exception(f"Only three query algorithms are supported: {algorithms}")
|
||||||
|
if algorithm == 'memory':
|
||||||
|
result = self.memory_search(ip)
|
||||||
|
elif algorithm == 'binary':
|
||||||
|
result = self.binary_search(ip)
|
||||||
|
else:
|
||||||
|
result = self.btree_search(ip)
|
||||||
|
addr_list = result.get('region').split('|')
|
||||||
|
addr = ''.join(filter(lambda x: x != '0', addr_list[:-1]))
|
||||||
|
isp = addr_list[-1]
|
||||||
|
if isp == '0':
|
||||||
|
isp = '未知'
|
||||||
|
info = {'addr': addr, 'isp': isp}
|
||||||
|
return info
|
||||||
|
|||||||
+17
-13
@@ -22,9 +22,9 @@ class Module(object):
|
|||||||
self.cookie = None
|
self.cookie = None
|
||||||
self.header = dict()
|
self.header = dict()
|
||||||
self.proxy = None
|
self.proxy = None
|
||||||
self.delay = settings.request_delay # 请求睡眠时延
|
self.delay = 1 # 请求睡眠时延
|
||||||
self.timeout = settings.request_timeout # 请求超时时间
|
self.timeout = settings.request_timeout_second # 请求超时时间
|
||||||
self.verify = settings.request_verify # 请求SSL验证
|
self.verify = settings.request_ssl_verify # 请求SSL验证
|
||||||
self.domain = str() # 当前进行子域名收集的主域
|
self.domain = str() # 当前进行子域名收集的主域
|
||||||
self.subdomains = set() # 存放发现的子域
|
self.subdomains = set() # 存放发现的子域
|
||||||
self.infos = dict() # 存放子域有关信息
|
self.infos = dict() # 存放子域有关信息
|
||||||
@@ -73,7 +73,7 @@ class Module(object):
|
|||||||
:param dict params: request parameters
|
:param dict params: request parameters
|
||||||
:param bool check: check response
|
:param bool check: check response
|
||||||
:param kwargs: other params
|
:param kwargs: other params
|
||||||
:return: requests's response object
|
:return: response object
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
resp = requests.head(url,
|
resp = requests.head(url,
|
||||||
@@ -93,16 +93,20 @@ class Module(object):
|
|||||||
return resp
|
return resp
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def get(self, url, params=None, check=True, **kwargs):
|
def get(self, url, params=None, check=True, ignore=False, **kwargs):
|
||||||
"""
|
"""
|
||||||
Custom get request
|
Custom get request
|
||||||
|
|
||||||
:param str url: request url
|
:param str url: request url
|
||||||
:param dict params: request parameters
|
:param dict params: request parameters
|
||||||
:param bool check: check response
|
:param bool check: check response
|
||||||
|
:param bool ignore: ignore error
|
||||||
:param kwargs: other params
|
:param kwargs: other params
|
||||||
:return: requests's response object
|
:return: response object
|
||||||
"""
|
"""
|
||||||
|
level = 'ERROR'
|
||||||
|
if ignore:
|
||||||
|
level = 'DEBUG'
|
||||||
try:
|
try:
|
||||||
resp = requests.get(url,
|
resp = requests.get(url,
|
||||||
params=params,
|
params=params,
|
||||||
@@ -113,7 +117,7 @@ class Module(object):
|
|||||||
verify=self.verify,
|
verify=self.verify,
|
||||||
**kwargs)
|
**kwargs)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.log('ERROR', e.args)
|
logger.log(level, e.args)
|
||||||
return None
|
return None
|
||||||
if not check:
|
if not check:
|
||||||
return resp
|
return resp
|
||||||
@@ -129,7 +133,7 @@ class Module(object):
|
|||||||
:param dict data: request data
|
:param dict data: request data
|
||||||
:param bool check: check response
|
:param bool check: check response
|
||||||
:param kwargs: other params
|
:param kwargs: other params
|
||||||
:return: requests's response object
|
:return: response object
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
resp = requests.post(url,
|
resp = requests.post(url,
|
||||||
@@ -195,7 +199,7 @@ class Module(object):
|
|||||||
:param str module: module name
|
:param str module: module name
|
||||||
:return: proxy
|
:return: proxy
|
||||||
"""
|
"""
|
||||||
if not settings.enable_proxy:
|
if not settings.enable_request_proxy:
|
||||||
logger.log('TRACE', f'All modules do not use proxy')
|
logger.log('TRACE', f'All modules do not use proxy')
|
||||||
return self.proxy
|
return self.proxy
|
||||||
if settings.proxy_all_module:
|
if settings.proxy_all_module:
|
||||||
@@ -280,8 +284,8 @@ class Module(object):
|
|||||||
'cidr': None,
|
'cidr': None,
|
||||||
'asn': None,
|
'asn': None,
|
||||||
'org': None,
|
'org': None,
|
||||||
'ip2region': None,
|
'addr': None,
|
||||||
'ip2location': None,
|
'isp': None,
|
||||||
'resolver': None,
|
'resolver': None,
|
||||||
'module': self.module,
|
'module': self.module,
|
||||||
'source': self.source,
|
'source': self.source,
|
||||||
@@ -331,8 +335,8 @@ class Module(object):
|
|||||||
'cidr': info.get('cidr'),
|
'cidr': info.get('cidr'),
|
||||||
'asn': info.get('asn'),
|
'asn': info.get('asn'),
|
||||||
'org': info.get('org'),
|
'org': info.get('org'),
|
||||||
'ip2region': info.get('ip2region'),
|
'addr': info.get('addr'),
|
||||||
'ip2location': info.get('ip2location'),
|
'isp': info.get('isp'),
|
||||||
'resolver': info.get('resolver'),
|
'resolver': info.get('resolver'),
|
||||||
'module': self.module,
|
'module': self.module,
|
||||||
'source': self.source,
|
'source': self.source,
|
||||||
|
|||||||
@@ -5,6 +5,5 @@ class Query(Module):
|
|||||||
"""
|
"""
|
||||||
Query base class
|
Query base class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
Module.__init__(self)
|
Module.__init__(self)
|
||||||
|
|||||||
+130
-185
@@ -1,10 +1,10 @@
|
|||||||
import json
|
import json
|
||||||
import asyncio
|
from threading import Thread
|
||||||
import functools
|
from queue import Queue
|
||||||
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
import tqdm
|
import tqdm
|
||||||
from aiohttp import ClientSession
|
import requests
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
from common import utils
|
from common import utils
|
||||||
@@ -12,17 +12,17 @@ from config.log import logger
|
|||||||
from config import settings
|
from config import settings
|
||||||
|
|
||||||
|
|
||||||
def get_limit_conn():
|
def req_thread_count():
|
||||||
count = settings.limit_open_conn
|
count = settings.request_thread_count
|
||||||
if isinstance(count, int):
|
if isinstance(count, int):
|
||||||
count = max(16, count)
|
count = max(16, count)
|
||||||
else:
|
else:
|
||||||
count = utils.get_coroutine_count()
|
count = utils.get_request_count()
|
||||||
logger.log('DEBUG', f'Request coroutine {count}')
|
logger.log('DEBUG', f'Number of request threads {count}')
|
||||||
return count
|
return count
|
||||||
|
|
||||||
|
|
||||||
def get_ports(port):
|
def get_port_seq(port):
|
||||||
logger.log('DEBUG', 'Getting port range')
|
logger.log('DEBUG', 'Getting port range')
|
||||||
ports = set()
|
ports = set()
|
||||||
if isinstance(port, (set, list, tuple)):
|
if isinstance(port, (set, list, tuple)):
|
||||||
@@ -40,81 +40,33 @@ def get_ports(port):
|
|||||||
return set(ports)
|
return set(ports)
|
||||||
|
|
||||||
|
|
||||||
def gen_req_data(data, ports):
|
def gen_req_url(domain, port):
|
||||||
|
if str(port).endswith('443'):
|
||||||
|
url = f'https://{domain}:{port}'
|
||||||
|
if port == 443:
|
||||||
|
url = f'https://{domain}'
|
||||||
|
return url
|
||||||
|
url = f'http://{domain}:{port}'
|
||||||
|
if port == 80:
|
||||||
|
url = f'http://{domain}'
|
||||||
|
return url
|
||||||
|
|
||||||
|
|
||||||
|
def gen_req_urls(data, ports):
|
||||||
logger.log('INFOR', 'Generating request urls')
|
logger.log('INFOR', 'Generating request urls')
|
||||||
new_data = []
|
urls = set()
|
||||||
for data in data:
|
for info in data:
|
||||||
resolve = data.get('resolve')
|
resolve = info.get('resolve')
|
||||||
# 解析不成功的子域不进行http请求探测
|
# 解析不成功的子域不进行http请求探测
|
||||||
if resolve != 1:
|
if resolve != 1:
|
||||||
continue
|
continue
|
||||||
subdomain = data.get('subdomain')
|
subdomain = info.get('subdomain')
|
||||||
for port in ports:
|
for port in ports:
|
||||||
if str(port).endswith('443'):
|
urls.add(gen_req_url(subdomain, port))
|
||||||
url = f'https://{subdomain}:{port}'
|
return urls
|
||||||
if port == 443:
|
|
||||||
url = f'https://{subdomain}'
|
|
||||||
data['id'] = None
|
|
||||||
data['url'] = url
|
|
||||||
data['port'] = port
|
|
||||||
new_data.append(data)
|
|
||||||
data = dict(data) # 需要生成一个新的字典对象
|
|
||||||
else:
|
|
||||||
url = f'http://{subdomain}:{port}'
|
|
||||||
if port == 80:
|
|
||||||
url = f'http://{subdomain}'
|
|
||||||
data['id'] = None
|
|
||||||
data['url'] = url
|
|
||||||
data['port'] = port
|
|
||||||
new_data.append(data)
|
|
||||||
data = dict(data) # 需要生成一个新的字典对象
|
|
||||||
return new_data
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch(session, method, url):
|
def get_html_title(markup):
|
||||||
"""
|
|
||||||
请求
|
|
||||||
|
|
||||||
:param session: session对象
|
|
||||||
:param method: 请求方法
|
|
||||||
:param str url: url地址
|
|
||||||
:return: 响应对象和响应文本
|
|
||||||
"""
|
|
||||||
timeout = aiohttp.ClientTimeout(total=None,
|
|
||||||
connect=None,
|
|
||||||
sock_read=settings.sockread_timeout,
|
|
||||||
sock_connect=settings.sockconn_timeout)
|
|
||||||
try:
|
|
||||||
if method == 'HEAD':
|
|
||||||
async with session.head(url,
|
|
||||||
ssl=settings.verify_ssl,
|
|
||||||
allow_redirects=settings.allow_redirects,
|
|
||||||
timeout=timeout,
|
|
||||||
proxy=settings.aiohttp_proxy) as resp:
|
|
||||||
text = await resp.text()
|
|
||||||
else:
|
|
||||||
async with session.get(url,
|
|
||||||
ssl=settings.verify_ssl,
|
|
||||||
allow_redirects=settings.allow_redirects,
|
|
||||||
timeout=timeout,
|
|
||||||
proxy=settings.aiohttp_proxy) as resp:
|
|
||||||
|
|
||||||
try:
|
|
||||||
# 先尝试用utf-8解码
|
|
||||||
text = await resp.text(encoding='utf-8', errors='strict')
|
|
||||||
except UnicodeError:
|
|
||||||
try:
|
|
||||||
# 再尝试用gb18030解码
|
|
||||||
text = await resp.text(encoding='gb18030', errors='strict')
|
|
||||||
except UnicodeError:
|
|
||||||
# 最后尝试自动解码
|
|
||||||
text = await resp.text(encoding=None, errors='ignore')
|
|
||||||
return resp, text
|
|
||||||
except Exception as e:
|
|
||||||
return e, None
|
|
||||||
|
|
||||||
|
|
||||||
def get_title(markup):
|
|
||||||
"""
|
"""
|
||||||
获取标题
|
获取标题
|
||||||
|
|
||||||
@@ -161,107 +113,107 @@ def get_jump_urls(history):
|
|||||||
return urls
|
return urls
|
||||||
|
|
||||||
|
|
||||||
def request_callback(future, index, datas):
|
def get(url, resp_queue, session):
|
||||||
resp, text = future.result()
|
timeout = settings.request_timeout_second
|
||||||
if isinstance(resp, BaseException):
|
redirect = settings.request_allow_redirect
|
||||||
exception = resp
|
proxy = utils.get_proxy()
|
||||||
logger.log('TRACE', exception.args)
|
try:
|
||||||
name = utils.get_classname(exception)
|
resp = session.get(url, timeout=timeout, allow_redirects=redirect, proxies=proxy)
|
||||||
datas[index]['reason'] = name + ' ' + str(exception)
|
except Exception as e:
|
||||||
datas[index]['request'] = 0
|
logger.log('DEBUG', e.args)
|
||||||
datas[index]['alive'] = 0
|
return
|
||||||
|
resp_queue.put(resp)
|
||||||
|
|
||||||
|
|
||||||
|
def request(urls_queue, resp_queue, session):
|
||||||
|
while not urls_queue.empty():
|
||||||
|
url = urls_queue.get()
|
||||||
|
get(url, resp_queue, session)
|
||||||
|
urls_queue.task_done()
|
||||||
|
|
||||||
|
|
||||||
|
def progress(urls, resp_queue):
|
||||||
|
bar = tqdm.tqdm()
|
||||||
|
bar.total = len(urls)
|
||||||
|
bar.desc = 'Request Progress'
|
||||||
|
bar.ncols = 80
|
||||||
|
while True:
|
||||||
|
done = resp_queue.qsize()
|
||||||
|
bar.n = done
|
||||||
|
bar.update()
|
||||||
|
if done == bar.total:
|
||||||
|
break
|
||||||
|
|
||||||
|
|
||||||
|
def get_session():
|
||||||
|
header = utils.gen_fake_header()
|
||||||
|
verify = settings.request_ssl_verify
|
||||||
|
redirect_limit = settings.request_redirect_limit
|
||||||
|
session = requests.Session()
|
||||||
|
session.headers = header
|
||||||
|
session.verify = verify
|
||||||
|
session.max_redirects = redirect_limit
|
||||||
|
return session
|
||||||
|
|
||||||
|
|
||||||
|
def bulk_request(urls):
|
||||||
|
logger.log('INFOR', 'Requesting urls in bulk')
|
||||||
|
resp_list = list()
|
||||||
|
urls_queue = Queue()
|
||||||
|
resp_queue = Queue()
|
||||||
|
for url in urls:
|
||||||
|
urls_queue.put(url)
|
||||||
|
session = get_session()
|
||||||
|
thread_count = req_thread_count()
|
||||||
|
|
||||||
|
progress_thread = Thread(target=progress, args=(urls, resp_queue))
|
||||||
|
progress_thread.start()
|
||||||
|
|
||||||
|
for _ in range(thread_count):
|
||||||
|
request_thread = Thread(target=request, args=(urls_queue, resp_queue, session))
|
||||||
|
request_thread.start()
|
||||||
|
|
||||||
|
urls_queue.join()
|
||||||
|
|
||||||
|
while not resp_queue.empty():
|
||||||
|
resp = resp_queue.get()
|
||||||
|
resp_list.append(resp)
|
||||||
|
return resp_list
|
||||||
|
|
||||||
|
|
||||||
|
def gen_new_info(info, resp):
|
||||||
|
port = resp.raw._pool.port
|
||||||
|
info['port'] = port
|
||||||
|
info['url'] = resp.url
|
||||||
|
info['reason'] = resp.reason
|
||||||
|
code = resp.status_code
|
||||||
|
info['status'] = code
|
||||||
|
info['request'] = 1
|
||||||
|
if code == 400 or code >= 500:
|
||||||
|
info['alive'] = 0
|
||||||
else:
|
else:
|
||||||
datas[index]['reason'] = resp.reason
|
info['alive'] = 1
|
||||||
datas[index]['status'] = resp.status
|
|
||||||
datas[index]['request'] = 1
|
|
||||||
if resp.status == 400 or resp.status >= 500:
|
|
||||||
datas[index]['alive'] = 0
|
|
||||||
else:
|
|
||||||
datas[index]['alive'] = 1
|
|
||||||
headers = resp.headers
|
headers = resp.headers
|
||||||
if settings.enable_banner_identify:
|
if settings.enable_banner_identify:
|
||||||
datas[index]['banner'] = utils.get_sample_banner(headers)
|
info['banner'] = utils.get_sample_banner(headers)
|
||||||
datas[index]['header'] = json.dumps(dict(headers))
|
info['header'] = json.dumps(dict(headers))
|
||||||
history = resp.history
|
history = resp.history
|
||||||
datas[index]['history'] = json.dumps(get_jump_urls(history))
|
info['history'] = json.dumps(get_jump_urls(history))
|
||||||
if isinstance(text, str):
|
text = utils.decode_resp_text(resp)
|
||||||
title = get_title(text).strip()
|
title = get_html_title(text).strip()
|
||||||
datas[index]['title'] = utils.remove_invalid_string(title)
|
info['title'] = utils.remove_invalid_string(title)
|
||||||
datas[index]['response'] = utils.remove_invalid_string(text)
|
info['response'] = utils.remove_invalid_string(text)
|
||||||
|
return info
|
||||||
|
|
||||||
|
|
||||||
def get_connector():
|
def gen_new_data(data, resp_list):
|
||||||
count = get_limit_conn()
|
new_data = list()
|
||||||
return aiohttp.TCPConnector(ttl_dns_cache=300,
|
for resp in resp_list:
|
||||||
ssl=settings.verify_ssl,
|
subdomain = resp.raw._pool.host
|
||||||
limit=count,
|
for info in data:
|
||||||
limit_per_host=settings.limit_per_host)
|
if info.get('subdomain') == subdomain:
|
||||||
|
new_data.append(gen_new_info(info, resp))
|
||||||
|
return new_data
|
||||||
async def async_request(urls):
|
|
||||||
results = list()
|
|
||||||
connector = get_connector()
|
|
||||||
headers = utils.get_random_header()
|
|
||||||
session = ClientSession(connector=connector, headers=headers)
|
|
||||||
tasks = []
|
|
||||||
for i, url in enumerate(urls):
|
|
||||||
task = asyncio.ensure_future(fetch(session, 'GET', url))
|
|
||||||
tasks.append(task)
|
|
||||||
if tasks:
|
|
||||||
futures = asyncio.as_completed(tasks)
|
|
||||||
for future in tqdm.tqdm(futures,
|
|
||||||
total=len(tasks),
|
|
||||||
desc='Request Progress',
|
|
||||||
ncols=80):
|
|
||||||
result = await future
|
|
||||||
results.append(result)
|
|
||||||
await session.close()
|
|
||||||
return results
|
|
||||||
|
|
||||||
|
|
||||||
async def bulk_request(data, port):
|
|
||||||
ports = get_ports(port)
|
|
||||||
no_req_data = utils.get_filtered_data(data)
|
|
||||||
to_req_data = gen_req_data(data, ports)
|
|
||||||
method = settings.request_method.upper()
|
|
||||||
logger.log('INFOR', f'Use {method} method to request')
|
|
||||||
logger.log('INFOR', 'Async subdomains request in progress')
|
|
||||||
connector = get_connector()
|
|
||||||
headers = utils.get_random_header()
|
|
||||||
session = ClientSession(connector=connector, headers=headers)
|
|
||||||
tasks = []
|
|
||||||
for num, data in enumerate(to_req_data):
|
|
||||||
url = data.get('url')
|
|
||||||
task = asyncio.ensure_future(fetch(session, method, url))
|
|
||||||
task.add_done_callback(functools.partial(request_callback,
|
|
||||||
index=num,
|
|
||||||
datas=to_req_data))
|
|
||||||
tasks.append(task)
|
|
||||||
if tasks:
|
|
||||||
futures = asyncio.as_completed(tasks)
|
|
||||||
for future in tqdm.tqdm(futures,
|
|
||||||
total=len(tasks),
|
|
||||||
desc='Request Progress',
|
|
||||||
ncols=80):
|
|
||||||
await future
|
|
||||||
await session.close()
|
|
||||||
return to_req_data + no_req_data
|
|
||||||
|
|
||||||
|
|
||||||
def set_loop_policy():
|
|
||||||
try:
|
|
||||||
import uvloop
|
|
||||||
except ImportError:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
|
|
||||||
|
|
||||||
|
|
||||||
def set_loop():
|
|
||||||
set_loop_policy()
|
|
||||||
loop = asyncio.get_event_loop()
|
|
||||||
asyncio.set_event_loop(loop)
|
|
||||||
return loop
|
|
||||||
|
|
||||||
|
|
||||||
def run_request(domain, data, port):
|
def run_request(domain, data, port):
|
||||||
@@ -274,25 +226,18 @@ def run_request(domain, data, port):
|
|||||||
:return list: result
|
:return list: result
|
||||||
"""
|
"""
|
||||||
logger.log('INFOR', f'Start requesting subdomains of {domain}')
|
logger.log('INFOR', f'Start requesting subdomains of {domain}')
|
||||||
loop = set_loop()
|
|
||||||
data = utils.set_id_none(data)
|
data = utils.set_id_none(data)
|
||||||
request_coroutine = bulk_request(data, port)
|
ports = get_port_seq(port)
|
||||||
data = loop.run_until_complete(request_coroutine)
|
filtered_data = utils.get_filtered_data(data)
|
||||||
loop.run_until_complete(asyncio.sleep(0.25))
|
req_urls = gen_req_urls(data, ports)
|
||||||
|
resp_list = bulk_request(req_urls)
|
||||||
|
new_data = gen_new_data(data, resp_list)
|
||||||
|
data = new_data + filtered_data
|
||||||
count = utils.count_alive(data)
|
count = utils.count_alive(data)
|
||||||
logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
|
logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
def urls_request(urls):
|
|
||||||
logger.log('INFOR', 'Start urls request module')
|
|
||||||
loop = set_loop()
|
|
||||||
request_coroutine = async_request(urls)
|
|
||||||
data = loop.run_until_complete(request_coroutine)
|
|
||||||
loop.run_until_complete(asyncio.sleep(0.25))
|
|
||||||
return data
|
|
||||||
|
|
||||||
|
|
||||||
def save_db(name, data):
|
def save_db(name, data):
|
||||||
"""
|
"""
|
||||||
Save request results to database
|
Save request results to database
|
||||||
|
|||||||
+22
-30
@@ -5,14 +5,11 @@ from config.log import logger
|
|||||||
from config import settings
|
from config import settings
|
||||||
from common import utils
|
from common import utils
|
||||||
from common.ipasn import IPAsnInfo
|
from common.ipasn import IPAsnInfo
|
||||||
from common.ipgeo import IpGeoInfo
|
from common.ipreg import IpRegData
|
||||||
from common.ipreg import IpRegInfo
|
|
||||||
|
|
||||||
|
|
||||||
ip_asn = IPAsnInfo()
|
ip_asn = IPAsnInfo()
|
||||||
ip_geo = IpGeoInfo()
|
ip_reg = IpRegData()
|
||||||
db_path = settings.data_storage_dir.joinpath('ip2region.db')
|
|
||||||
ip_reg = IpRegInfo(db_path)
|
|
||||||
|
|
||||||
|
|
||||||
def filter_subdomain(data):
|
def filter_subdomain(data):
|
||||||
@@ -82,12 +79,12 @@ def gen_infos(data, qname, info, infos):
|
|||||||
cname = list()
|
cname = list()
|
||||||
ips = list()
|
ips = list()
|
||||||
public = list()
|
public = list()
|
||||||
ttls = list()
|
ttl = list()
|
||||||
cidrs = list()
|
cidr = list()
|
||||||
asns = list()
|
asn = list()
|
||||||
orgs = list()
|
org = list()
|
||||||
locs = list()
|
addr = list()
|
||||||
regs = list()
|
isp = list()
|
||||||
answers = data.get('answers')
|
answers = data.get('answers')
|
||||||
for answer in answers:
|
for answer in answers:
|
||||||
if answer.get('type') == 'A':
|
if answer.get('type') == 'A':
|
||||||
@@ -95,31 +92,26 @@ def gen_infos(data, qname, info, infos):
|
|||||||
cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号
|
cname.append(answer.get('name')[:-1]) # 去除最右边的`.`点号
|
||||||
ip = answer.get('data')
|
ip = answer.get('data')
|
||||||
ips.append(ip)
|
ips.append(ip)
|
||||||
ttl = answer.get('ttl')
|
ttl.append(str(answer.get('ttl')))
|
||||||
ttls.append(str(ttl))
|
public.append(str(utils.ip_is_public(ip)))
|
||||||
is_public = utils.ip_is_public(ip)
|
|
||||||
public.append(str(is_public))
|
|
||||||
asn_info = ip_asn.find(ip)
|
asn_info = ip_asn.find(ip)
|
||||||
cidrs.append(asn_info.get('cidr'))
|
cidr.append(asn_info.get('cidr'))
|
||||||
asns.append(asn_info.get('asn'))
|
asn.append(asn_info.get('asn'))
|
||||||
orgs.append(asn_info.get('org'))
|
org.append(asn_info.get('org'))
|
||||||
loc = f'{ip_geo.get_country_long(ip)} ' \
|
ip_info = ip_reg.query(ip)
|
||||||
f'{ip_geo.get_region(ip)} ' \
|
addr.append(ip_info.get('addr'))
|
||||||
f'{ip_geo.get_city(ip)}'
|
isp.append(ip_info.get('isp'))
|
||||||
locs.append(loc)
|
|
||||||
reg = ip_reg.memory_search(ip).get('region').decode('utf-8')
|
|
||||||
regs.append(reg)
|
|
||||||
info['resolve'] = 1
|
info['resolve'] = 1
|
||||||
info['reason'] = 'OK'
|
info['reason'] = 'OK'
|
||||||
info['cname'] = ','.join(cname)
|
info['cname'] = ','.join(cname)
|
||||||
info['content'] = ','.join(ips)
|
info['content'] = ','.join(ips)
|
||||||
info['public'] = ','.join(public)
|
info['public'] = ','.join(public)
|
||||||
info['ttl'] = ','.join(ttls)
|
info['ttl'] = ','.join(ttl)
|
||||||
info['cidr'] = ','.join(cidrs)
|
info['cidr'] = ','.join(cidr)
|
||||||
info['asn'] = ','.join(asns)
|
info['asn'] = ','.join(asn)
|
||||||
info['org'] = ','.join(orgs)
|
info['org'] = ','.join(org)
|
||||||
info['ip2location'] = ','.join(locs)
|
info['addr'] = ','.join(addr)
|
||||||
info['ip2region'] = ','.join(regs)
|
info['isp'] = ','.join(isp)
|
||||||
infos[qname] = info
|
infos[qname] = info
|
||||||
if not flag:
|
if not flag:
|
||||||
info['alive'] = 0
|
info['alive'] = 0
|
||||||
|
|||||||
@@ -0,0 +1,138 @@
|
|||||||
|
"""
|
||||||
|
根据网页结构判断页面相似性(Determine page similarity based on HTML page structure)
|
||||||
|
判断方法：根据网页的DOM树确定网页的模板特征向量，对模板特征向量计算网页结构相似性。
|
||||||
|
来源地址：https://github.com/SPuerBRead/HTMLSimilarity
|
||||||
|
计算参考: https://patents.google.com/patent/CN101694668B/zh
|
||||||
|
"""
|
||||||
|
from treelib import Tree
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import bs4
|
||||||
|
|
||||||
|
|
||||||
|
class DOMTree(object):
|
||||||
|
def __init__(self, label, attrs):
|
||||||
|
self.label = label
|
||||||
|
self.attrs = attrs
|
||||||
|
|
||||||
|
|
||||||
|
class HTMLParser(object):
|
||||||
|
def __init__(self, html):
|
||||||
|
self.dom_id = 1
|
||||||
|
self.dom_tree = Tree()
|
||||||
|
self.bs_html = BeautifulSoup(html, 'html.parser')
|
||||||
|
|
||||||
|
def get_dom_structure_tree(self):
|
||||||
|
for content in self.bs_html.contents:
|
||||||
|
if isinstance(content, bs4.element.Tag):
|
||||||
|
self.bs_html = content
|
||||||
|
self.recursive_descendants(self.bs_html, 1)
|
||||||
|
return self.dom_tree
|
||||||
|
|
||||||
|
def recursive_descendants(self, descendants, parent_id):
|
||||||
|
if self.dom_id == 1:
|
||||||
|
self.dom_tree.create_node(descendants.name, self.dom_id,
|
||||||
|
data=DOMTree(descendants.name, descendants.attrs))
|
||||||
|
self.dom_id = self.dom_id + 1
|
||||||
|
for child in descendants.contents:
|
||||||
|
if isinstance(child, bs4.element.Tag):
|
||||||
|
self.dom_tree.create_node(child.name, self.dom_id, parent_id,
|
||||||
|
data=DOMTree(child.name, child.attrs))
|
||||||
|
self.dom_id = self.dom_id + 1
|
||||||
|
self.recursive_descendants(child, self.dom_id - 1)
|
||||||
|
|
||||||
|
|
||||||
|
class Converter(object):
|
||||||
|
def __init__(self, dom_tree, dimension):
|
||||||
|
self.dom_tree = dom_tree
|
||||||
|
self.node_info_list = []
|
||||||
|
self.dimension = dimension
|
||||||
|
self.initial_weight = 1
|
||||||
|
self.attenuation_ratio = 0.6
|
||||||
|
self.dom_eigenvector = {}.fromkeys(range(0, dimension), 0)
|
||||||
|
|
||||||
|
def get_eigenvector(self):
|
||||||
|
for node_id in range(1, self.dom_tree.size() + 1):
|
||||||
|
node = self.dom_tree.get_node(node_id)
|
||||||
|
node_feature = self.create_feature(node)
|
||||||
|
feature_hash = self.feature_hash(node_feature)
|
||||||
|
node_weight = self.calculate_weight(node, node_id, feature_hash)
|
||||||
|
self.construct_eigenvector(feature_hash, node_weight)
|
||||||
|
return self.dom_eigenvector
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def create_feature(node):
|
||||||
|
node_attr_list = []
|
||||||
|
node_feature = node.data.label + '|'
|
||||||
|
for attr in node.data.attrs.keys():
|
||||||
|
node_attr_list.append(attr + ':' + str(node.data.attrs[attr]))
|
||||||
|
node_feature += '|'.join(node_attr_list)
|
||||||
|
return node_feature
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def feature_hash(node_feature):
|
||||||
|
return abs(hash(node_feature)) % (10 ** 8)
|
||||||
|
|
||||||
|
def calculate_weight(self, node, node_id, feature_hash):
|
||||||
|
brother_node_count = 0
|
||||||
|
depth = self.dom_tree.depth(node)
|
||||||
|
for brother_node in self.dom_tree.siblings(node_id):
|
||||||
|
brother_node_feature = self.create_feature(brother_node)
|
||||||
|
brother_node_feature_hash = self.feature_hash(brother_node_feature)
|
||||||
|
if brother_node_feature_hash == feature_hash:
|
||||||
|
brother_node_count = brother_node_count + 1
|
||||||
|
if brother_node_count:
|
||||||
|
node_weight = self.initial_weight * self.attenuation_ratio ** depth \
|
||||||
|
* self.attenuation_ratio ** brother_node_count
|
||||||
|
else:
|
||||||
|
node_weight = self.initial_weight * self.attenuation_ratio ** depth
|
||||||
|
return node_weight
|
||||||
|
|
||||||
|
def construct_eigenvector(self, feature_hash, node_weight):
|
||||||
|
feature_hash = feature_hash % self.dimension
|
||||||
|
self.dom_eigenvector[feature_hash] += node_weight
|
||||||
|
|
||||||
|
|
||||||
|
def calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension):
|
||||||
|
a, b = 0, 0
|
||||||
|
for i in range(dimension):
|
||||||
|
a += dom1_eigenvector[i]-dom2_eigenvector[i]
|
||||||
|
if dom1_eigenvector[i] and dom2_eigenvector[i]:
|
||||||
|
b += dom1_eigenvector[i] + dom2_eigenvector[i]
|
||||||
|
pseudodistance = abs(a)/b
|
||||||
|
return pseudodistance
|
||||||
|
|
||||||
|
|
||||||
|
def get_pseudodistance(html_doc1, html_doc2, dimension=5000):
|
||||||
|
"""
|
||||||
|
获取html文档结构相似度
|
||||||
|
|
||||||
|
:param str html_doc1: html文档
|
||||||
|
:param str html_doc2: html文档
|
||||||
|
:param int dimension: 降维后的维数
|
||||||
|
:return 伪距离
|
||||||
|
"""
|
||||||
|
hp1 = HTMLParser(html_doc1)
|
||||||
|
html_doc1_dom_tree = hp1.get_dom_structure_tree()
|
||||||
|
hp2 = HTMLParser(html_doc2)
|
||||||
|
html_doc2_dom_tree = hp2.get_dom_structure_tree()
|
||||||
|
converter = Converter(html_doc1_dom_tree, dimension)
|
||||||
|
dom1_eigenvector = converter.get_eigenvector()
|
||||||
|
converter = Converter(html_doc2_dom_tree, dimension)
|
||||||
|
dom2_eigenvector = converter.get_eigenvector()
|
||||||
|
return calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension)
|
||||||
|
|
||||||
|
|
||||||
|
def is_similar(html_doc1, html_doc2, dimension=5000):
|
||||||
|
"""
|
||||||
|
根据计算出的伪距离来判断是否html页面结构相似
|
||||||
|
|
||||||
|
:param str html_doc1: html文档
|
||||||
|
:param str html_doc2: html文档
|
||||||
|
:param int dimension: 降维后的维数
|
||||||
|
:return 是否相似（伪距离value<0.2时相似，value>0.2时不相似）
|
||||||
|
"""
|
||||||
|
value = get_pseudodistance(html_doc1, html_doc2, dimension)
|
||||||
|
if value > 0.2:
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
return True
|
||||||
+36
-49
@@ -10,7 +10,6 @@ import subprocess
|
|||||||
from ipaddress import IPv4Address, ip_address
|
from ipaddress import IPv4Address, ip_address
|
||||||
from stat import S_IXUSR
|
from stat import S_IXUSR
|
||||||
|
|
||||||
import psutil
|
|
||||||
import tenacity
|
import tenacity
|
||||||
import requests
|
import requests
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -49,10 +48,10 @@ def gen_fake_header():
|
|||||||
"""
|
"""
|
||||||
Generate fake request headers
|
Generate fake request headers
|
||||||
"""
|
"""
|
||||||
headers = settings.headers
|
headers = settings.request_default_headers
|
||||||
if not isinstance(headers, dict):
|
if not isinstance(headers, dict):
|
||||||
headers = dict()
|
headers = dict()
|
||||||
if settings.random_user_agent:
|
if settings.enable_random_ua:
|
||||||
ua = random.choice(user_agents)
|
ua = random.choice(user_agents)
|
||||||
headers['User-Agent'] = ua
|
headers['User-Agent'] = ua
|
||||||
headers['Accept-Encoding'] = 'gzip, deflate'
|
headers['Accept-Encoding'] = 'gzip, deflate'
|
||||||
@@ -74,7 +73,7 @@ def get_random_proxy():
|
|||||||
Get random proxy
|
Get random proxy
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
return random.choice(settings.proxy_pool)
|
return random.choice(settings.request_proxy_pool)
|
||||||
except IndexError:
|
except IndexError:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -83,7 +82,7 @@ def get_proxy():
|
|||||||
"""
|
"""
|
||||||
Get proxy
|
Get proxy
|
||||||
"""
|
"""
|
||||||
if settings.enable_proxy:
|
if settings.enable_request_proxy:
|
||||||
return get_random_proxy()
|
return get_random_proxy()
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -112,26 +111,21 @@ def match_main_domain(domain):
|
|||||||
|
|
||||||
|
|
||||||
def read_target_file(target):
|
def read_target_file(target):
|
||||||
domains = set()
|
domains = list()
|
||||||
with open(target, encoding='utf-8', errors='ignore') as file:
|
with open(target, encoding='utf-8', errors='ignore') as file:
|
||||||
for line in file:
|
for line in file:
|
||||||
domain = match_main_domain(line)
|
domain = match_main_domain(line)
|
||||||
if not domain:
|
if not domain:
|
||||||
continue
|
continue
|
||||||
domain = get_main_domain(domain)
|
domains.append(domain)
|
||||||
if not domain:
|
sorted_domains = sorted(set(domains), key=domains.index)
|
||||||
continue
|
return sorted_domains
|
||||||
domains.add(domain)
|
|
||||||
return domains
|
|
||||||
|
|
||||||
|
|
||||||
def get_from_target(target):
|
def get_from_target(target):
|
||||||
domains = set()
|
domains = set()
|
||||||
if isinstance(target, str):
|
if isinstance(target, str):
|
||||||
domain = match_main_domain(target)
|
domain = match_main_domain(target)
|
||||||
if not domain:
|
|
||||||
return domains
|
|
||||||
domain = get_main_domain(domain)
|
|
||||||
if not domain:
|
if not domain:
|
||||||
return domains
|
return domains
|
||||||
domains.add(domain)
|
domains.add(domain)
|
||||||
@@ -507,27 +501,8 @@ def get_process_num():
|
|||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|
||||||
def get_coroutine_count():
|
def get_request_count():
|
||||||
"""
|
return os.cpu_count() * 10
|
||||||
根据内存大小获取并发数量
|
|
||||||
"""
|
|
||||||
mem = psutil.virtual_memory()
|
|
||||||
total_mem = mem.total
|
|
||||||
g_size = 1024 * 1024 * 1024
|
|
||||||
if total_mem < 1 * g_size:
|
|
||||||
return 16
|
|
||||||
elif total_mem < 2 * g_size:
|
|
||||||
return 32
|
|
||||||
elif total_mem < 4 * g_size:
|
|
||||||
return 64
|
|
||||||
elif total_mem < 8 * g_size:
|
|
||||||
return 128
|
|
||||||
elif total_mem < 16 * g_size:
|
|
||||||
return 256
|
|
||||||
elif total_mem < 32 * g_size:
|
|
||||||
return 512
|
|
||||||
else:
|
|
||||||
return 1024
|
|
||||||
|
|
||||||
|
|
||||||
def uniq_dict_list(dict_list):
|
def uniq_dict_list(dict_list):
|
||||||
@@ -567,7 +542,6 @@ def check_net():
|
|||||||
|
|
||||||
def check_pre():
|
def check_pre():
|
||||||
logger.log('INFOR', 'Checking dependent environment')
|
logger.log('INFOR', 'Checking dependent environment')
|
||||||
system = platform.system()
|
|
||||||
implementation = platform.python_implementation()
|
implementation = platform.python_implementation()
|
||||||
version = platform.python_version()
|
version = platform.python_version()
|
||||||
if implementation != 'CPython':
|
if implementation != 'CPython':
|
||||||
@@ -576,16 +550,6 @@ def check_pre():
|
|||||||
if version < '3.6':
|
if version < '3.6':
|
||||||
logger.log('FATAL', 'OneForAll requires Python 3.6 or higher')
|
logger.log('FATAL', 'OneForAll requires Python 3.6 or higher')
|
||||||
exit(1)
|
exit(1)
|
||||||
if system == 'Windows' and implementation == 'CPython' and version < '3.8':
|
|
||||||
logger.log('FATAL', 'OneForAll requires Python 3.8 '
|
|
||||||
'or higher when running on Windows')
|
|
||||||
exit(1)
|
|
||||||
if system in {"Linux", "Darwin"}:
|
|
||||||
try:
|
|
||||||
import uvloop
|
|
||||||
except ImportError:
|
|
||||||
logger.log('ALERT', f'Please install the uvloop library manually '
|
|
||||||
f'to accelerate subdomain requests')
|
|
||||||
|
|
||||||
|
|
||||||
def check_env():
|
def check_env():
|
||||||
@@ -604,8 +568,8 @@ def check_version(local):
|
|||||||
api = 'https://api.github.com/repos/shmilylty/OneForAll/releases/latest'
|
api = 'https://api.github.com/repos/shmilylty/OneForAll/releases/latest'
|
||||||
header = get_random_header()
|
header = get_random_header()
|
||||||
proxy = get_proxy()
|
proxy = get_proxy()
|
||||||
timeout = settings.request_timeout
|
timeout = settings.request_timeout_second
|
||||||
verify = settings.request_verify
|
verify = settings.request_ssl_verify
|
||||||
try:
|
try:
|
||||||
resp = requests.get(url=api, headers=header, proxies=proxy,
|
resp = requests.get(url=api, headers=header, proxies=proxy,
|
||||||
timeout=timeout, verify=verify)
|
timeout=timeout, verify=verify)
|
||||||
@@ -613,7 +577,7 @@ def check_version(local):
|
|||||||
latest = resp_json['tag_name']
|
latest = resp_json['tag_name']
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.log('ERROR', 'An error occurred while checking the latest version')
|
logger.log('ERROR', 'An error occurred while checking the latest version')
|
||||||
logger.log('ERROR', e.args)
|
logger.log('DEBUG', e.args)
|
||||||
return
|
return
|
||||||
if latest > local:
|
if latest > local:
|
||||||
change = resp_json.get("body")
|
change = resp_json.get("body")
|
||||||
@@ -740,3 +704,26 @@ def check_random_subdomain(subdomains):
|
|||||||
if subdomain:
|
if subdomain:
|
||||||
logger.log('ALERT', f'Please check whether {subdomain} is correct or not')
|
logger.log('ALERT', f'Please check whether {subdomain} is correct or not')
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def get_url_resp(url):
|
||||||
|
timeout = settings.request_timeout
|
||||||
|
verify = settings.request_verify
|
||||||
|
try:
|
||||||
|
resp = requests.get(url, params=None, timeout=timeout, verify=verify)
|
||||||
|
except Exception as e:
|
||||||
|
logger.log('DEBUG', f'Error request {url}')
|
||||||
|
logger.log('DEBUG', e.args)
|
||||||
|
return None
|
||||||
|
return resp
|
||||||
|
|
||||||
|
|
||||||
|
def decode_resp_text(resp):
|
||||||
|
try:
|
||||||
|
text = resp.text(encoding='utf-8', errors='strict') # 先尝试用utf-8严格解码
|
||||||
|
except UnicodeError:
|
||||||
|
try:
|
||||||
|
text = resp.text(encoding='gb18030', errors='strict') # 再尝试用gb18030严格解码
|
||||||
|
except UnicodeError:
|
||||||
|
text = resp.text(encoding=None, errors='ignore') # 最后尝试自动解码
|
||||||
|
return text
|
||||||
|
|||||||
+31
-45
@@ -6,6 +6,9 @@ OneForAll默认配置
|
|||||||
import pathlib
|
import pathlib
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
|
# 禁用所有警告信息
|
||||||
|
warnings.filterwarnings("ignore")
|
||||||
|
|
||||||
# 路径设置
|
# 路径设置
|
||||||
relative_directory = pathlib.Path(__file__).parent.parent # OneForAll代码相对路径
|
relative_directory = pathlib.Path(__file__).parent.parent # OneForAll代码相对路径
|
||||||
module_dir = relative_directory.joinpath('modules') # OneForAll模块目录
|
module_dir = relative_directory.joinpath('modules') # OneForAll模块目录
|
||||||
@@ -40,7 +43,7 @@ enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_mo
|
|||||||
# 只使用ask和baidu搜索引擎收集子域的示例
|
# 只使用ask和baidu搜索引擎收集子域的示例
|
||||||
# enable_partial_module = [('modules.search', 'ask')
|
# enable_partial_module = [('modules.search', 'ask')
|
||||||
# ('modules.search', 'baidu')]
|
# ('modules.search', 'baidu')]
|
||||||
module_thread_timeout = 120.0 # 每个收集模块线程超时时间(默认2分钟)
|
module_thread_timeout = 90.0 # 每个收集模块线程超时时间(默认90秒)
|
||||||
|
|
||||||
# 爆破模块设置
|
# 爆破模块设置
|
||||||
enable_wildcard_check = True # 开启泛解析检测(默认True)
|
enable_wildcard_check = True # 开启泛解析检测(默认True)
|
||||||
@@ -51,7 +54,7 @@ brute_status_format = 'ansi' # 爆破时状态输出格式（默认asni，可
|
|||||||
brute_process_num = 1 # 默认1
|
brute_process_num = 1 # 默认1
|
||||||
brute_concurrent_num = 2000 # 并发查询数量(默认2000，最大推荐10000)
|
brute_concurrent_num = 2000 # 并发查询数量(默认2000，最大推荐10000)
|
||||||
brute_socket_num = 1 # 爆破时每个进程下的socket数量
|
brute_socket_num = 1 # 爆破时每个进程下的socket数量
|
||||||
brute_resolve_num = 10 # 解析失败时尝试换名称服务器重查次数
|
brute_resolve_num = 15 # 解析失败时尝试换名称服务器重查次数
|
||||||
# 爆破所使用的字典路径 默认data/subdomains.txt
|
# 爆破所使用的字典路径 默认data/subdomains.txt
|
||||||
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
|
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
|
||||||
# 爆破所使用的字典路径 默认data/cn_nameservers.txt
|
# 爆破所使用的字典路径 默认data/cn_nameservers.txt
|
||||||
@@ -79,24 +82,39 @@ ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标
|
|||||||
banner_process_number = 4 # 识别进程数量(默认4)
|
banner_process_number = 4 # 识别进程数量(默认4)
|
||||||
|
|
||||||
# 代理设置
|
# 代理设置
|
||||||
enable_proxy = False # 是否使用代理(全局开关)
|
enable_request_proxy = False # 是否使用代理(全局开关，默认False)
|
||||||
proxy_all_module = False # 代理所有模块
|
proxy_all_module = False # 代理所有模块
|
||||||
proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
|
proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
|
||||||
'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
|
'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
|
||||||
'YandexSearch', 'CrossDomainXml',
|
'YandexSearch', 'CrossDomainXml',
|
||||||
'ContentSecurityPolicy'] # 代理自定义的模块
|
'ContentSecurityPolicy'] # 代理自定义的模块
|
||||||
proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
||||||
'https': 'https://127.0.0.1:1080'}] # 代理池
|
'https': 'https://127.0.0.1:1080'}] # 代理池
|
||||||
# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
||||||
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
||||||
|
|
||||||
|
|
||||||
# 网络请求设置
|
# 请求设置
|
||||||
request_delay = 1 # 请求时延
|
request_thread_count = None # 请求线程数量(默认None，则根据内存大小设置)
|
||||||
request_timeout = 60 # 请求超时
|
request_timeout_second = 30 # 请求超时秒数(默认30秒)
|
||||||
request_verify = False # 请求SSL验证
|
request_ssl_verify = False # 请求SSL验证(默认False)
|
||||||
# 禁用所有警告信息
|
request_allow_redirect = True # 请求允许重定向(默认True)
|
||||||
warnings.filterwarnings("ignore")
|
request_redirect_limit = 10 # 请求跳转限制(默认10次)
|
||||||
|
# 默认请求头 可以在headers里添加自定义请求头
|
||||||
|
request_default_headers = {
|
||||||
|
'Accept': 'text/html,application/xhtml+xml,'
|
||||||
|
'application/xml;q=0.9,*/*;q=0.8',
|
||||||
|
'Accept-Encoding': 'gzip, deflate',
|
||||||
|
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
|
||||||
|
'Cache-Control': 'max-age=0',
|
||||||
|
'DNT': '1',
|
||||||
|
'Referer': 'https://www.google.com/',
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||||||
|
'(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
|
||||||
|
'Upgrade-Insecure-Requests': '1',
|
||||||
|
'X-Forwarded-For': '127.0.0.1'
|
||||||
|
}
|
||||||
|
enable_random_ua = True # 使用随机UA(默认True，开启可以覆盖request_default_headers的UA)
|
||||||
|
|
||||||
# 搜索模块设置
|
# 搜索模块设置
|
||||||
# 开启全量搜索会尽量去获取搜索引擎搜索的全部结果，不过搜索耗时可能会过长
|
# 开启全量搜索会尽量去获取搜索引擎搜索的全部结果，不过搜索耗时可能会过长
|
||||||
@@ -112,9 +130,8 @@ resolver_nameservers = [
|
|||||||
'8.8.8.8', # Google DNS
|
'8.8.8.8', # Google DNS
|
||||||
'1.1.1.1' # CloudFlare DNS
|
'1.1.1.1' # CloudFlare DNS
|
||||||
] # 指定查询的DNS域名服务器
|
] # 指定查询的DNS域名服务器
|
||||||
resolver_timeout = 5.0 # 解析超时时间
|
resolver_timeout = 5.0 # 解析超时时间(默认5.0秒)
|
||||||
resolver_lifetime = 60.0 # 解析存活时间
|
resolver_lifetime = 10.0 # 解析存活时间(默认10.0秒)
|
||||||
limit_resolve_conn = 500 # 限制同一时间解析的数量(默认500)
|
|
||||||
|
|
||||||
# 请求端口探测设置
|
# 请求端口探测设置
|
||||||
# 你可以在端口列表添加自定义端口
|
# 你可以在端口列表添加自定义端口
|
||||||
@@ -140,42 +157,11 @@ large_ports = [80, 81, 280, 300, 443, 591, 593, 832, 888, 901, 981, 1010, 1080,
|
|||||||
18092, 20000, 20720, 24465, 28017, 28080, 30821, 43110, 61600]
|
18092, 20000, 20720, 24465, 28017, 28080, 30821, 43110, 61600]
|
||||||
ports = {'small': small_ports, 'medium': medium_ports, 'large': large_ports}
|
ports = {'small': small_ports, 'medium': medium_ports, 'large': large_ports}
|
||||||
|
|
||||||
# aiohttp有关配置
|
|
||||||
verify_ssl = False
|
|
||||||
# aiohttp 支持 HTTP/HTTPS形式的代理
|
|
||||||
aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com"
|
|
||||||
allow_redirects = True # 允许请求跳转
|
|
||||||
# 为了保证请求质量 请谨慎更改以下设置
|
|
||||||
# request_method只能是HEAD或GET,HEAD请求方法更快，但是不能获取响应体并提取从中title及banner
|
|
||||||
request_method = 'GET' # 使用请求方法，默认GET
|
|
||||||
sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒
|
|
||||||
sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒
|
|
||||||
limit_open_conn = None # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置
|
|
||||||
# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
|
|
||||||
limit_per_host = 10 # 0表示不限制,默认10
|
|
||||||
|
|
||||||
common_subnames = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
|
common_subnames = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
|
||||||
'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
|
'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
|
||||||
'help', 'live', 'mall', 'blogs', 'files', 'forum',
|
'help', 'live', 'mall', 'blogs', 'files', 'forum',
|
||||||
'store', 'mobile'}
|
'store', 'mobile'}
|
||||||
|
|
||||||
# 请求头设置
|
|
||||||
# 可以在headers里添加自定义请求头
|
|
||||||
headers = {
|
|
||||||
'Accept': 'text/html,application/xhtml+xml,'
|
|
||||||
'application/xml;q=0.9,*/*;q=0.8',
|
|
||||||
'Accept-Encoding': 'gzip, deflate',
|
|
||||||
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
|
|
||||||
'Cache-Control': 'max-age=0',
|
|
||||||
'DNT': '1',
|
|
||||||
'Referer': 'https://www.google.com/',
|
|
||||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
|
||||||
'(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
|
|
||||||
'Upgrade-Insecure-Requests': '1',
|
|
||||||
'X-Forwarded-For': '127.0.0.1'
|
|
||||||
}
|
|
||||||
random_user_agent = True # 使用随机UA(默认True，开启可以覆盖header的设置)
|
|
||||||
|
|
||||||
# 模块API配置
|
# 模块API配置
|
||||||
# Censys可以免费注册获取API：https://censys.io/api
|
# Censys可以免费注册获取API：https://censys.io/api
|
||||||
censys_api_id = ''
|
censys_api_id = ''
|
||||||
|
|||||||
+1
-1
@@ -35,4 +35,4 @@ logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')
|
|||||||
# 命令终端日志级别默认为INFOR
|
# 命令终端日志级别默认为INFOR
|
||||||
logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True)
|
logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True)
|
||||||
# 日志文件默认为级别为DEBUG
|
# 日志文件默认为级别为DEBUG
|
||||||
logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True)
|
logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True, encoding='utf-8')
|
||||||
|
|||||||
+3
-24
@@ -35,11 +35,8 @@ enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_mo
|
|||||||
# 只使用ask和baidu搜索引擎收集子域的示例
|
# 只使用ask和baidu搜索引擎收集子域的示例
|
||||||
# enable_partial_module = [('modules.search', 'ask')
|
# enable_partial_module = [('modules.search', 'ask')
|
||||||
# ('modules.search', 'baidu')]
|
# ('modules.search', 'baidu')]
|
||||||
module_thread_timeout = 120.0 # 每个收集模块线程超时时间(默认2分钟)
|
|
||||||
|
|
||||||
# 爆破模块设置
|
# 爆破模块设置
|
||||||
brute_wildcard_check = True # 开启泛解析检测(默认True)
|
|
||||||
brute_wildcard_deal = True # 开启泛解析处理(默认True)
|
|
||||||
brute_concurrent_num = 2000 # 爆破时并发查询数量(默认2000，最大推荐10000)
|
brute_concurrent_num = 2000 # 爆破时并发查询数量(默认2000，最大推荐10000)
|
||||||
# 爆破所使用的字典路径 默认data/subdomains.txt
|
# 爆破所使用的字典路径 默认data/subdomains.txt
|
||||||
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
|
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
|
||||||
@@ -66,37 +63,19 @@ fuzz_rule = None # fuzz域名的正则 示例：'[a-z][0-9]' 表示第一位是
|
|||||||
brute_ip_blacklist = {'0.0.0.0', '0.0.0.1'} # IP黑名单 子域解析到IP黑名单则标记为非法子域
|
brute_ip_blacklist = {'0.0.0.0', '0.0.0.1'} # IP黑名单 子域解析到IP黑名单则标记为非法子域
|
||||||
ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标记为非法(泛解析)子域
|
ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标记为非法(泛解析)子域
|
||||||
|
|
||||||
# banner识别模块设置
|
|
||||||
banner_process_number = 4 # 识别进程数量(默认4)
|
|
||||||
|
|
||||||
# 代理设置
|
# 代理设置
|
||||||
enable_proxy = False # 是否使用代理(全局开关)
|
enable_request_proxy = False # 是否使用代理(全局开关)
|
||||||
proxy_all_module = False # 代理所有模块
|
proxy_all_module = False # 代理所有模块
|
||||||
proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
|
proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
|
||||||
'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
|
'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
|
||||||
'YandexSearch', 'CrossDomainXml',
|
'YandexSearch', 'CrossDomainXml',
|
||||||
'ContentSecurityPolicy'] # 代理自定义的模块
|
'ContentSecurityPolicy'] # 代理自定义的模块
|
||||||
proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
||||||
'https': 'https://127.0.0.1:1080'}] # 代理池
|
'https': 'https://127.0.0.1:1080'}] # 代理池
|
||||||
# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
||||||
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
||||||
|
|
||||||
|
|
||||||
# 网络请求设置
|
|
||||||
request_delay = 1 # 请求时延
|
|
||||||
request_timeout = 60 # 请求超时
|
|
||||||
request_verify = False # 请求SSL验证
|
|
||||||
|
|
||||||
# 搜索模块设置
|
# 搜索模块设置
|
||||||
enable_recursive_search = False # 递归搜索子域
|
enable_recursive_search = False # 递归搜索子域
|
||||||
search_recursive_times = 2 # 递归搜索层数
|
search_recursive_times = 2 # 递归搜索层数
|
||||||
|
|
||||||
# aiohttp有关配置
|
|
||||||
# aiohttp 支持 HTTP/HTTPS形式的代理
|
|
||||||
aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com"
|
|
||||||
# 为了保证请求质量 请谨慎更改以下设置
|
|
||||||
sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒
|
|
||||||
sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒
|
|
||||||
limit_open_conn = None # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置
|
|
||||||
# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
|
|
||||||
limit_per_host = 10 # 0表示不限制,默认10
|
|
||||||
|
|||||||
Binary file not shown.
@@ -29,7 +29,6 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f
|
|||||||
python3 dbexport.py --target table_name --tb True --show False
|
python3 dbexport.py --target table_name --tb True --show False
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
--alive True/False Only export alive subdomains or not (default False)
|
|
||||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||||
--path Result directory (default directory is ./results)
|
--path Result directory (default directory is ./results)
|
||||||
|
|
||||||
@@ -82,4 +81,3 @@ def export(target, type='target', db=None, alive=False, limit=None, path=None, f
|
|||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
fire.Fire(export)
|
fire.Fire(export)
|
||||||
# export('example.com')
|
|
||||||
|
|||||||
+1
-18
@@ -4,7 +4,7 @@
|
|||||||
[](https://codecov.io/gh/shmilylty/OneForAll)
|
[](https://codecov.io/gh/shmilylty/OneForAll)
|
||||||
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
|
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
|
||||||
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
|
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
|
||||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||||
[](https://github.com/shmilylty/OneForAll/releases)
|
[](https://github.com/shmilylty/OneForAll/releases)
|
||||||
|
|
||||||
👊**OneForAll is a powerful subdomain integration tool** 📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
|
👊**OneForAll is a powerful subdomain integration tool** 📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
|
||||||
@@ -15,23 +15,6 @@
|
|||||||
|
|
||||||
📢 Please read this document to help you start quickly!
|
📢 Please read this document to help you start quickly!
|
||||||
|
|
||||||
<details>
|
|
||||||
<summary><b>🐍Installation requirements</b></summary>
|
|
||||||
|
|
||||||
OneForAll is developed and tested based on [Python 3.8.0](https://www.python.org/downloads/release/python-380/). Recommend use release higher than Python 3.8.0 (Windows platform must use Python 3.8.0 or later). For more information on installing the Python environment, please read [Python 3 installation Guide](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3).
|
|
||||||
|
|
||||||
After installation python, run the following command to check the Python and pip3 versions:
|
|
||||||
```bash
|
|
||||||
python -V
|
|
||||||
pip3 -V
|
|
||||||
```
|
|
||||||
If you see the following output, there is no problem with the Python environment:
|
|
||||||
```bash
|
|
||||||
Python 3.8.0
|
|
||||||
pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.8)
|
|
||||||
```
|
|
||||||
</details>
|
|
||||||
|
|
||||||
<details>
|
<details>
|
||||||
<summary><b>✔Installation steps (for Git)</b></summary>
|
<summary><b>✔Installation steps (for Git)</b></summary>
|
||||||
|
|
||||||
|
|||||||
+111
-38
@@ -39,7 +39,6 @@ The OneForAll command line interface is based on [Fire](https://github.com/googl
|
|||||||
|
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
--alive True/False Only export alive subdomains or not (default False)
|
|
||||||
--port small/medium/large See details in ./config/setting.py(default small)
|
--port small/medium/large See details in ./config/setting.py(default small)
|
||||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||||
--path Result path (default None, automatically generated)
|
--path Result path (default None, automatically generated)
|
||||||
@@ -87,64 +86,138 @@ The OneForAll command line interface is based on [Fire](https://github.com/googl
|
|||||||
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python aiobrute.py --help
|
python brute.py --help
|
||||||
```
|
```
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
NAME
|
NAME
|
||||||
aiobrute.py - OneForAll multi-process multi-correlation asynchronous subdomain blasting module
|
brute.py - OneForAll subdomain brute module
|
||||||
|
|
||||||
SYNOPSIS
|
SYNOPSIS
|
||||||
aiobrute.py --target=TARGET <flags>
|
brute.py <flags>
|
||||||
|
|
||||||
DESCRIPTION
|
DESCRIPTION
|
||||||
Example：
|
Example：
|
||||||
python3 aiobrute.py --target example.com run
|
brute.py --target domain.com --word True run
|
||||||
python3 aiobrute.py --target ./domains.txt run
|
brute.py --targets ./domains.txt --word True run
|
||||||
python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
|
brute.py --target domain.com --word True --concurrent 2000 run
|
||||||
python3 aiobrute.py --target example.com --wordlist subdomains.txt run
|
brute.py --target domain.com --word True --wordlist subnames.txt run
|
||||||
python3 aiobrute.py --target example.com --recursive True --depth 2 run
|
brute.py --target domain.com --word True --recursive True --depth 2 run
|
||||||
python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
|
brute.py --target d.com --fuzz True --place m.*.d.com --rule '[a-z]' run
|
||||||
|
brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
Parameter valid optional value 1, 0, none indicates that the export is
|
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||||
valid, invalid, and all subdomains, respectively.
|
--path Result path (default None, automatically generated)
|
||||||
|
|
||||||
Parameter format have optional values 'txt', 'rst', 'csv', 'tsv', 'json',
|
|
||||||
'yaml', 'html', 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'.
|
|
||||||
If the parameter path is None, the appropriate file is generated in the
|
|
||||||
project result directory based on the format parameter and the domain
|
|
||||||
name.
|
|
||||||
|
|
||||||
ARGUMENTS
|
|
||||||
TARGET
|
|
||||||
Single domain name or file path for one domain name per line (required)
|
|
||||||
|
|
||||||
FLAGS
|
FLAGS
|
||||||
|
--target=TARGET
|
||||||
|
One domain (target or targets must be provided)
|
||||||
|
--targets=TARGETS
|
||||||
|
File path of one domain per line
|
||||||
--process=PROCESS
|
--process=PROCESS
|
||||||
Number of processes blasted (default CPU core count)
|
Number of processes (default 1)
|
||||||
--coroutine=COROUTINE
|
--concurrent=CONCURRENT
|
||||||
Number of coroutines per blasting process (default 1024)
|
Number of concurrent (default 2000)
|
||||||
|
--word=WORD
|
||||||
|
Use word mode generate dictionary (default False)
|
||||||
--wordlist=WORDLIST
|
--wordlist=WORDLIST
|
||||||
Specify the dictionary path used for blasting (config.py is used by default)
|
Dictionary path used in word mode (default use ./config/default.py)
|
||||||
--recursive=RECURSIVE
|
--recursive=RECURSIVE
|
||||||
Whether to use recursive blasting (default False)
|
Use recursion (default False)
|
||||||
--depth=DEPTH
|
--depth=DEPTH
|
||||||
Depth of recursive blasting (default 2)
|
Recursive depth (default 2)
|
||||||
--namelist=NAMELIST
|
--nextlist=NEXTLIST
|
||||||
Specifies the dictionary path used by recursive blasting (configured by default using config.py)
|
Dictionary file path used by recursive (default use ./config/default.py)
|
||||||
--fuzz=FUZZ
|
--fuzz=FUZZ
|
||||||
Whether to use the fuzz mode for blasting (default False, you must specify the fuzz regular rule)
|
Use fuzz mode generate dictionary (default False)
|
||||||
|
--place=PLACE
|
||||||
|
Designated fuzz position (required if use fuzz mode)
|
||||||
--rule=RULE
|
--rule=RULE
|
||||||
Regular rules used by fuzz mode (configured by default using config.py)
|
Specify the regexp rules used in fuzz mode (required if use fuzz mode)
|
||||||
|
--fuzzlist=FUZZLIST
|
||||||
|
Dictionary path used in fuzz mode (default use ./config/default.py)
|
||||||
--export=EXPORT
|
--export=EXPORT
|
||||||
Whether to export the blast result (default True)
|
Export the results (default True)
|
||||||
--valid=VALID
|
--alive=ALIVE
|
||||||
Export validity of subdomains (default None)
|
Only export alive subdomains (default False)
|
||||||
--format=FORMAT
|
--format=FORMAT
|
||||||
Export format (default xls)
|
Result format (default csv)
|
||||||
--path=PATH
|
--path=PATH
|
||||||
Export path (default None)
|
Result directory (default None)
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **takeover.py help**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python takeover.py --help
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
NAME
|
||||||
|
takeover.py - OneForAll subdomain takeover module SYNOPSIS takeover.py <flags> DESCRIPTION
|
||||||
|
|
||||||
|
Example:
|
||||||
|
python3 takeover.py --target www.example.com --format csv run
|
||||||
|
python3 takeover.py --targets ./subdomains.txt --thread 10 run
|
||||||
|
|
||||||
|
Note:
|
||||||
|
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||||
|
--path Result directory (default directory is ./results)
|
||||||
|
|
||||||
|
FLAGS
|
||||||
|
--target=TARGET
|
||||||
|
One domain (target or targets must be provided)
|
||||||
|
--targets=TARGETS
|
||||||
|
File path of one domain per line
|
||||||
|
--thread=THREAD
|
||||||
|
threads number (default 20)
|
||||||
|
--path=PATH
|
||||||
|
Result directory (default None)
|
||||||
|
--format=FORMAT
|
||||||
|
Result format (default csv)
|
||||||
|
```
|
||||||
|
|
||||||
|
4. **dbexport.py help**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python dbexport.py --help
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
NAME
|
||||||
|
dbexport.py - OneForAll export from database module
|
||||||
|
|
||||||
|
SYNOPSIS
|
||||||
|
dbexport.py TARGET <flags>
|
||||||
|
|
||||||
|
DESCRIPTION
|
||||||
|
Example:
|
||||||
|
python3 dbexport.py --target name --format csv --dir= ./result.csv
|
||||||
|
python3 dbexport.py --db result.db --target name --show False
|
||||||
|
python3 dbexport.py --target table_name --tb True --show False
|
||||||
|
|
||||||
|
Note:
|
||||||
|
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||||
|
--path Result directory (default directory is ./results)
|
||||||
|
|
||||||
|
POSITIONAL ARGUMENTS
|
||||||
|
TARGET
|
||||||
|
Table to be exported
|
||||||
|
|
||||||
|
FLAGS
|
||||||
|
--type=TYPE
|
||||||
|
Type of target
|
||||||
|
--db=DB
|
||||||
|
Database path to be exported (default ./results/result.sqlite3)
|
||||||
|
--alive=ALIVE
|
||||||
|
Only export the results of alive subdomains (default False)
|
||||||
|
--limit=LIMIT
|
||||||
|
Export limit (default None)
|
||||||
|
--path=PATH
|
||||||
|
Result directory (default None)
|
||||||
|
--format=FORMAT
|
||||||
|
Result format (default csv)
|
||||||
--show=SHOW
|
--show=SHOW
|
||||||
Terminal display exported data (default False)
|
Displays the exported data in terminal (default False)
|
||||||
```
|
```
|
||||||
+5
-5
@@ -84,19 +84,19 @@ DNS解析返回的TTL值
|
|||||||
|
|
||||||
### cidr
|
### cidr
|
||||||
|
|
||||||
IP2LOCATION-ASN库查询出的CIDR
|
ip2location库查询出的CIDR
|
||||||
|
|
||||||
### asn
|
### asn
|
||||||
|
|
||||||
IP2LOCATION-ASN库查询出的ASN
|
ip2location库查询出的ASN
|
||||||
|
|
||||||
### ip2region
|
### addr
|
||||||
|
|
||||||
ip2region库查询出的物理地址
|
ip2region库查询出的物理地址
|
||||||
|
|
||||||
### ip2location
|
### isp
|
||||||
|
|
||||||
ip2location库查询出的物理地址
|
ip2region库查询出的网络服务提供商
|
||||||
|
|
||||||
### resolver
|
### resolver
|
||||||
|
|
||||||
|
|||||||
+34
-40
@@ -38,7 +38,6 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
|
|||||||
python3 oneforall.py --target example.com --show True run
|
python3 oneforall.py --target example.com --show True run
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
--alive True/False 只导出存活子域(默认False)
|
|
||||||
--port small/medium/large 详见./config/setting.py(默认small)
|
--port small/medium/large 详见./config/setting.py(默认small)
|
||||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (结果格式，默认CSV)
|
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (结果格式，默认CSV)
|
||||||
--path 结果路径(默认None，自动生成)
|
--path 结果路径(默认None，自动生成)
|
||||||
@@ -90,63 +89,56 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
NAME
|
NAME
|
||||||
aiobrute.py - OneForAll多进程多协程异步子域爆破模块
|
brute.py - OneForAll子域爆破模块
|
||||||
|
|
||||||
SYNOPSIS
|
SYNOPSIS
|
||||||
aiobrute.py --target=TARGET <flags>
|
brute.py <flags>
|
||||||
|
|
||||||
DESCRIPTION
|
DESCRIPTION
|
||||||
Example：
|
Example：
|
||||||
python3 aiobrute.py --target example.com run
|
brute.py --target domain.com --word True run
|
||||||
python3 aiobrute.py --target ./domains.txt run
|
brute.py --targets ./domains.txt --word True run
|
||||||
python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
|
brute.py --target domain.com --word True --coroutine 2000 run
|
||||||
python3 aiobrute.py --target example.com --wordlist subdomains.txt run
|
brute.py --target domain.com --word True --wordlist subnames.txt run
|
||||||
python3 aiobrute.py --target example.com --recursive True --depth 2 run
|
brute.py --target domain.com --word True --recursive True --depth 2 run
|
||||||
python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
|
brute.py --target d.com --fuzz True --place m.*.d.com --rule '[a-z]' run
|
||||||
|
brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为任务组，
|
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (结果格式，默认CSV)
|
||||||
当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
|
--path 导出路径(默认None，自动生成)
|
||||||
十万字典建议设置为5000，百万字典设置为50000
|
|
||||||
参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
|
|
||||||
参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
|
|
||||||
'dbf', 'latex', 'ods'
|
|
||||||
参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
|
|
||||||
|
|
||||||
ARGUMENTS
|
|
||||||
TARGET
|
|
||||||
单个域名或者每行一个域名的文件路径
|
|
||||||
|
|
||||||
FLAGS
|
FLAGS
|
||||||
|
--target=TARGET
|
||||||
|
单个域名(必须提供target或targets参数)
|
||||||
|
--targets=TARGETS
|
||||||
|
每行一个域名的文件路径
|
||||||
--process=PROCESS
|
--process=PROCESS
|
||||||
爆破的进程数(默认CPU核心数)
|
爆破的进程数(默认CPU核心数)
|
||||||
--coroutine=COROUTINE
|
--coroutine=COROUTINE
|
||||||
每个爆破进程下的协程数(默认64)
|
每个爆破进程下的协程数(默认2000)
|
||||||
--wordlist=WORDLIST
|
--wordlist=WORDLIST
|
||||||
指定爆破所使用的字典路径(默认使用config.py配置)
|
指定爆破所使用的字典路径(默认使用config.py配置)
|
||||||
--segment=SEGMENT
|
|
||||||
爆破任务分割(默认500)
|
|
||||||
--recursive=RECURSIVE
|
--recursive=RECURSIVE
|
||||||
是否使用递归爆破(默认False)
|
是否使用递归爆破(默认False)
|
||||||
--depth=DEPTH
|
--depth=DEPTH
|
||||||
递归爆破的深度(默认2)
|
递归爆破的深度(默认2)
|
||||||
--namelist=NAMELIST
|
--nextlist=NEXTLIST
|
||||||
指定递归爆破所使用的字典路径(默认使用config.py配置)
|
指定递归爆破所使用的字典路径(默认使用config.py配置)
|
||||||
--fuzz=FUZZ
|
--fuzz=FUZZ
|
||||||
是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
|
是否使用fuzz模式进行爆破(默认False)
|
||||||
--rule=RULE
|
--rule=RULE
|
||||||
fuzz模式使用的正则规则(默认使用config.py配置)
|
fuzz模式使用的正则规则(默认使用config.py配置)
|
||||||
|
--fuzzlist=FUZZLIST
|
||||||
|
指定fuzz模式所使用的字典路径(默认使用config.py配置)
|
||||||
--export=EXPORT
|
--export=EXPORT
|
||||||
是否导出爆破结果(默认True)
|
是否导出爆破结果(默认True)
|
||||||
--valid=VALID
|
--alive=ALIVE
|
||||||
导出子域的有效性(默认None)
|
只导出存活子域(默认False)
|
||||||
--format=FORMAT
|
--format=FORMAT
|
||||||
导出格式(默认xlsx)
|
导出格式(默认csv)
|
||||||
--path=PATH
|
--path=PATH
|
||||||
导出路径(默认None)
|
导出路径(默认None)
|
||||||
--show=SHOW
|
|
||||||
终端显示导出数据(默认False)
|
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
@@ -200,24 +192,26 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
|
|||||||
dbexport.py - OneForAll数据库导出模块
|
dbexport.py - OneForAll数据库导出模块
|
||||||
|
|
||||||
SYNOPSIS
|
SYNOPSIS
|
||||||
dbexport.py TABLE <flags>
|
dbexport.py TARGET <flags>
|
||||||
|
|
||||||
DESCRIPTION
|
DESCRIPTION
|
||||||
Example:
|
Example:
|
||||||
python3 dbexport.py --table name --format csv --path= ./result.csv
|
python3 dbexport.py --target name --format csv --dir= ./result.csv
|
||||||
python3 dbexport.py --db result.db --table name --show False
|
python3 dbexport.py --db result.db --target name --show False
|
||||||
|
python3 dbexport.py --target table_name --tb True --show False
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
|
--type target/table 要导出的目标类型(默认target)
|
||||||
参数format可选格式有'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
|
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (结果格式，默认CSV)
|
||||||
'dbf', 'latex', 'ods'
|
--path 结果路径(默认None，自动生成)
|
||||||
参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
|
|
||||||
|
|
||||||
POSITIONAL ARGUMENTS
|
POSITIONAL ARGUMENTS
|
||||||
TABLE
|
TARGET
|
||||||
要导出的表
|
要导出的目标类型
|
||||||
|
|
||||||
FLAGS
|
FLAGS
|
||||||
|
--type=TYPE
|
||||||
|
要导出的目标类型(默认target)
|
||||||
--db=DB
|
--db=DB
|
||||||
要导出的数据库路径(默认为results/result.sqlite3)
|
要导出的数据库路径(默认为results/result.sqlite3)
|
||||||
--valid=VALID
|
--valid=VALID
|
||||||
|
|||||||
Binary file not shown.
|
After Width: | Height: | Size: 43 KiB |
+6
-10
@@ -11,19 +11,15 @@ import dns.resolver
|
|||||||
import dns.zone
|
import dns.zone
|
||||||
|
|
||||||
from common import utils
|
from common import utils
|
||||||
from common.module import Module
|
from common.check import Check
|
||||||
from config.log import logger
|
from config.log import logger
|
||||||
|
|
||||||
|
|
||||||
class CheckAXFR(Module):
|
class AXFR(Check):
|
||||||
"""
|
def __init__(self, domain):
|
||||||
DNS zone transfer vulnerability base class
|
Check.__init__(self)
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, domain: str):
|
|
||||||
Module.__init__(self)
|
|
||||||
self.domain = domain
|
self.domain = domain
|
||||||
self.module = 'Check'
|
self.module = 'check'
|
||||||
self.source = 'AXFRCheck'
|
self.source = 'AXFRCheck'
|
||||||
self.results = []
|
self.results = []
|
||||||
|
|
||||||
@@ -92,7 +88,7 @@ def run(domain):
|
|||||||
|
|
||||||
:param str domain: 域名
|
:param str domain: 域名
|
||||||
"""
|
"""
|
||||||
check = CheckAXFR(domain)
|
check = AXFR(domain)
|
||||||
check.run()
|
check.run()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+9
-20
@@ -1,33 +1,22 @@
|
|||||||
"""
|
"""
|
||||||
检查crossdomain.xml文件收集子域名
|
检查crossdomain.xml文件收集子域名
|
||||||
"""
|
"""
|
||||||
|
from common.check import Check
|
||||||
from common.module import Module
|
|
||||||
|
|
||||||
|
|
||||||
class CheckCDX(Module):
|
class CrossDomain(Check):
|
||||||
"""
|
def __init__(self, domain):
|
||||||
检查crossdomain.xml文件收集子域名
|
Check.__init__(self)
|
||||||
"""
|
|
||||||
def __init__(self, domain: str):
|
|
||||||
Module.__init__(self)
|
|
||||||
self.domain = domain
|
self.domain = domain
|
||||||
self.module = 'Check'
|
self.module = 'check'
|
||||||
self.source = "CrossDomainXml"
|
self.source = "CrossDomainCheck"
|
||||||
|
|
||||||
def check(self):
|
def check(self):
|
||||||
"""
|
"""
|
||||||
检查crossdomain.xml收集子域名
|
检查crossdomain.xml收集子域名
|
||||||
"""
|
"""
|
||||||
urls = [f'http://{self.domain}/crossdomain.xml',
|
filenames = {'crossdomain.xml'}
|
||||||
f'https://{self.domain}/crossdomain.xml',
|
self.to_check(filenames)
|
||||||
f'http://www.{self.domain}/crossdomain.xml',
|
|
||||||
f'https://www.{self.domain}/crossdomain.xml']
|
|
||||||
for url in urls:
|
|
||||||
self.header = self.get_header()
|
|
||||||
self.proxy = self.get_proxy(self.source)
|
|
||||||
resp = self.get(url, check=False)
|
|
||||||
self.subdomains = self.collect_subdomains(resp)
|
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""
|
"""
|
||||||
@@ -47,7 +36,7 @@ def run(domain):
|
|||||||
|
|
||||||
:param domain: 域名
|
:param domain: 域名
|
||||||
"""
|
"""
|
||||||
check = CheckCDX(domain)
|
check = CrossDomain(domain)
|
||||||
check.run()
|
check.run()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+10
-12
@@ -1,21 +1,18 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
检查域名证书收集子域名
|
检查域名证书收集子域名
|
||||||
"""
|
"""
|
||||||
import socket
|
import socket
|
||||||
import ssl
|
import ssl
|
||||||
|
|
||||||
from common.module import Module
|
|
||||||
from config.log import logger
|
from config.log import logger
|
||||||
|
from common.check import Check
|
||||||
|
|
||||||
|
|
||||||
class CheckCert(Module):
|
class CertInfo(Check):
|
||||||
def __init__(self, domain):
|
def __init__(self, domain):
|
||||||
Module.__init__(self)
|
Check.__init__(self)
|
||||||
self.domain = domain
|
self.domain = domain
|
||||||
self.port = 443 # ssl port
|
self.module = 'check'
|
||||||
self.module = 'Check'
|
|
||||||
self.source = 'CertInfo'
|
self.source = 'CertInfo'
|
||||||
|
|
||||||
def check(self):
|
def check(self):
|
||||||
@@ -24,10 +21,11 @@ class CheckCert(Module):
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
ctx = ssl.create_default_context()
|
ctx = ssl.create_default_context()
|
||||||
sock = ctx.wrap_socket(socket.socket(),
|
sock = socket.socket()
|
||||||
server_hostname=self.domain)
|
sock.settimeout(10)
|
||||||
sock.connect((self.domain, self.port))
|
wrap_sock = ctx.wrap_socket(sock, server_hostname=self.domain)
|
||||||
cert_dict = sock.getpeercert()
|
wrap_sock.connect((self.domain, 443))
|
||||||
|
cert_dict = wrap_sock.getpeercert()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.log('DEBUG', e.args)
|
logger.log('DEBUG', e.args)
|
||||||
return
|
return
|
||||||
@@ -52,7 +50,7 @@ def run(domain):
|
|||||||
|
|
||||||
:param str domain: 域名
|
:param str domain: 域名
|
||||||
"""
|
"""
|
||||||
check = CheckCert(domain)
|
check = CertInfo(domain)
|
||||||
check.run()
|
check.run()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -3,19 +3,19 @@ Collect subdomains from ContentSecurityPolicy
|
|||||||
"""
|
"""
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
from common.module import Module
|
|
||||||
from config.log import logger
|
from config.log import logger
|
||||||
|
from common.check import Check
|
||||||
|
|
||||||
|
|
||||||
class CheckCSP(Module):
|
class CSP(Check):
|
||||||
"""
|
"""
|
||||||
Collect subdomains from ContentSecurityPolicy
|
Collect subdomains from ContentSecurityPolicy
|
||||||
"""
|
"""
|
||||||
def __init__(self, domain, header):
|
def __init__(self, domain, header):
|
||||||
Module.__init__(self)
|
Check.__init__(self)
|
||||||
self.domain = domain
|
self.domain = domain
|
||||||
self.module = 'Check'
|
self.module = 'check'
|
||||||
self.source = 'ContentSecurityPolicy'
|
self.source = 'CSPCheck'
|
||||||
self.csp_header = header
|
self.csp_header = header
|
||||||
|
|
||||||
def grab_header(self):
|
def grab_header(self):
|
||||||
@@ -34,8 +34,7 @@ class CheckCSP(Module):
|
|||||||
self.proxy = self.get_proxy(self.source)
|
self.proxy = self.get_proxy(self.source)
|
||||||
response = self.get(url, check=False)
|
response = self.get(url, check=False)
|
||||||
if response:
|
if response:
|
||||||
csp_header = response.headers
|
return response.headers
|
||||||
break
|
|
||||||
return csp_header
|
return csp_header
|
||||||
|
|
||||||
def check(self):
|
def check(self):
|
||||||
@@ -73,7 +72,7 @@ def run(domain, header=None):
|
|||||||
:param str domain: 域名
|
:param str domain: 域名
|
||||||
:param dict or None header: 响应头
|
:param dict or None header: 响应头
|
||||||
"""
|
"""
|
||||||
check = CheckCSP(domain, header)
|
check = CSP(domain, header)
|
||||||
check.run()
|
check.run()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,16 +1,16 @@
|
|||||||
# https://www.icann.org/resources/pages/dnssec-what-is-it-why-important-2019-03-20-zh
|
# https://www.icann.org/resources/pages/dnssec-what-is-it-why-important-2019-03-20-zh
|
||||||
# https://appsecco.com/books/subdomain-enumeration/active_techniques/zone_walking.html
|
# https://appsecco.com/books/subdomain-enumeration/active_techniques/zone_walking.html
|
||||||
|
|
||||||
from common.module import Module
|
|
||||||
from common import utils
|
from common import utils
|
||||||
|
from common.check import Check
|
||||||
|
|
||||||
|
|
||||||
class CheckNSEC(Module):
|
class NSEC(Check):
|
||||||
def __init__(self, domain):
|
def __init__(self, domain):
|
||||||
Module.__init__(self)
|
Check.__init__(self)
|
||||||
self.domain = domain
|
self.domain = domain
|
||||||
self.module = 'check'
|
self.module = 'check'
|
||||||
self.source = "CheckNSEC"
|
self.source = "NSECCheck"
|
||||||
|
|
||||||
def walk(self):
|
def walk(self):
|
||||||
domain = self.domain
|
domain = self.domain
|
||||||
@@ -47,7 +47,7 @@ def run(domain):
|
|||||||
|
|
||||||
:param str domain: 域名
|
:param str domain: 域名
|
||||||
"""
|
"""
|
||||||
brute = CheckNSEC(domain)
|
brute = NSEC(domain)
|
||||||
brute.run()
|
brute.run()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+8
-18
@@ -1,32 +1,22 @@
|
|||||||
"""
|
"""
|
||||||
检查内容安全策略收集子域名收集子域名
|
检查内容安全策略收集子域名收集子域名
|
||||||
"""
|
"""
|
||||||
from common.module import Module
|
from common.check import Check
|
||||||
|
|
||||||
|
|
||||||
class CheckRobots(Module):
|
class Robots(Check):
|
||||||
"""
|
|
||||||
检查robots.txt收集子域名
|
|
||||||
"""
|
|
||||||
def __init__(self, domain):
|
def __init__(self, domain):
|
||||||
Module.__init__(self)
|
Check.__init__(self)
|
||||||
self.domain = domain
|
self.domain = domain
|
||||||
self.module = 'Check'
|
self.module = 'check'
|
||||||
self.source = 'Robots'
|
self.source = 'RobotsCheck'
|
||||||
|
|
||||||
def check(self):
|
def check(self):
|
||||||
"""
|
"""
|
||||||
正则匹配域名的robots.txt文件中的子域
|
正则匹配域名的robots.txt文件中的子域
|
||||||
"""
|
"""
|
||||||
urls = [f'http://{self.domain}/robots.txt',
|
filenames = {'robots.txt'}
|
||||||
f'https://{self.domain}/robots.txt',
|
self.to_check(filenames)
|
||||||
f'http://www.{self.domain}/robots.txt',
|
|
||||||
f'https://www.{self.domain}/robots.txt']
|
|
||||||
for url in urls:
|
|
||||||
self.header = self.get_header()
|
|
||||||
self.proxy = self.get_proxy(self.source)
|
|
||||||
resp = self.get(url, check=False, allow_redirects=False)
|
|
||||||
self.subdomains = self.collect_subdomains(resp)
|
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""
|
"""
|
||||||
@@ -46,7 +36,7 @@ def run(domain):
|
|||||||
|
|
||||||
:param str domain: 域名
|
:param str domain: 域名
|
||||||
"""
|
"""
|
||||||
check = CheckRobots(domain)
|
check = Robots(domain)
|
||||||
check.run()
|
check.run()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,37 +1,22 @@
|
|||||||
"""
|
"""
|
||||||
检查内容安全策略收集子域名收集子域名
|
检查内容安全策略收集子域名收集子域名
|
||||||
"""
|
"""
|
||||||
from common.module import Module
|
from common.check import Check
|
||||||
|
|
||||||
|
|
||||||
class CheckRobots(Module):
|
class Sitemap(Check):
|
||||||
"""
|
|
||||||
检查sitemap收集子域名
|
|
||||||
"""
|
|
||||||
def __init__(self, domain):
|
def __init__(self, domain):
|
||||||
Module.__init__(self)
|
Check.__init__(self)
|
||||||
self.domain = domain
|
self.domain = domain
|
||||||
self.module = 'Check'
|
self.module = 'check'
|
||||||
self.source = 'Sitemap'
|
self.source = 'SitemapCheck'
|
||||||
|
|
||||||
def check(self):
|
def check(self):
|
||||||
"""
|
"""
|
||||||
正则匹配域名的sitemap文件中的子域
|
正则匹配域名的sitemap文件中的子域
|
||||||
"""
|
"""
|
||||||
urls = [f'http://{self.domain}/sitemap.xml',
|
filenames = {'sitemap.xml', 'sitemap.txt', 'sitemap.html', 'sitemapindex.xml'}
|
||||||
f'http://www.{self.domain}/sitemap.xml',
|
self.to_check(filenames)
|
||||||
f'http://{self.domain}/sitemap.txt',
|
|
||||||
f'http://www.{self.domain}/sitemap.txt',
|
|
||||||
f'http://{self.domain}/sitemap.html',
|
|
||||||
f'http://www.{self.domain}/sitemap.html',
|
|
||||||
f'http://{self.domain}/sitemap_index.xml',
|
|
||||||
f'http://www.{self.domain}/sitemap_index.xml']
|
|
||||||
for url in urls:
|
|
||||||
self.header = self.get_header()
|
|
||||||
self.proxy = self.get_proxy(self.source)
|
|
||||||
self.timeout = 10
|
|
||||||
resp = self.get(url, check=False)
|
|
||||||
self.subdomains = self.collect_subdomains(resp)
|
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""
|
"""
|
||||||
@@ -51,7 +36,7 @@ def run(domain):
|
|||||||
|
|
||||||
:param str domain: 域名
|
:param str domain: 域名
|
||||||
"""
|
"""
|
||||||
check = CheckRobots(domain)
|
check = Sitemap(domain)
|
||||||
check.run()
|
check.run()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+3
-28
@@ -1,38 +1,24 @@
|
|||||||
import time
|
|
||||||
import threading
|
import threading
|
||||||
import importlib
|
import importlib
|
||||||
|
|
||||||
import dbexport
|
|
||||||
from config.log import logger
|
from config.log import logger
|
||||||
from config import settings
|
from config import settings
|
||||||
from common import utils
|
|
||||||
|
|
||||||
|
|
||||||
class Collect(object):
|
class Collect(object):
|
||||||
"""
|
def __init__(self, domain):
|
||||||
Collect subdomains
|
self.domain = domain
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, domain, export=True):
|
|
||||||
self.domain = utils.get_main_domain(domain)
|
|
||||||
self.elapse = 0.0
|
|
||||||
self.modules = []
|
self.modules = []
|
||||||
self.collect_funcs = []
|
self.collect_funcs = []
|
||||||
self.path = None
|
|
||||||
self.export = export
|
|
||||||
self.format = 'csv'
|
|
||||||
|
|
||||||
def get_mod(self):
|
def get_mod(self):
|
||||||
"""
|
"""
|
||||||
Get modules
|
Get modules
|
||||||
"""
|
"""
|
||||||
if settings.enable_all_module:
|
if settings.enable_all_module:
|
||||||
# modules = ['brute', 'certificates', 'crawl',
|
|
||||||
# 'datasets', 'intelligence', 'search']
|
|
||||||
# The crawl module has some problems
|
# The crawl module has some problems
|
||||||
modules = ['certificates', 'check', 'datasets',
|
modules = ['certificates', 'check', 'datasets',
|
||||||
'dnsquery', 'intelligence', 'search']
|
'dnsquery', 'intelligence', 'search']
|
||||||
# modules = ['certificates']
|
|
||||||
for module in modules:
|
for module in modules:
|
||||||
module_path = settings.module_dir.joinpath(module)
|
module_path = settings.module_dir.joinpath(module)
|
||||||
for path in module_path.rglob('*.py'):
|
for path in module_path.rglob('*.py'):
|
||||||
@@ -55,7 +41,6 @@ class Collect(object):
|
|||||||
"""
|
"""
|
||||||
Class entrance
|
Class entrance
|
||||||
"""
|
"""
|
||||||
start = time.time()
|
|
||||||
logger.log('INFOR', f'Start collecting subdomains of {self.domain}')
|
logger.log('INFOR', f'Start collecting subdomains of {self.domain}')
|
||||||
self.get_mod()
|
self.get_mod()
|
||||||
self.import_func()
|
self.import_func()
|
||||||
@@ -66,8 +51,7 @@ class Collect(object):
|
|||||||
func_obj, func_name = collect_func
|
func_obj, func_name = collect_func
|
||||||
thread = threading.Thread(target=func_obj,
|
thread = threading.Thread(target=func_obj,
|
||||||
name=func_name,
|
name=func_name,
|
||||||
args=(self.domain,),
|
args=(self.domain,))
|
||||||
daemon=True)
|
|
||||||
threads.append(thread)
|
threads.append(thread)
|
||||||
# Start all threads
|
# Start all threads
|
||||||
for thread in threads:
|
for thread in threads:
|
||||||
@@ -82,15 +66,6 @@ class Collect(object):
|
|||||||
if thread.is_alive():
|
if thread.is_alive():
|
||||||
logger.log('ALERT', f'{thread.name} module thread timed out')
|
logger.log('ALERT', f'{thread.name} module thread timed out')
|
||||||
|
|
||||||
# Export
|
|
||||||
if self.export:
|
|
||||||
if not self.path:
|
|
||||||
name = f'{self.domain}.{self.format}'
|
|
||||||
self.path = settings.result_save_dir.joinpath(name)
|
|
||||||
dbexport.export(self.domain, type='table', path=self.path, format=self.format)
|
|
||||||
end = time.time()
|
|
||||||
self.elapse = round(end - start, 1)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
collect = Collect('example.com')
|
collect = Collect('example.com')
|
||||||
|
|||||||
@@ -1,95 +0,0 @@
|
|||||||
"""
|
|
||||||
通过枚举域名常见的SRV记录并做查询来发现子域
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
|
||||||
import queue
|
|
||||||
import threading
|
|
||||||
|
|
||||||
from common import utils
|
|
||||||
from common.module import Module
|
|
||||||
from config.setting import data_storage_dir
|
|
||||||
|
|
||||||
|
|
||||||
class BruteSRV(Module):
|
|
||||||
def __init__(self, domain):
|
|
||||||
Module.__init__(self)
|
|
||||||
self.domain = domain
|
|
||||||
self.module = 'dnsquery'
|
|
||||||
self.source = "BruteSRV"
|
|
||||||
self.qtype = 'SRV' # 利用的DNS记录的SRV记录查询子域
|
|
||||||
self.thread_num = 10
|
|
||||||
self.names_que = queue.Queue()
|
|
||||||
self.answers_que = queue.Queue()
|
|
||||||
|
|
||||||
def gen_names(self):
|
|
||||||
path = data_storage_dir.joinpath('srv_prefixes.json')
|
|
||||||
with open(path, encoding='utf-8', errors='ignore') as file:
|
|
||||||
prefixes = json.load(file)
|
|
||||||
names = map(lambda prefix: prefix + self.domain, prefixes)
|
|
||||||
|
|
||||||
for name in names:
|
|
||||||
self.names_que.put(name)
|
|
||||||
|
|
||||||
def brute(self):
|
|
||||||
"""
|
|
||||||
枚举域名的SRV记录
|
|
||||||
"""
|
|
||||||
self.gen_names()
|
|
||||||
|
|
||||||
for num in range(self.thread_num):
|
|
||||||
thread = BruteThread(self.names_que, self.answers_que)
|
|
||||||
thread.name = f'BruteThread-{num}'
|
|
||||||
thread.daemon = True
|
|
||||||
thread.start()
|
|
||||||
|
|
||||||
self.names_que.join()
|
|
||||||
|
|
||||||
while not self.answers_que.empty():
|
|
||||||
answer = self.answers_que.get()
|
|
||||||
if answer is None:
|
|
||||||
continue
|
|
||||||
for item in answer:
|
|
||||||
record = str(item)
|
|
||||||
subdomains = self.match_subdomains(record)
|
|
||||||
self.subdomains.update(subdomains)
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
"""
|
|
||||||
类执行入口
|
|
||||||
"""
|
|
||||||
self.begin()
|
|
||||||
self.brute()
|
|
||||||
self.finish()
|
|
||||||
self.save_json()
|
|
||||||
self.gen_result()
|
|
||||||
self.save_db()
|
|
||||||
|
|
||||||
|
|
||||||
class BruteThread(threading.Thread):
|
|
||||||
def __init__(self, names_que, answers_que):
|
|
||||||
threading.Thread.__init__(self)
|
|
||||||
self.names_que = names_que
|
|
||||||
self.answers_que = answers_que
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
while True:
|
|
||||||
name = self.names_que.get()
|
|
||||||
answer = utils.dns_query(name, 'SRV')
|
|
||||||
self.answers_que.put(answer)
|
|
||||||
self.names_que.task_done()
|
|
||||||
|
|
||||||
|
|
||||||
def run(domain):
|
|
||||||
"""
|
|
||||||
类统一调用入口
|
|
||||||
|
|
||||||
:param str domain: 域名
|
|
||||||
"""
|
|
||||||
brute = BruteSRV(domain)
|
|
||||||
brute.run()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
run('zonetransfer.me')
|
|
||||||
# run('example.com')
|
|
||||||
+4
-3
@@ -190,9 +190,10 @@ def find_subdomains(domain, data):
|
|||||||
subdomains.update(find_in_resp(domain, req_url, rsp_html))
|
subdomains.update(find_in_resp(domain, req_url, rsp_html))
|
||||||
js_urls.update(find_js_urls(domain, req_url, rsp_html))
|
js_urls.update(find_js_urls(domain, req_url, rsp_html))
|
||||||
|
|
||||||
resp_data = request.urls_request(js_urls)
|
resp_data = request.bulk_request(js_urls)
|
||||||
for resp, text in resp_data:
|
for resp in resp_data:
|
||||||
if not text:
|
if not resp:
|
||||||
continue
|
continue
|
||||||
|
text = utils.decode_resp_text(resp)
|
||||||
subdomains.update(find_in_resp(domain, resp.url, text))
|
subdomains.update(find_in_resp(domain, resp.url, text))
|
||||||
return subdomains
|
return subdomains
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
"""
|
||||||
|
通过枚举域名常见的SRV记录并做查询来发现子域
|
||||||
|
"""
|
||||||
|
|
||||||
|
import queue
|
||||||
|
import threading
|
||||||
|
|
||||||
|
from common import utils
|
||||||
|
from common.module import Module
|
||||||
|
from config.setting import data_storage_dir
|
||||||
|
|
||||||
|
|
||||||
|
class BruteSRV(Module):
|
||||||
|
def __init__(self, domain):
|
||||||
|
Module.__init__(self)
|
||||||
|
self.domain = domain
|
||||||
|
self.module = 'BruteSRV'
|
||||||
|
self.source = "BruteSRV"
|
||||||
|
self.qtype = 'SRV'
|
||||||
|
self.thread_num = 20
|
||||||
|
self.names_queue = queue.Queue()
|
||||||
|
self.answers_queue = queue.Queue()
|
||||||
|
|
||||||
|
def fill_queue(self):
|
||||||
|
path = data_storage_dir.joinpath('srv_prefixes.json')
|
||||||
|
prefixes = utils.load_json(path)
|
||||||
|
for prefix in prefixes:
|
||||||
|
self.names_queue.put(prefix + self.domain)
|
||||||
|
|
||||||
|
def do_brute(self):
|
||||||
|
for num in range(self.thread_num):
|
||||||
|
thread = BruteThread(self.names_queue, self.answers_queue)
|
||||||
|
thread.name = f'BruteThread-{num}'
|
||||||
|
thread.daemon = True
|
||||||
|
thread.start()
|
||||||
|
self.names_queue.join()
|
||||||
|
|
||||||
|
def deal_answers(self):
|
||||||
|
while not self.answers_queue.empty():
|
||||||
|
answer = self.answers_queue.get()
|
||||||
|
if answer is None:
|
||||||
|
continue
|
||||||
|
for item in answer:
|
||||||
|
record = str(item)
|
||||||
|
subdomains = self.match_subdomains(record)
|
||||||
|
self.subdomains.update(subdomains)
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
self.begin()
|
||||||
|
self.fill_queue()
|
||||||
|
self.do_brute()
|
||||||
|
self.deal_answers()
|
||||||
|
self.finish()
|
||||||
|
self.save_json()
|
||||||
|
self.gen_result()
|
||||||
|
self.save_db()
|
||||||
|
|
||||||
|
|
||||||
|
class BruteThread(threading.Thread):
|
||||||
|
def __init__(self, names_queue, answers_queue):
|
||||||
|
threading.Thread.__init__(self)
|
||||||
|
self.names_queue = names_queue
|
||||||
|
self.answers_queue = answers_queue
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
while True:
|
||||||
|
name = self.names_queue.get()
|
||||||
|
answer = utils.dns_query(name, 'SRV')
|
||||||
|
self.answers_queue.put(answer)
|
||||||
|
self.names_queue.task_done()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
brute = BruteSRV('zonetransfer.me')
|
||||||
|
brute.run()
|
||||||
+7
-3
@@ -17,6 +17,7 @@ from brute import Brute
|
|||||||
from common import utils, resolve, request
|
from common import utils, resolve, request
|
||||||
from common.database import Database
|
from common.database import Database
|
||||||
from modules.collect import Collect
|
from modules.collect import Collect
|
||||||
|
from modules.srv import BruteSRV
|
||||||
from modules.finder import Finder
|
from modules.finder import Finder
|
||||||
from modules import iscdn
|
from modules import iscdn
|
||||||
from config import settings
|
from config import settings
|
||||||
@@ -65,7 +66,6 @@ class OneForAll(object):
|
|||||||
python3 oneforall.py --target example.com --show True run
|
python3 oneforall.py --target example.com --show True run
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
--alive True/False Only export alive subdomains or not (default False)
|
|
||||||
--port small/medium/large See details in ./config/setting.py(default small)
|
--port small/medium/large See details in ./config/setting.py(default small)
|
||||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||||
--path Result path (default None, automatically generated)
|
--path Result path (default None, automatically generated)
|
||||||
@@ -188,8 +188,12 @@ class OneForAll(object):
|
|||||||
self.origin_table = self.domain + '_origin_result'
|
self.origin_table = self.domain + '_origin_result'
|
||||||
self.resolve_table = self.domain + '_resolve_result'
|
self.resolve_table = self.domain + '_resolve_result'
|
||||||
|
|
||||||
collect = Collect(self.domain, export=False)
|
collect = Collect(self.domain)
|
||||||
collect.run()
|
collect.run()
|
||||||
|
|
||||||
|
srv = BruteSRV(self.domain)
|
||||||
|
srv.run()
|
||||||
|
|
||||||
if self.brute:
|
if self.brute:
|
||||||
# Due to there will be a large number of dns resolution requests,
|
# Due to there will be a large number of dns resolution requests,
|
||||||
# may cause other network tasks to be error
|
# may cause other network tasks to be error
|
||||||
@@ -240,7 +244,7 @@ class OneForAll(object):
|
|||||||
# Scan subdomain takeover
|
# Scan subdomain takeover
|
||||||
if self.takeover:
|
if self.takeover:
|
||||||
subdomains = utils.get_subdomains(self.data)
|
subdomains = utils.get_subdomains(self.data)
|
||||||
takeover = Takeover(subdomains)
|
takeover = Takeover(targets=subdomains)
|
||||||
takeover.run()
|
takeover.run()
|
||||||
return self.data
|
return self.data
|
||||||
|
|
||||||
|
|||||||
+8
-13
@@ -1,24 +1,19 @@
|
|||||||
aiohttp==3.6.2
|
|
||||||
async-timeout==3.0.1
|
|
||||||
attrs==19.3.0
|
|
||||||
beautifulsoup4==4.9.1
|
beautifulsoup4==4.9.1
|
||||||
bs4==0.0.1
|
bs4==0.0.1
|
||||||
certifi==2020.6.20
|
certifi==2020.6.20
|
||||||
chardet==3.0.4
|
chardet==3.0.4
|
||||||
cloudscraper==1.2.42
|
cloudscraper==1.2.46
|
||||||
colorama==0.4.3
|
colorama==0.4.3
|
||||||
dnspython==1.16.0
|
dnspython==2.0.0
|
||||||
docopt==0.6.2
|
docopt==0.6.2
|
||||||
et-xmlfile==1.0.1
|
et-xmlfile==1.0.1
|
||||||
exrex==0.10.5
|
exrex==0.10.5
|
||||||
fire==0.3.1
|
fire==0.3.1
|
||||||
|
future==0.18.2
|
||||||
idna==2.10
|
idna==2.10
|
||||||
IP2Location==8.4.1
|
|
||||||
jdcal==1.4.1
|
jdcal==1.4.1
|
||||||
loguru==0.5.1
|
loguru==0.5.1
|
||||||
multidict==4.7.6
|
|
||||||
openpyxl==2.4.11
|
openpyxl==2.4.11
|
||||||
psutil==5.7.0
|
|
||||||
pyparsing==2.4.7
|
pyparsing==2.4.7
|
||||||
PySocks==1.7.1
|
PySocks==1.7.1
|
||||||
records==0.5.3
|
records==0.5.3
|
||||||
@@ -27,12 +22,12 @@ requests-file==1.5.1
|
|||||||
requests-toolbelt==0.9.1
|
requests-toolbelt==0.9.1
|
||||||
six==1.15.0
|
six==1.15.0
|
||||||
soupsieve==2.0.1
|
soupsieve==2.0.1
|
||||||
SQLAlchemy==1.3.18
|
SQLAlchemy==1.3.19
|
||||||
tablib==2.0.0
|
tablib==2.0.0
|
||||||
tenacity==6.2.0
|
tenacity==6.2.0
|
||||||
termcolor==1.1.0
|
termcolor==1.1.0
|
||||||
tldextract==2.2.2
|
tldextract==2.2.3
|
||||||
tqdm==4.47.0
|
tqdm==4.48.2
|
||||||
urllib3==1.25.9
|
treelib==1.6.1
|
||||||
|
urllib3==1.25.10
|
||||||
win32-setctime==1.0.1
|
win32-setctime==1.0.1
|
||||||
yarl==1.4.2
|
|
||||||
|
|||||||
+12
-14
@@ -20,7 +20,6 @@ from config.log import logger
|
|||||||
from config import settings
|
from config import settings
|
||||||
from common import utils
|
from common import utils
|
||||||
from common.module import Module
|
from common.module import Module
|
||||||
from common.domain import Domain
|
|
||||||
|
|
||||||
|
|
||||||
def get_fingerprint():
|
def get_fingerprint():
|
||||||
@@ -41,34 +40,32 @@ def get_cname(subdomain):
|
|||||||
return answer.to_text() # 一个子域只有一个CNAME记录
|
return answer.to_text() # 一个子域只有一个CNAME记录
|
||||||
|
|
||||||
|
|
||||||
def get_maindomain(subdomain):
|
|
||||||
return Domain(subdomain).registered()
|
|
||||||
|
|
||||||
|
|
||||||
class Takeover(Module):
|
class Takeover(Module):
|
||||||
"""
|
"""
|
||||||
OneForAll subdomain takeover module
|
OneForAll subdomain takeover module
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
python3 takeover.py --target www.example.com --format csv run
|
python3 takeover.py --target www.example.com --format csv run
|
||||||
python3 takeover.py --target ./subdomains.txt --thread 10 run
|
python3 takeover.py --targets ./subdomains.txt --thread 10 run
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
--format rst/csv/tsv/json/yaml/html/jira/xls/xlsx/dbf/latex/ods (result format)
|
||||||
--path Result directory (default directory is ./results)
|
--path Result directory (default directory is ./results)
|
||||||
|
|
||||||
:param any target: One domain or File path of one domain per line (required)
|
:param str target: One domain (target or targets must be provided)
|
||||||
:param int thread: threads number (default 100)
|
:param str targets: File path of one domain per line
|
||||||
|
:param int thread: threads number (default 20)
|
||||||
:param str format: Result format (default csv)
|
:param str format: Result format (default csv)
|
||||||
:param str path: Result directory (default None)
|
:param str path: Result directory (default None)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, target, thread=100, path=None, format='csv'):
|
def __init__(self, target=None, targets=None, thread=20, path=None, format='csv'):
|
||||||
Module.__init__(self)
|
Module.__init__(self)
|
||||||
self.subdomains = set()
|
self.subdomains = set()
|
||||||
self.module = 'Check'
|
self.module = 'Check'
|
||||||
self.source = 'Takeover'
|
self.source = 'Takeover'
|
||||||
self.target = target
|
self.target = target
|
||||||
|
self.targets = targets
|
||||||
self.thread = thread
|
self.thread = thread
|
||||||
self.path = path
|
self.path = path
|
||||||
self.format = format
|
self.format = format
|
||||||
@@ -101,10 +98,10 @@ class Takeover(Module):
|
|||||||
cname = get_cname(subdomain)
|
cname = get_cname(subdomain)
|
||||||
if cname is None:
|
if cname is None:
|
||||||
return
|
return
|
||||||
maindomain = get_maindomain(cname)
|
main_domain = utils.get_main_domain(cname)
|
||||||
for fingerprint in self.fingerprints:
|
for fingerprint in self.fingerprints:
|
||||||
cnames = fingerprint.get('cname')
|
cnames = fingerprint.get('cname')
|
||||||
if maindomain not in cnames:
|
if main_domain not in cnames:
|
||||||
continue
|
continue
|
||||||
responses = fingerprint.get('response')
|
responses = fingerprint.get('response')
|
||||||
self.compare(subdomain, cname, responses)
|
self.compare(subdomain, cname, responses)
|
||||||
@@ -116,7 +113,6 @@ class Takeover(Module):
|
|||||||
self.subdomainq.task_done()
|
self.subdomainq.task_done()
|
||||||
|
|
||||||
def progress(self):
|
def progress(self):
|
||||||
# 设置进度
|
|
||||||
bar = tqdm()
|
bar = tqdm()
|
||||||
bar.total = len(self.subdomains)
|
bar.total = len(self.subdomains)
|
||||||
bar.desc = 'Check Progress'
|
bar.desc = 'Check Progress'
|
||||||
@@ -127,12 +123,14 @@ class Takeover(Module):
|
|||||||
bar.update()
|
bar.update()
|
||||||
if done == bar.total: # 完成队列中所有子域的检查退出
|
if done == bar.total: # 完成队列中所有子域的检查退出
|
||||||
break
|
break
|
||||||
# bar.close()
|
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
start = time.time()
|
start = time.time()
|
||||||
logger.log('INFOR', f'Start running {self.source} module')
|
logger.log('INFOR', f'Start running {self.source} module')
|
||||||
self.subdomains = utils.get_domains(self.target)
|
if isinstance(self.targets, set):
|
||||||
|
self.subdomains = self.targets
|
||||||
|
else:
|
||||||
|
self.subdomains = utils.get_domains(self.target, self.targets)
|
||||||
self.format = utils.check_format(self.format, len(self.subdomains))
|
self.format = utils.check_format(self.format, len(self.subdomains))
|
||||||
timestamp = utils.get_timestamp()
|
timestamp = utils.get_timestamp()
|
||||||
name = f'takeover_check_result_{timestamp}'
|
name = f'takeover_check_result_{timestamp}'
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ Example
|
|||||||
from oneforall import OneForAll
|
from oneforall import OneForAll
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
test = OneForAll(target='hackfun.org')
|
test = OneForAll(target='mi.com')
|
||||||
test.brute = True
|
test.brute = True
|
||||||
test.req = True
|
test.req = True
|
||||||
test.takeover = True
|
test.takeover = True
|
||||||
|
|||||||
Reference in New Issue
Block a user