v0.0.6

重命名
重写标题获取
2026-08-26 04:47:48 +08:00 · 2019-08-27 17:56:37 +08:00 · 2019-08-27 17:39:54 +08:00 · 2019-08-27 17:36:19 +08:00 · 2019-08-26 18:27:10 +08:00 · 2019-08-25 18:01:50 +08:00
86 changed files with 25855 additions and 24147 deletions
@@ -1,25 +1,24 @@
 ---
-name: Bug报告
-about: 填写好报告有助于快速解决bug
-title: BUG
+name: 提交Bug
+about: "请务必按照模板提交Bug\U0001F64F"
+title: 请填写BUG标题
 labels: bug
 assignees: shmilylty

 ---

 **是否使用了最新代码**
-是或否
-（如果不是的话尝试克隆最新的代码再跑一下）
+是或否（如果不是的话尝试克隆最新的代码再跑一下）

-**bug描述**
+**Bug描述**
 清晰而简洁的Bug描述

 **如何复现**
-复现步骤
+复现步骤（可不写）
 复现命令

 **预期结果**
-清晰而简洁的预期结果描述
+清晰而简洁的预期结果描述（可不写）

 **实际结果**
 清晰而简洁的实际结果描述（如出现什么错误）
@@ -30,6 +29,7 @@ assignees: shmilylty
 **运行环境**
 - 系统：[例如Windows 10 x64]
 - Python版本：[例如3.7.1]
+- OneForAll版本：[例如0.0.6]


 **报错文本**
@@ -6,6 +6,23 @@ OneForAll的更新日志格式基于[Keep a Changelog](https://keepachangelog.co
 OneForAll遵守[语义化版本格式](https://semver.org/)。

 ## Unreleased
+## [0.0.6](https://github.com/shmilylty/oneforall/releases/tag/v0.0.6) - 2019-08-27
+- 修复一些已知问题
+- 添加PassiveDNS查询和Github子域搜索模块
+- 优化FoFa和BufferOver收集模块
+- 更新有关文档
+- 更新依赖
+
+## [0.0.5](https://github.com/shmilylty/oneforall/releases/tag/v0.0.5) - 2019-08-19
+- 修复一些已知Bugs
+- 优化各子域收集接口并添加新的子域收集接口
+- 添加子域DNS解析和子域HTTP探测进度条
+- 添加子域接管风险检查模块及其使用说明
+- 更新OneForAll依赖
+
+## [0.0.4](https://github.com/shmilylty/oneforall/releases/tag/v0.0.4) - 2019-08-11
+### 修复
+- 修复一些已知Bugs

 ## [0.0.3](https://github.com/shmilylty/oneforall/releases/tag/v0.0.3) - 2019-08-08
 ### 修改
@@ -6,6 +6,12 @@
 * **[Black Star](https://github.com/blackstar24)**
  * 模块贡献

-* [**iceMatcha**](https://github.com/iceMatcha)
-  * bug调试
+* **[Echocipher](https://github.com/Echocipher)**
+  * 模块贡献
+
+* **[iceMatcha](https://github.com/iceMatcha)**
+  * 工具测试
+
+* **Anyone**
+  * 工具反馈

@@ -16,11 +16,15 @@ records = "*"
 tldextract = "*"
 exrex = "*"
 aiohttp = "*"
-fire = "==0.2.1"
+fire = "*"
 bs4 = "*"
 cchardet = "*"
 lxml = "*"
 pysocks = "*"
+cloudscraper = "*"
+js2py = "*"
+tablib = "*"
+brotlipy = "*"

 [requires]
 python_version = "3.7"
@@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "d95e37615ca48fe39a66e8c24eff76910c66b01b0739c0ab00ffa0f4df022107"
+            "sha256": "9a29beced25222ff7c32a32aaa99c292ee43986700b280c6153e01245a5a71bc"
        },
        "pipfile-spec": 6,
        "requires": {
@@ -54,10 +54,17 @@
        },
        "aiomultiprocess": {
            "hashes": [
-                "sha256:fd9b616d5145ac2b01f315725277231ffad7e56eb6675885598f6fe861ef9fa3"
+                "sha256:c704383be74ccb806e2617cb3bb6a5faa1774b1c6643d62785c332eb6da9e742"
            ],
            "index": "pypi",
-            "version": "==0.6.0"
+            "version": "==0.6.1"
+        },
+        "asn1crypto": {
+            "hashes": [
+                "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87",
+                "sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49"
+            ],
+            "version": "==0.24.0"
        },
        "async-timeout": {
            "hashes": [
@@ -88,6 +95,74 @@
            ],
            "version": "==4.8.0"
        },
+        "brotli": {
+            "hashes": [
+                "sha256:0538dc1744fd17c314d2adc409ea7d1b779783b89fd95bcfb0c2acc93a6ea5a7",
+                "sha256:0970a47f471782912d7705160b2b0a9306e68e6fadf9cffcaeb42d8f0951e26c",
+                "sha256:113f51658e6fe548dce4b3749f6ef6c24de4184ba9c10a909cbee4261c2a5da0",
+                "sha256:1e1aa9c4d1558889f42749c8baf846007953bfd32c8209230cf1cd1f5ef33495",
+                "sha256:2f2f4f78f29ac4a45d15b3d9fc3fd9705e0ad313a44b129f6e1d0c6916bad0e2",
+                "sha256:3269f6de1dd150fd0cce1c158b61ff5ac06d627fd3ae9c6ea03aed26fbbff7ea",
+                "sha256:50dd9ad2a2bb12da4e9002a438672d182f98e546e99952de80280a1e1729664f",
+                "sha256:5519a4b01b1a4f965083cbfa2ef2b9774c5a5f352341c47b50776ad109423d72",
+                "sha256:5eb27722d320370315971c427eb8aa7cc0791f2a458840d357ac653bd0ad3a14",
+                "sha256:5f06b4d5b6f58e5b5c220c2f23cad034dc5efa51b01fde2351ced1605bd980e2",
+                "sha256:72848d25a5f9e736db4af4512e0c3feecc094d57d241f8f1ae959115a2c39756",
+                "sha256:743001bca75f4a6b4454be3510feca46f9d61a0c782a9bc2bc684bdb245e279e",
+                "sha256:9d1c2dd27a1083fefd05b1b2f8df4a6bc2aaa6c21dd82cd41c8ae5e7c23a87f8",
+                "sha256:a13ce9b419fe9f277c63f700efb0e444331509d1881b5610d2ba7e9080606967",
+                "sha256:a19ef0952b9d2803df88dff07f45a6c92d5676afb9b8d69cf32232d684036d11",
+                "sha256:ad766ca8b8c1419b71a22756b45264f45725c86133dc80a7cbe30b6b78c75620",
+                "sha256:ad7963f261988ee0883816b6b9f206f11461c9b3cb5cfbca0c9ab5adc406d395",
+                "sha256:c16201060c5a3f8742e3deae759014251ac92f382f82bc2a41dc079ff18c3f24",
+                "sha256:c43b202f65891861a9a336984a103de25de235f756de69e32db893156f767013",
+                "sha256:c675c6cce4295cb1a692f3de7416aacace7314e064b94bc86e93aceefce7fd3e",
+                "sha256:d17cec0b992b1434f5f9df9986563605a4d1b1acd5574c87fc2ac014bcbd3316",
+                "sha256:dc91f6129953861a73d9a65c52a8dd682b561a9ebaf65283541645cab6489917",
+                "sha256:e2f4cbd1760d2bf2f30e396c2301999aab0191aec031a6a8a04950b2f575a536",
+                "sha256:f192e6d3556714105c10486bbd6d045e38a0c04d9da3cef21e0a8dfd8e162df4",
+                "sha256:f775b07026af2b1b0b5a8b05e41571cdcf3a315a67df265d60af301656a5425b",
+                "sha256:f969ec7f56ba9636679e69ca07fba548312ccaca37412ee823c7f413541ad7e0",
+                "sha256:f9dc52cd70907aafb99a773b66b156f2f995c7a0d284397c487c8b71ddbef2f9",
+                "sha256:fc7212e36ebeb81aebf7949c92897b622490d7c0e333a479c0395591e7994600"
+            ],
+            "version": "==1.0.7"
+        },
+        "brotlipy": {
+            "hashes": [
+                "sha256:07194f4768eb62a4f4ea76b6d0df6ade185e24ebd85877c351daa0a069f1111a",
+                "sha256:091b299bf36dd6ef7a06570dbc98c0f80a504a56c5b797f31934d2ad01ae7d17",
+                "sha256:09ec3e125d16749b31c74f021aba809541b3564e5359f8c265cbae442810b41a",
+                "sha256:0be698678a114addcf87a4b9496c552c68a2c99bf93cf8e08f5738b392e82057",
+                "sha256:0fa6088a9a87645d43d7e21e32b4a6bf8f7c3939015a50158c10972aa7f425b7",
+                "sha256:1ea4e578241504b58f2456a6c69952c88866c794648bdc74baee74839da61d44",
+                "sha256:2699945a0a992c04fc7dc7fa2f1d0575a2c8b4b769f2874a08e8eae46bef36ae",
+                "sha256:2a80319ae13ea8dd60ecdc4f5ccf6da3ae64787765923256b62c598c5bba4121",
+                "sha256:2e5c64522364a9ebcdf47c5744a5ddeb3f934742d31e61ebfbbc095460b47162",
+                "sha256:36def0b859beaf21910157b4c33eb3b06d8ce459c942102f16988cca6ea164df",
+                "sha256:3a3e56ced8b15fbbd363380344f70f3b438e0fd1fcf27b7526b6172ea950e867",
+                "sha256:3c1d5e2cf945a46975bdb11a19257fa057b67591eb232f393d260e7246d9e571",
+                "sha256:50ca336374131cfad20612f26cc43c637ac0bfd2be3361495e99270883b52962",
+                "sha256:5de6f7d010b7558f72f4b061a07395c5c3fd57f0285c5af7f126a677b976a868",
+                "sha256:637847560d671657f993313ecc6c6c6666a936b7a925779fd044065c7bc035b9",
+                "sha256:653faef61241bf8bf99d73ca7ec4baa63401ba7b2a2aa88958394869379d67c7",
+                "sha256:786afc8c9bd67de8d31f46e408a3386331e126829114e4db034f91eacb05396d",
+                "sha256:79aaf217072840f3e9a3b641cccc51f7fc23037496bd71e26211856b93f4b4cb",
+                "sha256:7e31f7adcc5851ca06134705fcf3478210da45d35ad75ec181e1ce9ce345bb38",
+                "sha256:8b39abc3256c978f575df5cd7893153277216474f303e26f0e43ba3d3969ef96",
+                "sha256:9448227b0df082e574c45c983fa5cd4bda7bfb11ea6b59def0940c1647be0c3c",
+                "sha256:96bc59ff9b5b5552843dc67999486a220e07a0522dddd3935da05dc194fa485c",
+                "sha256:a07647886e24e2fb2d68ca8bf3ada398eb56fd8eac46c733d4d95c64d17f743b",
+                "sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2",
+                "sha256:b4c98b0d2c9c7020a524ca5bbff42027db1004c6571f8bc7b747f2b843128e7a",
+                "sha256:c6cc0036b1304dd0073eec416cb2f6b9e37ac8296afd9e481cac3b1f07f9db25",
+                "sha256:d2c1c724c4ac375feb2110f1af98ecdc0e5a8ea79d068efb5891f621a5b235cb",
+                "sha256:dc6c5ee0df9732a44d08edab32f8a616b769cc5a4155a12d2d010d248eb3fb07",
+                "sha256:fd1d1c64214af5d90014d82cee5d8141b13d44c92ada7a0c0ec0679c6f15a471"
+            ],
+            "index": "pypi",
+            "version": "==0.7.0"
+        },
        "bs4": {
            "hashes": [
                "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"
@@ -171,6 +246,14 @@
            ],
            "version": "==3.0.4"
        },
+        "cloudscraper": {
+            "hashes": [
+                "sha256:7080a4c9ea6fe244422551172eb28a1d6c042da4c9d56d15ff319dbbac2ab6d7",
+                "sha256:79d04cd3c7d782035b404b5093a0cdbacd219cb883134017f4f7ba703f12a93c"
+            ],
+            "index": "pypi",
+            "version": "==1.1.40"
+        },
        "colorama": {
            "hashes": [
                "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d",
@@ -179,6 +262,27 @@
            "markers": "sys_platform == 'win32'",
            "version": "==0.4.1"
        },
+        "cryptography": {
+            "hashes": [
+                "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c",
+                "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643",
+                "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216",
+                "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799",
+                "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a",
+                "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9",
+                "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc",
+                "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8",
+                "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53",
+                "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1",
+                "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609",
+                "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292",
+                "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e",
+                "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6",
+                "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed",
+                "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d"
+            ],
+            "version": "==2.7"
+        },
        "defusedxml": {
            "hashes": [
                "sha256:6687150770438374ab581bb7a1b327a847dd9c5749e396102de3fad4e8a3ef93",
@@ -234,6 +338,14 @@
            ],
            "version": "==1.4.1"
        },
+        "js2py": {
+            "hashes": [
+                "sha256:6e5628abfff2fb4051e8e77a353e44831f474e2ceb865278271897f7f326aeb6",
+                "sha256:bf87cb4432944470f11fed9c1cb8d0312dd505e7b867362f55102f24379ab94f"
+            ],
+            "index": "pypi",
+            "version": "==0.66"
+        },
        "loguru": {
            "hashes": [
                "sha256:b6fad0d7aed357b5c147edcc6982606b933754338950b72d8123f48c150c5a4f",
@@ -244,31 +356,31 @@
        },
        "lxml": {
            "hashes": [
-                "sha256:06e5599b9c54f797a3c0f384c67705a0d621031007aa2400a6c7d17300fdb995",
-                "sha256:092237cfe4ece074401b75001a2e525fa6e1fb9d40fee8b7b132b1947d3bd2f8",
-                "sha256:0b6d49d0a26fe8207df8dd27c40b75be4deb2277173903aa76ec3e82df77cbe7",
-                "sha256:0f77061c20b4f32b1cf39e8f661c74e966344084c996e7b23c3a94e472461df0",
-                "sha256:0fef86edfa2f146b4b0ae2c6c05c3e4a8f3388b3655eafbc4aab3247f4dabb24",
-                "sha256:2f163c8844db4ed06a230ef092e2461ad01830972a896b8f3cf8b5bac70ae85d",
-                "sha256:350333190052bbfbc3222b1805b59b7979d7276e57af2257367e15a2db27082d",
-                "sha256:3b57dc5ed7b6a7d852c961f2389ca99404c2b59fd2088baec6fbaca02f688be4",
-                "sha256:3e86e5df4a8edd6f725f3c76f1d45e046d4f3aa40478092e4f5f373ad1f526e2",
-                "sha256:43dac60d10341d3e56be089cd0798b70e70d45ce32279f4c3190d8cbd71350e4",
-                "sha256:4665ee84ac8ba11d58f1ed517e29ea8536b4ae4e0c6fb6c7d3dce70abcd279f0",
-                "sha256:5033cf606a7cb559db967689b1b2e743994000f783607ba4c484e90917395ad7",
-                "sha256:75d731af05bf40f808d7716e0d26b4b02913402f861c032ce8c36efca350ae72",
-                "sha256:7720174604c7647e357566ac9e4d135c137caed5e7b01223551a4c81c8dc8b9a",
-                "sha256:b33ec641309bcea40c76c1b105f988e4e8f9a2f1ee1486aa5c0eeef33956c9bb",
-                "sha256:d1135dc0ac197242028ede085b693ba1f2bff7f0f9b91080e2540348312bfa53",
-                "sha256:d5a61e9c2322b45f259909a02b76bc98c4641214e22a37191d00c151aa9cdb9a",
-                "sha256:da22c4b17bc17dad9c8faf6d94c8fe568ac71c867a56631ab874da418fc7f8f7",
-                "sha256:da5c48ec9f8d8b5df42d328b6d1fb8d9413cd664a2367ef4f6f7cc48ee5b82c0",
-                "sha256:db2794bad21b7b30b6849b4e1537171cae8a7087711d958d69c233470dc612e7",
-                "sha256:f1c2f67df727034f94ccb590142d1d110f3dd38f638a4f1567fdd9f39892ba05",
-                "sha256:f840dddded8b046edc774c88ed8d2442cdb231a68894c42c74e3a809450fae76"
+                "sha256:02ca7bf899da57084041bb0f6095333e4d239948ad3169443f454add9f4e9cb4",
+                "sha256:096b82c5e0ea27ce9138bcbb205313343ee66a6e132f25c5ed67e2c8d960a1bc",
+                "sha256:0a920ff98cf1aac310470c644bc23b326402d3ef667ddafecb024e1713d485f1",
+                "sha256:17cae1730a782858a6e2758fd20dd0ef7567916c47757b694a06ffafdec20046",
+                "sha256:17e3950add54c882e032527795c625929613adbd2ce5162b94667334458b5a36",
+                "sha256:1f4f214337f6ee5825bf90a65d04d70aab05526c08191ab888cb5149501923c5",
+                "sha256:2e8f77db25b0a96af679e64ff9bf9dddb27d379c9900c3272f3041c4d1327c9d",
+                "sha256:4dffd405390a45ecb95ab5ab1c1b847553c18b0ef8ed01e10c1c8b1a76452916",
+                "sha256:6b899931a5648862c7b88c795eddff7588fb585e81cecce20f8d9da16eff96e0",
+                "sha256:726c17f3e0d7a7200718c9a890ccfeab391c9133e363a577a44717c85c71db27",
+                "sha256:760c12276fee05c36f95f8040180abc7fbebb9e5011447a97cdc289b5d6ab6fc",
+                "sha256:796685d3969815a633827c818863ee199440696b0961e200b011d79b9394bbe7",
+                "sha256:891fe897b49abb7db470c55664b198b1095e4943b9f82b7dcab317a19116cd38",
+                "sha256:a471628e20f03dcdfde00770eeaf9c77811f0c331c8805219ca7b87ac17576c5",
+                "sha256:a63b4fd3e2cabdcc9d918ed280bdde3e8e9641e04f3c59a2a3109644a07b9832",
+                "sha256:b0b84408d4eabc6de9dd1e1e0bc63e7731e890c0b378a62443e5741cfd0ae90a",
+                "sha256:be78485e5d5f3684e875dab60f40cddace2f5b2a8f7fede412358ab3214c3a6f",
+                "sha256:c27eaed872185f047bb7f7da2d21a7d8913457678c9a100a50db6da890bc28b9",
+                "sha256:c81cb40bff373ab7a7446d6bbca0190bccc5be3448b47b51d729e37799bb5692",
+                "sha256:d11874b3c33ee441059464711cd365b89fa1a9cf19ae75b0c189b01fbf735b84",
+                "sha256:e9c028b5897901361d81a4718d1db217b716424a0283afe9d6735fe0caf70f79",
+                "sha256:fe489d486cd00b739be826e8c1be188ddb74c7a1ca784d93d06fda882a6a1681"
            ],
            "index": "pypi",
-            "version": "==4.4.0"
+            "version": "==4.4.1"
        },
        "multidict": {
            "hashes": [
@@ -340,6 +452,20 @@
            ],
            "version": "==2.19"
        },
+        "pyjsparser": {
+            "hashes": [
+                "sha256:2b12842df98d83f65934e0772fa4a5d8b123b3bc79f1af1789172ac70265dd21",
+                "sha256:be60da6b778cc5a5296a69d8e7d614f1f870faf94e1b1b6ac591f2ad5d729579"
+            ],
+            "version": "==2.7.1"
+        },
+        "pyopenssl": {
+            "hashes": [
+                "sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200",
+                "sha256:c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6"
+            ],
+            "version": "==19.0.0"
+        },
        "pysocks": {
            "hashes": [
                "sha256:15d38914b60dbcb231d276f64882a20435c049450160e953ca7d313d1405f16f",
@@ -349,6 +475,13 @@
            "index": "pypi",
            "version": "==1.7.0"
        },
+        "pytz": {
+            "hashes": [
+                "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32",
+                "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7"
+            ],
+            "version": "==2019.2"
+        },
        "pyyaml": {
            "hashes": [
                "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9",
@@ -390,6 +523,13 @@
            ],
            "version": "==1.4.3"
        },
+        "requests-toolbelt": {
+            "hashes": [
+                "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f",
+                "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"
+            ],
+            "version": "==0.9.1"
+        },
        "six": {
            "hashes": [
                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
@@ -399,23 +539,24 @@
        },
        "soupsieve": {
            "hashes": [
-                "sha256:72b5f1aea9101cf720a36bb2327ede866fd6f1a07b1e87c92a1cc18113cbc946",
-                "sha256:e4e9c053d59795e440163733a7fec6c5972210e1790c507e4c7b051d6c5259de"
+                "sha256:8662843366b8d8779dec4e2f921bebec9afd856a5ff2e82cd419acc5054a1a92",
+                "sha256:a5a6166b4767725fd52ae55fee8c8b6137d9a51e9f1edea461a062a759160118"
            ],
-            "version": "==1.9.2"
+            "version": "==1.9.3"
        },
        "sqlalchemy": {
            "hashes": [
-                "sha256:217e7fc52199a05851eee9b6a0883190743c4fb9c8ac4313ccfceaffd852b0ff"
+                "sha256:0459bf0ea6478f3e904de074d65769a11d74cdc34438ab3159250c96d089aef0"
            ],
            "markers": "python_version >= '3.0'",
-            "version": "==1.3.6"
+            "version": "==1.3.7"
        },
        "tablib": {
            "hashes": [
                "sha256:0f88a9cebdaa1a2cc29ae57387082ee81015d1149ecd34e48a8c8d3b4dd21670",
                "sha256:5f33c079b07eb10cf9c4b4696add2ecf32c89db7729240546ecdcd5c92f67e13"
            ],
+            "index": "pypi",
            "version": "==0.13.0"
        },
        "termcolor": {
@@ -434,11 +575,18 @@
        },
        "tqdm": {
            "hashes": [
-                "sha256:14a285392c32b6f8222ecfbcd217838f88e11630affe9006cd0e94c7eff3cb61",
-                "sha256:25d4c0ea02a305a688e7e9c2cdc8f862f989ef2a4701ab28ee963295f5b109ab"
+                "sha256:438d6a735167099d75e5fd9a55175c6727c4dbba345ae406b2886c2728fe3e80",
+                "sha256:ebc205051d79b49989140f5f6c73ec23fce5f590cbc4d9cd6e4c47f168fa0f10"
            ],
            "index": "pypi",
-            "version": "==4.32.2"
+            "version": "==4.34.0"
+        },
+        "tzlocal": {
+            "hashes": [
+                "sha256:11c9f16e0a633b4b60e1eede97d8a46340d042e67b670b290ca526576e039048",
+                "sha256:949b9dd5ba4be17190a80c0268167d7e6c92c62b30026cf9764caf3e308e5590"
+            ],
+            "version": "==2.0.0"
        },
        "urllib3": {
            "hashes": [
@@ -3,16 +3,20 @@
 [![Build Status](https://travis-ci.org/shmilylty/OneForAll.svg?branch=master)](https://travis-ci.org/shmilylty/OneForAll)
 [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
-[![License](https://img.shields.io/github/license/shmilylty/OneForAll)](./LICENSE)
-[![python](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue)](./)
-[![python](https://img.shields.io/badge/release-v0.0.3-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
+[![python](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/release-v0.0.6-brightgreen)](https://github.com/shmilylty/OneForAll/tree/master/releases)

-👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](./README.en.md)
+👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/README.en.md)

-![](./images/All_Might.jpg)
+![](https://raw.githubusercontent.com/shmilylty/OneForAll/master/images/All_Might.jpg)

 ## 🎉项目简介

+项目主页：[https://shmilylty.github.io/OneForAll/](https://shmilylty.github.io/OneForAll/)
+
+项目地址：[https://github.com/shmilylty/OneForAll](https://github.com/shmilylty/OneForAll)
+
 在渗透测试中信息收集的重要性不言而喻，子域收集是信息收集中必不可少且非常重要的一环，目前网上也开源了许多子域收集的工具，但是总是存在以下部分问题：

 * **不够强大**，子域收集的接口不够多，不能做到对批量子域自动收集，没有自动子域解析，验证，FUZZ以及信息拓展等功能。
@@ -28,42 +32,72 @@

 ## 👍功能特性

-* **收集能力强大**，详细模块请阅读[收集模块说明](./docs/collection_modules.md)。
+* **收集能力强大**，详细模块请阅读[收集模块说明](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md)。
  1. 利用证书透明度收集子域（目前有6个模块：`censys_api`，`certdb_api`，`certspotter`，`crtsh`，`entrust`，`google`）

-  2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，后续会添加检查NSEC记录，NSEC记录等模块）
+  2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，检查robots文件`robots`，检查sitemap文件`sitemap`，后续会添加检查NSEC记录，NSEC3记录等模块）

  3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善）

-  4. 利用DNS数据集收集子域（目前有16个模块：`binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `ptrarchive`, `sitedossier`）
+  4. 利用DNS数据集收集子域（目前有18个模块：`binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`,  `ptrarchive`, `sitedossier`,`threatcrowd`）

  5. 利用DNS查询收集子域（目前有1个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，该模块还有待添加和完善）

  6. 利用威胁情报平台数据收集子域（目前有5个模块：`riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）

-  7. 利用搜索引擎发现子域（目前有15个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
-
-* **处理功能强大**，发现的子域结果支持自动去除，自动DNS解析，HTTP请求探测，自动移除无效子域，拓展子域的Banner信息，最终支持的导出格式有`csv`, `tsv`, `json`, `yaml`, `html`, `xls`, `xlsx`, `dbf`, `latex`, `ods`。
-
-* **速度极速**，[收集模块](./oneforall/collect.py)使用多线程调用，[爆破模块](./oneforall/aiobrute.py)使用异步多进程多协程，DNS解析和HTTP请求使用异步多协程。
+  7. 利用搜索引擎发现子域（目前有16个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
+* **支持子域爆破**，该模块有常规的字典爆破，也有自定义的fuzz模式，支持批量爆破和递归爆破，自动判断泛解析并处理。
+* **支持子域验证**，默认开启子域验证，自动解析子域DNS，自动请求子域获取title和banner，并综合判断子域存活情况。
+* **支持子域接管**，默认开启子域接管风险检查，支持子域自动接管（目前只有Github，有待完善），支持批量检查。
+* **处理功能强大**，发现的子域结果支持自动去除，自动DNS解析，HTTP请求探测，自动筛选出有效子域，拓展子域的Banner信息，最终支持的导出格式有`txt`, `rst`, `csv`, `tsv`, `json`, `yaml`, `html`, `xls`, `xlsx`, `dbf`, `latex`, `ods`。
+* **速度极快**，[收集模块](https://github.com/shmilylty/OneForAll/tree/master/oneforall/collect.py)使用多线程调用，[爆破模块](https://github.com/shmilylty/OneForAll/tree/master/oneforall/aiobrute.py)使用异步多进程多协程，子域验证中DNS解析和HTTP请求使用异步多协程，多线程检查[子域接管](https://github.com/shmilylty/OneForAll/tree/master/oneforall/takeover.py)风险。
+* **体验良好**，日志和终端输出全使用中文，各大模块都有进度条，异步保存各模块结果。

 ## 🚀上手指南

-由于该项目**处于开发中**，会不断进行更新迭代，下载使用最好**克隆**最新项目。
+📢由于该项目**处于开发中**，会不断进行更新迭代，下载使用最好**克隆**最新项目，请务必花一点时间阅读此文档，有助于你快速熟悉OneForAll！

 **🐍安装要求**

-1. Python 3.6-3.7
+OneForAll是基于CPython开发的，所以你需要Python环境才能运行，如果你的系统还没有Python环境你可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)，理论上Python 3.6，3.7和3.8都可以正常运行OneForAll，**但是**许多测试都是在Python 3.7上进行的，所以**推荐**你使用**Python 3.7**版本运行OneForAll。运行以下命令检查Python和pip3版本：
+```bash
+python -V
+pip3 -V
+```
+如果你看到以下类似输出便说明Python环境没有问题：
+```bash
+Python 3.7.4
+pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.7)
+```

 **✔安装步骤**

-1. 下载更新
+1. **下载**
+   本项目已经在[码云](https://gitee.com/shmilylty/OneForAll.git)(Gitee)镜像了一份，国内推荐使用码云进行克隆比较快：

+   ```bash
+   git clone https://gitee.com/shmilylty/OneForAll.git
+   ```
+   或者：
   ```bash
   git clone https://github.com/shmilylty/OneForAll.git
   ```

-   ❗如果你之前已经克隆了项目运行之前请**备份**自己修改过的文件到项目外的地方（如**config.py**），然后执行以下命令**更新**项目：
+2. **安装**
+   首先运行以下命令
+   你可以通过pip3安装OneForAll的依赖（如果你熟悉[pipenv](https://docs.pipenv.org/en/latest/)，那么推荐你使用[pipenv安装依赖]((https://github.com/shmilylty/OneForAll/tree/master/docs/Installation_dependency.md))），以下为**Windows系统**下使用**pip3**安装依赖的示例：（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python37`，那么请以管理员身份运行命令提示符cmd执行以下命令！）
+
+   ```bash
+   cd OneForAll/
+   python -m pip install --user  -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+   pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+   cd oneforall/
+   python oneforall.py --help
+   ```
+其他系统平台的请参考[依赖安装](https://github.com/shmilylty/OneForAll/tree/master/docs/installation_dependency.md)，如果在安装依赖过程中发现编译某个依赖库失败时可以参考[Q&A](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md)中解决方法，如果还没有解决欢迎加群反馈。
+
+3. **更新**
+   ❗注意：如果你之前已经克隆了项目运行之前请**备份**自己修改过的文件到项目外的地方（如**config.py**），然后执行以下命令**更新**项目：

   ```bash
   git fetch --all
@@ -71,49 +105,36 @@
   git pull
   ```

-2. 安装依赖
-
-   * 使用pipenv
-
-     ```bash
-     pip3 install pipenv
-     cd OneForAll/
-     pipenv install --python 3.7
-     cd oneforall
-     pipenv run python3 oneforall.py --help
-     ```
-
-   * 使用pip3
-   
-     ```bash
-     cd OneForAll/oneforall
-     pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
-     python3 oneforall.py --help
-     ```
-
-
-如果在安装依赖过程中发现编译某个依赖库失败时可以参考[编译失败解决方法](./docs/building_fail_solution.md)，如果还没有解决欢迎加群反馈。
-
 **✨使用演示**

-```bash
-python3 oneforall.py --target example.com run
-```
+1. 如果你是通过pip3安装的依赖则使用以下命令运行示例：   
+    ```bash
+    cd oneforall/
+    python3 oneforall.py --target example.com run
+    ```

-![Example](./docs/usage_example.svg)
+    ![Example](./docs/usage_example.svg)
+
+2. 如果你通过pipenv安装的依赖则使用以下命令运行示例：
+   ```bash
+   cd oneforall/
+   pipenv run python oneforall.py --target example.com run
+   ```

 **🤔使用帮助**

-命令行参数只提供了一些常用参数，更多详细的参数配置请见[config.py](./oneforall/config.py)，如果你认为有些参数是命令界面经常使用到的或缺少了什么参数等问题非常欢迎反馈。由于众所周知的原因，如果要使用一些被墙的收集接口请先到[config.py](./oneforall/config.py)配置代理，有些收集模块需要提供API（大多都是可以注册账号免费获取），如果需要使用请到[config.py](./oneforall/config.py)配置API信息，如果不使用请忽略有关报错提示。（详细模块请阅读[收集模块说明](./docs/collection_modules.md)）
+命令行参数只提供了一些常用参数，更多详细的参数配置请见[config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py)，如果你认为有些参数是命令界面经常使用到的或缺少了什么参数等问题非常欢迎反馈。由于众所周知的原因，如果要使用一些被墙的收集接口请先到[config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py)配置代理，有些收集模块需要提供API（大多都是可以注册账号免费获取），如果需要使用请到[config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py)配置API信息，如果不使用请忽略有关报错提示。（详细模块请阅读[收集模块说明](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md)）

 OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实现，有关Fire更高级使用方法请参阅[使用Fire CLI](https://github.com/google/python-fire/blob/master/docs/using-cli.md)，有任何使用疑惑欢迎加群交流。

-oneforall.py是主程序入口，oneforall.py里有调用aiobrute.py和dbexport.py，为了方便进行子域爆破和数据库导出独立出了aiobrute.py和dbexport.py，这两个文件可以单独运行，并且所接受参数要更丰富一点。
+[oneforall.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/oneforall.py)是主程序入口，oneforall.py可以调用[aiobrute.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/aiobrute.py)，[takerover.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/takerover.py)及[dbexport.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/dbexport.py)等模块，为了方便进行子域爆破独立出了aiobrute.py，为了方便进行子域接管风险检查独立出了takerover.py，为了方便数据库导出独立出了dbexport.py，这些模块都可以单独运行，并且所接受参数要更丰富一点。

-1. oneforall.py使用帮助
+❗注意：当你在使用过程中遇到一些问题或者疑惑时，请先到[Issues](https://github.com/shmilylty/OneForAll/tree/master/issues)里使用搜索找找答案，还可以参阅[常见问题与回答](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md)。
+
+1. **oneforall.py使用帮助**

   ```bash
-   pipenv run python oneforall.py --help
+   python oneforall.py --help
   ```
   ```bash
   NAME
@@ -123,18 +144,83 @@ oneforall.py是主程序入口，oneforall.py里有调用aiobrute.py和dbexport.
       oneforall.py --target=TARGET <flags>
   
   DESCRIPTION
-       Version: 0.0.2
-       Project: https://github.com/shmilylty/OneForAll/
+       Version: 0.0.6
+       Project: https://git.io/fjHT1
   
       Example:
-           python oneforall.py --target example.com run
-           python oneforall.py --target example.com --brute True --port medium valid 1 run
-           python oneforall.py --target ./domains.txt --format csv --path= ./result.csv  --output True run
+           python3 oneforall.py --target example.com run
+           python3 oneforall.py --target ./domains.txt run
+           python3 oneforall.py --target example.com --brute True run
+           python3 oneforall.py --target example.com --verify False run
+           python3 oneforall.py --target example.com --valid None run
+           python3 oneforall.py --target example.com --port medium run
+           python3 oneforall.py --target example.com --format csv run
+           python3 oneforall.py --target example.com --show True run
   
       Note:
-           参数valid可选值有1，0，None，分别表示导出有效，无效，全部子域
+           参数valid可选值1，0，None分别表示导出有效，无效，全部子域
+           参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
           参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
-           参数format可选格式有'csv','tsv','json','yaml','html','xls','xlsx','dbf','latex','ods'
+           参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                             'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   ARGUMENTS
+       TARGET
+           单个域名或者每行一个域名的文件路径(必需参数)
+   
+   FLAGS
+       --brute=BRUTE
+           使用爆破模块(默认False)
+       --verify=VERIFY
+           验证子域有效性(默认True)
+       --port=PORT
+           请求验证的端口范围(默认medium)
+       --valid=VALID
+           导出子域的有效性(默认1)
+       --path=PATH
+           导出路径(默认None)
+       --format=FORMAT
+           导出格式(默认xlsx)
+       --show=SHOW
+           终端显示导出数据(默认False)
+   ```
+
+2. **aiobrute.py使用帮助**
+
+   关于泛解析问题处理程序首先会访问一个随机的子域判断是否泛解析，如果使用了泛解析则是通过以下判断处理：
+   - 一是主要是与泛解析的IP集合和TTL值做对比，可以参考[这篇文章](http://sh3ll.me/archives/201704041222.txt)。
+   - 二是多次解析到同一IP集合次数（默认设置为10，可以在config.py设置大小）。
+   - 三是考虑爆破效率问题目前还没有加上HTTP响应体相似度对比和响应体内容判断，如果有必要后续添加。
+   经过不严谨测试在16核心的CPU，使用16进程64协程，100M带宽的环境下，设置任务分割为50000，跑两百万字典大概10分钟左右跑完，大概3333个子域每秒。
+
+   ```bash
+   python aiobrute.py --help
+   ```
+
+   ```bash
+   NAME
+       aiobrute.py - OneForAll多进程多协程异步子域爆破模块
+   
+   SYNOPSIS
+       aiobrute.py --target=TARGET <flags>
+   
+   DESCRIPTION
+       Example：
+           python3 aiobrute.py --target example.com run
+           python3 aiobrute.py --target ./domains.txt run
+           python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
+           python3 aiobrute.py --target example.com --wordlist subdomains.txt run
+           python3 aiobrute.py --target example.com --recursive True --depth 2 run
+           python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
+   
+       Note:
+           参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为任务组，
+           当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+           十万字典建议设置为5000，百万字典设置为50000
+           参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+           参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                             'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
   
   ARGUMENTS
@@ -142,121 +228,38 @@ oneforall.py是主程序入口，oneforall.py里有调用aiobrute.py和dbexport.
           单个域名或者每行一个域名的文件路径
   
   FLAGS
-       --brute=BRUTE
-           是否使用爆破模块（默认禁用）
-       --port=PORT
-           HTTP请求验证的端口范围（默认medium）
-       --valid=VALID
-           导出子域的有效性（默认1）
-       --path=PATH
-           导出路径(默认None)
-       --format=FORMAT
-           导出格式（默认xlsx）
-       --output=OUTPUT
-           是否将导出数据输出到终端（默认False）
-   ```
-
-2. aiobrute.py使用帮助
-
-   关于泛解析问题处理程序首先会访问一个随机的子域判断是否泛解析，如果使用了泛解析则是通过以下判断处理：
-   - 一是主要是与泛解析的IP集合和TTL值做对比，可以参考[这篇文章](http://sh3ll.me/archives/201704041222.txt)。
-   - 二是多次解析到同一IP集合次数（默认设置为10，可以在config.py设置大小）
-   - 考虑爆破效率问题目前还没有加上HTTP响应体相似度对比和响应体内容判断
-   经过测试在16核心的CPU，使用16进程64协程，100M带宽的环境下，设置任务分割为50000，跑两百万字典大概10分钟左右跑完，大概3333个子域每秒。
-
-   ```bash
-   pipenv run python aiobrute.py --help
-   ```
-
-   ```bash
-    NAME
-        aiobrute.py - OneForAll多进程多协程异步子域爆破模块
-
-    SYNOPSIS
-        aiobrute.py COMMAND | --target=TARGET <flags>
-
-    DESCRIPTION
-        Example：
-            python aiobrute.py --target example.com run
-            python aiobrute.py --target ./domains.txt run
-            python aiobrute.py --target example.com --processes 4 --coroutine 64 --wordlist data/subdomains.txt run
-            python aiobrute.py --target example.com --recursive True --depth 2 --namelist data/next_subdomains.txt run
-            python aiobrute.py --target www.{fuzz}.example.com --fuzz True --rule [a-z][0-9] run
-
-        Note:
-            参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为一任务组，
-            当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
-            十万字典建议设置为5000，百万字典设置为50000
-
-    ARGUMENTS
-        TARGET
-            单个域名或者每行一个域名的文件路径
-
-    FLAGS
-        --processes=PROCESSES
-            爆破的进程数(默认CPU核心数)
-        --coroutine=COROUTINE
-            每个爆破进程下的协程数(默认16)
-        --wordlist=WORDLIST
-            指定爆破所使用的字典路径(默认使用config.py配置)
-        --segment=SEGMENT
-            爆破任务分割(默认500)
-        --recursive=RECURSIVE
-            是否使用递归爆破(默认False)
-        --depth=DEPTH
-            递归爆破的深度(默认2)
-        --namelist=NAMELIST
-            指定递归爆破所使用的字典路径(默认使用config.py配置)
-        --fuzz=FUZZ
-            是否使用fuzz模式进行爆破(默认False，开启必须指定fuzz正则规则)
-        --rule=RULE
-            fuzz模式使用的正则规则(默认使用config.py配置)
-
-   ```
-
-3. dbexport.py使用帮助
-
-   ```bash
-   pipenv run python dbexport.py --help
-   ```
-
-   ```bash
-   NAME
-       dbexport.py - OneForAll数据库导出模块
-   
-   SYNOPSIS
-       dbexport.py TABLE <flags>
-   
-   DESCRIPTION
-       Example:
-           python dbexport.py --db result.db --table name --format csv --output False
-           python dbexport.py --db result.db --table name --format csv --path= ./result.csv
-   
-       Note:
-           参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
-           参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
-           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
-   
-   POSITIONAL ARGUMENTS
-       TABLE
-           要导出的表
-   
-   FLAGS
-       --db=DB
-           要导出的数据库路径(默认为results/result.sqlite3)
+       --process=PROCESS
+           爆破的进程数(默认CPU核心数)
+       --coroutine=COROUTINE
+           每个爆破进程下的协程数(默认64)
+       --wordlist=WORDLIST
+           指定爆破所使用的字典路径(默认使用config.py配置)
+       --segment=SEGMENT
+           爆破任务分割(默认500)
+       --recursive=RECURSIVE
+           是否使用递归爆破(默认False)
+       --depth=DEPTH
+           递归爆破的深度(默认2)
+       --namelist=NAMELIST
+           指定递归爆破所使用的字典路径(默认使用config.py配置)
+       --fuzz=FUZZ
+           是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
+       --rule=RULE
+           fuzz模式使用的正则规则(默认使用config.py配置)
+       --export=EXPORT
+           是否导出爆破结果(默认True)
       --valid=VALID
           导出子域的有效性(默认None)
-       --path=PATH
-           导出路径(默认None)
       --format=FORMAT
           导出格式(默认xlsx)
-       --output=OUTPUT
-           是否将导出数据输出到终端(默认False)
-   
-   NOTES
-       You can also use flags syntax for POSITIONAL ARGUMENTS
+       --path=PATH
+           导出路径(默认None)
+       --show=SHOW
+           终端显示导出数据(默认False)
   ```

+3. 其他模块使用请参考[使用帮助](https://github.com/shmilylty/OneForAll/tree/master/docs/using_help.md)
+
 ## 👏主要框架

 * [aiodns](https://github.com/saghul/aiodns) - 简单DNS异步解析库。
@@ -313,18 +316,18 @@ D:.

 ## ⌛后续计划

- [ ] 子域收集模块优化
- [ ] 子域接管功能实现
- [ ] 子域收集爬虫实现
- [ ] 操作强大交互人性的前端界面实现
+- [ ] 各模块支持优化和完善
+- [ ] 子域监控（标记每次新发现的子域）
+- [ ] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
+- [ ] 操作强大交互人性的前端界面实现（暂定：前端：Element + 后端：Flask）

-更多详细信息请阅读[TODO.md](./TODO.md)。
+更多详细信息请阅读[TODO.md](https://github.com/shmilylty/OneForAll/tree/master/TODO.md)。

 ## 🔖版本控制

 该项目使用[SemVer](https://semver.org/)语言化版本格式进行版本管理，你可以在[Releases](https://github.com/shmilylty/OneForAll/releases)查看可用版本。

-## 👨‍💻作者
+## 👨‍💻作者及贡献者

 * **[Jing Ling](https://github.com/shmilylty)**
  * 核心开发
@@ -332,14 +335,20 @@ D:.
 * **[Black Star](https://github.com/blackstar24)**
  * 模块贡献

-* [**iceMatcha**](https://github.com/iceMatcha)
-  * bug调试
+* **[Echocipher](https://github.com/Echocipher)**
+  * 模块贡献

-*你也可以在[CONTRIBUTORS.md](./CONTRIBUTORS.md)中参看所有参与该项目的开发者。*
+* **[iceMatcha](https://github.com/iceMatcha)**
+  * 工具测试
+
+* **Anyone**
+  * 工具反馈
+
+*你也可以在[CONTRIBUTORS.md](https://github.com/shmilylty/OneForAll/tree/master/CONTRIBUTORS.md)中参看所有参与该项目的开发者。*

 ## 📄版权

-该项目签署了GPL-3.0授权许可，详情请参阅[LICENSE.md](./LICENSE.md)。
+该项目签署了GPL-3.0授权许可，详情请参阅[LICENSE.md](https://github.com/shmilylty/OneForAll/tree/master/LICENSE.md)。

 ## 😘鸣谢

@@ -349,4 +358,7 @@ D:.

 ## 📜免责声明 ##

-本工具仅限于安全研究与教学使用，用户使用本工具所造成的所有后果，由用户承担全部法律及连带责任，本项目所有作者和贡献者不承担任何法律及连带责任。
+本工具仅限于合法授权的企业安全建设，在使用本工具过程中，您应确保自己所有行为符合当地的法律法规，并且已经取得了足够的授权。
+如您在使用本工具的过程中存在任何非法行为，您需自行承担所有后果，本工具所有作者和所有贡献者不承担任何法律及连带责任。
+除非您已充分阅读、完全理解并接受本协议所有条款，否则，请您不要安装并使用本工具。
+您的使用行为或者您以其他任何明示或者默示方式表示接受本协议的，即视为您已阅读并同意本协议的约束。
@@ -2,8 +2,8 @@

 ## 下一步计划

- [ ] 子域收集模块优化
- [ ] 子域接管功能实现
- [ ] 子域收集爬虫实现
- [ ] 操作强大交互人性的前端界面实现
+- [ ] 各模块支持优化和完善
+- [ ] 子域监控（标记每次新发现的子域）
+- [ ] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
+- [ ] 操作强大交互人性的前端界面实现（暂定：Element+Flask）

@@ -0,0 +1 @@
+theme: jekyll-theme-architect
@@ -0,0 +1,31 @@
+# 常见问题与回答
+
+## 依赖问题
+
+**Q: 在安装依赖过程遇到编译某个依赖库失败，怎么解决？**
+
+A: 可以尝试以下方法:
+
+1. 到提供编译好的whl文件的第三方平台，找到对应库手动下载安装。第三方平台平台有： 
+ * [https://www.lfd.uci.edu/~gohlke/pythonlibs](https://www.lfd.uci.edu/~gohlke/pythonlibs)
+ * [https://pythonwheels.com/](https://pythonwheels.com/)
+
+选择好对应版本执行以下命令手动安装。举个例子，当编译pycares时失败时，找到[https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares](https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares)，由于我的系统是Windows 10 64位，使用的Python 3.7便下载`pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl`（一般来说下载最新版本的），然后手动安装：
+
+```bash
+pip3 install pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl
+```
+
+2. 到库的项目地址issues和wiki等找找有没有解决方法，如果没有就给他们提issues发邮件😜。
+
+## 使用问题
+
+**Q: 为什么运行OneForAll之后最终结果为空？**
+
+A: 有几种可能性：第一可能目标域名没有子域。第二由于OneForAll默认会自动验证子域，在导出是只会有效子域，所以存在导出时没有有效子域的情况，你可以在运行OneForAll使用--valid=None指定导出所有发现的子域，你也可以使用--verify=False指定不验证子域的有效性。
+
+**Q: 安装依赖时出现以下类似报错，怎么解决？**
+   Cannot uninstall 'PyYAML'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall.
+
+A: 安装依赖时尝试加上--ignore-installed参数
+
@@ -1,13 +0,0 @@
-如果在安装依赖过程遇到编译某个依赖库失败时可以尝试以下方法：
-
-1. 到提供编译好的whl文件的第三方平台，找到对应库手动下载安装。第三方平台平台有： 
- * [https://www.lfd.uci.edu/~gohlke/pythonlibs](https://www.lfd.uci.edu/~gohlke/pythonlibs)
- * [https://pythonwheels.com/](https://pythonwheels.com/)
-
-选择好对应版本执行以下命令手动安装。举个例子，当编译pycares时失败时，找到[https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares](https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares)，由于我的系统是Windows 10 64位，使用的Python 3.7便下载`pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl`（一般来说下载最新版本的），然后手动安装：
-
-```bash
-pip3 install pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl
-```
-
-2. 到库的项目地址issues和wiki等找找有没有解决方法，如果没有就给他们提issues发邮件😜。
@@ -33,12 +33,14 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As

  2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，后续会添加检查NSEC记录，NSEC3记录等模块）

-      | 模块名称 | 是否需要代理 | 是否需要API | 其他说明         |
-      | -------- | ------------ | ----------- | ---------------- |
-      | axfr     | 否           | 否          | 域传送漏洞利用   |
-      | cdx      | 由域名决定   | 否          | 检查跨域策略文件 |
-      | cert     | 否           | 否          | 检查HTTPS证书    |
-      | csp      | 由域名决定   | 否          | 检查内容安全策略 |
+      | 模块名称 | 是否需要代理           | 是否需要API | 其他说明           |
+      | -------- | ---------------------- | ----------- | ------------------ |
+      | axfr     | 否                     | 否          | 域传送漏洞利用     |
+      | cdx      | 手动设置（默认不使用） | 否          | 检查跨域策略文件   |
+      | cert     | 否                     | 否          | 检查HTTPS证书      |
+      | csp      | 手动设置（默认不使用） | 否          | 检查内容安全策略   |
+      | robots   | 手动设置（默认不使用） | 否          | 检查robots.txt文件 |
+      | sitemap  | 手动设置（默认不使用） | 否          | 检查sitemap文件    |
  3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善）

      | 模块名称     | 是否需要代理 | 是否需要API | 其他说明 |
@@ -46,7 +48,7 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
      | archivecrawl | 否           | 否          |          |
      | commoncrawl  | 否           | 否          |          |

-  4. 利用DNS数据集收集子域（目前有16个模块：`binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `ptrarchive`, `sitedossier`）
+  4. 利用DNS数据集收集子域（目前有18个模块：`binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`,  `ptrarchive`, `sitedossier`,`threatcrowd`）

      | 模块名称           | 是否需要代理 | 是否需要API | 其他说明                                           |
      | ------------------ | ------------ | ----------- | -------------------------------------------------- |
@@ -61,11 +63,13 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
      | hackertarget       | 否           | 否          |                                                    |
      | ipv4info           | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
      | netcraft           | 否           | 否          |                                                    |
+      | passivedns_api     | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
      | ptrarchive         | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
      | riddler            | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
      | robtex             | 否           | 否          |                                                    |
      | securitytrails_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
      | sitedossier        | 否           | 否          |                                                    |
+      | threatcrowd        | 否           | 否          |                                                    |
  5. 利用DNS查询收集子域（目前有1个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，该模块还有待添加和完善）

      | 模块名称 | 是否需要代理 | 是否需要API | 其他说明                      |
@@ -80,24 +84,25 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
      | threatminer    | 否           | 否          |                                                    |
      | virustotal     | 否           | 否          |                                                    |
      | virustotal_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
-  7. 利用搜索引擎发现子域（目前有15个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`）
+  7. 利用搜索引擎发现子域（目前有16个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`）

     除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。

-     | 模块        | 是否需要代理           | 是否需要API | 其他说明                                           |
-     | ----------- | ---------------------- | ----------- | -------------------------------------------------- |
-     | ask         | 是                     | 否          |                                                    |
-     | baidu       | 否                     | 否          |                                                    |
-     | bing        | 否                     | 否          |                                                    |
-     | bing_api    | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py) |
-     | duckduckgo  | 是                     | 否          |                                                    |
-     | exalead     | 否，最好使用国外代理。 | 否          |                                                    |
-     | fofa_api    | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py) |
-     | google      | 是                     | 否          |                                                    |
-     | google_api  | 是                     | 是          | API使用和申请见[config.py](../oneforall/config.py) |
-     | shodan_api  | 否，最好使用国外代理。 | 是          | API使用和申请见[config.py](../oneforall/config.py) |
-     | so          | 否                     | 否          |                                                    |
-     | sogou       | 否                     | 否          |                                                    |
-     | yahoo       | 是                     | 否          |                                                    |
-     | yandex      | 是                     | 否          |                                                    |
-     | zoomeye_api | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+     | 模块        | 是否需要代理           | 是否需要API | 其他说明                                                    |
+     | ----------- | ---------------------- | ----------- | ----------------------------------------------------------- |
+     | ask         | 是                     | 否          |                                                             |
+     | baidu       | 否                     | 否          |                                                             |
+     | bing        | 否                     | 否          |                                                             |
+     | bing_api    | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
+     | duckduckgo  | 是                     | 否          |                                                             |
+     | exalead     | 否，最好使用国外代理。 | 否          |                                                             |
+     | fofa_api    | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
+     | github      | 否                     | 否          | 在[config.py](../oneforall/config.py)设置Github邮件名和密码 |
+     | google      | 是                     | 否          |                                                             |
+     | google_api  | 是                     | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
+     | shodan_api  | 否，最好使用国外代理。 | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
+     | so          | 否                     | 否          |                                                             |
+     | sogou       | 否                     | 否          |                                                             |
+     | yahoo       | 是                     | 否          |                                                             |
+     | yandex      | 是                     | 否          |                                                             |
+     | zoomeye_api | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
@@ -0,0 +1,64 @@
+# 安装依赖
+
+你可以通过pip3和pipenv两种方法安装OneForAll的依赖（如果你熟悉[pipenv](https://docs.pipenv.org/en/latest/)，那么推荐使用你使用pipenv）：
+
+* **Windows系统**（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python37`，那么请以管理员身份运行命令提示符cmd执行以下命令！）
+
+    1. 使用pipenv
+
+    ```bash
+    cd OneForAll/
+    python -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
+    pipenv install --user --python 3.7
+    cd oneforall
+    pipenv run python oneforall.py --help
+    ```
+
+    2. 使用pip3
+
+    ```bash
+    cd OneForAll/
+    python -m pip install --user  -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+    cd oneforall/
+    python oneforall.py --help
+    ```
+* **Linux系统**
+
+    1. 使用pipenv
+    ```bash
+    cd OneForAll/
+    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    sudo pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
+    sudo pipenv install --user --python 3.7
+    cd oneforall
+    pipenv run python3 oneforall.py --help
+    ```
+    2. 使用pip3
+    ```bash
+    cd OneForAll/
+    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+    cd oneforall/
+    python3 oneforall.py --help
+    ```
+* **Darwin系统**
+
+    1. 使用pipenv
+    ```bash
+    cd OneForAll/
+    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
+    pipenv install --user --python 3.7
+    cd oneforall
+    pipenv run python3 oneforall.py --help
+    ```
+    2. 使用pip3
+    ```bash
+    cd OneForAll/
+    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+    cd oneforall/
+    python3 oneforall.py --help
+    ```
@@ -0,0 +1,212 @@
+# 使用帮助
+
+oneforall.py是主程序入口，oneforall.py里有调用aiobrute.py和dbexport.py，为了方便进行子域爆破和数据库导出独立出了aiobrute.py和dbexport.py，这两个文件可以单独运行，并且所接受参数要更丰富一点。
+
+1. oneforall.py使用帮助
+
+   ```bash
+   python oneforall.py --help
+   ```
+   ```bash
+   NAME
+       oneforall.py - OneForAll是一款功能强大的子域收集工具
+   
+   SYNOPSIS
+       oneforall.py --target=TARGET <flags>
+   
+   DESCRIPTION
+       Version: 0.0.4
+       Project: https://git.io/fjHT1
+   
+       Example:
+           python3 oneforall.py --target example.com run
+           python3 oneforall.py --target ./domains.txt run
+           python3 oneforall.py --target example.com --brute True run
+           python3 oneforall.py --target example.com --verify False run
+           python3 oneforall.py --target example.com --valid None run
+           python3 oneforall.py --target example.com --port medium run
+           python3 oneforall.py --target example.com --format csv run
+           python3 oneforall.py --target example.com --show True run
+   
+       Note:
+           参数valid可选值1，0，None分别表示导出有效，无效，全部子域
+           参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
+           参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+           参数format可选格式有'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
+                             'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   ARGUMENTS
+       TARGET
+           单个域名或者每行一个域名的文件路径(必需参数)
+   
+   FLAGS
+       --brute=BRUTE
+           使用爆破模块(默认False)
+       --verify=VERIFY
+           验证子域有效性(默认True)
+       --port=PORT
+           请求验证的端口范围(默认medium)
+       --valid=VALID
+           导出子域的有效性(默认1)
+       --path=PATH
+           导出路径(默认None)
+       --format=FORMAT
+           导出格式(默认xlsx)
+       --show=SHOW
+           终端显示导出数据(默认False)
+   ```
+
+2. aiobrute.py使用帮助
+
+   关于泛解析问题处理程序首先会访问一个随机的子域判断是否泛解析，如果使用了泛解析则是通过以下判断处理：
+   - 一是主要是与泛解析的IP集合和TTL值做对比，可以参考[这篇文章](http://sh3ll.me/archives/201704041222.txt)。
+   - 二是多次解析到同一IP集合次数（默认设置为10，可以在config.py设置大小）
+   - 考虑爆破效率问题目前还没有加上HTTP响应体相似度对比和响应体内容判断
+   经过测试在16核心的CPU，使用16进程64协程，100M带宽的环境下，设置任务分割为50000，跑两百万字典大概10分钟左右跑完，大概3333个子域每秒。
+
+   ```bash
+   python aiobrute.py --help
+   ```
+
+   ```bash
+   NAME
+       aiobrute.py - OneForAll多进程多协程异步子域爆破模块
+   
+   SYNOPSIS
+       aiobrute.py --target=TARGET <flags>
+   
+   DESCRIPTION
+       Example：
+           python3 aiobrute.py --target example.com run
+           python3 aiobrute.py --target ./domains.txt run
+           python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
+           python3 aiobrute.py --target example.com --wordlist subdomains.txt run
+           python3 aiobrute.py --target example.com --recursive True --depth 2 run
+           python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
+   
+       Note:
+           参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为任务组，
+           当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+           十万字典建议设置为5000，百万字典设置为50000
+           参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+           参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
+                             'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   ARGUMENTS
+       TARGET
+           单个域名或者每行一个域名的文件路径
+   
+   FLAGS
+       --process=PROCESS
+           爆破的进程数(默认CPU核心数)
+       --coroutine=COROUTINE
+           每个爆破进程下的协程数(默认64)
+       --wordlist=WORDLIST
+           指定爆破所使用的字典路径(默认使用config.py配置)
+       --segment=SEGMENT
+           爆破任务分割(默认500)
+       --recursive=RECURSIVE
+           是否使用递归爆破(默认False)
+       --depth=DEPTH
+           递归爆破的深度(默认2)
+       --namelist=NAMELIST
+           指定递归爆破所使用的字典路径(默认使用config.py配置)
+       --fuzz=FUZZ
+           是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
+       --rule=RULE
+           fuzz模式使用的正则规则(默认使用config.py配置)
+       --export=EXPORT
+           是否导出爆破结果(默认True)
+       --valid=VALID
+           导出子域的有效性(默认None)
+       --format=FORMAT
+           导出格式(默认xlsx)
+       --path=PATH
+           导出路径(默认None)
+       --show=SHOW
+           终端显示导出数据(默认False)
+   
+   ```
+
+
+3. takeover.py使用帮助
+
+   ```bash
+   python takeover.py --help
+   ```
+   
+   ```bash
+   NAME
+    takeover.py - OneForAll多线程子域接管风险检查模块
+   
+   
+    SYNOPSIS
+        takeover.py COMMAND | --target=TARGET <flags>
+   
+    DESCRIPTION
+        Example:
+            python3 takeover.py --target www.example.com  --format csv run
+            python3 takeover.py --target ./subdomains.txt --thread 10 run
+   
+        Note:
+            参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                              'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+            参数dpath为None默认使用OneForAll结果目录
+   
+    ARGUMENTS
+        TARGET
+            单个子域或者每行一个子域的文件路径(必需参数)
+   
+    FLAGS
+        --thread=THREAD
+            线程数(默认100)
+        --dpath=DPATH
+            导出目录(默认None)
+        --format=FORMAT
+            导出格式(默认xls)
+   
+   ```
+
+
+4. dbexport.py使用帮助
+
+   ```bash
+   python dbexport.py --help
+   ```
+
+   ```bash
+   NAME
+       dbexport.py - OneForAll数据库导出模块
+   
+   SYNOPSIS
+       dbexport.py TABLE <flags>
+   
+   DESCRIPTION
+       Example:
+           python3 dbexport.py --table name --format csv --path= ./result.csv
+           python3 dbexport.py --db result.db --table name --show False
+   
+       Note:
+           参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+           参数format可选格式有'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
+                             'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   POSITIONAL ARGUMENTS
+       TABLE
+           要导出的表
+   
+   FLAGS
+       --db=DB
+           要导出的数据库路径(默认为results/result.sqlite3)
+       --valid=VALID
+           导出子域的有效性(默认None)
+       --path=PATH
+           导出路径(默认None)
+       --format=FORMAT
+           导出格式(默认xlsx)
+       --show=SHOW
+           终端显示导出数据(默认False)
+   ```
@@ -14,14 +14,16 @@ import secrets
 import signal
 import time

-import aiomultiprocess
+import aiomultiprocess as aiomp
 import exrex
 import fire
 import tqdm

 import config
-from common import database, resolve, utils
+import dbexport
+from common import resolve, utils
 from common.module import Module
+from common.database import Database
 from config import logger


@@ -130,20 +132,24 @@ class AIOBrute(Module):
    OneForAll多进程多协程异步子域爆破模块

    Example：
-        python3 aiobrute.py --target example.com run
-        python3 aiobrute.py --target ./domains.txt run
-        python3 aiobrute.py --target example.com --processes 4 --coroutine 64
-        python3 aiobrute.py --target example.com --wordlist subdomains.txt run
+        python3 aiobrute.py --target subdomain.com run
+        python3 aiobrute.py --target ./subdomains.txt run
+        python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
+        python3 aiobrute.py --target example.com --wordlist subnames.txt run
        python3 aiobrute.py --target example.com --recursive True --depth 2 run
        python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run

    Note:
-        参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为任务组，
-        当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+        参数segment的设置受CPU性能，网络带宽，运营商限制等限制，默认500个子域为任务组，
+        当你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
        十万字典建议设置为5000，百万字典设置为50000
+        参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件

    :param str target:       单个域名或者每行一个域名的文件路径
-    :param int processes:    爆破的进程数(默认CPU核心数)
+    :param int process:      爆破的进程数(默认CPU核心数)
    :param int coroutine:    每个爆破进程下的协程数(默认64)
    :param str wordlist:     指定爆破所使用的字典路径(默认使用config.py配置)
    :param int segment:      爆破任务分割(默认500)
@@ -152,18 +158,24 @@ class AIOBrute(Module):
    :param str namelist:     指定递归爆破所使用的字典路径(默认使用config.py配置)
    :param bool fuzz:        是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
    :param str rule:         fuzz模式使用的正则规则(默认使用config.py配置)
+    :param bool export:      是否导出爆破结果(默认True)
+    :param int valid:        导出子域的有效性(默认None)
+    :param str format:       导出格式(默认csv)
+    :param str path:         导出路径(默认None)
+    :param bool show:        终端显示导出数据(默认False)
    """

-    def __init__(self, target, processes=None, coroutine=64, wordlist=None,
+    def __init__(self, target, process=None, coroutine=64, wordlist=None,
                 segment=500, recursive=False, depth=2, namelist=None,
-                 fuzz=False, rule=None):
+                 fuzz=False, rule=None, export=True, valid=None, format='csv',
+                 path=None, show=False):
        Module.__init__(self)
        self.domains = set()
        self.domain = str()
        self.module = 'Brute'
        self.source = 'AIOBrute'
        self.target = target
-        self.processes = processes or config.brute_processes_num
+        self.process = process or config.brute_process_num
        self.coroutine = coroutine or config.brute_coroutine_num
        self.wordlist = wordlist or config.brute_wordlist_path
        self.segment = segment or config.brute_task_segment
@@ -172,6 +184,11 @@ class AIOBrute(Module):
        self.recursive_namelist = namelist or config.recursive_namelist_path
        self.fuzz = fuzz or config.enable_fuzz
        self.rule = rule or config.fuzz_rule
+        self.export = export
+        self.valid = valid
+        self.format = format
+        self.path = path
+        self.show = show
        self.nameservers = config.resolver_nameservers
        self.ips_times = dict()  # IP集合出现次数
        self.enable_wildcard = False  # 当前域名是否使用泛解析
@@ -179,7 +196,7 @@ class AIOBrute(Module):
        self.wildcard_ttl = int()  # 泛解析TTL整型值

    def gen_tasks(self, domain):
-        # 如果domain不是self.domain，而是self.domain的子域 生成递归爆破字典
+        # 如果domain不是self.subdomain，而是self.domain的子域 生成递归爆破字典
        if self.domain != domain:
            logger.log('INFOR', f'使用{self.recursive_namelist}字典')
            domains = gen_brute_domains(domain, self.recursive_namelist)
@@ -194,6 +211,8 @@ class AIOBrute(Module):

    def deal_results(self, results):
        for result in results:
+            if result is None:
+                continue
            if isinstance(result, Exception):
                # logger.log('DEBUG', f'爆破{subdomain}时出错 {str(answers)}')
                continue
@@ -207,7 +226,8 @@ class AIOBrute(Module):
                self.ips_times[str(ips)] = value + 1
                ttl = answers[0].ttl
                if self.enable_wildcard:
-                    if wildcard_by_compare(ips, ttl,
+                    if wildcard_by_compare(ips,
+                                           ttl,
                                           self.wildcard_ips,
                                           self.wildcard_ttl):
                        continue
@@ -220,15 +240,17 @@ class AIOBrute(Module):

    async def main(self, domain, rx_queue):
        if not self.fuzz:  # fuzz模式不探测域名是否使用泛解析
-            self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl = detect_wildcard(domain)
+            self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl \
+                = detect_wildcard(domain)
        tasks = self.gen_tasks(domain)
        logger.log('INFOR', f'正在爆破{domain}的域名')
-        for task in tqdm.tqdm(tasks, desc='Progress',
-                              smoothing=1.0, ncols=True):
-            async with aiomultiprocess.Pool(processes=self.processes,
-                                            initializer=init_worker,
-                                            childconcurrency=self.coroutine)\
-                    as pool:
+        for task in tqdm.tqdm(tasks,
+                              desc='Progress',
+                              smoothing=1.0,
+                              ncols=True):
+            async with aiomp.Pool(processes=self.process,
+                                  initializer=init_worker,
+                                  childconcurrency=self.coroutine) as pool:
                try:
                    results = await pool.map(resolve.aiodns_query_a, task)
                except KeyboardInterrupt:
@@ -248,13 +270,12 @@ class AIOBrute(Module):
        while self.domains:
            self.domain = self.domains.pop()
            start = time.time()
-            db_conn = database.connect_db()
-            table_name = self.domain.replace('.', '_')
-            database.create_table(db_conn, table_name)
+            db = Database()
+            db.create_table(self.domain)
            if not rx_queue:
                rx_queue = queue.Queue()
            logger.log('INFOR', f'开始执行{self.source}模块爆破域名{self.domain}')
-            logger.log('INFOR', f'使用{self.processes}进程乘{self.coroutine}协程')
+            logger.log('INFOR', f'使用{self.process}进程乘{self.coroutine}协程')
            # fuzz模式不使用递归爆破
            if self.recursive_brute and not self.fuzz:
                logger.log('INFOR', f'开始递归爆破{self.domain}的第1层子域')
@@ -271,17 +292,15 @@ class AIOBrute(Module):
                                        f'第{layer_num + 1}层子域')
                    for subdomain in self.subdomains.copy():
                        # 进行下一层子域爆破的限制条件
-                        if subdomain.count('.') - self.domain.count('.') == layer_num:
+                        if subdomain.count('.') - self.domain.count('.') \
+                                == layer_num:
                            loop.run_until_complete(self.main(subdomain,
                                                              rx_queue))
            # 队列不空就一直取数据存数据库
            while not rx_queue.empty():
                source, results = rx_queue.get()
                # 将结果存入数据库中
-                database.save_db(db_conn, table_name, results, source)
-            database.copy_table(db_conn, table_name)
-            database.deduplicate_subdomain(db_conn, table_name)
-            database.remove_invalid(db_conn, table_name)
+                db.save_db(self.domain, results, source)

            end = time.time()
            self.elapsed = round(end - start, 1)
@@ -291,6 +310,16 @@ class AIOBrute(Module):
                                f'发现{self.domain}的域名{length}个')
            logger.log('DEBUG', f'{self.source}模块发现{self.domain}的域名:\n'
                                f'{self.subdomains}')
+            # 数据库导出
+            if self.export:
+                if not self.path:
+                    name = f'{self.domain}_brute.{self.format}'
+                    self.path = config.result_save_path.joinpath(name)
+                dbexport.export(self.domain,
+                                valid=self.valid,
+                                dpath=self.path,
+                                format=self.format,
+                                show=self.show)


 def do(domain, result):  # 统一入口名字 方便多线程调用
@@ -1,14 +1,8 @@
-# coding=utf-8
-"""
-被动收集类
-"""
 import time
-import queue
 import threading
 import importlib
 import config
 import dbexport
-from common import database
 from config import logger


@@ -23,12 +17,11 @@ class Collect(object):
        self.collect_func = []
        self.path = None
        self.export = export
-        self.format = 'xlsx'
+        self.format = 'csv'

    def get_mod(self):
        """
        获取要运行的模块
-        :return: None
        """
        if config.enable_all_module:
            # modules = ['brute', 'certificates', 'crawl',
@@ -54,7 +47,7 @@ class Collect(object):
            import_object = importlib.import_module('.'+name, package)
            self.collect_func.append(getattr(import_object, 'do'))

-    def run(self, rx_queue=None):
+    def run(self):
        """
        类运行入口
        """
@@ -77,19 +70,12 @@ class Collect(object):
        for thread in threads:
            thread.join()

-        db_conn = database.connect_db()
-        table_name = self.domain.replace('.', '_')
-        database.create_table(db_conn, table_name)
-        database.copy_table(db_conn, table_name)
-        database.deduplicate_subdomain(db_conn, table_name)
-        database.remove_invalid(db_conn, table_name)
-        db_conn.close()
        # 数据库导出
        if self.export:
            if not self.path:
                name = f'{self.domain}.{self.format}'
                self.path = config.result_save_path.joinpath(name)
-            dbexport.export(table_name, path=self.path, format=self.format)
+            dbexport.export(self.domain, dpath=self.path, format=self.format)
        end = time.time()
        self.elapsed = round(end - start, 1)

@@ -11,160 +11,200 @@ from records import Connection
 from config import logger


-def connect_db(db_path=None):
-    """
-    获取数据库对象
+class Database(object):
+    def __init__(self, db_path=None):
+        self.conn = self.get_conn(db_path)

-    :param db_path: 数据库连接或路径
-    :return: SQLite数据库
-    """
-    logger.log('DEBUG', f'正在获取数据库连接')
-    if isinstance(db_path, Connection):
-        return db_path
-    protocol = 'sqlite:///'
-    if not db_path:  # 数据库路径为空连接默认数据库
-        db_path = f'{protocol}{config.result_save_path}/result.sqlite3'
-    else:
-        db_path = protocol + db_path
-    db = records.Database(db_path)  # 不存在数据库时会新建一个数据库
-    logger.log('DEBUG', f'使用数据库: {db_path}')
-    return db.get_connection()
+    @staticmethod
+    def get_conn(db_path):
+        """
+        获取数据库对象

+        :param db_path: 数据库连接或路径
+        :return: SQLite数据库
+        """
+        logger.log('DEBUG', f'正在获取数据库连接')
+        if isinstance(db_path, Connection):
+            return db_path
+        protocol = 'sqlite:///'
+        if not db_path:  # 数据库路径为空连接默认数据库
+            db_path = f'{protocol}{config.result_save_path}/result.sqlite3'
+        else:
+            db_path = protocol + db_path
+        db = records.Database(db_path)  # 不存在数据库时会新建一个数据库
+        logger.log('DEBUG', f'使用数据库: {db_path}')
+        return db.get_connection()

-def create_table(db_conn, table_name):
-    """
-    初始化数据库
+    def create_table(self, table_name):
+        """
+        初始化数据库

-    :param db_conn: 数据库连接
-    :param str table_name: 要创建的表名
-    """
-    logger.log('DEBUG', f'正在创建{table_name}表')
-    try:
-        db_conn.query(f'create table if not exists "{table_name}" ('
-                      f'id integer primary key,'
-                      f'url text,'
-                      f'subdomain text,'
-                      f'port int,'
-                      f'ips text,'
-                      f'status int,'
-                      f'reason text,'
-                      f'valid int,'
-                      f'title text,'
-                      f'banner text,'
-                      f'module text,'
-                      f'source text,'
-                      f'elapsed float,'
-                      f'count int)')
-    except Exception as e:
-        logger.log('ERROR', e)
-
-
-def save_db(db_conn, table_name, results, module_name=None):
-    """
-    将各模块结果存入数据库
-
-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    :param list results: 结果列表
-    :param str module_name: 模块名
-    """
-    logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域结果存入数据库')
-    if results:
+        :param str table_name: 要创建的表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在创建{table_name}表')
        try:
-            db_conn.bulk_query(f'insert into "{table_name}" (id, url, subdomain, port, ips, status,'
-                               f'reason, valid, title, banner, module, source, elapsed, count)'
-                               f'values (:id, :url, :subdomain, :port, :ips, :status, :reason, :valid,'
-                               f':title, :banner, :module, :source, :elapsed, :count)', results)
+            self.conn.query(f'create table if not exists "{table_name}" ('
+                            f'id integer primary key,'
+                            f'url text,'
+                            f'subdomain text,'
+                            f'port int,'
+                            f'ips text,'
+                            f'status int,'
+                            f'reason text,'
+                            f'valid int,'
+                            f'title text,'
+                            f'banner text,'
+                            f'module text,'
+                            f'source text,'
+                            f'elapsed float,'
+                            f'count int)')
        except Exception as e:
            logger.log('ERROR', e)

+    def save_db(self, table_name, results, module_name=None):
+        """
+        将各模块结果存入数据库

-def copy_table(db_conn, table_name):
-    """
-    复制表创建备份
+        :param str table_name: 表名
+        :param list results: 结果列表
+        :param str module_name: 模块名
+        """
+        logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域结果存入数据库')
+        table_name = table_name.replace('.', '_')
+        if results:
+            try:
+                self.conn.bulk_query(
+                    f'insert into "{table_name}" ('
+                    f'id, url, subdomain, port, ips, status, reason, valid,'
+                    f'title, banner, module, source, elapsed, count)'
+                    f'values (:id, :url, :subdomain, :port, :ips, :status,'
+                    f':reason, :valid, :title, :banner, :module, :source,'
+                    f':elapsed, :count)',
+                    results)
+            except Exception as e:
+                logger.log('ERROR', e)

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    new_table_name = table_name + '_bak'
-    logger.log('DEBUG', f'正在将{table_name}表复制到{new_table_name}新表')
-    try:
-        db_conn.query(f'drop table if exists "{new_table_name}"')
-        db_conn.query(f'create table "{new_table_name}" as select * from "{table_name}"')
-    except Exception as e:
-        logger.log('ERROR', e)
+    def copy_table(self, table_name, bak_table_name):
+        """
+        复制表创建备份

+        :param str table_name: 表名
+        :param str bak_table_name: 新表名
+        """
+        table_name = table_name.replace('.', '_')
+        bak_table_name = bak_table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在将{table_name}表复制到{bak_table_name}新表')
+        try:
+            self.conn.query(f'drop table if exists "{bak_table_name}"')
+            self.conn.query(f'create table "{bak_table_name}" '
+                            f'as select * from "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

-def clear_table(db_conn, table_name):
-    """
-    清空表中数据
+    def clear_table(self, table_name):
+        """
+        清空表中数据

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    logger.log('DEBUG', f'正在清空{table_name}表中的数据')
-    try:
-        db_conn.query(f'delete from "{table_name}"')
-    except Exception as e:
-        logger.log('ERROR', e)
-
-
-def deduplicate_subdomain(db_conn, table_name):
-    """
-    去重表中的子域并删除空值和无效值
-
-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    logger.log('DEBUG', f'正在去重{table_name}表中的子域')
-    try:
-        db_conn.query(f'delete from "{table_name}" where id not in (select min(id) from "{table_name}" group by subdomain)')
-    except Exception as e:
-        logger.log('ERROR', e)
-
-
-def remove_invalid(db_conn, table_name):
-    """
-        去除表中的空值或无效子域
-
-        :param db_conn: 数据库连接
        :param str table_name: 表名
        """
-    logger.log('DEBUG', f'正在去除{table_name}表中的无效子域')
-    try:
-        db_conn.query(f'delete from "{table_name}" where subdomain is null or valid == 0')
-    except Exception as e:
-        logger.log('ERROR', e)
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在清空{table_name}表中的数据')
+        try:
+            self.conn.query(f'delete from "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

+    def drop_table(self, table_name):
+        """
+        删除表

-def get_data(db_conn, table_name):
-    """
-    获取表中的所有数据
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在删除{table_name}表')
+        try:
+            self.conn.query(f'drop table if exists "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    logger.log('DEBUG', f'获取{table_name}表中的所有数据')
-    try:
-        rows = db_conn.query(f'select * from "{table_name}"')
-    except Exception as e:
-        logger.log('ERROR', e)
-    else:
-        return rows
+    def rename_table(self, table_name, new_table_name):
+        """
+        复制表创建备份

+        :param str table_name: 表名
+        :param str new_table_name: 新表名
+        """
+        table_name = table_name.replace('.', '_')
+        new_table_name = new_table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在将{table_name}表重命名为{table_name}表')
+        try:
+            self.conn.query(f'alter table "{table_name}" '
+                            f'rename to "{new_table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

-def get_subdomain(db_conn, table_name, valid):
-    """
-    获取表中的子域数据
+    def deduplicate_subdomain(self, table_name):
+        """
+        去重表中的子域

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    :param int valid: 是否有效
-    """
-    logger.log('DEBUG', f'获取{table_name}表中的所有数据')
-    try:
-        rows = db_conn.query(f'select * from "{table_name}" where valid = {valid}')
-    except Exception as e:
-        logger.log('ERROR', e)
-    else:
-        return rows
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在去重{table_name}表中的子域')
+        try:
+            self.conn.query(
+                f'delete from "{table_name}" where id not in (select min(id) '
+                f'from "{table_name}" group by subdomain)')
+        except Exception as e:
+            logger.log('ERROR', e)
+
+    def remove_invalid(self, table_name):
+        """
+        去除表中的空值或无效子域
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在去除{table_name}表中的无效子域')
+        try:
+            self.conn.query(
+                f'delete from "{table_name}" where '
+                f'subdomain is null or valid == 0')
+        except Exception as e:
+            logger.log('ERROR', e)
+
+    def get_data(self, table_name):
+        """
+        获取表中的所有数据
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'获取{table_name}表中的所有数据')
+        try:
+            rows = self.conn.query(f'select * from "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)
+        else:
+            return rows
+
+    def get_subdomain(self, table_name, valid):
+        """
+        获取表中的子域数据
+
+        :param str table_name: 表名
+        :param int valid: 是否有效
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'获取{table_name}表中的所有数据')
+        try:
+            rows = self.conn.query(
+                f'select * from "{table_name}" where valid = {valid}')
+        except Exception as e:
+            logger.log('ERROR', e)
+        else:
+            return rows
+
+    def close(self):
+        self.conn.close()
@@ -31,9 +31,9 @@ class Domain(object):
        """
        域名导出

-        >>>d = Domain('www.example.com')
+        >>> d = Domain('www.example.com')
        <domain.Domain object>
-        >>>d.extract()
+        >>> d.extract()
        ExtractResult(subdomain='www', domain='example', suffix='com')

        :return: 导出结果
@@ -50,9 +50,9 @@ class Domain(object):
        """
        获取注册域名

-        >>>d = Domain('www.example.com')
+        >>> d = Domain('www.example.com')
        <domain.Domain object>
-        >>>d.registered()
+        >>> d.registered()
        example.com

        :return: 注册域名
@@ -13,7 +13,7 @@ import config
 from config import logger
 from . import utils
 from .domain import Domain
-from common import database
+from common.database import Database


 lock = threading.Lock()
@@ -35,7 +35,7 @@ class Module(object):
        self.results = list()  # 存放模块结果
        self.start = time.time()  # 模块开始执行时间
        self.end = None
-        self.elapsed = time.time() - self.start  # 模块执行耗时
+        self.elapsed = None  # 模块执行耗时

    def check(self, *apis):
        """
@@ -66,12 +66,13 @@ class Module(object):
        logger.log('DEBUG', f'{self.source}模块发现{self.domain}的子域\n'
                   f'{self.subdomains}')

-    def get(self, url, params=None, **kwargs):
+    def get(self, url, params=None, check=True, **kwargs):
        """
        自定义get请求

        :param str url: 请求地址
        :param dict params: 请求参数
+        :param bool check: 检查响应
        :param kwargs: 其他参数
        :return: requests响应对象
        """
@@ -87,24 +88,19 @@ class Module(object):
        except Exception as e:
            logger.log('ERROR', e)
            return None
-        # 状态码非200或者响应体为空
-        if resp.status_code != 200 or not resp.content:
-            logger.log('ALERT', f'GET {resp.url} {resp.status_code} - '
-                       f'{resp.reason} {len(resp.content)}')
-            content_type = resp.headers.get('Content-Type')
-            if content_type and 'json' in content_type and resp.content:
-                logger.log('ALERT', resp.json())
-            return None
-        logger.log('DEBUG', f'GET {resp.url} {resp.status_code} - '
-                   f'{resp.reason} {len(resp.content)}')
-        return resp
+        if not check:
+            return resp
+        if utils.check_response('GET', resp):
+            return resp
+        return None

-    def post(self, url, data=None, **kwargs):
+    def post(self, url, data=None, check=True, **kwargs):
        """
        自定义post请求

        :param str url: 请求地址
        :param dict data: 请求数据
+        :param bool check: 检查响应
        :param kwargs: 其他参数
        :return: requests响应对象
        """
@@ -120,15 +116,11 @@ class Module(object):
        except Exception as e:
            logger.log('ERROR', e)
            return None
-        # 状态码非200或者响应体为空
-        if resp.status_code != 200 or not resp.content:
-            content_type = resp.headers.get('Content-Type')
-            if content_type and 'json' in content_type and resp.content:
-                logger.log('ALERT', resp.json())
-            return None
-        logger.log('DEBUG', f'POST {resp.url} {resp.status_code} - '
-                   f'{resp.reason} {len(resp.content)}')
-        return resp
+        if not check:
+            return resp
+        if utils.check_response('GET', resp):
+            return resp
+        return None

    def get_header(self):
        """
@@ -227,7 +219,7 @@ class Module(object):
                      'ips': None,
                      'status': None,
                      'reason': None,
-                      'valid': 1,
+                      'valid': None,
                      'title': None,
                      'banner': None,
                      'module': self.module,
@@ -247,7 +239,7 @@ class Module(object):
                          'ips': ips,
                          'status': None,
                          'reason': None,
-                          'valid': 1,
+                          'valid': None,
                          'title': None,
                          'banner': None,
                          'module': self.module,
@@ -259,10 +251,10 @@ class Module(object):

    def save_db(self):
        lock.acquire()
-        db_conn = database.connect_db()
-        table_name = self.domain.replace('.', '_')
-        database.create_table(db_conn, table_name)
+        db = Database()
+        db.create_table(self.domain)
        source, results = self.results
        # 将结果存入数据库中
-        database.save_db(db_conn, table_name, results, source)
+        db.save_db(self.domain, results, source)
+        db.close()
        lock.release()
@@ -1,7 +1,10 @@
 # coding=utf-8

 import asyncio
+import functools
+
 import aiohttp
+import tqdm
 from aiohttp import ClientSession
 from aiohttp.resolver import AsyncResolver
 from bs4 import BeautifulSoup
@@ -30,13 +33,19 @@ def get_ports(port):
 def gen_new_datas(datas, ports):
    logger.log('INFOR', f'正在生成请求地址')
    new_datas = []
-    protocols = ['http://', 'https://']
+    protocols = ['http://']
    for data in datas:
-        if data.get('valid'):  # 有效的子域才进行http请求探测
+        valid = data.get('valid')
+        if valid is None:  # 子域有效性未知的才进行http请求探测
            subdomain = data.get('subdomain')
            for port in ports:
                for protocol in protocols:
-                    url = f'{protocol}{subdomain}:{port}'
+                    if port == 443:
+                        url = f'https://{subdomain}:{port}'
+                    elif port == 8443:
+                        url = f'https://{subdomain}:{port}'
+                    else:
+                        url = f'{protocol}{subdomain}:{port}'
                    data['id'] = None
                    data['url'] = url
                    data['port'] = port
@@ -51,7 +60,7 @@ async def fetch(session, url, semaphore):

    :param session: session对象
    :param url: url地址
-    :param semaphore: 同步对象(控制并发量)
+    :param semaphore: 并发信号量
    :return: 响应对象和响应文本
    """
    timeout = aiohttp.ClientTimeout(total=config.get_timeout)
@@ -61,65 +70,114 @@ async def fetch(session, url, semaphore):
                               allow_redirects=config.get_redirects,
                               timeout=timeout,
                               proxy=config.get_proxy) as resp:
-            text = await resp.text()
-            return resp, text
+
+            try:
+                text = await resp.text(encoding='gb2312')  # 先尝试用fb2312解码
+            except UnicodeDecodeError:
+                text = await resp.text()
+        return resp, text


-def deal_results(datas, results):
-    for index, result in enumerate(results):
-        if isinstance(result, Exception):
-            logger.log('DEBUG', result.args)
-            datas[index]['reason'] = str(result.args)
+def get_title(markup):
+    """
+    获取标题
+
+    :param markup: html标签
+    :return: 标题
+    """
+    soup = BeautifulSoup(markup, 'lxml')
+
+    title = soup.title
+    if title:
+        return title.text.strip()
+
+    h1 = soup.h1
+    if h1:
+        return h1.text.strip()
+
+    h2 = soup.h2
+    if h2:
+        return h2.text.strip()
+
+    desc = soup.find('meta', attrs={'name': 'description'})
+    if desc:
+        return desc['content'].strip()
+
+    word = soup.find('meta', attrs={'name': 'keywords'})
+    if word:
+        return word['content'].strip()
+
+    if len(markup) <= 200:
+        return markup.strip()
+
+    return soup.text.strip()
+
+
+def request_callback(future, index, datas):
+    try:
+        result = future.result()
+    except BaseException as e:
+        logger.log('DEBUG', e.args)
+        datas[index]['reason'] = str(e.args)
+        datas[index]['valid'] = 0
+    else:
+        resp, text = result
+        datas[index]['reason'] = resp.reason
+        datas[index]['status'] = resp.status
+        if resp.status >= 500:
            datas[index]['valid'] = 0
-            continue
-        if isinstance(result, tuple):
-            resp, text = result
-            datas[index]['reason'] = resp.reason
-            datas[index]['status'] = resp.status
-            if resp.status == 400 or resp.status >= 500:
-                datas[index]['valid'] = 0
-            else:
-                headers = resp.headers
-                banner = str({'Server': headers.get('Server'),
-                              'Via': headers.get('Via'),
-                              'X-Powered-By': headers.get('X-Powered-By')})
-                datas[index]['banner'] = banner
-                soup = BeautifulSoup(text, 'lxml')
-                title = soup.title
-                head = soup.head
-                if title:
-                    datas[index]['title'] = title.text
-                elif head:
-                    datas[index]['title'] = head.text
-                else:
-                    datas[index]['title'] = text
-    return datas
+        else:
+            datas[index]['valid'] = 1
+            headers = resp.headers
+            banner = str({'Server': headers.get('Server'),
+                          'Via': headers.get('Via'),
+                          'X-Powered-By': headers.get('X-Powered-By')})
+            datas[index]['banner'] = banner
+            datas[index]['title'] = get_title(text)


 async def bulk_get_request(datas, port):
-    logger.log('INFOR', f'正在异步进行子域的GET请求')
    ports = get_ports(port)
    new_datas = gen_new_datas(datas, ports)
-    header = None
-    if config.fake_header:
-        header = utils.gen_fake_header()
+    logger.log('INFOR', f'正在异步进行子域的GET请求')
+
+    limit_open_conn = config.limit_open_conn
+    if limit_open_conn is None:  # 默认情况
+        limit_open_conn = utils.get_semaphore()
+    elif not isinstance(limit_open_conn, int):  # 如果传入不是数字的情况
+        limit_open_conn = utils.get_semaphore()
    # 使用异步域名解析器 自定义域名服务器
    resolver = AsyncResolver(nameservers=config.resolver_nameservers)
    conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
-                                limit=config.limit_open_conn,
+                                limit=limit_open_conn,
                                limit_per_host=config.limit_per_host,
                                resolver=resolver)
-    semaphore = asyncio.Semaphore(utils.get_semaphore())
+
+    semaphore = asyncio.Semaphore(limit_open_conn)
+    header = None
+    if config.fake_header:
+        header = utils.gen_fake_header()
    async with ClientSession(connector=conn, headers=header) as session:
        tasks = []
        for i, data in enumerate(new_datas):
            url = data.get('url')
            task = asyncio.ensure_future(fetch(session, url, semaphore))
+            task.add_done_callback(functools.partial(request_callback,
+                                                     index=i,
+                                                     datas=new_datas))
            tasks.append(task)
        if tasks:  # 任务列表里有任务不空时才进行解析
            # 等待所有task完成 错误聚合到结果列表里
-            results = await asyncio.gather(*tasks, return_exceptions=True)
-            new_datas = deal_results(new_datas, results)
+            futures = asyncio.as_completed(tasks)
+            for future in tqdm.tqdm(futures,
+                                    total=len(tasks),
+                                    desc='Progress',
+                                    smoothing=1.0,
+                                    ncols=True):
+                try:
+                    await future
+                except:
+                    pass

    logger.log('INFOR', f'完成异步进行子域的GET请求')
    return new_datas
@@ -4,6 +4,8 @@ import functools

 import dns.resolver
 import aiodns
+import tqdm
+
 import config
 from common import utils
 from config import logger
@@ -49,26 +51,43 @@ async def aiodns_query_a(hostname, semaphore=None):
    """
    if semaphore is None:
        resolver = aiodns_resolver()
-        answers = await resolver.query(hostname, 'A')
+        try:
+            answers = await resolver.query(hostname, 'A')
+        except BaseException as e:
+            logger.log('DEBUG', e.args)
+            answers = None
        return hostname, answers
    else:
        async with semaphore:
            resolver = aiodns_resolver()
-            answers = await resolver.query(hostname, 'A')
+            try:
+                answers = await resolver.query(hostname, 'A')
+            except BaseException as e:
+                logger.log('DEBUG', e.args)
+                answers = None
            return hostname, answers


 def resolve_callback(future, index, datas):
+    """
+    解析结果回调处理
+    :param future: future对象
+    :param index: 下标
+    :param datas: 结果集
+    """
    try:
        result = future.result()
-    except aiodns.error.DNSError as e:
+    except BaseException as e:
        datas[index]['ips'] = str(e.args)
        datas[index]['valid'] = 0
    else:
        if isinstance(result, tuple):
            _, answers = result
-            ips = {record.host for record in answers}
-            datas[index]['ips'] = str(ips)
+            if answers:
+                ips = {record.host for record in answers}
+                datas[index]['ips'] = str(ips)
+            else:
+                datas[index]['ips'] = 'No answers'


 async def bulk_query_a(datas):
@@ -80,7 +99,7 @@ async def bulk_query_a(datas):
    """
    logger.log('INFOR', '正在异步查询子域的A记录')
    tasks = []
-    semaphore = asyncio.Semaphore(utils.get_semaphore())
+    semaphore = asyncio.Semaphore(config.limit_resolve_conn)
    for i, data in enumerate(datas):
        if not data.get('ips'):
            subdomain = data.get('subdomain')
@@ -90,6 +109,16 @@ async def bulk_query_a(datas):
                                                     datas=datas))  # 回调
            tasks.append(task)
    if tasks:  # 任务列表里有任务不空时才进行解析
-        await asyncio.wait(tasks)  # 等待所有task完成
+        futures = asyncio.as_completed(tasks)
+        for future in tqdm.tqdm(futures,
+                                total=len(tasks),
+                                desc='Progress',
+                                smoothing=1.0,
+                                ncols=True):
+            try:
+                await future
+            except:
+                pass
+        # await asyncio.wait(tasks)  # 等待所有task完成
    logger.log('INFOR', '完成异步查询子域的A记录')
    return datas
@@ -1,10 +1,10 @@
 # coding=utf-8
 import re
-import pathlib
 import random
 import ipaddress
 import platform
 import config
+from pathlib import Path
 from common.domain import Domain
 from config import logger

@@ -109,20 +109,22 @@ def get_domains(target):
    :param set or str target:
    :return: 域名集合
    """
-    domains = set()
-    logger.log('INFOR', f'正在获取域名')
-    if isinstance(target, set):
+    domains = list()
+    logger.log('DEBUG', f'正在获取域名')
+    if isinstance(target, (set, tuple)):
+        domains = list(target)
+    elif isinstance(target, list):
        domains = target
    elif isinstance(target, str):
-        path = pathlib.Path(target)
+        path = Path(target)
        if path.is_file():
            with open(target) as file:
                for line in file:
                    domain = Domain(line.strip()).match()
                    if domain:
-                        domains.add(domain)
-        if Domain(target).match():
-            domains = {target}
+                        domains.append(domain)
+        elif Domain(target).match():
+            domains = [target]
    logger.log('INFOR', f'获取到{len(domains)}个域名')
    return domains

@@ -135,8 +137,73 @@ def get_semaphore():
    """
    system = platform.system()
    if system == 'Windows':
-        return 500
+        return 300
    elif system == 'Linux':
-        return 1000
+        return 800
    elif system == 'Darwin':
-        return 1000
+        return 800
+
+
+def check_dpath(dpath):
+    """
+    检查目录路径
+
+    :param dpath: 传入的目录路径
+    :return: 目录路径
+    """
+    if isinstance(dpath, str):
+        dpath = Path(dpath)
+    else:
+        dpath = config.result_save_path
+    if not dpath.is_dir():
+        logger.log('FATAL', f'{dpath}不是目录')
+    if not dpath.exists():
+        logger.log('ALERT', f'不存在{dpath}将会新建此目录')
+        dpath.mkdir(parents=True, exist_ok=True)
+    return dpath
+
+
+def check_format(format):
+    """
+    检查导出格式
+
+    :param format: 传入的导出格式
+    :return: 导出格式
+    """
+    formats = ['txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+               'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods']
+    if format in formats:
+        return format
+    else:
+        logger.log('ALERT', f'不支持{format}格式导出')
+        logger.log('ALERT', '默认使用csv格式导出')
+        return 'xls'
+
+
+def save_data(fpath, data):
+    try:
+        with open(fpath, 'w', encoding="utf-8", newline='') as file:
+            file.write(data)
+            logger.log('ALERT', fpath)
+    except TypeError:
+        with open(fpath, 'wb') as file:
+            file.write(data)
+            logger.log('ALERT', fpath)
+    except Exception as e:
+        logger.log('ERROR', e)
+
+
+def check_response(method, resp):
+    if resp.status_code == 200 and resp.content:
+        return True
+    logger.log('ALERT', f'{method} {resp.url} {resp.status_code} - '
+                        f'{resp.reason} {len(resp.content)}')
+    content_type = resp.headers.get('Content-Type')
+    if content_type and 'json' in content_type and resp.content:
+        try:
+            msg = resp.json()
+        except Exception as e:
+            logger.log('DEBUG', e.args)
+        else:
+            logger.log('ALERT', msg)
+    return False
@@ -5,6 +5,8 @@ OneForAll配置
 import os
 import sys
 import pathlib
+
+import requests
 from loguru import logger

 # 路径设置
@@ -15,8 +17,8 @@ data_storage_path = oneforall_relpath.joinpath('data')  # 数据存放目录
 result_save_path = oneforall_relpath.joinpath('results')  # 结果保存目录

 # 模块设置
-save_module_result = True  # 保存模块中各脚本结果 默认保存
-enable_all_module = True  # 启用所有模块 默认启用
+save_module_result = True  # 保存模块中各脚本结果(默认True)
+enable_all_module = True  # 启用所有模块(默认True)
 enable_partial_module = []  # 启用部分模块 必须禁用enable_all_module才能生效
 # 只使用ask和baidu搜索引擎收集子域
 # enable_partial_module = [('modules.search', 'ask')
@@ -25,26 +27,27 @@ enable_partial_module = []  # 启用部分模块 必须禁用enable_all_module

 # 爆破模块设置
 enable_brute_module = False  # 使用爆破模块(默认禁用)
+enable_verify_subdomain = True  # 验证子域有效性(默认True)
 enable_wildcard_check = True  # 开启泛解析检测 会去掉泛解析的子域
 # 爆破时使用的进程数(根据系统中CPU数量情况设置 不宜大于CPU数量 默认为系统中的CPU数量)
-brute_processes_num = os.cpu_count()
+brute_process_num = os.cpu_count()
 brute_coroutine_num = 128  # 爆破时每个进程下的协程数(不宜大于1000)
 # 爆破所使用的字典路径 默认data/subdomains.txt
-brute_wordlist_path = 'data/subdomains.txt'
+brute_wordlist_path = data_storage_path.joinpath('subnames.txt')
 brute_task_segment = 500
-# 参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为一任务组，
+# 参数segment的设置受CPU性能，网络带宽，运营商限制等限制，默认500个子域为一任务组，
 # 当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
 # 十万字典建议设置为5000，百万字典设置为50000
 enable_recursive_brute = False  # 是否使用递归爆破(默认禁用)
 brute_recursive_depth = 2  # 递归爆破深度(默认2层)
 # 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt
-recursive_namelist_path = 'data/next_subdomains.txt'
+recursive_namelist_path = data_storage_path.joinpath('next_subnames.txt')
 enable_fuzz = False  # 是否使用fuzz模式枚举域名
 fuzz_rule = ''  # fuzz域名的正则 示例：[a-z][0-9] 第一位是字母 第二位是数字
 ips_appear_maximum = 10  # 同一IP集合出现次数超过10认为是泛解析

 # 代理设置
-enable_proxy = False  # 是否使用代理 全局开关
+enable_proxy = False  # 是否使用代理(全局开关)
 proxy_all_module = False  # 代理所有模块
 proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
                        'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
@@ -52,8 +55,8 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
                        'ContentSecurityPolicy']  # 代理自定义的模块
 proxy_pool = [{'http': 'http://127.0.0.1:1080',
               'https': 'https://127.0.0.1:1080'}]  # 代理池
-# proxy_pool = [{'http': 'socks5://127.0.0.1:10808',
-#                'https': 'socks5://127.0.0.1:10808'}]  # 代理池
+# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
+#                'https': 'socks5h://127.0.0.1:10808'}]  # 代理池


 # 网络请求设置
@@ -61,6 +64,7 @@ enable_fake_header = True  # 启用伪造请求头
 request_delay = 1  # 请求时延
 request_timeout = 30  # 请求超时
 request_verify = True  # 请求SSL验证
+requests.packages.urllib3.disable_warnings()  # 禁用安全警告信息

 # 搜索模块设置
 enable_recursive_search = False  # 递归搜索子域
@@ -78,6 +82,7 @@ resolver_nameservers = [
 ]  # 指定查询的DNS域名服务器
 resolver_timeout = 5.0  # 解析超时时间
 resolver_lifetime = 30.0  # 解析存活时间
+limit_resolve_conn = 50  # 限制同一时间解析的数量(默认50)

 # http探测设置
 small_ports = {80, 443}
@@ -97,10 +102,11 @@ ports = {'small': small_ports, 'medium': medium_ports,
 verify_ssl = False
 # aiohttp 支持 HTTP/HTTPS形式的代理
 get_proxy = None  # proxy="http://user:pass@some.proxy.com"
-get_timeout = 10  # http请求探测总超时时间 None或者0则表示不检测超时
+get_timeout = 120  # http请求探测总超时时间 None或者0则表示不检测超时
 get_redirects = True  # 允许请求跳转
 fake_header = True  # 使用伪造请求头
-limit_open_conn = 100  # 限制同一时间打开的连接数(默认100)，0表示不限制
+# 限制同一时间打开的连接数(默认None，根据系统不同设置，Windows系统400 其他系统800)
+limit_open_conn = None
 # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
 limit_per_host = 0  # 默认0表示不限制

@@ -114,7 +120,7 @@ censys_api_secret = ''
 # 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
 binaryedge_api = ''

-# Binaryedge可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
+# Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
 chinaz_api = ''

 # Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/
@@ -166,21 +172,37 @@ dnsdb_api_key = ''
 # 免费的API有效期只有2天，到期之后可以再次生成，每天可以查询50次。
 ipv4info_api_key = ''

+# https://github.com/360netlab/flint
+# passivedns_api_addr默认空使用http://api.passivedns.cn
+# passivedns_api_token可为空
+passivedns_api_addr = ''
+passivedns_api_token = ''
+
+# Github Token可以访问https://github.com/settings/tokens生成,user为Github用户名
+github_api_user = ''
+github_api_token = ''
+# github子域收集模块使用
+github_email = ''
+github_password = ''
+
 subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
                     'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
                     'help', 'live', 'mall', 'blogs', 'files', 'forum',
                     'store', 'mobile'}

 # 日志配置
+# 终端日志输出格式
 stdout_fmt = '<cyan>{time:HH:mm:ss,SSS}</cyan> ' \
          '[<level>{level: <5}</level>] ' \
          '<blue>{module}</blue>:<cyan>{line}</cyan> - ' \
          '<level>{message}</level>'
+# 日志文件记录格式
 logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
          '[<level>{level: <5}</level>] ' \
-          '<cyan>{process.name}</cyan>:<cyan>{thread.name: <10}</cyan> | ' \
-          '<blue>{module}</blue>.<blue>{function}</blue>:<blue>{line}</blue> - ' \
-          '<level>{message}</level>'
+          '<cyan>{process.name}({process.id})</cyan>:' \
+          '<cyan>{thread.name: <10}({thread.id: <5})</cyan> | ' \
+          '<blue>{module}</blue>.<blue>{function}</blue>:' \
+          '<blue>{line}</blue> - <level>{message}</level>'

 log_path = result_save_path.joinpath('oneforall.log')

@@ -196,5 +218,5 @@ if not os.environ.get('PYTHONIOENCODING'):  # 设置编码
    os.environ['PYTHONIOENCODING'] = 'utf-8'

 logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True)
-logger.add(log_path, level='TRACE', format=logfile_fmt, enqueue=True,
+logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True,
           encoding='utf-8')
@@ -0,0 +1,249 @@
+[
+    {
+        "name":"github", 
+        "cname":["github.io", "github.map.fastly.net"], 
+        "response":["There isn't a GitHub Pages site here.", "For root URLs (like http://example.com/) you must provide an index.html file"]
+    },
+    {
+        "name":"heroku", 
+        "cname":["herokudns.com", "herokussl.com", "herokuapp.com"], 
+        "response":["There's nothing here, yet.", "herokucdn.com/error-pages/no-such-app.html", "<title>No such app</title>"]
+    },
+    {
+        "name":"unbounce",
+        "cname":["unbouncepages.com"],
+        "response":["Sorry, the page you were looking for doesn’t exist.", "The requested URL was not found on this server"]
+    },
+    {
+        "name":"tumblr",
+        "cname":["tumblr.com"],
+        "response":["There's nothing here.", "Whatever you were looking for doesn't currently exist at this address."]
+    },
+    {
+        "name":"shopify",
+        "cname":["myshopify.com"],
+        "response":["Sorry, this shop is currently unavailable.", "Only one step left!"]
+    },
+    {
+        "name":"instapage",
+        "cname":["pageserve.co", "secure.pageserve.co", "https://instapage.com/"],
+        "response":["Looks Like You're Lost","The page you're looking for is no longer available."]
+    },
+    {
+        "name":"desk",
+        "cname":["desk.com"],
+        "response":["Please try again or try Desk.com free for 14 days.", "Sorry, We Couldn't Find That Page"]
+    },
+    {
+        "name":"campaignmonitor",
+        "cname":["createsend.com", "name.createsend.com"],
+        "response":["Double check the URL", "<strong>Trying to access your account?</strong>"]
+    },
+    {
+        "name":"cargocollective",
+        "cname":["cargocollective.com"],
+        "response":["404 Not Found"]
+    },
+    {
+        "name":"statuspage",
+        "cname":["statuspage.io"],
+        "response":["Better Status Communication", "You are being <a href=\"https://www.statuspage.io\">redirected"]
+    },
+    {
+        "name":"amazonaws",
+        "cname":["amazonaws.com"],
+        "response":["NoSuchBucket", "The specified bucket does not exist"]
+    },
+    {
+        "name":"bitbucket",
+        "cname":["bitbucket.org"],  
+        "response":["The page you have requested does not exist","Repository not found"]
+    },
+    {
+        "name":"smartling",
+        "cname":["smartling.com"],
+        "response":["Domain is not configured"]
+    },
+    {
+        "name":"acquia",
+        "cname":["acquia.com"],
+        "response":["If you are an Acquia Cloud customer and expect to see your site at this address","The site you are looking for could not be found."]
+    },
+    {
+        "name":"fastly",
+        "cname":["fastly.net"],
+        "response":["Please check that this domain has been added to a service", "Fastly error: unknown domain"]
+    },
+    {
+        "name":"pantheon",
+        "cname":["pantheonsite.io"],
+        "response":["The gods are wise", "The gods are wise, but do not know of the site which you seek."]
+    },
+    {
+        "name":"zendesk",
+        "cname":["zendesk.com"],
+        "response":["Help Center Closed"]
+    },
+    {
+        "name":"uservoice",
+        "cname":["uservoice.com"],
+        "response":["This UserVoice subdomain is currently available!"]
+    },
+    {
+        "name":"ghost",
+        "cname":["ghost.io"],
+        "response":["The thing you were looking for is no longer here", "The thing you were looking for is no longer here, or never was"]
+    },
+    {
+        "name":"pingdom",
+        "cname":["stats.pingdom.com"],
+        "response":["pingdom"]
+    },
+    {
+        "name":"tilda",
+        "cname":["tilda.ws"],
+        "response":["Domain has been assigned"]
+    },
+    {
+        "name":"wordpress",
+        "cname":["wordpress.com"],  
+        "response":["Do you want to register"]
+    },
+    {
+        "name":"teamwork",
+        "cname":["teamwork.com"],
+        "response":["Oops - We didn't find your site."]
+    },
+    {
+        "name":"helpjuice",
+        "cname":["helpjuice.com"],
+        "response":["We could not find what you're looking for."]
+    },
+    {
+        "name":"helpscout",
+        "cname":["helpscoutdocs.com"],
+        "response":["No settings were found for this company:"]
+    },
+    {
+        "name":"cargo",
+        "cname":["cargocollective.com"],
+        "response":["If you're moving your domain away from Cargo you must make this configuration through your registrar's DNS control panel."]
+    },
+    {
+        "name":"feedpress",
+        "cname":["redirect.feedpress.me"],
+        "response":["The feed has not been found."]
+    },
+    {
+        "name":"surge",
+        "cname":["surge.sh"],
+        "response":["project not found"]
+    },
+    {
+        "name":"surveygizmo",
+        "cname":["privatedomain.sgizmo.com", "privatedomain.surveygizmo.eu", "privatedomain.sgizmoca.com"],
+        "response":["data-html-name"]
+    },
+    {
+        "name":"mashery",
+        "cname":["mashery.com"],
+        "response":["Unrecognized domain <strong>"]
+    },
+    {
+        "name":"intercom",
+        "cname":["custom.intercom.help"],
+        "response":["This page is reserved for artistic dogs.","<h1 class=\"headline\">Uh oh. That page doesn’t exist.</h1>"]
+    },
+    {
+        "name":"webflow",
+        "cname":["proxy.webflow.io"],
+        "response":["<p class=\"description\">The page you are looking for doesn't exist or has been moved.</p>"]
+    },
+    {
+        "name":"kajabi",
+        "cname":["endpoint.mykajabi.com"],
+        "response":["<h1>The page you were looking for doesn't exist.</h1>"]
+    },
+    {
+        "name":"thinkific",
+        "cname":["thinkific.com"],
+        "response":["You may have mistyped the address or the page may have moved."]
+    },
+    {
+        "name":"tave",
+        "cname":["clientaccess.tave.com"],
+        "response":["<h1>Error 404: Page Not Found</h1>"]
+    },
+    {
+        "name":"wishpond",
+        "cname":["wishpond.com"],
+        "response":["https://www.wishpond.com/404?campaign=true"]
+    },
+    {
+        "name":"aftership",
+        "cname":["aftership.com"],
+        "response":["Oops.</h2><p class=\"text-muted text-tight\">The page you're looking for doesn't exist."]
+    },
+    {
+        "name":"aha",
+        "cname":["ideas.aha.io"],
+        "response":["There is no portal here ... sending you back to Aha!"]
+    },
+    {
+        "name":"brightcove",
+        "cname":["brightcovegallery.com", "gallery.video", "bcvp0rtal.com"],
+        "response":["<p class=\"bc-gallery-error-code\">Error Code: 404</p>"]
+    },
+    {
+        "name":"bigcartel",
+        "cname":["bigcartel.com"],
+        "response":["<h1>Oops! We couldn&#8217;t find that page.</h1>"]
+    },
+    {
+        "name":"activecompaign",
+        "cname":["activehosted.com"],
+        "response":["alt=\"LIGHTTPD - fly light.\""]
+    },
+    {
+        "name":"compaignmonitor",
+        "cname":["createsend.com"],
+        "response":["Double check the URL or <a href=\"mailto:help@createsend.com"]
+    },
+    {
+        "name":"simplebooklet",
+        "cname":["simplebooklet.com"],
+        "response":["We can't find this <a href=\"https://simplebooklet.com"]
+    },
+    {
+        "name":"getresponse",
+        "cname":[".gr8.com"],
+        "response":["With GetResponse Landing Pages, lead generation has never been easier"]
+    },
+    {
+        "name":"vend",
+        "cname":["vendecommerce.com"],
+        "response":["Looks like you've traveled too far into cyberspace."]
+    },
+    {
+        "name":"jetbrains",
+        "cname":["myjetbrains.com"],
+        "response":["is not a registered InCloud YouTrack.","is not a registered InCloud YouTrack."]
+    },
+    {
+        "name":"azure",
+        "cname":["azurewebsites.net",
+            ".cloudapp.net",
+            ".cloudapp.azure.com",
+            ".trafficmanager.net",
+            ".blob.core.windows.net",
+            ".azure-api.net",
+            ".azurehdinsight.net",
+            ".azureedge.net"],
+        "response":["404 Web Site not found"]
+    },
+    {
+        "name":"readme",
+        "cname":["readme.io"],
+        "response":["Project doesnt exist... yet!"]
+    }
+]
@@ -1,163 +1,163 @@
-test
-test2
-t
-dev
-1
-2
-3
-s1
-s2
-s3
-admin
-adm
-a
-b
-c
-m
-ht
-adminht
-webht
-web
-gm
-sys
-system
-manage
-manager
-mgr
-passport
-bata
-wei
-weixin
-wechat
-wx
-wiki
-upload
-ftp
-pic
-jira
-zabbix
-nagios
-bug
-bugzilla
-sql
-mysql
-db
-stmp
-pop
-imap
-mail
-zimbra
-exchange
-forum
-bbs
-list
-count
-counter
-img
-img01
-img02
-img03
-img04
-api
-cache
-js
-css
-app
-apps
-wap
-sms
-zip
-monitor
-proxy
-update
-upgrade
-stat
-stats
-data
-portal
-blog
-autodiscover
-en
-search
-so
-oa
-database
-home
-sso
-help
-vip
-s
-w
-down
-download
-downloads
-dl
-svn
-git
-log
-staff
-vpn
-sslvpn
-ssh
-scanner
-sandbox
-ldap
-lab
-go
-demo
-console
-cms
-auth
-crm
-erp
-res
-static
-old
-new
-beta
-image
-service
-login
-3g
-docs
-it
-e
-live
-library
-files
-i
-d
-cp
-connect
-gateway
-lib
-preview
-backup
-share
-status
-assets
-user
-vote
-bugs
-cas
-feedback
-id
-edm
-survey
-union
-ceshi
-dev1
-updates
-phpmyadmin
-pma
-edit
-master
-xml
-control
-profile
-zhidao
-tool
-toolbox
-boss
-activity
-www
+test
+test2
+t
+dev
+1
+2
+3
+s1
+s2
+s3
+admin
+adm
+a
+b
+c
+m
+ht
+adminht
+webht
+web
+gm
+sys
+system
+manage
+manager
+mgr
+passport
+bata
+wei
+weixin
+wechat
+wx
+wiki
+upload
+ftp
+pic
+jira
+zabbix
+nagios
+bug
+bugzilla
+sql
+mysql
+db
+stmp
+pop
+imap
+mail
+zimbra
+exchange
+forum
+bbs
+list
+count
+counter
+img
+img01
+img02
+img03
+img04
+api
+cache
+js
+css
+app
+apps
+wap
+sms
+zip
+monitor
+proxy
+update
+upgrade
+stat
+stats
+data
+portal
+blog
+autodiscover
+en
+search
+so
+oa
+database
+home
+sso
+help
+vip
+s
+w
+down
+download
+downloads
+dl
+svn
+git
+log
+staff
+vpn
+sslvpn
+ssh
+scanner
+sandbox
+ldap
+lab
+go
+demo
+console
+cms
+auth
+crm
+erp
+res
+static
+old
+new
+beta
+image
+service
+login
+3g
+docs
+it
+e
+live
+library
+files
+i
+d
+cp
+connect
+gateway
+lib
+preview
+backup
+share
+status
+assets
+user
+vote
+bugs
+cas
+feedback
+id
+edm
+survey
+union
+ceshi
+dev1
+updates
+phpmyadmin
+pma
+edit
+master
+xml
+control
+profile
+zhidao
+tool
+toolbox
+boss
+activity
+www
@@ -9,55 +9,51 @@ OneForAll数据库导出模块
 """

 import fire
-from common import database
-from config import logger
+from common import utils
+from common.database import Database


-def export(table, db=None, valid=None, path=None, format='xlsx', output=False):
+def export(table, db=None, valid=None, dpath=None, format='csv', show=False):
    """
    OneForAll数据库导出模块

    Example:
-        python dbexport.py --db result.db --table name --format csv --output False
-        python dbexport.py --db result.db --table name --format csv --path= ./result.csv
+        python3 dbexport.py --table name --format csv --dir= ./result.csv
+        python3 dbexport.py --db result.db --table name --show False

    Note:
-        参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
-        参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
-        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+        参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+        参数dpath为None默认使用OneForAll结果目录

    :param str table:   要导出的表
    :param str db:      要导出的数据库路径(默认为results/result.sqlite3)
    :param int valid:   导出子域的有效性(默认None)
-    :param str format:  导出格式(默认xlsx)
-    :param str path:    导出路径(默认None)
-    :param bool output: 是否将导出数据输出到终端(默认False)
+    :param str format:  导出格式(默认xls)
+    :param str dpath:    导出目录(默认None)
+    :param bool show:   终端显示导出数据(默认False)
    """
-    db_conn = database.connect_db(db)
+    format = utils.check_format(format)
+    dpath = utils.check_dpath(dpath)
+    database = Database(db)
    if valid is None:
-        rows = database.get_data(db_conn, table)
+        rows = database.get_data(table)
    elif isinstance(valid, int):
-        rows = database.get_subdomain(db_conn, table, valid)
+        rows = database.get_subdomain(table, valid)
    else:
-        rows = database.get_data(db_conn, table)  # 意外情况导出全部子域
-    if output:
+        rows = database.get_data(table)  # 意外情况导出全部子域
+    if show:
        print(rows.dataset)
-    if not path:
-        path = 'export.' + format
-    logger.log('INFOR', f'正在将数据库中{table}表导出')
-    try:
-        with open(path, 'w') as file:
-            file.write(rows.export(format))
-            logger.log('INFOR', '成功完成导出')
-            logger.log('INFOR', path)
-    except TypeError:
-        with open(path, 'wb') as file:
-            file.write(rows.export(format))
-            logger.log('INFOR', '成功完成导出')
-            logger.log('INFOR', path)
-    except Exception as e:
-        logger.log('ERROR', e)
+    if format == 'txt':
+        data = str(rows.dataset)
+    else:
+        data = rows.export(format)
+    database.close()
+    fpath = dpath.joinpath(f'{table}.{format}')
+    utils.save_data(fpath, data)


 if __name__ == '__main__':
    fire.Fire(export)
+    # save('example_com_last', format='txt')
@@ -1 +1,2 @@
-example.com
+example.com
+hackfun.org
@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+"""
+github自动接管
+"""
+
+import json
+import base64
+import requests
+import config
+
+HEADERS = {
+    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "Accept-Language": "zh-CN,zh;q=0.9",
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
+}
+
+
+def github_takeover(url):
+    # 读取config配置文件
+    repo_name = url
+    print('[*]正在读取配置文件...')
+    user = config.github_api_user
+    token = config.github_api_token
+    CHECK_HEADERS = {
+        "Authorization": 'token ' + token,
+        "Accept": "application/vnd.github.switcheroo-preview+json"
+    }
+    repos_url = 'https://api.github.com/repos/' + user + '/' + repo_name
+    repos_r = requests.get(url=repos_url, headers=CHECK_HEADERS)
+    # 验证token是否正确
+    if 'message' in repos_r.json():
+        if repos_r.json()['message'] == 'Bad credentials':
+            print('[*]请检查Token是否正确')
+        elif repos_r.json()['message'] == 'Not Found':
+            print('[*]正在生成接管库...')  # 生成接管库
+            creat_repo_dict = {
+                "name": repo_name,
+                "description": "This is a subdomain takeover Repository",
+            }
+            creat_repo_url = 'https://api.github.com/user/repos'
+            creat_repo_r = requests.post(url=creat_repo_url,
+                                         headers=CHECK_HEADERS,
+                                         data=json.dumps(creat_repo_dict))
+            creat_repo_status = creat_repo_r.status_code
+            if creat_repo_status == 201:
+                print('[*]创建接管库' + repo_name + '成功，正在进行自动接管...')
+                # 接管文件生成
+                # index.html文件
+                html = b'''
+                <html>
+                    <p>Subdomain Takerover Test!</>
+                </html>
+                '''
+                html64 = base64.b64encode(html).decode('utf-8')
+                html_dict = {
+                    "message": "my commit message",
+                    "committer": {
+                        "name": "user",  # 提交id，非必改项
+                        "email": "user@163.com"  # 同上
+                    },
+                    "content": html64
+                }
+                # CNAME文件
+                cname_url = bytes(url, encoding='utf-8')
+                cname_url64 = base64.b64encode(cname_url).decode('utf-8')
+                url_dict = {
+                    "message": "my commit message",
+                    "committer": {
+                        "name": "user",
+                        "email": "user@163.com"
+                    },
+                    "content": cname_url64
+                }
+                html_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/index.html'
+                url_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/CNAME'
+                html_r = requests.put(url=html_url, data=json.dumps(html_dict),
+                                      headers=CHECK_HEADERS)  # 上传index.html
+                cname_r = requests.put(url=url_url, data=json.dumps(url_dict),
+                                       headers=CHECK_HEADERS)  # 上传CNAME
+                rs = cname_r.status_code
+                if rs == 201:
+                    print('[*]生成接管库成功，正在开启Github pages...')
+                    page_url = "https://api.github.com/repos/" + user + "/" + url + "/pages"
+                    page_dict = {
+                        "source": {
+                            "branch": "master"
+                        }
+                    }
+                    page_r = requests.post(url=page_url,
+                                           data=json.dumps(page_dict),
+                                           headers=CHECK_HEADERS)  # 开启page
+                    if page_r.status_code == 201:
+                        print('[+]自动接管成功，请稍后访问http://' + str(url) + '查看结果')
+                    else:
+                        print('[+]开启Github pages失败，请检查网络或稍后重试...')
+                else:
+                    print('[+]生成接管库失败，请检查网络或稍后重试...')
+    elif url in repos_r.json()['name']:
+        print('[*]生成接管库失败，请检查https://github.com/' + user +
+              '?tab=repositories是否存在同名接管库...')
@@ -22,29 +22,29 @@ class CensysAPI(Query):
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        data = {
-            'query': 'parsed.names: example.com',
+            'query': f'parsed.names: {self.domain}',
            'page': 1,
            'fields': ['parsed.subject_dn'],
            'flatten': True}
        resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
        if not resp:
            return
-        resp_json = resp.json()
-        status = resp_json.get('status')
+        data = resp.json()
+        status = data.get('status')
        if status != 'ok':
            logger.log('ALERT', status)
            return
-        subdomains_find = self.match(self.domain, str(resp_json))
-        self.subdomains = self.subdomains.union(subdomains_find)
-        pages = resp_json.get('metadata').get('pages')
+        subdomains = self.match(self.domain, str(data))
+        self.subdomains = self.subdomains.union(subdomains)
+        pages = data.get('metadata').get('pages')
        for page in range(2, pages + 1):
            time.sleep(self.delay)
            data['page'] = page
            resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
            if not resp:
                return
-            subdomains_find = self.match(self.domain, str(resp.json()))
-            self.subdomains = self.subdomains.union(subdomains_find)
+            subdomains = self.match(self.domain, str(resp.json()))
+            self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -54,10 +54,10 @@ class CensysAPI(Query):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -9,7 +9,7 @@ class CertDBAPI(Query):
        Query.__init__(self)
        self.domain = domain
        self.module = 'Certificate'
-        self.source = 'CertDBQuery'
+        self.source = 'CertDBAPIQuery'
        self.addr = 'https://api.spyse.com/v1/subdomains'
        self.token = config.certdb_api_token

@@ -29,9 +29,11 @@ class CertDBAPI(Query):
            if not resp:
                return
            json = resp.json()
-            subdomains_find = utils.match_subdomain(self.domain, str(json))
+            subdomains = utils.match_subdomain(self.domain, str(json))
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains_find)
+            self.subdomains = self.subdomains.union(subdomains)
            page_num += 1
            # 默认每次查询最多返回30条 当前条数小于30条说明已经查完
            if json.get('count') < 30:
@@ -45,10 +47,10 @@ class CertDBAPI(Query):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -18,14 +18,15 @@ class CertSpotter(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'domain': self.domain, 'include_subdomains': 'true',
+        params = {'domain': self.domain,
+                  'include_subdomains': 'true',
                  'expand': 'dns_names'}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
+        subdomains = utils.match_subdomain(self.domain, str(resp.json()))
        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains_find)
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -33,10 +34,10 @@ class CertSpotter(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -22,8 +22,8 @@ class Crtsh(Query):
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = utils.match_subdomain(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -31,10 +31,10 @@ class Crtsh(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -18,13 +18,14 @@ class Entrust(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'fields': 'subjectDN', 'domain': self.domain,
+        params = {'fields': 'subjectDN',
+                  'domain': self.domain,
                  'includeExpired': 'true'}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = utils.match_subdomain(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -32,10 +33,10 @@ class Entrust(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -19,14 +19,15 @@ class Google(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'include_expired': 'true', 'include_subdomains': 'true',
+        params = {'include_expired': 'true',
+                  'include_subdomains': 'true',
                  'domain': self.domain}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, resp.text)
+        subdomains = utils.match_subdomain(self.domain, resp.text)
        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains_find)
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -34,10 +35,10 @@ class Google(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -1,10 +1,11 @@
 """
-查询域名的NS记录(域名服务器记录，记录该域名由哪台域名服务器解析)，
-检查查出的域名服务器是否开启DNS域传送，如果开启且没做访问控制和身份验证便加以利用获取域名的所有记录
+查询域名的NS记录(域名服务器记录，记录该域名由哪台域名服务器解析)，检查查出的域名服务器是
+否开启DNS域传送，如果开启且没做访问控制和身份验证便加以利用获取域名的所有记录。

-DNS域传送(DNS zone transfer)指的是一台备用域名服务器使用来自主域名服务器的数据刷新自己的域数据库，
-目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
-当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获取某个域的所有记录。
+DNS域传送(DNS zone transfer)指的是一台备用域名服务器使用来自主域名服务器的数据刷新自己
+的域数据库，目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
+当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获
+取某个域的所有记录。
 """
 import dns.resolver
 import dns.zone
@@ -23,9 +24,34 @@ class CheckAXFR(Module):
        self.domain = self.register(domain)
        self.module = 'Check'
        self.source = 'AXFRCheck'
-        self.nsservers = []
        self.results = []

+    def axfr(self, server):
+        """
+        执行域传送
+
+        :param server: 域名服务器
+        """
+        logger.log('DEBUG', f'尝试对{self.domain}的域名服务器{server}进行域传送')
+        try:
+            xfr = dns.query.xfr(server, self.domain, timeout=30.0)
+            zone = dns.zone.from_xfr(xfr)
+        except Exception as e:
+            logger.log('DEBUG', str(e))
+            logger.log('DEBUG', f'对{self.domain}的域名服务器{server}进行域传送失败')
+            return
+        names = zone.nodes.keys()
+        for name in names:
+            full_domain = str(name) + '.' + self.domain
+            subdomain = utils.match_subdomain(self.domain, full_domain)
+            self.subdomains = self.subdomains.union(subdomain)
+            record = zone[name].to_text(name)
+            self.results.append(record)
+        if self.results:
+            logger.log('DEBUG', f'发现{self.domain}在{server}上的域传送记录')
+            logger.log('DEBUG', '\n'.join(self.results))
+            self.results = []
+
    def check(self):
        """
        正则匹配响应头中的内容安全策略字段以发现子域名
@@ -36,43 +62,23 @@ class CheckAXFR(Module):
        except Exception as e:
            logger.log('ERROR', e)
            return
-        self.nsservers = [str(answer) for answer in answers]
-        if not len(self.nsservers):
+        nsservers = [str(answer) for answer in answers]
+        if not len(nsservers):
            logger.log('ALERT', f'没有找到{self.domain}的NS域名服务器记录')
            return
-        for nsserver in self.nsservers:
-            logger.log('DEBUG', f'正在尝试对{self.domain}的域名服务器{nsserver}进行域传送')
-            try:
-                xfr = dns.query.xfr(nsserver, self.domain)
-                zone = dns.zone.from_xfr(xfr)
-            except Exception as e:
-                logger.log('DEBUG', str(e))
-                logger.log('DEBUG', f'对{self.domain}的域名服务器{nsserver}进行域传送失败')
-                continue
-            else:
-                names = zone.nodes.keys()
-                for name in names:
-                    subdomain = utils.match_subdomain(self.domain, str(name) + '.' + self.domain)
-                    self.subdomains = self.subdomains.union(subdomain)
-                    record = zone[name].to_text(name)
-                    self.results.append(record)
-            if self.results:
-                logger.log('DEBUG', f'发现{self.domain}在{nsserver}上的域传送记录')
-                logger.log('DEBUG', '\n'.join(self.results))
-                self.results = []
+        for nsserver in nsservers:
+            self.axfr(nsserver)

    def run(self):
        """
        类执行入口
        """
        self.begin()
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}的域传送漏洞')
        self.check()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}的域传送漏洞')
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -87,3 +93,4 @@ def do(domain):  # 统一入口名字 方便多线程调用

 if __name__ == '__main__':
    do('ZoneTransfer.me')
+    do('example.com')
@@ -3,15 +3,13 @@
 """

 from common.module import Module
-from common.utils import match_subdomain
-from config import logger
+from common import utils


 class CheckCDX(Module):
    """
    检查crossdomain.xml文件收集子域名
    """
-
    def __init__(self, domain: str):
        Module.__init__(self)
        self.domain = self.register(domain)
@@ -21,7 +19,6 @@ class CheckCDX(Module):
    def check(self):
        """
        检查crossdomain.xml收集子域名
-        :return:
        """
        urls = [f'http://{self.domain}/crossdomain.xml',
                f'https://{self.domain}/crossdomain.xml',
@@ -36,20 +33,18 @@ class CheckCDX(Module):
                break
        if not response:
            return
-        self.subdomains = match_subdomain(self.domain, response.text)
+        self.subdomains = utils.match_subdomain(self.domain, response.text)

    def run(self):
        """
        类执行入口
        """
        self.begin()
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的跨域策略')
        self.check()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的跨域策略')
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -31,20 +31,19 @@ class CheckCert(Module):
        except Exception as e:
            logger.log('ERROR', e)
            return
-        subdomains_find = utils.match_subdomain(self.domain, str(cert_dict))
-        self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = utils.match_subdomain(self.domain, str(cert_dict))
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的证书中的子域')
+        self.begin()
        self.check()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的证书中的子域')
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -25,8 +25,10 @@ class CheckCSP(Module):
        正则匹配响应头中的内容安全策略字段以发现子域名
        """
        if not self.header:
-            urls = [f'http://{self.domain}', f'https://{self.domain}',
-                    f'http://www.{self.domain}', f'https://www.{self.domain}']
+            urls = [f'http://{self.domain}',
+                    f'https://{self.domain}',
+                    f'http://www.{self.domain}',
+                    f'https://www.{self.domain}']
            response = None
            for url in urls:
                self.header = self.get_header()
@@ -48,13 +50,12 @@ class CheckCSP(Module):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
+        self.begin()
        self.check()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
-        self.finish()


 def do(domain, header=None):  # 统一入口名字 方便多线程调用
@@ -0,0 +1,64 @@
+"""
+检查内容安全策略收集子域名收集子域名
+"""
+import requests
+
+from common.module import Module
+from common import utils
+from config import logger
+
+
+class CheckRobots(Module):
+    """
+    检查robots.txt收集子域名
+    """
+
+    def __init__(self, domain):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Check'
+        self.source = 'Robots'
+
+    def check(self):
+        """
+        正则匹配域名的robots.txt文件中的子域
+        """
+        urls = [f'http://{self.domain}/robots.txt',
+                f'https://{self.domain}/robots.txt',
+                f'http://www.{self.domain}/robots.txt',
+                f'https://www.{self.domain}/robots.txt']
+        response = None
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            response = self.get(url, allow_redirects=False)
+            if response:
+                break
+        if not response:
+            return
+        self.subdomains = utils.match_subdomain(self.domain, response.text)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    check = CheckRobots(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    do('qq.com')
@@ -0,0 +1,77 @@
+"""
+检查内容安全策略收集子域名收集子域名
+"""
+import requests
+
+from common.module import Module
+from common import utils
+from config import logger
+
+
+class CheckRobots(Module):
+    """
+    检查sitemap收集子域名
+    """
+
+    def __init__(self, domain):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Check'
+        self.source = 'Sitemap'
+
+    def check(self):
+        """
+        正则匹配域名的sitemap文件中的子域
+        """
+        urls = [f'http://{self.domain}/sitemap.xml',
+                f'https://{self.domain}/sitemap.xml',
+                f'http://www.{self.domain}/sitemap.xml',
+                f'https://www.{self.domain}/sitemap.xml',
+                f'http://{self.domain}/sitemap.txt',
+                f'https://{self.domain}/sitemap.txt',
+                f'http://www.{self.domain}/sitemap.txt',
+                f'https://www.{self.domain}/sitemap.txt',
+                f'http://{self.domain}/sitemap.html',
+                f'https://{self.domain}/sitemap.html',
+                f'http://www.{self.domain}/sitemap.html',
+                f'https://www.{self.domain}/sitemap.html',
+                f'http://{self.domain}/sitemap_index.xml',
+                f'https://{self.domain}/sitemap_index.xml',
+                f'http://www.{self.domain}/sitemap_index.xml',
+                f'https://www.{self.domain}/sitemap_index.xml']
+        response = None
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            self.timeout = 10
+            response = self.get(url, allow_redirects=False)
+            if response:
+                break
+        if not response:
+            return
+        self.subdomains = utils.match_subdomain(self.domain, response.text)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    check = CheckRobots(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    do('qq.com')
@@ -26,9 +26,10 @@ class ArchiveCrawl(Crawl):
        for resp in cdx.iter(url, limit=limit):
            if resp.data.get('status') not in ['301', '302']:
                url = resp.data.get('url')
-                subdomains_find = self.match(self.register(domain), url + resp.text)
+                subdomains = self.match(self.register(domain),
+                                             url + resp.text)
                # 合并搜索子域名搜索结果
-                self.subdomains = self.subdomains.union(subdomains_find)
+                self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -40,10 +41,10 @@ class ArchiveCrawl(Crawl):
        for subdomain in self.subdomains:
            if subdomain != self.domain:
                self.crawl(subdomain, 10)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -27,9 +27,9 @@ class CommonCrawl(Crawl):

        for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
            if resp.data.get('status') not in ['301', '302']:
-                subdomains_find = self.match(self.register(domain), resp.text)
+                subdomains = self.match(self.register(domain), resp.text)
                # 合并搜索子域名搜索结果
-                self.subdomains = self.subdomains.union(subdomains_find)
+                self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -41,10 +41,10 @@ class CommonCrawl(Crawl):
        for subdomain in self.subdomains:
            if subdomain != self.domain:
                self.crawl(subdomain, 10)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -24,8 +24,8 @@ class BinaryEdgeAPI(Query):
        resp = self.get(url)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = self.match(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -35,10 +35,10 @@ class BinaryEdgeAPI(Query):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -1,5 +1,7 @@
 import time
+import cloudscraper
 from common.query import Query
+from config import logger


 class BufferOver(Query):
@@ -8,22 +10,28 @@ class BufferOver(Query):
        self.domain = self.register(domain)
        self.module = 'Dataset'
        self.source = 'BufferOverQuery'
-        self.addr = 'https://dns.bufferover.run/dns'
+        self.addr = 'https://dns.bufferover.run/dns?q='

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
        time.sleep(self.delay)
-        self.header = self.get_header()
-        self.proxy = self.get_proxy(self.source)
-        params = {'q': self.domain}
-        resp = self.get(self.addr, params)
+        # 绕过cloudFlare验证
+        scraper = cloudscraper.create_scraper()
+        scraper.interpreter = 'js2py'
+        scraper.proxies = self.get_proxy(self.source)
+        url = self.addr + self.domain
+        try:
+            resp = scraper.get(url, timeout=self.timeout)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return
        if not resp:
            return
-        subdomains_find = self.match(self.domain, resp.text)
+        subdomains = self.match(self.domain, str(resp.json()))
        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains_find)
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -31,10 +39,10 @@ class BufferOver(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -21,9 +21,9 @@ class Chinaz(Query):
        resp = self.get(self.addr)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, resp.text)
+        subdomains = self.match(self.domain, resp.text)
        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains_find)
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -31,10 +31,10 @@ class Chinaz(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -23,9 +23,9 @@ class ChinazAPI(Query):
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, str(resp.json()))
+        subdomains = self.match(self.domain, str(resp.json()))
        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains_find)
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -35,10 +35,10 @@ class ChinazAPI(Query):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -23,9 +23,9 @@ class CirclAPI(Query):
        resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
        if not resp:
            return
-        subdomains_find = self.match(self.domain, str(resp.json()))
+        subdomains = self.match(self.domain, str(resp.json()))
        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains_find)
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -35,10 +35,10 @@ class CirclAPI(Query):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -1,7 +1,9 @@
-import random
 import time
+import random
+import cloudscraper
 from bs4 import BeautifulSoup
 from common.query import Query
+from config import logger


 class DNSdb(Query):
@@ -10,37 +12,59 @@ class DNSdb(Query):
        self.domain = self.register(domain)
        self.module = 'Dataset'
        self.source = 'DNSdbQuery'
-        self.addr = 'https://www.dnsdb.org/'
+        self.addr = 'http://www.dnsdb.org/'
+        self.url = f'{self.addr}{self.domain}/'
+
+    def get_tokens(self):
+        """
+        绕过cloudFlare验证并获取taken
+
+        :return: 绕过失败返回None 成功返回tokens
+        """
+        scraper = cloudscraper.create_scraper()
+        scraper.interpreter = 'js2py'
+        scraper.proxies = self.get_proxy(self.source)
+        scraper.timeout = 10
+        try:
+            tokens = scraper.get_tokens(self.url)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return None
+        if len(tokens) != 2:
+            return None
+        return tokens

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        self.header = self.get_header()
-        self.header.update({'Referer': self.addr})
-        self.proxy = self.get_proxy(self.source)
-        url = self.addr + self.domain + '/'
-        resp = self.get(url)
+        tokens = self.get_tokens()
+        if not tokens:
+            logger.log('ALERT', f'{self.source}模块绕过cloudFlare检查失败')
+            return False
+        self.cookie = tokens[0]
+        self.header = {'User-Agent': tokens[1]}
+        self.timeout = 10
+        resp = self.get(self.url)
        if not resp:
            return
-        if resp.status_code == 200:
-            if 'index' in resp.text:
-                soup = BeautifulSoup(resp.text, features='lxml')
-                index_urls = set(map(lambda x: self.addr + self.domain + x.text, soup.find_all('a')))
-                for url in index_urls:
-                    # 休眠绕过CloudFlare的DDoS保护
-                    self.delay = random.randint(2, 5)
-                    time.sleep(self.delay)
-                    resp = self.get(url)
-                    if not resp:
-                        return
-                    subdomains_find = self.match(self.domain, resp.text)
-                    # 合并搜索子域名搜索结果
-                    self.subdomains = self.subdomains.union(subdomains_find)
-            else:
-                subdomains_find = self.match(self.domain, resp.text)
+        if 'index' in resp.text:
+            soup = BeautifulSoup(resp.text, features='lxml')
+            base = self.addr+self.domain
+            urls = list(map(lambda a: base + '/' + a.get('href'),
+                            soup.find_all('a')))
+            urls = urls[:-1]  # idn域名暂时不考虑
+            for url in urls:
+                resp = self.get(url)
+                if not resp:
+                    return
+                subdomains = self.match(self.domain, resp.text)
                # 合并搜索子域名搜索结果
-                self.subdomains = self.subdomains.union(subdomains_find)
+                self.subdomains = self.subdomains.union(subdomains)
+        else:
+            subdomains = self.match(self.domain, resp.text)
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -48,10 +72,10 @@ class DNSdb(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -59,12 +83,10 @@ def do(domain):  # 统一入口名字 方便多线程调用
    类统一调用入口

    :param str domain: 域名
-
    """
    query = DNSdb(domain)
    query.run()


 if __name__ == '__main__':
-
    do('example.com')
@@ -25,10 +25,9 @@ class DNSdbAPI(Query):
        resp = self.get(url)
        if not resp:
            return
-        if resp.status_code == 200:
-            subdomains_find = utils.match_subdomain(self.domain, resp.text)
-            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = utils.match_subdomain(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -38,10 +37,10 @@ class DNSdbAPI(Query):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -23,14 +23,15 @@ class DNSdumpster(Query):
        if not resp:
            return
        self.cookie = resp.cookies
-        data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'), 'targetip': self.domain}
+        data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'),
+                'targetip': self.domain}
        resp = self.post(self.addr, data)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, resp.text)
-        if subdomains_find:
+        subdomains = utils.match_subdomain(self.domain, resp.text)
+        if subdomains:
            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains_find)
+            self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -38,10 +39,10 @@ class DNSdumpster(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -21,10 +21,10 @@ class HackerTarget(Query):
        if not resp:
            return
        if resp.status_code == 200:
-            subdomains_find = utils.match_subdomain(self.domain, resp.text)
-            if subdomains_find:
+            subdomains = utils.match_subdomain(self.domain, resp.text)
+            if subdomains:
                # 合并搜索子域名搜索结果
-                self.subdomains = self.subdomains.union(subdomains_find)
+                self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -32,10 +32,10 @@ class HackerTarget(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -26,17 +26,18 @@ class IPv4InfoAPI(Query):
                return
            if resp.status_code != 200:
                break  # 请求不正常通常网络是有问题，不再继续请求下去
-            resp_json = resp.json()
-            subdomains_find = self.match(self.domain, str(resp_json))
-            if not subdomains_find:
+            data = resp.json()
+            subdomains = self.match(self.domain, str(data))
+            if not subdomains:
                break
            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains_find)
+            self.subdomains = self.subdomains.union(subdomains)
            # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
-            subdomains = resp_json.get('Subdomains')
-            # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
-            if len(subdomains) < 300:
-                break
+            subdomains = data.get('Subdomains')
+            if subdomains:
+                # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
+                if len(subdomains) < 300:
+                    break
            page += 1
            if page >= 50:  # ipv4info子域查询接口最多允许查询50页
                break
@@ -47,10 +48,10 @@ class IPv4InfoAPI(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -25,7 +25,7 @@ class NetCraft(Query):
        resp = self.get(self.init)
        if not resp:
            return None
-        self.cookie = self.get(self.init).cookies
+        self.cookie = resp.cookies
        cookie_value = self.cookie['netcraft_js_verification_challenge']
        cookie_encode = parse.unquote(cookie_value).encode('utf-8')
        verify_taken = hashlib.sha1(cookie_encode).hexdigest()
@@ -46,11 +46,11 @@ class NetCraft(Query):
            resp = self.get(self.addr + last, params)
            if not resp:
                return
-            subdomains_find = self.match(self.domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains_find)
+            self.subdomains = self.subdomains.union(subdomains)
            if 'Next page' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                break
            last = re.search(r'&last=.*' + self.domain, resp.text).group(0)
@@ -62,10 +62,10 @@ class NetCraft(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -0,0 +1,56 @@
+import time
+import config
+from common.query import Query
+
+
+class PassiveDnsAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'PassiveDnsQuery'
+        self.addr = config.passivedns_api_addr or 'http://api.passivedns.cn'
+        self.token = config.passivedns_api_token
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.header.update({'X-AuthToken': self.token})
+        self.proxy = self.get_proxy(self.source)
+        url = self.addr + '/flint/rrset/*.' + self.domain
+        resp = self.get(url)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.addr):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = PassiveDnsAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -10,7 +10,7 @@ class PTRArchive(Query):
        self.domain = self.register(domain)
        self.module = 'Dataset'
        self.source = "PTRArchiveQuery"
-        self.addr = 'http://ptrarchive.com/tools/search3.htm'
+        self.addr = 'http://ptrarchive.com/tools/search4.htm'

    def query(self):
        """
@@ -25,10 +25,10 @@ class PTRArchive(Query):
        if not resp:
            return
        if resp.status_code == 200:
-            subdomains_find = utils.match_subdomain(self.domain, resp.text)
-            if subdomains_find:
+            subdomains = utils.match_subdomain(self.domain, resp.text)
+            if subdomains:
                # 合并搜索子域名搜索结果
-                self.subdomains = self.subdomains.union(subdomains_find)
+                self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -36,10 +36,10 @@ class PTRArchive(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -21,9 +21,9 @@ class Riddler(Query):
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, resp.text)
+        subdomains = self.match(self.domain, resp.text)
        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains_find)
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -31,10 +31,10 @@ class Riddler(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -32,10 +32,10 @@ class Robtex(Query):
                resp = self.get(url)
                if not resp:
                    return
-                subdomains_find = self.match(self.domain, resp.text)
-                if subdomains_find:
+                subdomains = self.match(self.domain, resp.text)
+                if subdomains:
                    # 合并搜索子域名搜索结果
-                    self.subdomains = self.subdomains.union(subdomains_find)
+                    self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -43,10 +43,10 @@ class Robtex(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -26,10 +26,10 @@ class SecurityTrailsAPI(Query):
        if not resp:
            return
        prefixs = resp.json()['subdomains']
-        subdomains_find = [f'{prefix}.{self.domain}' for prefix in prefixs]
-        if subdomains_find:
+        subdomains = [f'{prefix}.{self.domain}' for prefix in prefixs]
+        if subdomains:
            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains_find)
+            self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -39,10 +39,10 @@ class SecurityTrailsAPI(Query):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -26,11 +26,11 @@ class SiteDossier(Query):
            resp = self.get(url)
            if not resp:
                return
-            subdomains_find = self.match(self.domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains_find)
+            self.subdomains = self.subdomains.union(subdomains)
            # 搜索页面没有出现下一页时停止搜索
            if 'Show next 100 items' not in resp.text:
                break
@@ -42,10 +42,10 @@ class SiteDossier(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -0,0 +1,55 @@
+import cloudscraper
+
+from common.query import Query
+from config import logger
+
+
+class ThreatCrowd(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.source = 'ThreatCrowdQuery'
+        self.addr = 'https://www.threatcrowd.org/searchApi' \
+                    '/v2/domain/report?domain='
+
+    def query(self):
+        # 绕过cloudFlare验证
+        scraper = cloudscraper.create_scraper()
+        scraper.interpreter = 'js2py'
+        scraper.proxies = self.get_proxy(self.source)
+        url = self.addr + self.domain
+        try:
+            resp = scraper.get(url, timeout=self.timeout)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = ThreatCrowd(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -60,9 +60,9 @@ class BruteSRV(Module):
        for result in results:
            if result:
                for answer in result:
-                    subdomain = utils.match_subdomain(self.domain, answer.host)
-                    if subdomain:
-                        self.subdomains = self.subdomains.union(subdomain)
+                    subdomains = utils.match_subdomain(self.domain, answer.host)
+                    if subdomains:
+                        self.subdomains = self.subdomains.union(subdomains)
                    else:
                        logger.log('DEBUG', f'{answer.host}不是{self.domain}的子域')
        if not len(self.subdomains):
@@ -73,13 +73,11 @@ class BruteSRV(Module):
        类执行入口
        """
        self.begin()
-        logger.log('DEBUG', f'开始枚举{self.domain}域的SRV记录')
        self.brute()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        logger.log('DEBUG', f'结束枚举{self.domain}域的SRV记录')
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -7,7 +7,7 @@ class RiskIQ(Query):
        Query.__init__(self)
        self.domain = self.register(domain)
        self.module = 'Intelligence'
-        self.source = 'RiskIQQuery'
+        self.source = 'RiskIQAPIQuery'
        self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
        self.user = config.riskiq_api_username
        self.key = config.riskiq_api_key
@@ -19,13 +19,14 @@ class RiskIQ(Query):
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        params = {'query': self.domain}
-        resp = self.get(url=self.addr, params=params, auth=(self.user, self.key))
+        resp = self.get(url=self.addr,
+                        params=params,
+                        auth=(self.user, self.key))
        if not resp:
            return
-        resp_json = resp.json()
-        subdomains_find = resp_json.get('subdomains')
-        if subdomains_find:
-            self.subdomains = set(map(lambda x: x + '.' + self.domain, subdomains_find))
+        data = resp.json()
+        names = data.get('subdomains')
+        self.subdomains = set(map(lambda sub: f'{sub}.{self.domain}', names))

    def run(self):
        """
@@ -35,10 +36,10 @@ class RiskIQ(Query):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -11,18 +11,20 @@ class ThreatBookAPI(Query):
        self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
        self.key = config.threatbook_api_key

-    def query(self, domain):
+    def query(self):
        """
        向接口查询子域并做子域匹配
        """
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'apikey': self.key, 'domain': domain, 'field': 'sub_domains'}
+        params = {'apikey': self.key,
+                  'domain': self.domain,
+                  'field': 'sub_domains'}
        resp = self.post(self.addr, params)
        if not resp:
            return
-        subdomain_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomain_find)
+        subdomains = self.match(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -31,11 +33,11 @@ class ThreatBookAPI(Query):
        if not self.check(self.key):
            return
        self.begin()
-        self.query(self.domain)
+        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -23,9 +23,9 @@ class ThreatMiner(Query):
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, resp.text)
+        subdomains = self.match(self.domain, resp.text)
        # 合并搜索子域名搜索结果
-        self.subdomains = self.subdomains.union(subdomains_find)
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -33,10 +33,10 @@ class ThreatMiner(Query):
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -15,7 +15,7 @@ class VirusTotal(Query):
        self.addr = 'https://www.virustotal.com/ui/domains/{}/subdomains'
        self.domain = self.register(domain)

-    def query(self, domain):
+    def query(self):
        """
        向接口查询子域并做子域匹配
        """
@@ -27,22 +27,22 @@ class VirusTotal(Query):
                                'TE': 'Trailers'})
            self.proxy = self.get_proxy(self.source)
            params = {'limit': '40', 'cursor': next_cursor}
-            resp = self.get(url=self.addr.format(domain), params=params)
+            resp = self.get(url=self.addr.format(self.domain), params=params)
            if not resp:
                return
-            resp_json = resp.json()
-            subdomain_find = set()
-            datas = resp_json.get('data')
+            data = resp.json()
+            subdomains = set()
+            datas = data.get('data')

            if datas:
                for data in datas:
                    subdomain = data.get('id')
                    if subdomain:
-                        subdomain_find.add(subdomain)
+                        subdomains.add(subdomain)
            else:
                break
-            self.subdomains = self.subdomains.union(subdomain_find)
-            meta = resp_json.get('meta')
+            self.subdomains = self.subdomains.union(subdomains)
+            meta = data.get('meta')
            if meta:
                next_cursor = meta.get('cursor')
            else:
@@ -53,11 +53,11 @@ class VirusTotal(Query):
        类执行入口
        """
        self.begin()
-        self.query(self.domain)
+        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -11,19 +11,19 @@ class VirusTotalAPI(Query):
        self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'
        self.key = config.virustotal_api_key

-    def query(self, domain):
+    def query(self):
        """
        向接口查询子域并做子域匹配
        """
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'apikey': self.key, 'domain': domain}
+        params = {'apikey': self.key, 'domain': self.domain}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        resp_json = resp.json()
-        subdomain_find = set(resp_json.get('subdomains'))
-        self.subdomains = self.subdomains.union(subdomain_find)
+        data = resp.json()
+        subdomains = set(data.get('subdomains'))
+        self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -32,11 +32,11 @@ class VirusTotalAPI(Query):
        if not self.check(self.key):
            return
        self.begin()
-        self.query(self.domain)
+        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -30,14 +30,14 @@ class Ask(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += 1
            if '>Next<' not in resp.text:
                break
@@ -61,10 +61,10 @@ class Ask(Search):
                    count = subdomain.count('.') - self.domain.count('.')
                    if count == layer_num:
                        self.search(subdomain)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -50,17 +50,17 @@ class Baidu(Search):
                return
            if len(domain) > 12:  # 解决百度搜索结果中域名过长会显示不全的问题
                # 获取百度跳转URL响应头的Location字段获取直链
-                subdomains_find = self.redirect_match(domain, resp.text)
+                subdomains = self.redirect_match(domain, resp.text)
            else:
-                subdomains_find = self.match(domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+                subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
-                if subdomains_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += self.per_page_num
            # 搜索页面没有出现下一页时停止搜索
            if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:
@@ -89,11 +89,10 @@ class Baidu(Search):
                    count = subdomain.count('.') - self.domain.count('.')
                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -36,15 +36,15 @@ class Bing(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomains_find = self.match(domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
-                if subdomains_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains_find)
+            self.subdomains = self.subdomains.union(subdomains)
            # 搜索页面没有出现下一页时停止搜索
            if '<div class="sw_next>' not in resp.text:
                break
@@ -72,11 +72,10 @@ class Bing(Search):
                    count = subdomain.count('.') - self.domain.count('.')
                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -8,8 +8,9 @@ class BingAPI(Search):
        Search.__init__(self)
        self.domain = domain
        self.module = 'Search'
-        self.source = 'BingCustomSearch'
-        self.addr = 'https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search'
+        self.source = 'BingAPISearch'
+        self.addr = 'https://api.cognitive.microsoft.com/' \
+                    'bingcustomsearch/v7.0/search'
        self.id = config.bing_api_id
        self.key = config.bing_api_key
        self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数
@@ -35,15 +36,15 @@ class BingAPI(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomains_find = self.match(domain, str(resp.json()))
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(domain, str(resp.json()))
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
-                if subdomains_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += self.per_page_num
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break
@@ -70,11 +71,10 @@ class BingAPI(Search):
                    count = subdomain.count('.') - self.domain.count('.')
                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -30,14 +30,14 @@ class DuckDuckGO(Search):
            resp = self.post(self.addr, data)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            try:
                s = re.findall(r'name="s" value="(\d.*)"', resp.text)[-1]
                dc = re.findall(r'name="dc" value="(\d.*)"', resp.text)
@@ -68,11 +68,10 @@ class DuckDuckGO(Search):
                    count = subdomain.count('.') - self.domain.count('.')
                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -32,13 +32,13 @@ class Exalead(Search):
            resp = self.get(url=self.addr, params=params)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += self.per_page_num
            if self.page_num > 1999:
                break
@@ -67,11 +67,10 @@ class Exalead(Search):
                    count = subdomain.count('.') - self.domain.count('.')
                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -11,7 +11,7 @@ class FoFa(Search):
        Search.__init__(self)
        self.domain = domain
        self.module = 'Search'
-        self.source = 'FoFaSearch'
+        self.source = 'FoFaAPISearch'
        self.addr = 'https://fofa.so/api/v1/search/all'
        self.delay = 1
        self.email = config.fofa_api_email
@@ -22,18 +22,28 @@ class FoFa(Search):
        发送搜索请求并做子域匹配
        """
        self.page_num = 1
-        query_data = base64.b64encode(f'domain={self.domain}'.encode('utf-8'))
+        subdomain_encode = f'domain={self.domain}'.encode('utf-8')
+        query_data = base64.b64encode(subdomain_encode)
        while True:
            time.sleep(self.delay)
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
-            query = {'email': self.email, 'key': self.key,
-                     'qbase64': query_data, 'page': self.page_num}
+            query = {'email': self.email,
+                     'key': self.key,
+                     'qbase64': query_data,
+                     'page': self.page_num,
+                     'size': 10000}
            resp = self.get(self.addr, query)
            if not resp:
                return
-            subdomain_find = self.match(self.domain, resp.text)
-            self.subdomains = self.subdomains.union(subdomain_find)
+            resp_json = resp.json()
+            subdomains = self.match(self.domain, str(resp_json))
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains = self.subdomains.union(subdomains)
+            size = resp_json.get('size')
+            if size < 10000:
+                break
            self.page_num += 1

    def run(self):
@@ -44,10 +54,10 @@ class FoFa(Search):
            return
        self.begin()
        self.search()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -0,0 +1,122 @@
+import re
+import time
+import requests
+import config
+from bs4 import BeautifulSoup
+from common.search import Search
+from config import logger
+
+
+class Github(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.source = 'GithubSearch'
+        self.module = 'Search'
+        self.addr = 'https://github.com/search'
+        self.domain = self.register(domain)
+        self.header = self.get_header()
+        self.session = requests.Session()
+        self.login_url = 'https://github.com/login'
+        self.post_url = 'https://github.com/session'
+        self.email = config.github_email
+        self.password = config.github_password
+
+    def login_github(self):
+        """
+        登录github
+
+        :return: 登录失败返回False 成功返回True
+        """
+        token = self.get_token()
+        if token is None:
+            logger.log('ERROR', f'{self.source}模块获取token失败')
+            return False
+        post_data = {
+            'commit': 'Sign in',
+            'utf8': '✓',
+            'authenticity_token': token,
+            'login': self.email,
+            'password': self.password
+        }
+        resp = self.session.post(self.post_url, data=post_data)
+        if resp.status_code != 200:
+            return False
+        match = re.search(r'"user-login" content="(.*?)"', resp.text)
+        if match:
+            return True
+
+    def get_token(self):
+        """
+        获取github登录token
+
+        :return: 获取失败返回None，成功返回token
+        """
+        resp = self.session.get(self.login_url)
+        if resp.status_code != 200:
+            return None
+        match = re.search(
+            r'name="authenticity_token" value="(.*?)"', resp.text)
+        if not match:
+            return None
+        return match.group(1)
+
+    def search(self, full_search=True):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.session.headers = self.get_header()
+        self.session.proxies = self.get_proxy(self.source)
+        self.session.verify = self.verify
+        if not self.login_github():
+            logger.log('ERROR', f'{self.session}模块登录失败')
+            return
+        page_num = 1
+        while True:
+            time.sleep(self.delay)
+            params = {'p': page_num, 'q': f'"{self.domain}"', 'type': 'Code'}
+            resp = self.session.get(self.addr, params=params)
+            if resp.status_code != 200:
+                logger.log('ERROR', f'{self.session}模块搜索出错')
+                break
+            soup = BeautifulSoup(resp.text, 'lxml')
+            subdomains = self.match(self.domain, soup.text)
+            print(subdomains)
+            self.subdomains = self.subdomains.union(subdomains)
+            if not subdomains:
+                break
+            if not full_search:
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
+                    break
+            if 'class="next_page disabled"' in resp.text:
+                break
+            if page_num > 100:
+                break
+            page_num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.email, self.password):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Github(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('mi.com')
@@ -40,13 +40,13 @@ class Google(Search):
            resp = self.get(url=self.addr, params=payload)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            page_num += per_page_num
            if 'start=' + str(page_num) not in resp.text:
                break
@@ -74,11 +74,10 @@ class Google(Search):
                    count = subdomain.count('.') - self.domain.count('.')
                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -35,13 +35,13 @@ class GoogleAPI(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomain_find = self.match(domain, str(resp.json()))
-            if not subdomain_find:
+            subdomains = self.match(domain, str(resp.json()))
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += self.per_page_num
            if self.page_num > 100:  # 免费的API只能查询前100条结果
                break
@@ -68,11 +68,10 @@ class GoogleAPI(Search):
                    count = subdomain.count('.') - self.domain.count('.')
                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -7,7 +7,7 @@ class ShodanAPI(Search):
        Search.__init__(self)
        self.domain = self.register(domain)
        self.module = 'Search'
-        self.source = 'ShodanSearch'
+        self.source = 'ShodanAPISearch'
        self.addr = 'https://api.shodan.io/shodan/host/search'
        self.key = config.shodan_api_key

@@ -25,9 +25,11 @@ class ShodanAPI(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomain_find = self.match(self.domain, resp.text)
-            if subdomain_find:
-                self.subdomains = self.subdomains.union(subdomain_find)
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            if subdomains:
+                self.subdomains = self.subdomains.union(subdomains)
            page += 1

    def run(self):
@@ -38,10 +40,10 @@ class ShodanAPI(Search):
            return
        self.begin()
        self.search()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -31,14 +31,14 @@ class So(Search):
            resp = self.get(url=self.addr, params=payload)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            page_num += 1
            # 搜索页面没有出现下一页时停止搜索
            if '<a id="snext"' not in resp.text:
@@ -67,11 +67,10 @@ class So(Search):
                    count = subdomain.count('.') - self.domain.count('.')
                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -28,14 +28,14 @@ class Sogou(Search):
            resp = self.get(self.addr, payload)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += 1
            # 搜索页面没有出现下一页时停止搜索
            if '<a id="sogou_next"' not in resp.text:
@@ -65,11 +65,10 @@ class Sogou(Search):
                    count = subdomain.count('.') - self.domain.count('.')
                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -36,15 +36,15 @@ class Yahoo(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomains_find = self.match(domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
-                if subdomains_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains_find)
+            self.subdomains = self.subdomains.union(subdomains)
            if '>Next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                break
            self.page_num += self.per_page_num
@@ -72,11 +72,10 @@ class Yahoo(Search):
                    count = subdomain.count('.') - self.domain.count('.')
                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -36,15 +36,15 @@ class Yandex(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomains_find = self.match(domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
-                if subdomains_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains_find)
+            self.subdomains = self.subdomains.union(subdomains)
            if '>next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                break
            self.page_num += 1
@@ -72,11 +72,10 @@ class Yandex(Search):
                    count = subdomain.count('.') - self.domain.count('.')
                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -25,12 +25,12 @@ class ZoomEyeAPI(Search):
        if not resp:
            logger.log('FETAL', f'登录失败无法获取{self.source}的访问token')
            return
-        resp_json = resp.json()
+        data = resp.json()
        if resp.status_code == 200:
            logger.log('DEBUG', f'{self.source}模块登录成功')
-            return resp_json.get('access_token')
+            return data.get('access_token')
        else:
-            logger.log('ALERT', resp_json.get('message'))
+            logger.log('ALERT', data.get('message'))
            exit(1)

    def search(self):
@@ -48,8 +48,10 @@ class ZoomEyeAPI(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomain_find = self.match(self.domain, resp.text)
-            self.subdomains = self.subdomains.union(subdomain_find)
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains = self.subdomains.union(subdomains)
            page_num += 1
            if page_num > 500:
                break
@@ -64,10 +66,10 @@ class ZoomEyeAPI(Search):
            return
        self.begin()
        self.search()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        self.finish()


 def do(domain):  # 统一入口名字 方便多线程调用
@@ -9,6 +9,7 @@ OneForAll是一款功能强大的子域收集工具
 """

 import asyncio
+
 import fire
 import config
 import dbexport
@@ -16,55 +17,136 @@ from datetime import datetime
 from config import logger
 from collect import Collect
 from aiobrute import AIOBrute
-from common import utils, database, resolve, request
+from common import utils, resolve, request
+from common.database import Database
+from takeover import Takeover

+yellow = '\033[01;33m'
+white = '\033[01;37m'
+green = '\033[01;32m'
+blue = '\033[01;34m'
+red = '\033[1;31m'
+end = '\033[0m'

-banner = """\033[01;33m
+version = white + '{' + red + 'v0.0.6#dev' + white + '}'
+
+banner = f"""{yellow}
             ___             _ _ 
- ___ ___ ___|  _|___ ___ ___| | | \033[01;37m{\033[1;31mv0.0.3#dev\033[01;37m}\033[01;32m
-| . |   | -_|  _| . |  _| .'| | | \033[01;34m
-|___|_|_|___|_| |___|_| |__,|_|_| \033[0m\033[4;37mgit.io/fjHT1\033[0m\n
-        """
+ ___ ___ ___|  _|___ ___ ___| | | {version}{green}
+| . |   | -_|  _| . |  _| .'| | | {blue}
+|___|_|_|___|_| |___|_| |__,|_|_| {white}git.io/fjHT1{end}
+"""


 class OneForAll(object):
    """
    OneForAll是一款功能强大的子域收集工具

-    Version: 0.0.3
-    Project: https://github.com/shmilylty/OneForAll/
+    Version: 0.0.6
+    Project: https://git.io/fjHT1

    Example:
-        python oneforall.py --target example.com run
-        python oneforall.py --target example.com --brute True --port medium valid 1 run
-        python oneforall.py --target ./domains.txt --format csv --path= ./result.csv  --output True run
+        python3 oneforall.py --target example.com run
+        python3 oneforall.py --target ./subdomains.txt run
+        python3 oneforall.py --target example.com --valid None run
+        python3 oneforall.py --target example.com --brute True run
+        python3 oneforall.py --target example.com --port medium run
+        python3 oneforall.py --target example.com --format csv run
+        python3 oneforall.py --target example.com --verify False run
+        python3 oneforall.py --target example.com --takeover False run
+        python3 oneforall.py --target example.com --show True run

    Note:
-        参数valid可选值有1，0，None，分别表示导出有效，无效，全部子域
+        参数valid可选值1，0，None分别表示导出有效，无效，全部子域
+        参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
        参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
-        参数format可选格式有'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
-        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'

-    :param str target:  单个域名或者每行一个域名的文件路径
-    :param bool brute:  是否使用爆破模块(默认禁用)
-    :param str port:    HTTP请求验证的端口范围(默认medium)
+    :param str target:  单个域名或者每行一个域名的文件路径(必需参数)
+    :param bool brute:  使用爆破模块(默认False)
+    :param bool verify: 验证子域有效性(默认True)
+    :param str port:    请求验证的端口范围(默认medium)
    :param int valid:   导出子域的有效性(默认1)
-    :param str format:  导出格式(默认xlsx)
-    :param str path:    导出路径(默认None)
-    :param bool output: 是否将导出数据输出到终端(默认False)
+    :param str format:  导出格式(默认csv)
+    :param bool show:   终端显示导出数据(默认False)
    """
-    def __init__(self, target, brute=False, port='medium', valid=1, path=None,
-                 format='xlsx', output=False):
+    def __init__(self, target, brute=None, verify=None, port='medium', valid=1,
+                 format='csv', takeover=True, show=False):
        self.target = target
        self.port = port
        self.domains = set()
-        self.domain = ''
+        self.domain = str()
        self.datas = list()
-        self.brute = brute or config.enable_brute_module
+        self.brute = brute
+        self.verify = verify
+        self.takeover = takeover
        self.valid = valid
-        self.path = path
        self.format = format
-        self.output = output
+        self.show = show
+
+    def main(self):
+        if self.brute is None:
+            self.brute = config.enable_brute_module
+        if self.verify is None:
+            self.verify = config.enable_verify_subdomain
+        rename_table = self.domain + '_last'
+        collect = Collect(self.domain, export=False)
+        collect.run()
+        if self.brute:
+            # 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
+            brute = AIOBrute(self.domain, export=False)
+            brute.run()
+
+        db = Database()
+        db.copy_table(self.domain, self.domain+'_ori')
+        db.remove_invalid(self.domain)
+        db.deduplicate_subdomain(self.domain)
+        # 不验证子域的情况
+        if not self.verify:
+            # 数据库导出
+            self.valid = None
+            dbexport.export(self.domain, valid=self.valid, format=self.format,
+                            show=self.show)
+            db.drop_table(rename_table)
+            db.rename_table(self.domain, rename_table)
+            return
+        # 开始验证子域工作
+        self.datas = db.get_data(self.domain).as_dict()
+        loop = asyncio.get_event_loop()
+        asyncio.set_event_loop(loop)
+
+        # 解析域名地址
+        task = resolve.bulk_query_a(self.datas)
+        self.datas = loop.run_until_complete(task)
+
+        # 保存解析结果
+        resolve_table = self.domain + '_res'
+        db.drop_table(resolve_table)
+        db.create_table(resolve_table)
+        db.save_db(resolve_table, self.datas, 'resolve')
+
+        # 请求域名地址
+        task = request.bulk_get_request(self.datas, self.port)
+        self.datas = loop.run_until_complete(task)
+        # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
+        loop.run_until_complete(asyncio.sleep(0.25))
+
+        db.clear_table(self.domain)
+        db.save_db(self.domain, self.datas)
+
+        # 数据库导出
+        dbexport.export(self.domain, valid=self.valid, format=self.format,
+                        show=self.show)
+        db.drop_table(rename_table)
+        db.rename_table(self.domain, rename_table)
+        db.close()
+        # 子域接管检查
+
+        if self.takeover:
+            subdomains = set(map(lambda x: x.get('subdomain'), self.datas))
+            takeover = Takeover(subdomains)
+            takeover.run()

    def run(self):
        print(banner)
@@ -74,29 +156,7 @@ class OneForAll(object):
        self.domains = utils.get_domains(self.target)
        if self.domains:
            for self.domain in self.domains:
-                collect = Collect(self.domain, export=False)
-                collect.run()
-                if self.brute:
-                    # 由于爆破会有大量dns解析请求 并发常常会导致其他任务中的网络请求超时
-                    brute = AIOBrute(self.domain)
-                    brute.run()
-                table_name = self.domain.replace('.', '_')
-                db_conn = database.connect_db()
-                self.datas = database.get_data(db_conn, table_name).as_dict()
-                loop = asyncio.get_event_loop()
-                asyncio.set_event_loop(loop)
-                self.datas = loop.run_until_complete(resolve.bulk_query_a(self.datas))
-                self.datas = loop.run_until_complete(request.bulk_get_request(self.datas, self.port))
-                # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
-                loop.run_until_complete(asyncio.sleep(0.25))
-                loop.close()
-                database.clear_table(db_conn, table_name)
-                database.save_db(db_conn, table_name, self.datas)
-                # 数据库导出
-                if not self.path:
-                    self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
-                dbexport.export(table_name, db_conn, self.valid, self.path, self.format, self.output)
-                db_conn.close()
+                self.main()
        else:
            logger.log('FATAL', f'获取域名失败')
        logger.log('INFOR', f'结束运行OneForAll')
@@ -105,3 +165,4 @@ class OneForAll(object):
 if __name__ == '__main__':
    fire.Fire(OneForAll)
    # OneForAll('example.com').run()
+    # OneForAll('./domains.txt').run()
@@ -0,0 +1,2 @@
+www.example.com
+www.hackfun.org
@@ -0,0 +1,169 @@
+#!/usr/bin/python3
+# coding=utf-8
+
+"""
+OneForAll子域接管模块
+
+:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
+:license: GNU General Public License v3.0, see LICENSE for more details.
+"""
+import time
+import json
+from threading import Thread
+from queue import Queue
+
+import fire
+from tablib import Dataset
+from tqdm import tqdm
+
+import config
+from config import logger
+from common import resolve, utils
+from common.module import Module
+from common.domain import Domain
+
+
+def get_fingerprint():
+    path = config.data_storage_path.joinpath('fingerprints.json')
+    with open(path) as file:
+        fingerprints = json.load(file)
+    return fingerprints
+
+
+def get_cname(subdomain):
+    resolver = resolve.dns_resolver()
+    try:
+        answers = resolver.query(subdomain, 'CNAME')
+    except Exception as e:
+        logger.log('DEBUG', e.args)
+        return None
+    for answer in answers:
+        return answer.to_text()  # 一个子域只有一个CNAME记录
+
+
+def get_maindomain(subdomain):
+    return Domain(subdomain).registered()
+
+
+class Takeover(Module):
+    """
+    OneForAll多线程子域接管风险检查模块
+
+    Example:
+        python3 takeover.py --target www.example.com  --format csv run
+        python3 takeover.py --target ./subdomains.txt --thread 10 run
+
+    Note:
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+        参数dpath为None默认使用OneForAll结果目录
+
+    :param str target:  单个子域或者每行一个子域的文件路径(必需参数)
+    :param int thread:  线程数(默认100)
+    :param str format:  导出格式(默认csv)
+    :param str dpath:   导出目录(默认None)
+    """
+    def __init__(self, target, thread=100, dpath=None, format='csv'):
+        Module.__init__(self)
+        self.subdomains = set()
+        self.module = 'Check'
+        self.source = 'Takeover'
+        self.target = target
+        self.thread = thread
+        self.dpath = dpath
+        self.format = format
+        self.fingerprints = None
+        self.subdomainq = Queue()
+        self.cnames = list()
+        self.results = Dataset()
+
+    def save(self):
+        logger.log('INFOR', '正在保存检查结果')
+        if self.format == 'txt':
+            data = str(self.results)
+        else:
+            data = self.results.export(self.format)
+        fpath = self.dpath.joinpath(f'takeover.{self.format}')
+        utils.save_data(fpath, data)
+
+    def compare(self, subdomain, cname, responses):
+        domain_resp = self.get('http://' + subdomain, check=False)
+        cname_resp = self.get('http://'+cname, check=False)
+        if domain_resp is None or cname_resp is None:
+            return
+
+        for resp in responses:
+            if resp in domain_resp.text and resp in cname_resp.text:
+                logger.log('ALERT', f'{subdomain}存在子域接管风险')
+                self.results.append([subdomain, cname])
+                break
+
+    def worker(self, subdomain):
+        cname = get_cname(subdomain)
+        if cname is None:
+            return
+        maindomain = get_maindomain(cname)
+        for fingerprint in self.fingerprints:
+            cnames = fingerprint.get('cname')
+            if maindomain not in cnames:
+                continue
+            responses = fingerprint.get('response')
+            self.compare(subdomain, cname, responses)
+
+    def check(self):
+        while not self.subdomainq.empty():  # 保证域名队列遍历结束后能退出线程
+            subdomain = self.subdomainq.get()  # 从队列中获取域名
+            self.worker(subdomain)
+            self.subdomainq.task_done()
+
+    def progress(self):
+        # 设置进度
+        bar = tqdm()
+        bar.total = len(self.subdomains)
+        bar.desc = 'Progress'
+        bar.ncols = True
+        while True:
+            done = bar.total - self.subdomainq.qsize()
+            bar.n = done
+            bar.update()
+            if done == bar.total:  # 完成队列中所有子域的检查退出
+                break
+        bar.close()
+
+    def run(self):
+        start = time.time()
+        logger.log('INFOR', f'开始执行{self.source}模块')
+        self.format = utils.check_format(self.format)
+        self.dpath = utils.check_dpath(self.dpath)
+        self.subdomains = utils.get_domains(self.target)
+        if self.subdomains:
+            logger.log('INFOR', f'正在检查子域接管风险')
+            self.fingerprints = get_fingerprint()
+            self.results.headers = ['subdomain', 'cname']
+            # 创建待检查的子域队列
+            for domain in self.subdomains:
+                self.subdomainq.put(domain)
+            # 检查线程
+            for _ in range(self.thread):
+                check_thread = Thread(target=self.check, daemon=True)
+                check_thread.start()
+            # 进度线程
+            progress_thread = Thread(target=self.progress, daemon=True)
+            progress_thread.start()
+
+            self.subdomainq.join()
+            self.save()
+        else:
+            logger.log('FATAL', f'获取域名失败')
+        end = time.time()
+        elapsed = round(end - start, 1)
+        logger.log('INFOR', f'{self.source}模块耗时{elapsed}秒'
+                            f'发现{len(self.results)}个子域存在接管风险')
+        logger.log('DEBUG', f'结束执行{self.source}模块')
+
+
+if __name__ == '__main__':
+    fire.Fire(Takeover)
+    # takeover = Takeover('www.example.com')
+    # takeover = Takeover('./subdomains.txt')
+    # takeover.run()
Author	SHA1	Message	Date
shmilylty	5d6b7c1b7b	v0.0.6	2019-08-27 17:56:37 +08:00
shmilylty	bc9b5b0afe	重命名	2019-08-27 17:39:54 +08:00
shmilylty	a332d3bdc4	重写标题获取	2019-08-27 17:36:19 +08:00
shmilylty	c3f6f0b08d	优化标题获取	2019-08-26 18:27:10 +08:00
shmilylty	515bd34c7e	修复问题15 https://github.com/shmilylty/OneForAll/issues/15	2019-08-25 18:01:50 +08:00
shmilylty	32330fa8b2	修复问题16 https://github.com/shmilylty/OneForAll/issues/16	2019-08-25 17:51:16 +08:00
shmilylty	16d3530ac3	修复问题16 https://github.com/shmilylty/OneForAll/issues/16	2019-08-25 17:48:34 +08:00
Jing Ling	cc10ce78d3	Update issue templates	2019-08-23 22:34:29 +08:00
shmilylty	f35239de40	v0.0.6	2019-08-23 18:55:16 +08:00
shmilylty	daaf7891d6	更新文档	2019-08-23 18:51:22 +08:00
Jing Ling	46d447392c	Update fofa_api.py	2019-08-23 18:35:47 +08:00
shmilylty	b4432dfd47	utf-8	2019-08-23 17:10:17 +08:00
shmilylty	1b9b9cb11d	添加brotlipy依赖	2019-08-23 17:04:13 +08:00
shmilylty	f4b4f06333	一直请求bug修复	2019-08-23 15:48:56 +08:00
shmilylty	fca97c7d5c	修复需要登录问题	2019-08-23 14:54:10 +08:00
shmilylty	818af18355	去掉无用打印	2019-08-23 10:18:03 +08:00
shmilylty	69f94193f6	默认导出格式修改为csv	2019-08-23 10:17:33 +08:00
shmilylty	403a3fefcb	添加Github子域搜索模块	2019-08-22 19:04:31 +08:00
shmilylty	8ce39a9127	修复编码问题	2019-08-22 14:42:19 +08:00
shmilylty	f3a1b2bada	更新文档	2019-08-22 11:27:09 +08:00
shmilylty	41422f4826	修改为正确的doctest格式	2019-08-21 18:24:36 +08:00
Jing Ling	6189bc73ac	错字纠正	2019-08-21 16:52:01 +08:00
Jing Ling	45d9ae2f2d	修复检查线程无限等待问题	2019-08-21 16:50:17 +08:00
Jing Ling	c329f6c77e	添加PassiveDNS查询	2019-08-21 16:45:40 +08:00
Jing Ling	9fa77a16b6	添加passivedns_api配置	2019-08-21 16:44:23 +08:00
Jing Ling	2c581c31aa	删除无用判断	2019-08-21 16:42:57 +08:00
shmilylty	76fc12c365	添加绕过cloudFlare验证	2019-08-20 21:44:30 +08:00
shmilylty	3413869bf1	禁用SSL安全警告信息	2019-08-20 21:29:13 +08:00
shmilylty	84aea67aa0	修复链接	2019-08-20 12:07:19 +08:00
shmilylty	73d92a92e1	文档更新	2019-08-20 11:59:00 +08:00
shmilylty	9c8d7e937e	更新文档	2019-08-20 11:47:28 +08:00
shmilylty	b165670ed3	合并building_fail_solution.md	2019-08-20 11:41:17 +08:00
shmilylty	4e6dd23129	使用推荐使用socks5h	2019-08-19 21:26:09 +08:00
shmilylty	f0a3b44c53	v0.0.5	2019-08-19 20:05:01 +08:00
shmilylty	472da35a36	tqdm修改	2019-08-19 19:58:00 +08:00
shmilylty	b60d86fdc3	tqdm配置	2019-08-19 19:45:11 +08:00
shmilylty	9b99b9fd35	添加不能正常安装依赖问题的解决方法	2019-08-19 19:00:29 +08:00
shmilylty	dde3da84fd	更新依赖	2019-08-19 18:56:08 +08:00
shmilylty	147f1030c3	添加takeover使用帮助	2019-08-19 18:44:23 +08:00
shmilylty	43d4b90782	pep8	2019-08-19 18:21:47 +08:00
shmilylty	dee7ee2d2c	添加github自动接管	2019-08-19 18:11:36 +08:00
shmilylty	bf8df3bc02	添加进度条修复线程退出卡死bug	2019-08-19 18:10:44 +08:00
shmilylty	82973c295c	标题信息获取处理	2019-08-19 02:24:00 +08:00
shmilylty	ac654f9bab	添加默认子域接管检查目标列表	2019-08-18 19:35:26 +08:00
shmilylty	310c40d7aa	添加子域接管检查功能	2019-08-18 19:34:16 +08:00
shmilylty	b3b528966c	重构	2019-08-18 19:33:23 +08:00
shmilylty	e31ff755f4	重构响应检查	2019-08-18 19:32:30 +08:00
shmilylty	101742de3c	添加部分公用函数	2019-08-18 19:31:50 +08:00
shmilylty	25e8541964	添加进度条	2019-08-18 19:31:22 +08:00
shmilylty	1251efd752	使用回调	2019-08-18 19:30:54 +08:00
shmilylty	61075f40ac	添加超时	2019-08-18 19:29:06 +08:00
shmilylty	39b92b79d9	pep8	2019-08-18 19:27:33 +08:00
shmilylty	0e88591a3b	修改默认导出格式为xls	2019-08-18 19:27:09 +08:00
shmilylty	32b06362bd	修改默认请求超时为30秒	2019-08-18 19:26:30 +08:00
shmilylty	298ef2072c	修改默认目前域名	2019-08-18 19:11:51 +08:00
shmilylty	7390e941a7	重命名	2019-08-18 19:02:54 +08:00
shmilylty	1fad2a413e	pep8	2019-08-18 19:01:43 +08:00
shmilylty	de96822c0d	添加请求超时	2019-08-18 02:45:32 +08:00
shmilylty	4c6af1a925	pep8	2019-08-18 02:12:40 +08:00
shmilylty	7ccddb93f7	修复误把domain替换为subdomain的问题	2019-08-18 02:12:12 +08:00
shmilylty	3883a14365	pep8	2019-08-18 00:03:08 +08:00
shmilylty	837b2f4d51	pep8	2019-08-18 00:02:25 +08:00
Jing Ling	03126872bb	Merge pull request #13 from Echocipher/master subdomain takeover	2019-08-17 14:57:30 +08:00
shmilylty	8eaf1bd60b	修改parsed.names为正确值，解决[问题14](https://github.com/shmilylty/OneForAll/issues/14 )。	2019-08-16 23:03:49 +08:00
shmilylty	7f9fbac68b	添加threatcrowd模块	2019-08-16 19:49:39 +08:00
shmilylty	335f6b6c0c	添加常见问题与问答文档	2019-08-16 19:41:21 +08:00
shmilylty	f52768632a	pep8	2019-08-16 19:13:19 +08:00
shmilylty	8d46ce0862	添加绕过cloudFlare检查	2019-08-16 18:33:11 +08:00
shmilylty	11cf46a006	添加cloudscraper和Js2Py依赖	2019-08-16 16:48:51 +08:00
Echocipher	f1e2b739e1	Update takeover.py	2019-08-16 16:44:21 +08:00
Echocipher	3e4a067c6b	Update takeover.py	2019-08-16 16:43:48 +08:00
shmilylty	d2dc83ebb2	添加ThreatCrowd接口	2019-08-16 16:22:33 +08:00
echocipher	c040501ee4	subdomain takeover	2019-08-16 15:35:40 +08:00
echocipher	70b1ddfda2	subdomain takeover	2019-08-16 15:31:45 +08:00
shmilylty	cf2e91bfcc	添加其他支持格式说明	2019-08-15 01:57:11 +08:00
Jing Ling	daed4d5a7c	Set theme jekyll-theme-architect	2019-08-14 23:58:17 +08:00
shmilylty	e99497fdc7	美化排版	2019-08-14 23:29:04 +08:00
shmilylty	cb7df51209	添加码云厂库链接和说明	2019-08-14 23:12:12 +08:00
shmilylty	a3c8834f05	美化排版	2019-08-14 22:22:30 +08:00
shmilylty	3f4336cca9	修改默认导出格式为xls，添加txt,rst,jira格式输出。	2019-08-14 19:07:52 +08:00
shmilylty	da4e3bbd2a	修复[问题10](https://github.com/shmilylty/OneForAll/issues/10 )	2019-08-14 16:35:29 +08:00
shmilylty	20e582135d	移除freebuf.com	2019-08-14 16:15:49 +08:00
shmilylty	bfb069509e	添加subdomains为None情况判断	2019-08-14 16:13:27 +08:00
shmilylty	d9c1cb3937	解决批量获取子域问题	2019-08-14 16:07:30 +08:00
shmilylty	f3540ee07b	8443端口使用https协议	2019-08-14 13:44:32 +08:00
shmilylty	794fec26db	修改使用帮助	2019-08-14 03:36:53 +08:00
shmilylty	46282e37a4	修复使用帮助链接	2019-08-14 03:26:12 +08:00
shmilylty	87749f1452	更友好的使用帮助	2019-08-14 03:25:15 +08:00
shmilylty	4428790766	添加使用帮助文档	2019-08-14 03:24:44 +08:00
shmilylty	51c42f2958	pep8	2019-08-14 03:22:00 +08:00
shmilylty	aaf5863d9a	对齐	2019-08-14 02:39:34 +08:00
shmilylty	88e084e695	添加详细的安装依赖文档	2019-08-14 02:02:31 +08:00
shmilylty	96487aec2e	模块优化	2019-08-13 19:15:13 +08:00
shmilylty	1179f5f118	添加limit_open_conn检查	2019-08-13 19:14:47 +08:00
shmilylty	ef3a1f5245	pep8	2019-08-13 19:13:56 +08:00
shmilylty	f223d9cf1b	参数优化	2019-08-13 19:13:08 +08:00
shmilylty	78832b4790	响应解码问题处理	2019-08-13 17:35:28 +08:00
shmilylty	3c065f56ae	修复字典路径问题	2019-08-12 22:31:17 +08:00
shmilylty	a65840b013	修复self.verify值的获取	2019-08-12 22:30:54 +08:00
shmilylty	49e4278647	修改output参数为show参数	2019-08-12 18:41:58 +08:00
shmilylty	5a324971ed	添加验证子域有效性参数	2019-08-12 18:41:18 +08:00
shmilylty	f3e76f7ee5	添加跳出判断	2019-08-12 11:26:50 +08:00
shmilylty	e1d5515c44	v0.0.4	2019-08-11 21:34:29 +08:00
shmilylty	bf10c1a652	v0.0.4	2019-08-11 21:32:04 +08:00
shmilylty	33d7b6cc02	参数调优	2019-08-11 20:49:38 +08:00
shmilylty	903b18a058	生成请求地址中443端口的特殊处理	2019-08-11 20:39:58 +08:00
shmilylty	38117f06d1	参数调优	2019-08-11 20:20:42 +08:00
shmilylty	f9117ff795	添加resp.json()异常处理	2019-08-11 20:10:23 +08:00
shmilylty	d2219bdf8c	添加搜索退出判断	2019-08-11 20:08:06 +08:00
shmilylty	3c8627035d	设置域传送超时为30秒	2019-08-11 19:58:10 +08:00
shmilylty	4bb52f4183	参数调优	2019-08-11 15:53:49 +08:00
shmilylty	f426b8f2f6	重构	2019-08-11 03:48:36 +08:00
shmilylty	ebd81b83e9	pep8	2019-08-10 14:54:08 +08:00
shmilylty	d70ebe637e	升级tqdm版本	2019-08-10 12:02:32 +08:00
shmilylty	d10bc9c8d9	默认不使用代理	2019-08-10 03:30:52 +08:00
shmilylty	a7b189f311	调整执行finish函数位置	2019-08-10 00:48:51 +08:00
shmilylty	98642da4f3	拆分出执行域传送函数	2019-08-09 23:46:09 +08:00
shmilylty	f17aea1f44	删除无用日志输出	2019-08-09 23:20:21 +08:00
shmilylty	611580aa97	修正模块名字	2019-08-09 21:02:27 +08:00
shmilylty	b21e31a223	database.py修改为类	2019-08-09 18:47:01 +08:00
shmilylty	4c63bd672f	修正valid参数使用描述	2019-08-09 16:56:03 +08:00
shmilylty	c7f86fdefa	新增2个模块	2019-08-09 16:54:10 +08:00
shmilylty	c026dc550e	新增检查sitemap收集子域名模块	2019-08-09 16:43:09 +08:00
shmilylty	f2b08f6b02	新增检查robots.txt收集子域名模块	2019-08-09 16:42:03 +08:00
Jing Ling	f0f9f9b75e	删除无用模板	2019-08-09 15:40:04 +08:00
Jing Ling	a9347b8ef0	修改提交Bug提示	2019-08-09 15:32:46 +08:00
shmilylty	c2679a6ceb	暂时不探测https	2019-08-09 15:13:22 +08:00
shmilylty	9197aaf52b	pep8	2019-08-09 14:25:21 +08:00
shmilylty	272662fa82	修改pip3安装命令	2019-08-09 14:24:50 +08:00
shmilylty	623a21a11e	移除fake-useragent依赖	2019-08-09 11:39:21 +08:00
shmilylty	05040d3b8d	resolve_callback函数添加说明并做容错处理	2019-08-09 01:12:52 +08:00