v0.0.8

实现新子域标记
去掉中括号
2026-08-26 04:47:48 +08:00 · 2019-10-30 00:46:06 +08:00 · 2019-10-29 22:47:44 +08:00 · 2019-10-29 16:00:25 +08:00 · 2019-10-28 20:06:16 +08:00 · 2019-10-28 18:45:52 +08:00
25 changed files with 3567 additions and 608 deletions
@@ -19,13 +19,13 @@ matrix:
      os: osx
      osx_image: xcode10.2  # Python 3.7 running on macOS 10.14.3
      language: shell       # 'language: python' is an error on Travis CI macOS
-    - name: "Python 3.7 on Windows"
+    - name: "Python 3.8 on Windows"
      os: windows           # Windows 10.0.17134 N/A Build 17134
      language: shell
      before_install:
        - choco install python
        - python -m pip install --upgrade pip
-      env: PATH=/c/Python37:/c/Python37/Scripts:$PATH
+      env: PATH=/c/Python38:/c/Python38/Scripts:$PATH

 install:
  - pip3 install -U pip
@@ -6,6 +6,12 @@ OneForAll的更新日志格式基于[Keep a Changelog](https://keepachangelog.co
 OneForAll遵守[语义化版本格式](https://semver.org/)。

 ## Unreleased
+## [0.0.8](https://github.com/shmilylty/oneforall/releases/tag/v0.0.8) - 2019-10-30
+- 添加新子域监控功能
+- 优化子域爆破字典和默认参数
+- 修复端口重复问题
+- 移除aiodns依赖
+
 ## [0.0.7](https://github.com/shmilylty/oneforall/releases/tag/v0.0.7) - 2019-10-18
 - 修复一些已知问题
 - 添加百度云观测接口
@@ -1,12 +1,11 @@
 [[source]]
 name = "pypi"
-url = "https://pypi.python.org/simple/"
+url = "https://mirrors.aliyun.com/pypi/simple/"
 verify_ssl = true

 [dev-packages]

 [packages]
-aiodns = "*"
 tqdm = "*"
 aiomultiprocess = "*"
 loguru = "*"
@@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "656e9f0f133ba443049ac692b2551b8a0daa69542d471dc3dff965f7cdb3a308"
+            "sha256": "ee0f44c186ca74e3344dfdf99d484042116eb11746a2ffe0245a1cdd9b2ff01c"
        },
        "pipfile-spec": 6,
        "requires": {
@@ -16,14 +16,6 @@
        ]
    },
    "default": {
-        "aiodns": {
-            "hashes": [
-                "sha256:815fdef4607474295d68da46978a54481dd1e7be153c7d60f9e72773cd38d77d",
-                "sha256:aaa5ac584f40fe778013df0aa6544bf157799bd3f608364b451840ed2c8688de"
-            ],
-            "index": "pypi",
-            "version": "==2.0.0"
-        },
        "aiohttp": {
            "hashes": [
                "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e",
@@ -49,13 +41,6 @@
            "index": "pypi",
            "version": "==0.6.1"
        },
-        "asn1crypto": {
-            "hashes": [
-                "sha256:7bb1cc02a5620b3d72da4ba070bda2f44f0e61b44dee910a302eddff802b6fb5",
-                "sha256:87620880a477123e01177a1f73d0f327210b43a3cdbd714efcd2fa49a8d7b384"
-            ],
-            "version": "==1.2.0"
-        },
        "async-timeout": {
            "hashes": [
                "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
@@ -70,13 +55,6 @@
            ],
            "version": "==19.3.0"
        },
-        "backports.csv": {
-            "hashes": [
-                "sha256:1277dfff73130b2e106bf3dd347adb3c5f6c4340882289d88f31240da92cbd6d",
-                "sha256:21f6e09bab589e6c1f877edbc40277b65e626262a86e69a70137db714eaac5ce"
-            ],
-            "version": "==1.0.7"
-        },
        "beautifulsoup4": {
            "hashes": [
                "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
@@ -200,40 +178,38 @@
        },
        "cffi": {
            "hashes": [
-                "sha256:08f99e8b38d5134d504aa7e486af8e4fde66a2f388bbecc270cdd1e00fa09ff8",
-                "sha256:1112d2fc92a867a6103bce6740a549e74b1d320cf28875609f6e93857eee4f2d",
-                "sha256:1b9ab50c74e075bd2ae489853c5f7f592160b379df53b7f72befcbe145475a36",
-                "sha256:24eff2997436b6156c2f30bed215c782b1d8fd8c6a704206053c79af95962e45",
-                "sha256:2eff642fbc9877a6449026ad66bf37c73bf4232505fb557168ba5c502f95999b",
-                "sha256:362e896cea1249ed5c2a81cf6477fabd9e1a5088aa7ea08358a4c6b0998294d2",
-                "sha256:40eddb3589f382cb950f2dcf1c39c9b8d7bd5af20665ce273815b0d24635008b",
-                "sha256:5ed40760976f6b8613d4a0db5e423673ca162d4ed6c9ed92d1f4e58a47ee01b5",
-                "sha256:632c6112c1e914c486f06cfe3f0cc507f44aa1e00ebf732cedb5719e6aa0466a",
-                "sha256:64d84f0145e181f4e6cc942088603c8db3ae23485c37eeda71cb3900b5e67cb4",
-                "sha256:6cb4edcf87d0e7f5bdc7e5c1a0756fbb37081b2181293c5fdf203347df1cd2a2",
-                "sha256:6f19c9df4785305669335b934c852133faed913c0faa63056248168966f7a7d5",
-                "sha256:719537b4c5cd5218f0f47826dd705fb7a21d83824920088c4214794457113f3f",
-                "sha256:7b0e337a70e58f1a36fb483fd63880c9e74f1db5c532b4082bceac83df1523fa",
-                "sha256:853376efeeb8a4ae49a737d5d30f5db8cdf01d9319695719c4af126488df5a6a",
-                "sha256:85bbf77ffd12985d76a69d2feb449e35ecdcb4fc54a5f087d2bd54158ae5bb0c",
-                "sha256:8978115c6f0b0ce5880bc21c967c65058be8a15f1b81aa5fdbdcbea0e03952d1",
-                "sha256:8f7eec920bc83692231d7306b3e311586c2e340db2dc734c43c37fbf9c981d24",
-                "sha256:8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226",
-                "sha256:92068ebc494b5f9826b822cec6569f1f47b9a446a3fef477e1d11d7fac9ea895",
-                "sha256:b57e1c8bcdd7340e9c9d09613b5e7fdd0c600be142f04e2cc1cc8cb7c0b43529",
-                "sha256:ba956c9b44646bc1852db715b4a252e52a8f5a4009b57f1dac48ba3203a7bde1",
-                "sha256:ca42034c11eb447497ea0e7b855d87ccc2aebc1e253c22e7d276b8599c112a27",
-                "sha256:dc9b2003e9a62bbe0c84a04c61b0329e86fccd85134a78d7aca373bbbf788165",
-                "sha256:dd308802beb4b2961af8f037becbdf01a1e85009fdfc14088614c1b3c383fae5",
-                "sha256:e77cd105b19b8cd721d101687fcf665fd1553eb7b57556a1ef0d453b6fc42faa",
-                "sha256:f56dff1bd81022f1c980754ec721fb8da56192b026f17f0f99b965da5ab4fbd2",
-                "sha256:fa4cc13c03ea1d0d37ce8528e0ecc988d2365e8ac64d8d86cafab4038cb4ce89",
-                "sha256:fa8cf1cb974a9f5911d2a0303f6adc40625c05578d8e7ff5d313e1e27850bd59",
-                "sha256:fb003019f06d5fc0aa4738492ad8df1fa343b8a37cbcf634018ad78575d185df",
-                "sha256:fd409b7778167c3bcc836484a8f49c0e0b93d3e745d975749f83aa5d18a5822f",
-                "sha256:fe5d65a3ee38122003245a82303d11ac05ff36531a8f5ce4bc7d4bbc012797e1"
+                "sha256:00d890313797d9fe4420506613384b43099ad7d2b905c0752dbcc3a6f14d80fa",
+                "sha256:0cf9e550ac6c5e57b713437e2f4ac2d7fd0cd10336525a27224f5fc1ec2ee59a",
+                "sha256:0ea23c9c0cdd6778146a50d867d6405693ac3b80a68829966c98dd5e1bbae400",
+                "sha256:193697c2918ecdb3865acf6557cddf5076bb39f1f654975e087b67efdff83365",
+                "sha256:1ae14b542bf3b35e5229439c35653d2ef7d8316c1fffb980f9b7647e544baa98",
+                "sha256:1e389e069450609c6ffa37f21f40cce36f9be7643bbe5051ab1de99d5a779526",
+                "sha256:263242b6ace7f9cd4ea401428d2d45066b49a700852334fd55311bde36dcda14",
+                "sha256:33142ae9807665fa6511cfa9857132b2c3ee6ddffb012b3f0933fc11e1e830d5",
+                "sha256:364f8404034ae1b232335d8c7f7b57deac566f148f7222cef78cf8ae28ef764e",
+                "sha256:47368f69fe6529f8f49a5d146ddee713fc9057e31d61e8b6dc86a6a5e38cecc1",
+                "sha256:4895640844f17bec32943995dc8c96989226974dfeb9dd121cc45d36e0d0c434",
+                "sha256:558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b",
+                "sha256:5ba86e1d80d458b338bda676fd9f9d68cb4e7a03819632969cf6d46b01a26730",
+                "sha256:63424daa6955e6b4c70dc2755897f5be1d719eabe71b2625948b222775ed5c43",
+                "sha256:6381a7d8b1ebd0bc27c3bc85bc1bfadbb6e6f756b4d4db0aa1425c3719ba26b4",
+                "sha256:6381ab708158c4e1639da1f2a7679a9bbe3e5a776fc6d1fd808076f0e3145331",
+                "sha256:6fd58366747debfa5e6163ada468a90788411f10c92597d3b0a912d07e580c36",
+                "sha256:728ec653964655d65408949b07f9b2219df78badd601d6c49e28d604efe40599",
+                "sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8",
+                "sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8",
+                "sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa",
+                "sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78",
+                "sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc",
+                "sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e",
+                "sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2",
+                "sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0",
+                "sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71",
+                "sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891",
+                "sha256:ec2fa3ee81707a5232bf2dfbd6623fdb278e070d596effc7e2d788f2ada71a05",
+                "sha256:fd82eb4694be712fcae03c717ca2e0fc720657ac226b80bbb597e971fc6928c2"
            ],
-            "version": "==1.13.0"
+            "version": "==1.13.1"
        },
        "chardet": {
            "hashes": [
@@ -244,11 +220,11 @@
        },
        "cloudscraper": {
            "hashes": [
-                "sha256:2316ddc0c00905536a3f1801917e68fb78610776b1b65f1b126627a7793c397b",
-                "sha256:a27d2452edbe3d77d089c71f74783edfd24802e4f100aef0fad0de4505a2b840"
+                "sha256:ea0a52a5c8af2645db4de3eccbf0e0b721a6daa721fb43afacbb51c6a6b8cf48",
+                "sha256:f64dff985d958837864ff24bd245353efbcee200e5e589efb7bbd84d8238b495"
            ],
            "index": "pypi",
-            "version": "==1.2.2"
+            "version": "==1.2.5"
        },
        "colorama": {
            "hashes": [
@@ -260,24 +236,29 @@
        },
        "cryptography": {
            "hashes": [
-                "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c",
-                "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643",
-                "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216",
-                "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799",
-                "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a",
-                "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9",
-                "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc",
-                "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8",
-                "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53",
-                "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1",
-                "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609",
-                "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292",
-                "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e",
-                "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6",
-                "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed",
-                "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d"
+                "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c",
+                "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595",
+                "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad",
+                "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651",
+                "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2",
+                "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff",
+                "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d",
+                "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42",
+                "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d",
+                "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e",
+                "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912",
+                "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793",
+                "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13",
+                "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7",
+                "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0",
+                "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879",
+                "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f",
+                "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9",
+                "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2",
+                "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf",
+                "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8"
            ],
-            "version": "==2.7"
+            "version": "==2.8"
        },
        "defusedxml": {
            "hashes": [
@@ -378,6 +359,12 @@
            "index": "pypi",
            "version": "==4.4.1"
        },
+        "markuppy": {
+            "hashes": [
+                "sha256:1adee2c0a542af378fe84548ff6f6b0168f3cb7f426b46961038a2bcfaad0d5f"
+            ],
+            "version": "==1.14"
+        },
        "multidict": {
            "hashes": [
                "sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f",
@@ -424,24 +411,6 @@
            ],
            "version": "==2.4.11"
        },
-        "pycares": {
-            "hashes": [
-                "sha256:2ca080db265ea238dc45f997f94effb62b979a617569889e265c26a839ed6305",
-                "sha256:6f79c6afb6ce603009db2042fddc2e348ad093ece9784cbe2daa809499871a23",
-                "sha256:70918d06eb0603016d37092a5f2c0228509eb4e6c5a3faacb4184f6ab7be7650",
-                "sha256:755187d28d24a9ea63aa2b4c0638be31d65fbf7f0ce16d41261b9f8cb55a1b99",
-                "sha256:7baa4b1f2146eb8423ff8303ebde3a20fb444a60db761fba0430d104fe35ddbf",
-                "sha256:90b27d4df86395f465a171386bc341098d6d47b65944df46518814ae298f6cc6",
-                "sha256:9e090dd6b2afa65cb51c133883b2bf2240fd0f717b130b0048714b33fb0f47ce",
-                "sha256:a11b7d63c3718775f6e805d6464cb10943780395ab042c7e5a0a7a9f612735dd",
-                "sha256:b253f5dcaa0ac7076b79388a3ac80dd8f3bd979108f813baade40d3a9b8bf0bd",
-                "sha256:c7f4f65e44ba35e35ad3febc844270665bba21cfb0fb7d749434e705b556e087",
-                "sha256:cdb342e6a254f035bd976d95807a2184038fc088d957a5104dcaab8be602c093",
-                "sha256:cf08e164f8bfb83b9fe633feb56f2754fae6baefcea663593794fa0518f8f98c",
-                "sha256:df9bc694cf03673878ea8ce674082c5acd134991d64d6c306d4bd61c0c1df98f"
-            ],
-            "version": "==3.0.0"
-        },
        "pycparser": {
            "hashes": [
                "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3"
@@ -549,11 +518,11 @@
        },
        "tablib": {
            "hashes": [
-                "sha256:0f88a9cebdaa1a2cc29ae57387082ee81015d1149ecd34e48a8c8d3b4dd21670",
-                "sha256:5f33c079b07eb10cf9c4b4696add2ecf32c89db7729240546ecdcd5c92f67e13"
+                "sha256:00654241e5beee437ba544e4fa4abef70ccec3668503aa95406c1250bb660770",
+                "sha256:6336e7aa3f0e5894b47270a3dc639cc2e78eb823c7ccb8c4512fb408cdc18d08"
            ],
            "index": "pypi",
-            "version": "==0.13.0"
+            "version": "==0.14.0"
        },
        "termcolor": {
            "hashes": [
@@ -4,8 +4,8 @@
 [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
-[![python](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.0.7-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/python-3.7%20%7C%203.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/release-v0.0.8-brightgreen)](https://github.com/shmilylty/OneForAll/releases)

 👊**OneForAll is a powerful subdomain collection tool**  📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)

@@ -59,15 +59,15 @@ At present, OneForAll is still under development, there must be a lot of problem

 **🐍Installation requirements**

-OneForAll is based on CPython, so you need a Python environment to run. If your system doesn't already have a Python environment, you can refer to the [Python 3 Installation Guide](https://pythonguidecn.readthedocs.io/en/latest/starting/installation.html#python-3), theoretically Python 3.6, 3.7 and 3.8 can run OneForAll, **but** many tests are done on Python 3.7, so **recommended** you use **Python 3.7** version runs OneForAll. Run the following command to check the Python and pip3 versions:
+OneForAll is based on CPython, so you need a Python environment to run. If your system doesn't already have a Python environment, you can refer to the [Python 3 Installation Guide](https://pythonguidecn.readthedocs.io/en/latest/starting/installation.html#python-3), theoretically Python 3.7 and 3.8 can run OneForAll, **but** many tests are done on Python 3.8, so **recommended** you use **Python 3.8** version runs OneForAll. Run the following command to check the Python and pip3 versions:
 ```bash
 python -V
 pip3 -V
 ```
 If you see the following output, there is no problem with the Python environment:
 ```bash
-Python 3.7.4
-pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.7)
+Python 3.8.0
+pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.8)
 ```

 **✔Installation steps**
@@ -89,8 +89,8 @@ pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages
   You can install OneForAll dependencies via pip3 (if you are familiar with [pipenv](https://docs.pipenv.org/en/latest/), then it is recommended that you use [pipenv install dependencies](https://github.com/shmilylty/OneForAll/tree/master/docs/Installation_dependency.md), the following is an example of using **pip3** to install dependencies under **Windows system**: (Note: If your Python3 is installed in the system Program Files In the directory, such as: `C:\Program Files\Python37`, then run the command prompt cmd as an administrator to execute the following command!)
 ```bash
   cd OneForAll/
-   python -m pip install --user  -U pip setuptools wheel
-   pip3 install --user -r requirements.txt
+   python -m pip install -U pip setuptools wheel
+   pip3 install -r requirements.txt
   cd oneforall/
   python oneforall.py --help
 ```
@@ -163,7 +163,7 @@ The OneForAll command line interface is based on [Fire](https://github.com/googl
           valid, invalid, and all subdomains, respectively.
           Parameter verify for True attempts to resolve and request the subdomain 
           and tag the validity of the subdomain based on the result.
-           Parameter port have optional values 'small', 'medium', 'large', 'xlarge', 
+           Parameter port have optional values 'default' 'small', 'medium', 'large',
           See config.py configuration for details.
           Parameter format have optional values 'txt', 'rst', 'csv', 'tsv', 'json', 
           'yaml', 'html', 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'.
@@ -181,7 +181,7 @@ The OneForAll command line interface is based on [Fire](https://github.com/googl
       --verify=VERIFY
           Verify the validity of subdomains (default True)
       --port=PORT
-           Port range for request authentication (default medium)
+           Port range for request authentication (default 80 port)
       --valid=VALID
           Export validity of subdomains (default 1)
       --path=PATH
@@ -334,7 +334,7 @@ Very warmly welcome all ace to improve the project together!
 ## ⌛Follow-up plan

 - [ ] Continuous optimization and improvement of each module
- [ ] Subdomain monitoring (marking each newly discovered subdomain)
+- [x] Subdomain monitoring (marking each newly discovered subdomain)
 - [ ] Subdomain collection crawler implementation (including collection of subdomains from static resource files such as JS)
 - [ ] Implementation of front-end interface for powerful interaction (tentative: front-end: Element + back-end: Flask)

@@ -375,4 +375,4 @@ Thanks ace of [A-Team](https://github.com/QAX-A-Team) for their enthusiastic and
 This tool is limited to legally authorized enterprise security construction. In the process of using this tool, you should ensure that all your actions comply with local laws and regulations and have obtained sufficient authorization.
 If you have any illegal behavior in the process of using this tool, you are responsible for all consequences, and all authors and all contributors of this tool do not assume any legal and joint responsibility.
 Unless you have fully read, fully understood and accepted all the terms of this Agreement, please do not install and use this tool.
-Your use or any other express or implied representation of you to this Agreement is deemed to have been read and agreed to be bound by this Agreement.
+Your use or any other express or implied representation of you to this Agreement is deemed to have been read and agreed to be bound by this Agreement.
@@ -4,8 +4,8 @@
 [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
-[![python](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.0.7-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/python-3.7%20%7C%203.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/release-v0.0.8-brightgreen)](https://github.com/shmilylty/OneForAll/releases)

 👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/README.en.md)

@@ -26,9 +26,9 @@

 * **效率问题**，没有利用多进程，多线程以及异步协程技术，速度较慢。

-为了解决以上痛点，此项目应用而生，OneForAll一词是来自我喜欢的一部日漫《[我的英雄学院](https://manhua.fzdm.com/131/)》，它是一种通过一代代的传承不断变强的潜力无穷的顶级个性，目前[番剧](https://www.bilibili.com/bangumi/media/md7452/)也更新到了第三季了，欢迎大佬们入坑😄。正如其名，我希望OneForAll是一款集百家之长，功能强大的全面快速子域收集终极神器🔨。
+为了解决以上痛点，此项目应用而生，OneForAll一词是来自我喜欢的一部日漫《[我的英雄学院](https://manhua.fzdm.com/131/)》，它是一种通过一代代的传承不断变强的潜力无穷的顶级个性，目前[番剧](https://www.bilibili.com/bangumi/media/md7452/)也更新到了第四季了，欢迎大佬们入坑😄。正如其名，我希望OneForAll是一款集百家之长，功能强大的全面快速子域收集终极神器🔨。

-目前OneForAll还在开发中，肯定有不少问题和需要改进的地方，欢迎大佬们提交[Issues](https://github.com/shmilylty/OneForAll/issues)和[PR](https://github.com/shmilylty/OneForAll/pulls)，用着还行给个小星星✨吧，目前有一个专门用于OneForAll交流和反馈QQ群👨‍👨‍👦‍👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)，也可以给我发邮件📧[admin@hackfun.org]。
+目前OneForAll还在开发中，肯定有不少问题和需要改进的地方，欢迎大佬们提交[Issues](https://github.com/shmilylty/OneForAll/issues)和[PR](https://github.com/shmilylty/OneForAll/pulls)，用着还行给个小星星✨吧，目前有一个专门用于OneForAll交流和反馈QQ群👨‍👨‍👦‍👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)（加群验证：我的英雄学院），也可以给我发邮件📧[admin@hackfun.org]。

 ## 👍功能特性

@@ -59,15 +59,15 @@

 **🐍安装要求**

-OneForAll是基于CPython开发的，所以你需要Python环境才能运行，如果你的系统还没有Python环境你可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)，理论上Python 3.6，3.7和3.8都可以正常运行OneForAll，**但是**许多测试都是在Python 3.7上进行的，所以**推荐**你使用**Python 3.7**版本运行OneForAll。运行以下命令检查Python和pip3版本：
+OneForAll是基于CPython开发的，所以你需要Python环境才能运行，如果你的系统还没有Python环境你可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)，理论上Python 3.7和3.8都可以正常运行OneForAll，**但是**许多测试都是在Python 3.8上进行的，所以**推荐**你使用**Python 3.8**版本运行OneForAll。运行以下命令检查Python和pip3版本：
 ```bash
 python -V
 pip3 -V
 ```
 如果你看到以下类似输出便说明Python环境没有问题：
 ```bash
-Python 3.7.4
-pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.7)
+Python 3.8.0
+pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python38\site-packages\pip (python 3.8)
 ```

 **✔安装步骤**
@@ -88,11 +88,11 @@ pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages
   
 ```bash
   cd OneForAll/
-   python -m pip install --user  -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-   pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+   python -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+   pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
   cd oneforall/
   python oneforall.py --help
-   ```
+```
   其他系统平台的请参考[依赖安装](https://github.com/shmilylty/OneForAll/tree/master/docs/installation_dependency.md)，如果在安装依赖过程中发现编译某个依赖库失败时可以参考[Q&A](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md)中解决方法，如果还没有解决欢迎加群反馈。

 3. **更新**
@@ -159,7 +159,7 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
       Note:
           参数valid可选值1，0，None分别表示导出有效，无效，全部子域
           参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
-           参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+           参数port可选值有'default', 'small', 'medium', 'large' 详见config.py配置
           参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
                             'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
@@ -174,7 +174,7 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
       --verify=VERIFY
           验证子域有效性(默认True)
       --port=PORT
-           请求验证的端口范围(默认medium)
+           请求验证的端口范围(默认只探测80端口)
       --valid=VALID
           导出子域的有效性(默认1)
       --path=PATH
@@ -316,7 +316,7 @@ D:.
 ## ⌛后续计划

 - [ ] 各模块持续优化和完善
- [ ] 子域监控（标记每次新发现的子域）
+- [x] 子域监控（标记每次新发现的子域）
 - [ ] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
 - [ ] 操作强大交互人性的前端界面实现（暂定：前端：Element + 后端：Flask）

@@ -10,10 +10,10 @@ A: 可以尝试以下方法:
 * [https://www.lfd.uci.edu/~gohlke/pythonlibs](https://www.lfd.uci.edu/~gohlke/pythonlibs)
 * [https://pythonwheels.com/](https://pythonwheels.com/)

-选择好对应版本执行以下命令手动安装。举个例子，当编译pycares时失败时，找到[https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares](https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares)，由于我的系统是Windows 10 64位，使用的Python 3.7便下载`pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl`（一般来说下载最新版本的），然后手动安装：
+选择好对应版本执行以下命令手动安装。举个例子，当编译 brotlipy 时失败时，找到[https://www.lfd.uci.edu/~gohlke/pythonlibs/#brotlipy](https://www.lfd.uci.edu/~gohlke/pythonlibs/#brotlipy)，由于我的系统是Windows 10 64位，使用的Python 3.8便下载`brotlipy‑0.7.0‑cp38‑cp38‑win_amd64.whl`（一般来说下载最新版本的），然后手动安装：

 ```bash
-pip3 install pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl
+pip3 install brotlipy‑0.7.0‑cp38‑cp38‑win_amd64.whl
 ```

 2. 到库的项目地址issues和wiki等找找有没有解决方法，如果没有就给他们提issues发邮件😜。
@@ -2,15 +2,15 @@

 你可以通过pip3和pipenv两种方法安装OneForAll的依赖（如果你熟悉[pipenv](https://docs.pipenv.org/en/latest/)，那么推荐使用你使用pipenv）：

-* **Windows系统**（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python37`，那么请以管理员身份运行命令提示符cmd执行以下命令！）
+* **Windows系统**（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python38`，那么请以管理员身份运行命令提示符cmd执行以下命令！）

    1. 使用pipenv

    ```bash
    cd OneForAll/
-    python -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-    pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
-    pipenv install --user --python 3.7
+    python -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install pipenv -i https://mirrors.aliyun.com/pypi/simple/
+    pipenv install --python 3.8
    cd oneforall
    pipenv run python oneforall.py --help
    ```
@@ -19,8 +19,8 @@

    ```bash
    cd OneForAll/
-    python -m pip install --user  -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+    python -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
    cd oneforall/
    python oneforall.py --help
    ```
@@ -29,17 +29,17 @@
    1. 使用pipenv
    ```bash
    cd OneForAll/
-    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-    sudo pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
-    sudo pipenv install --user --python 3.7
+    sudo python3 -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    sudo pip3 install pipenv -i https://mirrors.aliyun.com/pypi/simple/
+    sudo pipenv install --python 3.8
    cd oneforall
    pipenv run python3 oneforall.py --help
    ```
    2. 使用pip3
    ```bash
    cd OneForAll/
-    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+    sudo python3 -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    sudo pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
    cd oneforall/
    python3 oneforall.py --help
    ```
@@ -48,17 +48,17 @@
    1. 使用pipenv
    ```bash
    cd OneForAll/
-    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-    pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
-    pipenv install --user --python 3.7
+    python3 -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install pipenv -i https://mirrors.aliyun.com/pypi/simple/
+    pipenv install --python 3.8
    cd oneforall
    pipenv run python3 oneforall.py --help
    ```
    2. 使用pip3
    ```bash
    cd OneForAll/
-    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+    python3 -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
    cd oneforall/
    python3 oneforall.py --help
-    ```
+    ```
@@ -8,11 +8,11 @@ OneForAll多进程多协程异步子域爆破模块
 :license: GNU General Public License v3.0, see LICENSE for more details.
 """

-import asyncio
-import queue
-import secrets
-import signal
 import time
+import queue
+import signal
+import asyncio
+import secrets

 import aiomultiprocess as aiomp
 import exrex
@@ -39,19 +39,22 @@ def detect_wildcard(domain):
    :return: 如果没有使用泛解析返回False 反之返回泛解析的IP集合和ttl整型值
    """
    logger.log('INFOR', f'正在探测{domain}是否使用泛解析')
-    token = secrets.token_hex(16)
+    token = secrets.token_hex(4)
    random_subdomain = f'{token}.{domain}'
    try:
-        answers = resolve.dns_query_a(random_subdomain)
+        resolver = resolve.dns_resolver()
+        answers = resolver.query(random_subdomain, 'A')
    # 如果查询随机域名A记录出错 说明不存在随机子域的A记录 即没有开启泛解析
    except Exception as e:
        logger.log('DEBUG', e)
        logger.log('INFOR', f'{domain}没有使用泛解析')
        return False, None, None
    ttl = answers.ttl
+    name = answers.name
    ips = {item.address for item in answers}
    logger.log('ALERT', f'{domain}使用了泛解析')
-    logger.log('ALERT', f'{random_subdomain} 解析到IP: {ips} TTL: {ttl}')
+    logger.log('ALERT', f'{random_subdomain} 解析到域名: {name} '
+                        f'IP: {ips} TTL: {ttl}')
    return True, ips, ttl


@@ -210,29 +213,35 @@ class AIOBrute(Module):
        return utils.split_list(domains, self.segment)  # 分割任务组

    def deal_results(self, results):
-        for result in results:
-            if result is None:
+        for answer in results:
+            if answer is None:
                continue
-            if isinstance(result, Exception):
+            if isinstance(answer, Exception):
                # logger.log('DEBUG', f'爆破{subdomain}时出错 {str(answers)}')
                continue
-            if isinstance(result, tuple):
-                subdomain, answers = result
-                if not answers:
+            ips = {item.address for item in answer}
+            # 取值 如果是首次出现的IP集合 出现次数先赋值0
+            value = self.ips_times.setdefault(str(ips), 0)
+            self.ips_times[str(ips)] = value + 1
+            ttl = answer.rrset.ttl
+            subdomain = str(answer.rrset.name)
+            # 目前域名开启了泛解析
+            if self.enable_wildcard:
+                # 通过对比查询的子域和响应的子域来判断真实子域
+                # 去掉解析到CDN的情况
+                if not subdomain.endswith(self.domain + '.'):
                    continue
-                ips = {record.host for record in answers}
-                # 取值 如果是首次出现的IP集合 出现次数先赋值0
-                value = self.ips_times.setdefault(str(ips), 0)
-                self.ips_times[str(ips)] = value + 1
-                ttl = answers[0].ttl
-                if self.enable_wildcard:
-                    if wildcard_by_compare(ips,
-                                           ttl,
-                                           self.wildcard_ips,
-                                           self.wildcard_ttl):
-                        continue
+                # 通过对比解析到的IP集合和TTL确定子域来判断真实子域
+                if wildcard_by_compare(ips,
+                                       ttl,
+                                       self.wildcard_ips,
+                                       self.wildcard_ttl):
+                    continue
+                # 通过对比解析到的IP集合的次数来判断真实子域
                if wildcard_by_times(ips, self.ips_times):
                    continue
+            # 只添加没有出现过的子域
+            if subdomain not in self.subdomains:
                logger.log('INFOR', f'发现{self.domain}的子域: {subdomain} '
                                    f'解析IP: {ips} TTL: {ttl}')
                self.subdomains.add(subdomain)
@@ -252,7 +261,7 @@ class AIOBrute(Module):
                                  initializer=init_worker,
                                  childconcurrency=self.coroutine) as pool:
                try:
-                    results = await pool.map(resolve.aiodns_query_a, task)
+                    results = await pool.map(resolve.dns_query_a, task)
                except KeyboardInterrupt:
                    logger.log('ALERT', '爆破终止正在退出')
                    pool.terminate()  # 关闭pool，结束工作进程，不在处理未完成的任务。
@@ -37,7 +37,7 @@ class Database(object):

    def create_table(self, table_name):
        """
-        初始化数据库
+        创建表结构

        :param str table_name: 要创建的表名
        """
@@ -53,8 +53,11 @@ class Database(object):
                            f'status int,'
                            f'reason text,'
                            f'valid int,'
+                            f'new int,'
                            f'title text,'
                            f'banner text,'
+                            f'header text,'
+                            f'response text,'
                            f'module text,'
                            f'source text,'
                            f'elapsed float,'
@@ -78,14 +81,35 @@ class Database(object):
                self.conn.bulk_query(
                    f'insert into "{table_name}" ('
                    f'id, url, subdomain, port, ips, status, reason, valid,'
-                    f'title, banner, module, source, elapsed, count)'
+                    f'new, title, banner, header, response, module, source, '
+                    f'elapsed, count)'
                    f'values (:id, :url, :subdomain, :port, :ips, :status,'
-                    f':reason, :valid, :title, :banner, :module, :source,'
-                    f':elapsed, :count)',
+                    f':reason, :valid, :new, :title, :banner, :header,'
+                    f':response, :module, :source,:elapsed, :count)',
                    results)
            except Exception as e:
                logger.log('ERROR', e)

+    def exist_table(self, table_name):
+        """
+        判断是否存在某表
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在查询是否存在{table_name}表')
+        try:
+            result = self.conn.query(f'select count() from sqlite_master '
+                                     f'where type = "table" and '
+                                     f'name = "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)
+        else:
+            if len(result) != 0:
+                return True
+            else:
+                return False
+
    def copy_table(self, table_name, bak_table_name):
        """
        复制表创建备份
@@ -131,7 +155,7 @@ class Database(object):

    def rename_table(self, table_name, new_table_name):
        """
-        复制表创建备份
+        重命名表名

        :param str table_name: 表名
        :param str new_table_name: 新表名
@@ -190,18 +214,22 @@ class Database(object):
        else:
            return rows

-    def get_subdomain(self, table_name, valid):
+    def export_data(self, table_name, valid):
        """
-        获取表中的子域数据
+        获取表中的部分数据

        :param str table_name: 表名
-        :param int valid: 是否有效
+        :param any valid: 有效性
        """
        table_name = table_name.replace('.', '_')
+        query = f'select id, url, subdomain, port, ips, status, reason,' \
+                f'valid, new, title, banner from "{table_name}"'
+        if valid == 0 or valid == 1:
+            where = f' where valid = {valid}'
+            query += where
        logger.log('DEBUG', f'获取{table_name}表中的所有数据')
        try:
-            rows = self.conn.query(
-                f'select * from "{table_name}" where valid = {valid}')
+            rows = self.conn.query(query)
        except Exception as e:
            logger.log('ERROR', e)
        else:
@@ -220,8 +220,11 @@ class Module(object):
                      'status': None,
                      'reason': None,
                      'valid': None,
+                      'new': None,
                      'title': None,
                      'banner': None,
+                      'header': None,
+                      'response': None,
                      'module': self.module,
                      'source': self.source,
                      'elapsed': self.elapsed,
@@ -240,9 +243,12 @@ class Module(object):
                          'status': None,
                          'reason': None,
                          'valid': None,
+                          'new': None,
                          'title': None,
                          'banner': None,
                          'module': self.module,
+                          'header': None,
+                          'response': None,
                          'source': self.source,
                          'elapsed': self.elapsed,
                          'count': len(self.subdomains)}
@@ -1,70 +1,91 @@
-# coding=utf-8
-
 import asyncio
 import functools

 import aiohttp
 import tqdm
 from aiohttp import ClientSession
-from aiohttp.resolver import AsyncResolver
 from bs4 import BeautifulSoup
 import config
 from common import utils
 from config import logger


+def get_limit_conn():
+    limit_open_conn = config.limit_open_conn
+    if limit_open_conn is None:  # 默认情况
+        limit_open_conn = utils.get_semaphore()
+    elif not isinstance(limit_open_conn, int):  # 如果传入不是数字的情况
+        limit_open_conn = utils.get_semaphore()
+    return limit_open_conn
+
+
 def get_ports(port):
-    logger.log('INFOR', f'正在获取请求端口范围')
+    logger.log('DEBUG', f'正在获取请求探测端口范围')
    ports = set()
    if isinstance(port, set):
        ports = port
-    elif isinstance(port, str):
-        if port not in {'small', 'medium', 'large', 'xlarge'}:
-            logger.log('ERROR', f'不存在{port}等端口范围')
-            port = 'medium'
+    elif isinstance(port, list):
+        ports = set(port)
+    elif isinstance(port, tuple):
+        ports = set(port)
+    elif isinstance(port, int):
+        if 0 <= port <= 65535:
+            ports = {port}
+    elif port in {'default', 'small', 'medium', 'large'}:
+        logger.log('INFOR', f'探测{port}等端口范围')
        ports = config.ports.get(port)
-        logger.log('INFOR', f'使用{port}等端口范围')
-    if not ports:  # 意外情况 ports_range为空使用使用中等端口范围
-        logger.log('ALERT', f'使用medium等端口范围')
-        ports = config.ports.get('medium')
+    if not ports:  # 意外情况
+        logger.log('ERROR', f'指定探测端口范围有误')
+        ports = {80}
+    if ports == {80}:
+        logger.log('INFOR', f'探测默认端口范围')
    return ports


 def gen_new_datas(datas, ports):
    logger.log('INFOR', f'正在生成请求地址')
    new_datas = []
-    protocols = ['http://']
+    protocols = ['http://', 'https://']
    for data in datas:
        valid = data.get('valid')
        if valid is None:  # 子域有效性未知的才进行http请求探测
            subdomain = data.get('subdomain')
            for port in ports:
-                for protocol in protocols:
-                    if port == 443:
-                        url = f'https://{subdomain}:{port}'
-                    elif port == 8443:
-                        url = f'https://{subdomain}:{port}'
-                    else:
-                        url = f'{protocol}{subdomain}:{port}'
+                if port == 80:
+                    url = f'http://{subdomain}'
                    data['id'] = None
                    data['url'] = url
-                    data['port'] = port
+                    data['port'] = 80
                    new_datas.append(data)
                    data = dict(data)  # 需要生成一个新的字典对象
+                elif port == 443:
+                    url = f'https://{subdomain}'
+                    data['id'] = None
+                    data['url'] = url
+                    data['port'] = 443
+                    new_datas.append(data)
+                    data = dict(data)  # 需要生成一个新的字典对象
+                else:
+                    for protocol in protocols:
+                        url = f'{protocol}{subdomain}:{port}'
+                        data['id'] = None
+                        data['url'] = url
+                        data['port'] = port
+                        new_datas.append(data)
+                        data = dict(data)  # 需要生成一个新的字典对象
    return new_datas


-async def fetch(session, url, semaphore):
+async def fetch(session, url):
    """
    请求

    :param session: session对象
    :param url: url地址
-    :param semaphore: 并发信号量
    :return: 响应对象和响应文本
    """
    timeout = aiohttp.ClientTimeout(total=config.get_timeout)
-    async with semaphore:
+    try:
        async with session.get(url,
                               ssl=config.verify_ssl,
                               allow_redirects=config.get_redirects,
@@ -76,6 +97,8 @@ async def fetch(session, url, semaphore):
            except UnicodeDecodeError:
                text = await resp.text(errors='ignore')
        return resp, text
+    except BaseException as exception:
+        return exception


 def get_title(markup):
@@ -99,6 +122,10 @@ def get_title(markup):
    if h2:
        return h2.text.strip()

+    h3 = soup.h3
+    if h2:
+        return h3.text.strip()
+
    desc = soup.find('meta', attrs={'name': 'description'})
    if desc:
        return desc['content'].strip()
@@ -121,23 +148,29 @@ def request_callback(future, index, datas):
    try:
        result = future.result()
    except BaseException as e:
-        logger.log('DEBUG', e.args)
+        logger.log('TRACE', e.args)
        datas[index]['reason'] = str(e.args)
        datas[index]['valid'] = 0
    else:
-        resp, text = result
-        datas[index]['reason'] = resp.reason
-        datas[index]['status'] = resp.status
-        if resp.status >= 500:
-            datas[index]['valid'] = 0
+        if isinstance(result, tuple):
+            resp, text = result
+            datas[index]['reason'] = resp.reason
+            datas[index]['status'] = resp.status
+            if resp.status == 400 or resp.status >= 500:
+                datas[index]['valid'] = 0
+            else:
+                datas[index]['valid'] = 1
+                headers = resp.headers
+                banner = str({'Server': headers.get('Server'),
+                              'Via': headers.get('Via'),
+                              'X-Powered-By': headers.get('X-Powered-By')})
+                datas[index]['banner'] = banner[1:-1]
+                datas[index]['title'] = get_title(text)
+                datas[index]['header'] = str(dict(headers))[1:-1]
+                datas[index]['response'] = text
        else:
-            datas[index]['valid'] = 1
-            headers = resp.headers
-            banner = str({'Server': headers.get('Server'),
-                          'Via': headers.get('Via'),
-                          'X-Powered-By': headers.get('X-Powered-By')})
-            datas[index]['banner'] = banner
-            datas[index]['title'] = get_title(text)
+            datas[index]['reason'] = 'Something error'
+            datas[index]['valid'] = 0


 async def bulk_get_request(datas, port):
@@ -145,19 +178,11 @@ async def bulk_get_request(datas, port):
    new_datas = gen_new_datas(datas, ports)
    logger.log('INFOR', f'正在异步进行子域的GET请求')

-    limit_open_conn = config.limit_open_conn
-    if limit_open_conn is None:  # 默认情况
-        limit_open_conn = utils.get_semaphore()
-    elif not isinstance(limit_open_conn, int):  # 如果传入不是数字的情况
-        limit_open_conn = utils.get_semaphore()
-    # 使用异步域名解析器 自定义域名服务器
-    resolver = AsyncResolver(nameservers=config.resolver_nameservers)
-    conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
+    limit_open_conn = get_limit_conn()
+    conn = aiohttp.TCPConnector(ttl_dns_cache=300,
+                                ssl=config.verify_ssl,
                                limit=limit_open_conn,
-                                limit_per_host=config.limit_per_host,
-                                resolver=resolver)
-
-    semaphore = asyncio.Semaphore(limit_open_conn)
+                                limit_per_host=config.limit_per_host)
    header = None
    if config.fake_header:
        header = utils.gen_fake_header()
@@ -165,12 +190,13 @@ async def bulk_get_request(datas, port):
        tasks = []
        for i, data in enumerate(new_datas):
            url = data.get('url')
-            task = asyncio.ensure_future(fetch(session, url, semaphore))
+            task = asyncio.ensure_future(fetch(session, url))
            task.add_done_callback(functools.partial(request_callback,
                                                     index=i,
                                                     datas=new_datas))
            tasks.append(task)
-        if tasks:  # 任务列表里有任务不空时才进行解析
+        # 任务列表里有任务不空时才进行解析
+        if tasks:
            # 等待所有task完成 错误聚合到结果列表里
            futures = asyncio.as_completed(tasks)
            for future in tqdm.tqdm(futures,
@@ -178,10 +204,7 @@ async def bulk_get_request(datas, port):
                                    desc='Progress',
                                    smoothing=1.0,
                                    ncols=True):
-                try:
-                    await future
-                except:
-                    pass
+                await future

    logger.log('INFOR', f'完成异步进行子域的GET请求')
    return new_datas
@@ -1,13 +1,10 @@
-# coding=utf-8
 import asyncio
 import functools

-import dns.resolver
-import aiodns
 import tqdm
+import dns.resolver

 import config
-from common import utils
 from config import logger


@@ -22,7 +19,7 @@ def dns_resolver():
    return resolver


-def dns_query_a(hostname):
+async def dns_query_a(hostname):
    """
    查询A记录

@@ -30,42 +27,12 @@ def dns_query_a(hostname):
    :return: 查询结果
    """
    resolver = dns_resolver()
-    return resolver.query(hostname, 'A')
-
-
-def aiodns_resolver():
-    """
-    异步dns解析器
-    """
-    return aiodns.DNSResolver(nameservers=config.resolver_nameservers,
-                              timeout=config.resolver_timeout)
-
-
-async def aiodns_query_a(hostname, semaphore=None):
-    """
-    异步查询A记录
-
-    :param str hostname: 主机名
-    :param semaphore: 并发查询数量
-    :return: 主机名或查询结果或查询异常
-    """
-    if semaphore is None:
-        resolver = aiodns_resolver()
-        try:
-            answers = await resolver.query(hostname, 'A')
-        except BaseException as e:
-            logger.log('DEBUG', e.args)
-            answers = None
-        return hostname, answers
-    else:
-        async with semaphore:
-            resolver = aiodns_resolver()
-            try:
-                answers = await resolver.query(hostname, 'A')
-            except BaseException as e:
-                logger.log('DEBUG', e.args)
-                answers = None
-            return hostname, answers
+    try:
+        answer = resolver.query(hostname, 'A')
+    except BaseException as exception:
+        logger.log('TRACE', exception.args)
+        answer = exception
+    return answer


 def resolve_callback(future, index, datas):
@@ -76,18 +43,17 @@ def resolve_callback(future, index, datas):
    :param datas: 结果集
    """
    try:
-        result = future.result()
+        answer = future.result()
    except BaseException as e:
        datas[index]['ips'] = str(e.args)
        datas[index]['valid'] = 0
    else:
-        if isinstance(result, tuple):
-            _, answers = result
-            if answers:
-                ips = {record.host for record in answers}
-                datas[index]['ips'] = str(ips)
-            else:
-                datas[index]['ips'] = 'No answers'
+        if isinstance(answer, dns.resolver.Answer):
+            ips = {item.address for item in answer}
+            datas[index]['ips'] = str(ips)[1:-1]
+        else:
+            datas[index]['ips'] = 'Something error'
+            datas[index]['valid'] = 0


 async def bulk_query_a(datas):
@@ -99,11 +65,11 @@ async def bulk_query_a(datas):
    """
    logger.log('INFOR', '正在异步查询子域的A记录')
    tasks = []
-    semaphore = asyncio.Semaphore(config.limit_resolve_conn)
+    # semaphore = asyncio.Semaphore(config.limit_resolve_conn)
    for i, data in enumerate(datas):
        if not data.get('ips'):
            subdomain = data.get('subdomain')
-            task = asyncio.ensure_future(aiodns_query_a(subdomain, semaphore))
+            task = asyncio.ensure_future(dns_query_a(subdomain))
            task.add_done_callback(functools.partial(resolve_callback,
                                                     index=i,
                                                     datas=datas))  # 回调
@@ -115,10 +81,7 @@ async def bulk_query_a(datas):
                                desc='Progress',
                                smoothing=1.0,
                                ncols=True):
-            try:
-                await future
-            except:
-                pass
+            await future
        # await asyncio.wait(tasks)  # 等待所有task完成
    logger.log('INFOR', '完成异步查询子域的A记录')
    return datas
@@ -137,7 +137,7 @@ def get_semaphore():
    """
    system = platform.system()
    if system == 'Windows':
-        return 300
+        return 800
    elif system == 'Linux':
        return 800
    elif system == 'Darwin':
@@ -207,3 +207,29 @@ def check_response(method, resp):
        else:
            logger.log('ALERT', msg)
    return False
+
+
+def mark_subdomain(old_data, new_data):
+    """
+    标记新增子域并返回新的数据集
+
+    :param old_data: 之前数据集
+    :param new_data: 现在数据集
+    :return: 已标记的新的数据集
+    """
+    # 第一次收集子域的情况
+    if not old_data:
+        for index, item in enumerate(new_data):
+            item['new'] = 1
+            new_data[index] = item
+        return new_data
+    # 非第一次收集子域的情况
+    old_subdomains = {item.get('subdomain') for item in old_data}
+    for index, item in enumerate(new_data):
+        subdomain = item.get('subdomain')
+        if subdomain in old_subdomains:
+            item['new'] = 0
+        else:
+            item['new'] = 1
+        new_data[index] = item
+    return new_data
@@ -31,7 +31,7 @@ enable_verify_subdomain = True  # 验证子域有效性(默认True)
 enable_wildcard_check = True  # 开启泛解析检测 会去掉泛解析的子域
 # 爆破时使用的进程数(根据系统中CPU数量情况设置 不宜大于CPU数量 默认为系统中的CPU数量)
 brute_process_num = os.cpu_count()
-brute_coroutine_num = 128  # 爆破时每个进程下的协程数(不宜大于1000)
+brute_coroutine_num = 64  # 爆破时每个进程下的协程数(不宜大于500)
 # 爆破所使用的字典路径 默认data/subdomains.txt
 brute_wordlist_path = data_storage_path.joinpath('subnames.txt')
 brute_task_segment = 500
@@ -82,35 +82,34 @@ resolver_nameservers = [
 ]  # 指定查询的DNS域名服务器
 resolver_timeout = 5.0  # 解析超时时间
 resolver_lifetime = 30.0  # 解析存活时间
-limit_resolve_conn = 50  # 限制同一时间解析的数量(默认50)
+limit_resolve_conn = 500  # 限制同一时间解析的数量(默认500)

-# http探测设置
-small_ports = {80, 443}
-medium_ports = {80, 443, 8000, 8080, 8443}  # 默认使用
-large_ports = {80, 81, 443, 591, 2082, 2087, 2095, 2096, 3000, 8000, 8001,
-               8008, 8080, 8083, 8443, 8834, 8888}
-xlarge_ports = {80, 81, 300, 443, 591, 593, 832, 981, 1010, 1311, 2082,
-                2087, 2095, 2096, 2480, 3000, 3128, 3333, 4243, 4567, 4711,
-                4712, 4993, 5000, 5104, 5108, 5800, 6543, 7000, 7396, 7474,
-                8000, 8001, 8008, 8014, 8042, 8069, 8080, 8081, 8088, 8090,
-                8091, 8016, 8118, 8123, 8172, 8222, 8243, 8280, 8281, 8333,
-                8443, 8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080,
-                9090, 9091, 9200, 9443, 9800, 9981, 12443, 16080, 18091, 18092,
-                20720, 28017}
-ports = {'small': small_ports, 'medium': medium_ports,
-         'large': large_ports, 'xlarge': xlarge_ports}
+# 请求端口探测设置
+default_ports = {80}  # 默认使用
+small_ports = {80, 443, 8000, 8080, 8443}
+medium_ports = {80, 81, 443, 591, 2082, 2087, 2095, 2096, 3000, 8000, 8001,
+                8008, 8080, 8083, 8443, 8834, 8888}
+large_ports = {80, 81, 300, 443, 591, 593, 832, 888, 981, 1010, 1311, 2082,
+               2087, 2095, 2096, 2480, 3000, 3128, 3333, 4243, 4567, 4711,
+               4712, 4993, 5000, 5104, 5108, 5800, 6543, 7000, 7396, 7474,
+               8000, 8001, 8008, 8014, 8042, 8069, 8080, 8081, 8088, 8090,
+               8091, 8016, 8118, 8123, 8172, 8222, 8243, 8280, 8281, 8333,
+               8443, 8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080,
+               9090, 9091, 9200, 9443, 9800, 9981, 12443, 16080, 18091, 18092,
+               20720, 28017}  # 可以在这里面添加端口
+ports = {'default': default_ports, 'small': small_ports,
+         'medium': medium_ports, 'large': large_ports}
 verify_ssl = False
 # aiohttp 支持 HTTP/HTTPS形式的代理
 get_proxy = None  # proxy="http://user:pass@some.proxy.com"
-get_timeout = 120  # http请求探测总超时时间 None或者0则表示不检测超时
+get_timeout = 60  # http请求探测总超时时间 None或者0则表示不检测超时
 get_redirects = True  # 允许请求跳转
 fake_header = True  # 使用伪造请求头
 # 限制同一时间打开的连接数(默认None，根据系统不同设置，Windows系统400 其他系统800)
-limit_open_conn = None
+limit_open_conn = 200
 # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
 limit_per_host = 0  # 默认0表示不限制

-
 # 模块API配置
 # Censys可以免费注册获取API：https://censys.io/api
 censys_api_id = ''
@@ -193,16 +192,16 @@ subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
 # 日志配置
 # 终端日志输出格式
 stdout_fmt = '<cyan>{time:HH:mm:ss,SSS}</cyan> ' \
-          '[<level>{level: <5}</level>] ' \
-          '<blue>{module}</blue>:<cyan>{line}</cyan> - ' \
-          '<level>{message}</level>'
+             '[<level>{level: <5}</level>] ' \
+             '<blue>{module}</blue>:<cyan>{line}</cyan> - ' \
+             '<level>{message}</level>'
 # 日志文件记录格式
 logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
-          '[<level>{level: <5}</level>] ' \
-          '<cyan>{process.name}({process.id})</cyan>:' \
-          '<cyan>{thread.name: <10}({thread.id: <5})</cyan> | ' \
-          '<blue>{module}</blue>.<blue>{function}</blue>:' \
-          '<blue>{line}</blue> - <level>{message}</level>'
+              '[<level>{level: <5}</level>] ' \
+              '<cyan>{process.name}({process.id})</cyan>:' \
+              '<cyan>{thread.name: <10}({thread.id: <5})</cyan> | ' \
+              '<blue>{module}</blue>.<blue>{function}</blue>:' \
+              '<blue>{line}</blue> - <level>{message}</level>'

 log_path = result_save_path.joinpath('oneforall.log')

@@ -1,102 +1,102 @@
-[
-"_afpovertcp._tcp.",
-"_aix._tcp.",
-"_autodiscover._tcp.",
-"_caldav._tcp.",
-"_certificates._tcp.",
-"_client._smtp.",
-"_cmp._tcp.",
-"_crls._tcp.",
-"_crl._tcp.",
-"_finger._tcp.",
-"_ftp._tcp.",
-"_gc._tcp.",
-"_h323be._tcp.",
-"_h323be._udp.",
-"_h323cs._tcp.",
-"_h323cs._udp.",
-"_h323ls._tcp.",
-"_h323ls._udp.",
-"_h323rs._tcp.",
-"_hkps._tcp.",
-"_hkp._tcp.",
-"_http._tcp.",
-"_iax.udp.",
-"_imaps._tcp.",
-"_imap._tcp.",
-"_jabber-client._tcp.",
-"_jabber-client._udp.",
-"_jabber._tcp.",
-"_jabber._udp.",
-"_kerberos-adm._tcp.",
-"_kerberos._tcp.",
-"_kerberos._tcp.dc._msdcs.",
-"_kerberos._udp.",
-"_kpasswd._tcp.",
-"_kpasswd._udp.",
-"_ldap._tcp.",
-"_ldap._tcp.dc._msdcs.",
-"_ldap._tcp.gc._msdcs.",
-"_ldap._tcp.pdc._msdcs.",
-"_msdcs.",
-"_mysqlsrv._tcp.",
-"_nntp._tcp.",
-"_ntp._udp.",
-"_ocsp._tcp.",
-"_pgpkeys._tcp.",
-"_pgprevokations._tcp.",
-"_PKIXREP._tcp.",
-"_pop3s._tcp.",
-"_pop3._tcp.",
-"_sipfederationtls._tcp.",
-"_sipinternal._tcp.",
-"_sipinternaltls._tcp.",
-"_sips._tcp.",
-"_sip._tcp.",
-"_sip._tls.",
-"_sip._udp.",
-"_smtp._tcp.",
-"_ssh._tcp.",
-"_stun._tcp.",
-"_stun._udp.",
-"_svcp._tcp.",
-"_tcp.",
-"_telnet._tcp.",
-"_test._tcp.",
-"_tls.",
-"_udp.",
-"_vlmcs._tcp.",
-"_vlmcs._udp.",
-"_whois._tcp.",
-"_wpad._tcp.",
-"_xmpp-client._tcp.",
-"_xmpp-client._udp.",
-"_xmpp-server._tcp.",
-"_xmpp-server._udp.",
-"_https._tcp.",
-"_imap.tcp.",
-"_kerberos.tcp.dc._msdcs.",
-"_ldap._tcp.ForestDNSZones.",
-"_submission._tcp.",
-"_caldavs._tcp.",
-"_carddav._tcp.",
-"_carddavs._tcp.",
-"_x-puppet._tcp.",
-"_x-puppet-ca._tcp.",
-"_domainkey.",
-"_pkixrep._tcp.",
-"_cisco-phone-http.",
-"_cisco-phone-tftp.",
-"_cisco-uds._tcp.",
-"_ciscowtp._tcp.",
-"_collab-edge._tls.",
-"_cuplogin._tcp.",
-"_client._smtp._tcp.",
-"_sftp._tcp.",
-"_h323rs._udp.",
-"_sql._tcp.",
-"_sip._tcp.internal.",
-"_snmp._udp.",
-"_rdp._tcp.",
-"_xmpp-server._udp."
-]
+[
+"_afpovertcp._tcp.",
+"_aix._tcp.",
+"_autodiscover._tcp.",
+"_caldav._tcp.",
+"_certificates._tcp.",
+"_client._smtp.",
+"_cmp._tcp.",
+"_crls._tcp.",
+"_crl._tcp.",
+"_finger._tcp.",
+"_ftp._tcp.",
+"_gc._tcp.",
+"_h323be._tcp.",
+"_h323be._udp.",
+"_h323cs._tcp.",
+"_h323cs._udp.",
+"_h323ls._tcp.",
+"_h323ls._udp.",
+"_h323rs._tcp.",
+"_hkps._tcp.",
+"_hkp._tcp.",
+"_http._tcp.",
+"_iax.udp.",
+"_imaps._tcp.",
+"_imap._tcp.",
+"_jabber-client._tcp.",
+"_jabber-client._udp.",
+"_jabber._tcp.",
+"_jabber._udp.",
+"_kerberos-adm._tcp.",
+"_kerberos._tcp.",
+"_kerberos._tcp.dc._msdcs.",
+"_kerberos._udp.",
+"_kpasswd._tcp.",
+"_kpasswd._udp.",
+"_ldap._tcp.",
+"_ldap._tcp.dc._msdcs.",
+"_ldap._tcp.gc._msdcs.",
+"_ldap._tcp.pdc._msdcs.",
+"_msdcs.",
+"_mysqlsrv._tcp.",
+"_nntp._tcp.",
+"_ntp._udp.",
+"_ocsp._tcp.",
+"_pgpkeys._tcp.",
+"_pgprevokations._tcp.",
+"_PKIXREP._tcp.",
+"_pop3s._tcp.",
+"_pop3._tcp.",
+"_sipfederationtls._tcp.",
+"_sipinternal._tcp.",
+"_sipinternaltls._tcp.",
+"_sips._tcp.",
+"_sip._tcp.",
+"_sip._tls.",
+"_sip._udp.",
+"_smtp._tcp.",
+"_ssh._tcp.",
+"_stun._tcp.",
+"_stun._udp.",
+"_svcp._tcp.",
+"_tcp.",
+"_telnet._tcp.",
+"_test._tcp.",
+"_tls.",
+"_udp.",
+"_vlmcs._tcp.",
+"_vlmcs._udp.",
+"_whois._tcp.",
+"_wpad._tcp.",
+"_xmpp-client._tcp.",
+"_xmpp-client._udp.",
+"_xmpp-server._tcp.",
+"_xmpp-server._udp.",
+"_https._tcp.",
+"_imap.tcp.",
+"_kerberos.tcp.dc._msdcs.",
+"_ldap._tcp.ForestDNSZones.",
+"_submission._tcp.",
+"_caldavs._tcp.",
+"_carddav._tcp.",
+"_carddavs._tcp.",
+"_x-puppet._tcp.",
+"_x-puppet-ca._tcp.",
+"_domainkey.",
+"_pkixrep._tcp.",
+"_cisco-phone-http.",
+"_cisco-phone-tftp.",
+"_cisco-uds._tcp.",
+"_ciscowtp._tcp.",
+"_collab-edge._tls.",
+"_cuplogin._tcp.",
+"_client._smtp._tcp.",
+"_sftp._tcp.",
+"_h323rs._udp.",
+"_sql._tcp.",
+"_sip._tcp.internal.",
+"_snmp._udp.",
+"_rdp._tcp.",
+"_xmpp-server._udp."
+]
@@ -30,19 +30,14 @@ def export(table, db=None, valid=None, dpath=None, format='csv', show=False):
    :param str table:   要导出的表
    :param str db:      要导出的数据库路径(默认为results/result.sqlite3)
    :param int valid:   导出子域的有效性(默认None)
-    :param str format:  导出格式(默认xls)
+    :param str format:  导出格式(默认csv)
    :param str dpath:    导出目录(默认None)
    :param bool show:   终端显示导出数据(默认False)
    """
    format = utils.check_format(format)
    dpath = utils.check_dpath(dpath)
    database = Database(db)
-    if valid is None:
-        rows = database.get_data(table)
-    elif isinstance(valid, int):
-        rows = database.get_subdomain(table, valid)
-    else:
-        rows = database.get_data(table)  # 意外情况导出全部子域
+    rows = database.export_data(table, valid)  # 意外情况导出全部子域
    if show:
        print(rows.dataset)
    if format == 'txt':
@@ -50,7 +45,7 @@ def export(table, db=None, valid=None, dpath=None, format='csv', show=False):
    else:
        data = rows.export(format)
    database.close()
-    fpath = dpath.joinpath(f'{table}.{format}')
+    fpath = dpath.joinpath(f'{table}_subdomain.{format}')
    utils.save_data(fpath, data)


@@ -34,7 +34,7 @@ class CheckAXFR(Module):
        """
        logger.log('DEBUG', f'尝试对{self.domain}的域名服务器{server}进行域传送')
        try:
-            xfr = dns.query.xfr(server, self.domain, timeout=30.0)
+            xfr = dns.query.xfr(where=server, zone=self.domain)
            zone = dns.zone.from_xfr(xfr)
        except Exception as e:
            logger.log('DEBUG', str(e))
@@ -93,4 +93,4 @@ def do(domain):  # 统一入口名字 方便多线程调用

 if __name__ == '__main__':
    do('ZoneTransfer.me')
-    do('example.com')
+    # do('example.com')
@@ -24,7 +24,6 @@ class CeBaidu(Query):
        subdomains = self.match(self.domain, str(resp.json()))
        # 合并搜索子域名搜索结果
        self.subdomains = self.subdomains.union(subdomains)
-        print(self.subdomains)

    def run(self):
        """
@@ -1,92 +0,0 @@
-import time
-import random
-import cloudscraper
-from bs4 import BeautifulSoup
-from common.query import Query
-from config import logger
-
-
-class DNSdb(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = self.register(domain)
-        self.module = 'Dataset'
-        self.source = 'DNSdbQuery'
-        self.addr = 'http://www.dnsdb.org/'
-        self.url = f'{self.addr}{self.domain}/'
-
-    def get_tokens(self):
-        """
-        绕过cloudFlare验证并获取taken
-
-        :return: 绕过失败返回None 成功返回tokens
-        """
-        scraper = cloudscraper.create_scraper()
-        scraper.interpreter = 'js2py'
-        scraper.proxies = self.get_proxy(self.source)
-        scraper.timeout = 10
-        try:
-            tokens = scraper.get_tokens(self.url)
-        except Exception as e:
-            logger.log('ERROR', e.args)
-            return None
-        if len(tokens) != 2:
-            return None
-        return tokens
-
-    def query(self):
-        """
-        向接口查询子域并做子域匹配
-        """
-        tokens = self.get_tokens()
-        if not tokens:
-            logger.log('ALERT', f'{self.source}模块绕过cloudFlare检查失败')
-            return False
-        self.cookie = tokens[0]
-        self.header = {'User-Agent': tokens[1]}
-        self.timeout = 10
-        resp = self.get(self.url)
-        if not resp:
-            return
-        if 'index' in resp.text:
-            soup = BeautifulSoup(resp.text, features='lxml')
-            base = self.addr+self.domain
-            urls = list(map(lambda a: base + '/' + a.get('href'),
-                            soup.find_all('a')))
-            urls = urls[:-1]  # idn域名暂时不考虑
-            for url in urls:
-                resp = self.get(url)
-                if not resp:
-                    return
-                subdomains = self.match(self.domain, resp.text)
-                # 合并搜索子域名搜索结果
-                self.subdomains = self.subdomains.union(subdomains)
-        else:
-            subdomains = self.match(self.domain, resp.text)
-            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains)
-
-    def run(self):
-        """
-        类执行入口
-        """
-        self.begin()
-        self.query()
-        self.finish()
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-
-
-def do(domain):  # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    """
-    query = DNSdb(domain)
-    query.run()
-
-
-if __name__ == '__main__':
-    do('example.com')
@@ -4,69 +4,54 @@
 通过枚举域名常见的SRV记录并做查询来发现子域
 """

-import asyncio
 import json
-
-import aiodns
+import queue
+import threading

 from common import utils
+from common import resolve
 from common.module import Module
-from config import data_storage_path, logger, resolver_nameservers
+from config import data_storage_path, logger


 class BruteSRV(Module):
-    def __init__(self, domain: str):
+    def __init__(self, domain):
        Module.__init__(self)
        self.domain = self.register(domain)
        self.module = 'dnsquery'
        self.source = "BruteSRV"
-        self.loop = asyncio.new_event_loop()
-        self.nameservers = resolver_nameservers
-        self.resolver = aiodns.DNSResolver(self.nameservers, self.loop)
+        self.thread_num = 10
+        self.names_que = queue.Queue()
+        self.answers_que = queue.Queue()

-    async def query(self, name):
-        """
-        查询域名的SRV记录
-        :param str name: SRV记录
-        :return: 查询结果
-        """
-        logger.log('TRACE', f'尝试查询{name}的SRV记录')
-        try:
-            answers = await self.resolver.query(name, 'SRV')
-        except Exception as e:
-            logger.log('TRACE', e)
-            logger.log('TRACE', f'查询{name}的SRV记录失败')
-            return None
-        else:
-            logger.log('TRACE', f'查询{name}的SRV记录成功')
-            return answers
+    def gen_names(self):
+        path = data_storage_path.joinpath('srv_prefixes.json')
+        with open(path) as file:
+            prefixes = json.load(file)
+        names = map(lambda prefix: prefix + self.domain, prefixes)
+
+        for name in names:
+            self.names_que.put(name)

    def brute(self):
        """
        枚举域名的SRV记录
        """
-        names_path = data_storage_path.joinpath('srv_names.json')
-        with open(names_path) as fp:
-            names_dict = json.load(fp)
-        query_map = map(lambda name: name + self.domain, names_dict)
+        self.gen_names()

-        tasks = []
-        for query in query_map:
-            tasks.append(self.query(query))
-        task_group = asyncio.gather(*tasks, loop=self.loop)
-        self.loop.run_until_complete(asyncio.gather(task_group))
-        self.loop.close()
-        results = task_group.result()
-        for result in results:
-            if result:
-                for answer in result:
-                    subdomains = utils.match_subdomain(self.domain, answer.host)
-                    if subdomains:
-                        self.subdomains = self.subdomains.union(subdomains)
-                    else:
-                        logger.log('DEBUG', f'{answer.host}不是{self.domain}的子域')
-        if not len(self.subdomains):
-            logger.log('DEBUG', f'没有找到{self.domain}的SRV记录')
+        for i in range(self.thread_num):
+            thread = BruteThread(self.names_que, self.answers_que)
+            thread.daemon = True
+            thread.start()
+
+        self.names_que.join()
+
+        while not self.answers_que.empty():
+            answer = self.answers_que.get()
+            if answer is not None:
+                for item in answer:
+                    subdomains = utils.match_subdomain(self.domain, str(item))
+                    self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -80,6 +65,38 @@ class BruteSRV(Module):
        self.save_db()


+class BruteThread(threading.Thread):
+    def __init__(self, names_que, answers_que):
+        threading.Thread.__init__(self)
+        self.names_que = names_que
+        self.answers_que = answers_que
+        self.resolver = resolve.dns_resolver()
+
+    def query(self, name):
+        """
+        查询域名的SRV记录
+        :param str name: SRV记录
+        :return: 查询结果
+        """
+        logger.log('TRACE', f'尝试查询{name}的SRV记录')
+        try:
+            answer = self.resolver.query(name, 'SRV')
+        except Exception as exception:
+            logger.log('TRACE', exception.args)
+            logger.log('TRACE', f'查询{name}的SRV记录失败')
+            return None
+        else:
+            logger.log('TRACE', f'查询{name}的SRV记录成功')
+            return answer
+
+    def run(self):
+        while True:
+            name = self.names_que.get()
+            answer = self.query(name)
+            self.answers_que.put(answer)
+            self.names_que.task_done()
+
+
 def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口
@@ -91,5 +108,5 @@ def do(domain):  # 统一入口名字 方便多线程调用


 if __name__ == '__main__':
-
-    do('example.com')
+    do('zonetransfer.me')
+    # do('example.com')
@@ -28,7 +28,7 @@ blue = '\033[01;34m'
 red = '\033[1;31m'
 end = '\033[0m'

-version = white + '{' + red + 'v0.0.7#dev' + white + '}'
+version = white + '{' + red + 'v0.0.8#dev' + white + '}'

 banner = f"""{yellow}
             ___             _ _ 
@@ -42,7 +42,7 @@ class OneForAll(object):
    """
    OneForAll是一款功能强大的子域收集工具

-    Version: 0.0.7
+    Version: 0.0.8
    Project: https://git.io/fjHT1

    Example:
@@ -59,25 +59,25 @@ class OneForAll(object):
    Note:
        参数valid可选值1，0，None分别表示导出有效，无效，全部子域
        参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
-        参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+        参数port可选值有'default', 'small', 'medium', 'large', 详见config.py配置
        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'

    :param str target:  单个域名或者每行一个域名的文件路径(必需参数)
    :param bool brute:  使用爆破模块(默认False)
    :param bool verify: 验证子域有效性(默认True)
-    :param str port:    请求验证的端口范围(默认medium)
+    :param str port:    请求验证子域的端口范围(默认只探测80端口)
    :param int valid:   导出子域的有效性(默认1)
    :param str format:  导出格式(默认csv)
    :param bool show:   终端显示导出数据(默认False)
    """
-    def __init__(self, target, brute=None, verify=None, port='medium', valid=1,
-                 format='csv', takeover=True, show=False):
+    def __init__(self, target, brute=None, verify=None, port='default',
+                 valid=1, format='csv', takeover=True, show=False):
        self.target = target
        self.port = port
        self.domains = set()
        self.domain = str()
-        self.datas = list()
+        self.data = list()
        self.brute = brute
        self.verify = verify
        self.takeover = takeover
@@ -90,7 +90,8 @@ class OneForAll(object):
            self.brute = config.enable_brute_module
        if self.verify is None:
            self.verify = config.enable_verify_subdomain
-        rename_table = self.domain + '_last'
+        old_table = self.domain + '_last'
+        new_table = self.domain + '_now'
        collect = Collect(self.domain, export=False)
        collect.run()
        if self.brute:
@@ -102,49 +103,61 @@ class OneForAll(object):
        db.copy_table(self.domain, self.domain+'_ori')
        db.remove_invalid(self.domain)
        db.deduplicate_subdomain(self.domain)
+
+        old_data = []
+        # 非第一次收集子域的情况时数据库预处理
+        if db.exist_table(new_table):
+            db.drop_table(old_table)  # 如果存在上次收集结果表就先删除
+            db.rename_table(new_table, old_table)  # 新表重命名为旧表
+            old_data = db.get_data(old_table).as_dict()
+
        # 不验证子域的情况
        if not self.verify:
            # 数据库导出
            self.valid = None
-            dbexport.export(self.domain, valid=self.valid, format=self.format,
-                            show=self.show)
-            db.drop_table(rename_table)
-            db.rename_table(self.domain, rename_table)
+            dbexport.export(self.domain, valid=self.valid,
+                            format=self.format, show=self.show)
+            db.drop_table(new_table)
+            db.rename_table(self.domain, new_table)
            return
        # 开始验证子域工作
-        self.datas = db.get_data(self.domain).as_dict()
+        self.data = db.get_data(self.domain).as_dict()
+
+        # 标记新发现子域
+        self.data = utils.mark_subdomain(old_data, self.data)
+
        loop = asyncio.get_event_loop()
        asyncio.set_event_loop(loop)

        # 解析域名地址
-        task = resolve.bulk_query_a(self.datas)
-        self.datas = loop.run_until_complete(task)
+        task = resolve.bulk_query_a(self.data)
+        self.data = loop.run_until_complete(task)

        # 保存解析结果
        resolve_table = self.domain + '_res'
        db.drop_table(resolve_table)
        db.create_table(resolve_table)
-        db.save_db(resolve_table, self.datas, 'resolve')
+        db.save_db(resolve_table, self.data, 'resolve')

        # 请求域名地址
-        task = request.bulk_get_request(self.datas, self.port)
-        self.datas = loop.run_until_complete(task)
+        task = request.bulk_get_request(self.data, self.port)
+        self.data = loop.run_until_complete(task)
        # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
        loop.run_until_complete(asyncio.sleep(0.25))

        db.clear_table(self.domain)
-        db.save_db(self.domain, self.datas)
+        db.save_db(self.domain, self.data)

        # 数据库导出
-        dbexport.export(self.domain, valid=self.valid, format=self.format,
-                        show=self.show)
-        db.drop_table(rename_table)
-        db.rename_table(self.domain, rename_table)
+        dbexport.export(self.domain, valid=self.valid,
+                        format=self.format, show=self.show)
+        db.drop_table(new_table)
+        db.rename_table(self.domain, new_table)
        db.close()
        # 子域接管检查

        if self.takeover:
-            subdomains = set(map(lambda x: x.get('subdomain'), self.datas))
+            subdomains = set(map(lambda x: x.get('subdomain'), self.data))
            takeover = Takeover(subdomains)
            takeover.run()

@@ -34,8 +34,8 @@ def get_cname(subdomain):
    resolver = resolve.dns_resolver()
    try:
        answers = resolver.query(subdomain, 'CNAME')
-    except Exception as e:
-        logger.log('DEBUG', e.args)
+    except Exception as exception:
+        logger.log('TRACE', exception.args)
        return None
    for answer in answers:
        return answer.to_text()  # 一个子域只有一个CNAME记录
@@ -58,7 +58,7 @@ class Takeover(Module):
                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
        参数dpath为None默认使用OneForAll结果目录

-    :param str target:  单个子域或者每行一个子域的文件路径(必需参数)
+    :param any target:  单个子域或者每行一个子域的文件路径(必需参数)
    :param int thread:  线程数(默认100)
    :param str format:  导出格式(默认csv)
    :param str dpath:   导出目录(默认None)
@@ -83,12 +83,13 @@ class Takeover(Module):
            data = str(self.results)
        else:
            data = self.results.export(self.format)
-        fpath = self.dpath.joinpath(f'takeover.{self.format}')
+        ts = int(time.time())
+        fpath = self.dpath.joinpath(f'takeover_{ts}.{self.format}')
        utils.save_data(fpath, data)

    def compare(self, subdomain, cname, responses):
        domain_resp = self.get('http://' + subdomain, check=False)
-        cname_resp = self.get('http://'+cname, check=False)
+        cname_resp = self.get('http://' + cname, check=False)
        if domain_resp is None or cname_resp is None:
            return

@@ -1,7 +1,5 @@
-aiodns==2.0.0
 aiohttp==3.6.2
 aiomultiprocess==0.6.1
-asn1crypto==1.2.0
 async-timeout==3.0.1
 attrs==19.3.0
 backports.csv==1.0.7
@@ -11,11 +9,11 @@ brotlipy==0.7.0
 bs4==0.0.1
 cchardet==2.1.4
 certifi==2019.9.11
-cffi==1.13.0
+cffi==1.13.1
 chardet==3.0.4
-cloudscraper==1.2.2
+cloudscraper==1.2.5
 colorama==0.4.1
-cryptography==2.7
+cryptography==2.8
 defusedxml==0.6.0
 dnspython==1.16.0
 docopt==0.6.2
@@ -26,11 +24,11 @@ idna==2.8
 jdcal==1.4.1
 Js2Py==0.66
 loguru==0.3.2
-lxml==4.4.1
+lxml==4.3.5
+MarkupPy==1.14
 multidict==4.5.2
 odfpy==1.4.0
 openpyxl==2.4.11
-pycares==3.0.0
 pycparser==2.19
 pyjsparser==2.7.1
 pyOpenSSL==19.0.0
@@ -44,7 +42,7 @@ requests-toolbelt==0.9.1
 six==1.12.0
 soupsieve==1.9.4
 SQLAlchemy==1.3.10
-tablib==0.13.0
+tablib==0.14.0
 termcolor==1.1.0
 tldextract==2.2.2
 tqdm==4.36.1
Author	SHA1	Message	Date
shmilylty	e662f28407	v0.0.8	2019-10-30 00:46:06 +08:00
shmilylty	26d32d3cd9	实现新子域标记	2019-10-29 22:47:44 +08:00
shmilylty	55c4cf734a	去掉中括号	2019-10-29 16:00:25 +08:00
shmilylty	c568f73b51	去掉中括号	2019-10-28 20:06:16 +08:00
shmilylty	9c25b77aa6	新增new,header,response字段	2019-10-28 18:45:52 +08:00
shmilylty	9fbaf19f35	修正安装命令	2019-10-28 11:33:58 +08:00
shmilylty	e3c3965020	优化字典	2019-10-28 11:20:05 +08:00
shmilylty	40000472a5	由于dnsdb.org域名不在暂时删除此模块	2019-10-28 10:19:22 +08:00
shmilylty	cb8ac909d7	更改cname查询错误日志级别	2019-10-28 10:18:00 +08:00
shmilylty	7627b53650	使用多线程重写	2019-10-27 18:21:14 +08:00
shmilylty	f30f48aea2	修复解析没有结果的问题	2019-10-26 17:58:31 +08:00
shmilylty	e66c6d7e21	更新依赖	2019-10-25 21:40:24 +08:00
shmilylty	8ede7d9943	更新依赖	2019-10-25 21:40:04 +08:00
shmilylty	0d72aefb62	更改日志级别	2019-10-25 21:39:28 +08:00
shmilylty	893e6628e6	优化标题获取	2019-10-25 15:46:27 +08:00
shmilylty	baf394fc0b	参数优化	2019-10-24 22:30:49 +08:00
shmilylty	035963758d	修复端口重复问题	2019-10-24 21:56:07 +08:00
shmilylty	d760e02289	请求优化	2019-10-24 19:06:21 +08:00
shmilylty	e6b145d563	修正入口	2019-10-24 18:46:19 +08:00
shmilylty	7afb359918	调整参数	2019-10-24 18:44:58 +08:00
shmilylty	707a5eda93	请求优化	2019-10-24 18:44:35 +08:00
shmilylty	6d3c0ae873	请求优化	2019-10-24 16:08:55 +08:00
shmilylty	476e75869c	修改提示类型	2019-10-24 15:57:39 +08:00
shmilylty	e7110c24c5	默认请求只探测80端口	2019-10-23 01:31:51 +08:00
shmilylty	0e4c7961fe	请求子域返回状态码为400的暂时标记为无效子域	2019-10-22 23:49:29 +08:00
shmilylty	6f0cb70e02	推荐使用Python 3.8	2019-10-21 21:58:00 +08:00
shmilylty	4dec25d187	优化真实子域判断	2019-10-21 21:30:32 +08:00
shmilylty	e5e24804f6	更新番剧季数	2019-10-21 09:39:33 +08:00
shmilylty	f1b24e5674	不使用aiodns库并更新依赖库	2019-10-20 23:53:33 +08:00
shmilylty	d0edcb9a0b	不使用aiodns	2019-10-20 23:44:11 +08:00
shmilylty	2dbb5a1097	不使用aiodns	2019-10-20 23:42:24 +08:00
shmilylty	451992f07c	不使用aiodns	2019-10-20 23:24:16 +08:00
shmilylty	5cfbf885ca	不使用aiodns	2019-10-20 16:22:11 +08:00
shmilylty	ecc1cb53dc	简化命名	2019-10-19 21:26:48 +08:00
shmilylty	39d9adb6bc	正确判断查询结果	2019-10-19 21:15:09 +08:00
shmilylty	8675dc0202	移除aiodns依赖	2019-10-19 21:10:17 +08:00
shmilylty	de81a73097	目前加上超时会查不出结果，暂时去掉超时参数。	2019-10-19 18:38:03 +08:00
shmilylty	9d6f122254	删除无用输出	2019-10-19 16:48:19 +08:00
shmilylty	be11d5c3a2	更新Python 3.8在Win10上的配置	2019-10-18 15:29:36 +08:00