v0.0.9

更新文档
2026-08-26 04:47:48 +08:00 · 2020-02-20 15:05:46 +08:00 · 2020-02-20 15:04:17 +08:00 · 2020-02-20 14:49:55 +08:00 · 2020-02-20 14:37:05 +08:00 · 2020-02-20 14:27:38 +08:00
79 changed files with 2858681 additions and 4440 deletions
@@ -1,36 +0,0 @@
---
-name: 提交Bug
-about: "请务必按照模板提交Bug\U0001F64F"
-title: 请填写BUG标题
-labels: bug
-assignees: shmilylty
-
---
-
-**是否使用了最新代码**
-是或否（如果不是的话尝试克隆最新的代码再跑一下）
-
-**Bug描述**
-清晰而简洁的Bug描述
-
-**如何复现**
-复现步骤（可不写）
-复现命令
-
-**预期结果**
-清晰而简洁的预期结果描述（可不写）
-
-**实际结果**
-清晰而简洁的实际结果描述（如出现什么错误）
-
-**屏幕截图**
-
-
-**运行环境**
- - 系统：[例如Windows 10 x64]
- - Python版本：[例如3.7.1]
- OneForAll版本：[例如0.0.6]
-
-
-**报错文本**
-复制完整的报错文本
@@ -0,0 +1,39 @@
+---
+name: 请使用这个中文模板提交Bug
+about: "请务必按照模板提交Bug\U0001F64F"
+title: 请填写BUG标题
+labels: bug
+assignees: shmilylty
+
+---
+
+**是否使用了最新代码**
+是或否（如果不是的话尝试克隆最新的代码再跑一下！）
+
+**Bug描述**
+清晰而简洁的Bug描述（必写）
+
+**运行环境**
+ - 系统：[例如Windows 10 x64]（必写）
+ - Python版本：[例如3.7.1]（必写）
+ - OneForAll版本：[例如0.0.6]（必写）
+ 
+**如何复现**
+复现步骤（选写）
+
+复现命令（必写）
+
+**报错文本**
+复制完整的报错文本（必写）
+
+**预期结果**
+清晰而简洁的预期结果描述（选写，如正常情况应该是怎么样的）
+
+**实际结果**
+清晰而简洁的实际结果描述（选写，如出现什么错误）
+
+**屏幕截图**
+完整OneForAll执行流程截图（建议上传）
+
+**日志上传**
+上传oneforall.log日志文件（复杂问题建议上传）
@@ -0,0 +1,34 @@
+name: OneForAll test
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Lint with flake8
+      run: |
+        pip install flake8
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+#     - name: Test with pytest
+#       run: |
+#       pip install pytest
+#       pytest
+    - name: Test with example
+      run: |
+        pip install coverage
+        coverage run oneforall/example.py
@@ -19,13 +19,16 @@ matrix:
      os: osx
      osx_image: xcode10.2  # Python 3.7 running on macOS 10.14.3
      language: shell       # 'language: python' is an error on Travis CI macOS
-    - name: "Python 3.7 on Windows"
+    - name: "Python 3.8 on Windows"
      os: windows           # Windows 10.0.17134 N/A Build 17134
      language: shell
      before_install:
        - choco install python
        - python -m pip install --upgrade pip
-      env: PATH=/c/Python37:/c/Python37/Scripts:$PATH
+        - chcp.com 65001
+      env: 
+        - PATH=/c/Python38:/c/Python38/Scripts:$PATH
+        - PYTHONIOENCODING=UTF-8

 install:
  - pip3 install -U pip
@@ -5,7 +5,26 @@ OneForAll的更新日志格式基于[Keep a Changelog](https://keepachangelog.co

 OneForAll遵守[语义化版本格式](https://semver.org/)。

-## Unreleased
+# Unreleased
+
+# Released
+## [0.0.9](https://github.com/shmilylty/oneforall/releases/tag/v0.0.9) - 2020-02-20
+- 重构子域解析模块
+- 添加4个新的子域收集模块
+- 添加了Docker部署
+- 优化配置参数和收集模块
+- 优化子域爆破字典和默认参数
+- 优化响应体解码处理
+- 更新说明文档
+- 修复已知bug
+- 移除aiodns依赖
+
+## [0.0.8](https://github.com/shmilylty/oneforall/releases/tag/v0.0.8) - 2019-10-30
+- 添加新子域监控功能
+- 优化子域爆破字典和默认参数
+- 修复端口重复问题
+- 移除aiodns依赖
+
 ## [0.0.7](https://github.com/shmilylty/oneforall/releases/tag/v0.0.7) - 2019-10-18
 - 修复一些已知问题
 - 添加百度云观测接口
@@ -0,0 +1,13 @@
+FROM python:3.8-alpine3.10
+
+MAINTAINER milktea@vmoe.info
+
+RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
+#RUN apk --no-cache add git
+RUN apk --no-cache add git build-base libffi-dev libxml2-dev libxslt-dev libressl-dev
+RUN git clone https://github.com/shmilylty/OneForAll
+RUN pip install -r /OneForAll/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
+
+WORKDIR /OneForAll
+
+ENTRYPOINT ["/bin/ash"]
@@ -1,12 +1,12 @@
 [[source]]
 name = "pypi"
-url = "https://pypi.python.org/simple/"
+url = "https://mirrors.aliyun.com/pypi/simple/"
 verify_ssl = true

 [dev-packages]
+pylint = "*"

 [packages]
-aiodns = "*"
 tqdm = "*"
 aiomultiprocess = "*"
 loguru = "*"
@@ -18,13 +18,12 @@ exrex = "*"
 aiohttp = "*"
 fire = "*"
 bs4 = "*"
-cchardet = "*"
 lxml = "*"
 pysocks = "*"
 cloudscraper = "*"
 js2py = "*"
 tablib = "*"
-brotlipy = "*"
+brotli = "*"

 [requires]
 python_version = "3.8"
@@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "656e9f0f133ba443049ac692b2551b8a0daa69542d471dc3dff965f7cdb3a308"
+            "sha256": "8a83580d8db568c6dcf5fc16fc978b6b2997829ff648695fef305efc4a6859fb"
        },
        "pipfile-spec": 6,
        "requires": {
@@ -16,14 +16,6 @@
        ]
    },
    "default": {
-        "aiodns": {
-            "hashes": [
-                "sha256:815fdef4607474295d68da46978a54481dd1e7be153c7d60f9e72773cd38d77d",
-                "sha256:aaa5ac584f40fe778013df0aa6544bf157799bd3f608364b451840ed2c8688de"
-            ],
-            "index": "pypi",
-            "version": "==2.0.0"
-        },
        "aiohttp": {
            "hashes": [
                "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e",
@@ -44,17 +36,10 @@
        },
        "aiomultiprocess": {
            "hashes": [
-                "sha256:c704383be74ccb806e2617cb3bb6a5faa1774b1c6643d62785c332eb6da9e742"
+                "sha256:341d53af9b7fedf6425d9c09636d029035d63eecaa54caaff7354145a969c65e"
            ],
            "index": "pypi",
-            "version": "==0.6.1"
-        },
-        "asn1crypto": {
-            "hashes": [
-                "sha256:7bb1cc02a5620b3d72da4ba070bda2f44f0e61b44dee910a302eddff802b6fb5",
-                "sha256:87620880a477123e01177a1f73d0f327210b43a3cdbd714efcd2fa49a8d7b384"
-            ],
-            "version": "==1.2.0"
+            "version": "==0.7.0"
        },
        "async-timeout": {
            "hashes": [
@@ -70,20 +55,13 @@
            ],
            "version": "==19.3.0"
        },
-        "backports.csv": {
-            "hashes": [
-                "sha256:1277dfff73130b2e106bf3dd347adb3c5f6c4340882289d88f31240da92cbd6d",
-                "sha256:21f6e09bab589e6c1f877edbc40277b65e626262a86e69a70137db714eaac5ce"
-            ],
-            "version": "==1.0.7"
-        },
        "beautifulsoup4": {
            "hashes": [
-                "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
-                "sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931",
-                "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
+                "sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a",
+                "sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887",
+                "sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae"
            ],
-            "version": "==4.8.1"
+            "version": "==4.8.2"
        },
        "brotli": {
            "hashes": [
@@ -93,19 +71,23 @@
                "sha256:1e1aa9c4d1558889f42749c8baf846007953bfd32c8209230cf1cd1f5ef33495",
                "sha256:2f2f4f78f29ac4a45d15b3d9fc3fd9705e0ad313a44b129f6e1d0c6916bad0e2",
                "sha256:3269f6de1dd150fd0cce1c158b61ff5ac06d627fd3ae9c6ea03aed26fbbff7ea",
+                "sha256:3f4a1f6240916c7984c7f2542786710f622992508dafee0b1714e6d340fb9ffd",
                "sha256:50dd9ad2a2bb12da4e9002a438672d182f98e546e99952de80280a1e1729664f",
                "sha256:5519a4b01b1a4f965083cbfa2ef2b9774c5a5f352341c47b50776ad109423d72",
                "sha256:5eb27722d320370315971c427eb8aa7cc0791f2a458840d357ac653bd0ad3a14",
                "sha256:5f06b4d5b6f58e5b5c220c2f23cad034dc5efa51b01fde2351ced1605bd980e2",
+                "sha256:71ceee286ea7ec613f1c36f1c6181864a6ca24ebb55e371276f33d6af8742834",
                "sha256:72848d25a5f9e736db4af4512e0c3feecc094d57d241f8f1ae959115a2c39756",
                "sha256:743001bca75f4a6b4454be3510feca46f9d61a0c782a9bc2bc684bdb245e279e",
+                "sha256:7ac98c71a15648fd11bc1f32608b6110e396121280790082e32b9a3109048bc6",
                "sha256:9d1c2dd27a1083fefd05b1b2f8df4a6bc2aaa6c21dd82cd41c8ae5e7c23a87f8",
                "sha256:a13ce9b419fe9f277c63f700efb0e444331509d1881b5610d2ba7e9080606967",
                "sha256:a19ef0952b9d2803df88dff07f45a6c92d5676afb9b8d69cf32232d684036d11",
+                "sha256:a44d41e7a80cc832f04f4c1577a67bd311e28a6e40702a258590d8949a204d9d",
                "sha256:ad766ca8b8c1419b71a22756b45264f45725c86133dc80a7cbe30b6b78c75620",
                "sha256:ad7963f261988ee0883816b6b9f206f11461c9b3cb5cfbca0c9ab5adc406d395",
+                "sha256:af0451e23016631a2f52925a10d738ac4a0f794ac315c30380b22efc0c90cbc6",
                "sha256:c16201060c5a3f8742e3deae759014251ac92f382f82bc2a41dc079ff18c3f24",
-                "sha256:c43b202f65891861a9a336984a103de25de235f756de69e32db893156f767013",
                "sha256:c675c6cce4295cb1a692f3de7416aacace7314e064b94bc86e93aceefce7fd3e",
                "sha256:d17cec0b992b1434f5f9df9986563605a4d1b1acd5574c87fc2ac014bcbd3316",
                "sha256:dc91f6129953861a73d9a65c52a8dd682b561a9ebaf65283541645cab6489917",
@@ -114,46 +96,11 @@
                "sha256:f775b07026af2b1b0b5a8b05e41571cdcf3a315a67df265d60af301656a5425b",
                "sha256:f969ec7f56ba9636679e69ca07fba548312ccaca37412ee823c7f413541ad7e0",
                "sha256:f9dc52cd70907aafb99a773b66b156f2f995c7a0d284397c487c8b71ddbef2f9",
+                "sha256:f9ee88bb52352588ceb811d045b5c9bb1dc38927bc150fd156244f60ff3f59f1",
                "sha256:fc7212e36ebeb81aebf7949c92897b622490d7c0e333a479c0395591e7994600"
            ],
-            "version": "==1.0.7"
-        },
-        "brotlipy": {
-            "hashes": [
-                "sha256:07194f4768eb62a4f4ea76b6d0df6ade185e24ebd85877c351daa0a069f1111a",
-                "sha256:091b299bf36dd6ef7a06570dbc98c0f80a504a56c5b797f31934d2ad01ae7d17",
-                "sha256:09ec3e125d16749b31c74f021aba809541b3564e5359f8c265cbae442810b41a",
-                "sha256:0be698678a114addcf87a4b9496c552c68a2c99bf93cf8e08f5738b392e82057",
-                "sha256:0fa6088a9a87645d43d7e21e32b4a6bf8f7c3939015a50158c10972aa7f425b7",
-                "sha256:1379347337dc3d20b2d61456d44ccce13e0625db2611c368023b4194d5e2477f",
-                "sha256:1ea4e578241504b58f2456a6c69952c88866c794648bdc74baee74839da61d44",
-                "sha256:2699945a0a992c04fc7dc7fa2f1d0575a2c8b4b769f2874a08e8eae46bef36ae",
-                "sha256:2a80319ae13ea8dd60ecdc4f5ccf6da3ae64787765923256b62c598c5bba4121",
-                "sha256:2e5c64522364a9ebcdf47c5744a5ddeb3f934742d31e61ebfbbc095460b47162",
-                "sha256:36def0b859beaf21910157b4c33eb3b06d8ce459c942102f16988cca6ea164df",
-                "sha256:3a3e56ced8b15fbbd363380344f70f3b438e0fd1fcf27b7526b6172ea950e867",
-                "sha256:3c1d5e2cf945a46975bdb11a19257fa057b67591eb232f393d260e7246d9e571",
-                "sha256:4e4638b49835d567d447a2cfacec109f9a777f219f071312268b351b6839436d",
-                "sha256:50ca336374131cfad20612f26cc43c637ac0bfd2be3361495e99270883b52962",
-                "sha256:5de6f7d010b7558f72f4b061a07395c5c3fd57f0285c5af7f126a677b976a868",
-                "sha256:637847560d671657f993313ecc6c6c6666a936b7a925779fd044065c7bc035b9",
-                "sha256:653faef61241bf8bf99d73ca7ec4baa63401ba7b2a2aa88958394869379d67c7",
-                "sha256:786afc8c9bd67de8d31f46e408a3386331e126829114e4db034f91eacb05396d",
-                "sha256:79aaf217072840f3e9a3b641cccc51f7fc23037496bd71e26211856b93f4b4cb",
-                "sha256:7e31f7adcc5851ca06134705fcf3478210da45d35ad75ec181e1ce9ce345bb38",
-                "sha256:8b39abc3256c978f575df5cd7893153277216474f303e26f0e43ba3d3969ef96",
-                "sha256:9448227b0df082e574c45c983fa5cd4bda7bfb11ea6b59def0940c1647be0c3c",
-                "sha256:96bc59ff9b5b5552843dc67999486a220e07a0522dddd3935da05dc194fa485c",
-                "sha256:a07647886e24e2fb2d68ca8bf3ada398eb56fd8eac46c733d4d95c64d17f743b",
-                "sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2",
-                "sha256:b4c98b0d2c9c7020a524ca5bbff42027db1004c6571f8bc7b747f2b843128e7a",
-                "sha256:c6cc0036b1304dd0073eec416cb2f6b9e37ac8296afd9e481cac3b1f07f9db25",
-                "sha256:d2c1c724c4ac375feb2110f1af98ecdc0e5a8ea79d068efb5891f621a5b235cb",
-                "sha256:dc6c5ee0df9732a44d08edab32f8a616b769cc5a4155a12d2d010d248eb3fb07",
-                "sha256:fd1d1c64214af5d90014d82cee5d8141b13d44c92ada7a0c0ec0679c6f15a471"
-            ],
            "index": "pypi",
-            "version": "==0.7.0"
+            "version": "==1.0.7"
        },
        "bs4": {
            "hashes": [
@@ -162,78 +109,12 @@
            "index": "pypi",
            "version": "==0.0.1"
        },
-        "cchardet": {
-            "hashes": [
-                "sha256:079aa02a14072874d943a671ba778a9def5b0e3cedc2ac9f59308526cfb31472",
-                "sha256:3e048a21688dcb4c797f40c8deb3600887bcaf435620256fd8becd4252012750",
-                "sha256:41fced7a6f05ef859fe3eac89fc2120aca3cbbfd2b6c803bed3ee4bf02956903",
-                "sha256:440903d5dca3d326f4b841e7fa760b6af1be4f950ead1a6ff77b76eaa46f0cd3",
-                "sha256:50170f346527c5df4d3cb94648ca187c666e61c0db6e510b984e867c44709d8b",
-                "sha256:6c55a6e7bc7337671c9f1ad90746c0efb2b2979ff4305c7ca1d7d381f05174c1",
-                "sha256:7f581ea172b252034f745dfd49733966b73b73907bdef0b47ad5f2008b797d54",
-                "sha256:80f7b087198827e60c81574c321b12f89188eae626ae1567d66808928be42f88",
-                "sha256:8ba753ff73ca2f3554999a0e027eab9450f6ffdb7e92e1b4e13b52be89995349",
-                "sha256:9ad8f61d6d1ca37bd4b954ad92d461ea4f58d0dc413b0790a5abed7c09e54996",
-                "sha256:a35bd23cedbaa87cc9300af1dd10bb03fda41894045fbca7bfdf1d350b813f25",
-                "sha256:a8feb9a7def2310e18c27e485a21a38669abe8c2e36b93c6ce1a1363495d4cdf",
-                "sha256:aa9dd4cee8a5210a6d0a7b263b98dc50637e00401fc4a5ad3ce2dbef54fdfa02",
-                "sha256:ab9858a0673262e467619df91f425cfef0590dcf5deef5c0c7945e9dc4dbd7d8",
-                "sha256:b09a488bbb35be95f82845e3c4312be9025e8377975b027eee67e0b39445e070",
-                "sha256:b2893d558761b3534cddf5a49ba8d77df3d8f964d7b14680b925f4a85fc13476",
-                "sha256:b5a8f9b229a30cd2432572d15e169483bc47c24418772ff58d0585050631c2fd",
-                "sha256:bded54eeccd5f810bc69e076b3d9a35819a92e5e0559ad274b9ae9061b1b881d",
-                "sha256:cbc206061e69561af6e4cba11f99abd928346c6b5bcdc83eb32ae40e9fc23a5f",
-                "sha256:cc9745e0400da4cfb49f075e7819f22473b66443f953427058fee2c7b9547cc0",
-                "sha256:db30bf3825702c07fc55a290d41663fd8151f870642a15667bbabf81fff21e0b",
-                "sha256:eeeb1b95bb5851dda93ee522860a0e6066d47921cb1d540cb778346e37e5a524",
-                "sha256:f1c3919fb71ac5da3aeee42c5b731c99dcd2beed71db7fdc28ca993c173f0402"
-            ],
-            "index": "pypi",
-            "version": "==2.1.4"
-        },
        "certifi": {
            "hashes": [
-                "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50",
-                "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"
+                "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
+                "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
            ],
-            "version": "==2019.9.11"
-        },
-        "cffi": {
-            "hashes": [
-                "sha256:08f99e8b38d5134d504aa7e486af8e4fde66a2f388bbecc270cdd1e00fa09ff8",
-                "sha256:1112d2fc92a867a6103bce6740a549e74b1d320cf28875609f6e93857eee4f2d",
-                "sha256:1b9ab50c74e075bd2ae489853c5f7f592160b379df53b7f72befcbe145475a36",
-                "sha256:24eff2997436b6156c2f30bed215c782b1d8fd8c6a704206053c79af95962e45",
-                "sha256:2eff642fbc9877a6449026ad66bf37c73bf4232505fb557168ba5c502f95999b",
-                "sha256:362e896cea1249ed5c2a81cf6477fabd9e1a5088aa7ea08358a4c6b0998294d2",
-                "sha256:40eddb3589f382cb950f2dcf1c39c9b8d7bd5af20665ce273815b0d24635008b",
-                "sha256:5ed40760976f6b8613d4a0db5e423673ca162d4ed6c9ed92d1f4e58a47ee01b5",
-                "sha256:632c6112c1e914c486f06cfe3f0cc507f44aa1e00ebf732cedb5719e6aa0466a",
-                "sha256:64d84f0145e181f4e6cc942088603c8db3ae23485c37eeda71cb3900b5e67cb4",
-                "sha256:6cb4edcf87d0e7f5bdc7e5c1a0756fbb37081b2181293c5fdf203347df1cd2a2",
-                "sha256:6f19c9df4785305669335b934c852133faed913c0faa63056248168966f7a7d5",
-                "sha256:719537b4c5cd5218f0f47826dd705fb7a21d83824920088c4214794457113f3f",
-                "sha256:7b0e337a70e58f1a36fb483fd63880c9e74f1db5c532b4082bceac83df1523fa",
-                "sha256:853376efeeb8a4ae49a737d5d30f5db8cdf01d9319695719c4af126488df5a6a",
-                "sha256:85bbf77ffd12985d76a69d2feb449e35ecdcb4fc54a5f087d2bd54158ae5bb0c",
-                "sha256:8978115c6f0b0ce5880bc21c967c65058be8a15f1b81aa5fdbdcbea0e03952d1",
-                "sha256:8f7eec920bc83692231d7306b3e311586c2e340db2dc734c43c37fbf9c981d24",
-                "sha256:8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226",
-                "sha256:92068ebc494b5f9826b822cec6569f1f47b9a446a3fef477e1d11d7fac9ea895",
-                "sha256:b57e1c8bcdd7340e9c9d09613b5e7fdd0c600be142f04e2cc1cc8cb7c0b43529",
-                "sha256:ba956c9b44646bc1852db715b4a252e52a8f5a4009b57f1dac48ba3203a7bde1",
-                "sha256:ca42034c11eb447497ea0e7b855d87ccc2aebc1e253c22e7d276b8599c112a27",
-                "sha256:dc9b2003e9a62bbe0c84a04c61b0329e86fccd85134a78d7aca373bbbf788165",
-                "sha256:dd308802beb4b2961af8f037becbdf01a1e85009fdfc14088614c1b3c383fae5",
-                "sha256:e77cd105b19b8cd721d101687fcf665fd1553eb7b57556a1ef0d453b6fc42faa",
-                "sha256:f56dff1bd81022f1c980754ec721fb8da56192b026f17f0f99b965da5ab4fbd2",
-                "sha256:fa4cc13c03ea1d0d37ce8528e0ecc988d2365e8ac64d8d86cafab4038cb4ce89",
-                "sha256:fa8cf1cb974a9f5911d2a0303f6adc40625c05578d8e7ff5d313e1e27850bd59",
-                "sha256:fb003019f06d5fc0aa4738492ad8df1fa343b8a37cbcf634018ad78575d185df",
-                "sha256:fd409b7778167c3bcc836484a8f49c0e0b93d3e745d975749f83aa5d18a5822f",
-                "sha256:fe5d65a3ee38122003245a82303d11ac05ff36531a8f5ce4bc7d4bbc012797e1"
-            ],
-            "version": "==1.13.0"
+            "version": "==2019.11.28"
        },
        "chardet": {
            "hashes": [
@@ -244,47 +125,19 @@
        },
        "cloudscraper": {
            "hashes": [
-                "sha256:2316ddc0c00905536a3f1801917e68fb78610776b1b65f1b126627a7793c397b",
-                "sha256:a27d2452edbe3d77d089c71f74783edfd24802e4f100aef0fad0de4505a2b840"
+                "sha256:a50e366d6d5ae299ce1554fc025c1e756b2598bc0b2232efa76e36c3e8f79e63",
+                "sha256:b01bd3ab916d30624643d7c14569218024b3f81b20da8045af4a301129f3bc11"
            ],
            "index": "pypi",
-            "version": "==1.2.2"
+            "version": "==1.2.23"
        },
        "colorama": {
            "hashes": [
-                "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d",
-                "sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"
+                "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff",
+                "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"
            ],
            "markers": "sys_platform == 'win32'",
-            "version": "==0.4.1"
-        },
-        "cryptography": {
-            "hashes": [
-                "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c",
-                "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643",
-                "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216",
-                "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799",
-                "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a",
-                "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9",
-                "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc",
-                "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8",
-                "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53",
-                "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1",
-                "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609",
-                "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292",
-                "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e",
-                "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6",
-                "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed",
-                "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d"
-            ],
-            "version": "==2.7"
-        },
-        "defusedxml": {
-            "hashes": [
-                "sha256:6687150770438374ab581bb7a1b327a847dd9c5749e396102de3fad4e8a3ef93",
-                "sha256:f684034d135af4c6cbb949b8a4d2ed61634515257a67299e5f940fbaa34377f5"
-            ],
-            "version": "==0.6.0"
+            "version": "==0.4.3"
        },
        "dnspython": {
            "hashes": [
@@ -336,87 +189,74 @@
        },
        "js2py": {
            "hashes": [
-                "sha256:6e5628abfff2fb4051e8e77a353e44831f474e2ceb865278271897f7f326aeb6",
-                "sha256:bf87cb4432944470f11fed9c1cb8d0312dd505e7b867362f55102f24379ab94f"
+                "sha256:168952e3827198f68eacbf84a7b23c80a55794415291f821ec6c96f3fd9c3253",
+                "sha256:6c45cce8eb66b16f8dc692c6e6a2d509be9544b63cfb386eadf7309a931feb62"
            ],
            "index": "pypi",
-            "version": "==0.66"
+            "version": "==0.67"
        },
        "loguru": {
            "hashes": [
-                "sha256:b6fad0d7aed357b5c147edcc6982606b933754338950b72d8123f48c150c5a4f",
-                "sha256:e3138bfdee5f57481a2a6e078714be20f8c71ab1ff3f07f8fb1cfa25191fed2a"
+                "sha256:074b3caa6748452c1e4f2b302093c94b65d5a4c5a4d7743636b4121e06437b0e",
+                "sha256:a6101fd435ac89ba5205a105a26a6ede9e4ddbb4408a6e167852efca47806d11"
            ],
            "index": "pypi",
-            "version": "==0.3.2"
+            "version": "==0.4.1"
        },
        "lxml": {
            "hashes": [
-                "sha256:02ca7bf899da57084041bb0f6095333e4d239948ad3169443f454add9f4e9cb4",
-                "sha256:096b82c5e0ea27ce9138bcbb205313343ee66a6e132f25c5ed67e2c8d960a1bc",
-                "sha256:0a920ff98cf1aac310470c644bc23b326402d3ef667ddafecb024e1713d485f1",
-                "sha256:17cae1730a782858a6e2758fd20dd0ef7567916c47757b694a06ffafdec20046",
-                "sha256:17e3950add54c882e032527795c625929613adbd2ce5162b94667334458b5a36",
-                "sha256:1f4f214337f6ee5825bf90a65d04d70aab05526c08191ab888cb5149501923c5",
-                "sha256:2e8f77db25b0a96af679e64ff9bf9dddb27d379c9900c3272f3041c4d1327c9d",
-                "sha256:4dffd405390a45ecb95ab5ab1c1b847553c18b0ef8ed01e10c1c8b1a76452916",
-                "sha256:6b899931a5648862c7b88c795eddff7588fb585e81cecce20f8d9da16eff96e0",
-                "sha256:726c17f3e0d7a7200718c9a890ccfeab391c9133e363a577a44717c85c71db27",
-                "sha256:760c12276fee05c36f95f8040180abc7fbebb9e5011447a97cdc289b5d6ab6fc",
-                "sha256:796685d3969815a633827c818863ee199440696b0961e200b011d79b9394bbe7",
-                "sha256:891fe897b49abb7db470c55664b198b1095e4943b9f82b7dcab317a19116cd38",
-                "sha256:a471628e20f03dcdfde00770eeaf9c77811f0c331c8805219ca7b87ac17576c5",
-                "sha256:a63b4fd3e2cabdcc9d918ed280bdde3e8e9641e04f3c59a2a3109644a07b9832",
-                "sha256:b0b84408d4eabc6de9dd1e1e0bc63e7731e890c0b378a62443e5741cfd0ae90a",
-                "sha256:be78485e5d5f3684e875dab60f40cddace2f5b2a8f7fede412358ab3214c3a6f",
-                "sha256:c27eaed872185f047bb7f7da2d21a7d8913457678c9a100a50db6da890bc28b9",
-                "sha256:c81cb40bff373ab7a7446d6bbca0190bccc5be3448b47b51d729e37799bb5692",
-                "sha256:d11874b3c33ee441059464711cd365b89fa1a9cf19ae75b0c189b01fbf735b84",
-                "sha256:e9c028b5897901361d81a4718d1db217b716424a0283afe9d6735fe0caf70f79",
-                "sha256:fe489d486cd00b739be826e8c1be188ddb74c7a1ca784d93d06fda882a6a1681"
+                "sha256:06d4e0bbb1d62e38ae6118406d7cdb4693a3fa34ee3762238bcb96c9e36a93cd",
+                "sha256:0701f7965903a1c3f6f09328c1278ac0eee8f56f244e66af79cb224b7ef3801c",
+                "sha256:1f2c4ec372bf1c4a2c7e4bb20845e8bcf8050365189d86806bad1e3ae473d081",
+                "sha256:4235bc124fdcf611d02047d7034164897ade13046bda967768836629bc62784f",
+                "sha256:5828c7f3e615f3975d48f40d4fe66e8a7b25f16b5e5705ffe1d22e43fb1f6261",
+                "sha256:585c0869f75577ac7a8ff38d08f7aac9033da2c41c11352ebf86a04652758b7a",
+                "sha256:5d467ce9c5d35b3bcc7172c06320dddb275fea6ac2037f72f0a4d7472035cea9",
+                "sha256:63dbc21efd7e822c11d5ddbedbbb08cd11a41e0032e382a0fd59b0b08e405a3a",
+                "sha256:7bc1b221e7867f2e7ff1933165c0cec7153dce93d0cdba6554b42a8beb687bdb",
+                "sha256:8620ce80f50d023d414183bf90cc2576c2837b88e00bea3f33ad2630133bbb60",
+                "sha256:8a0ebda56ebca1a83eb2d1ac266649b80af8dd4b4a3502b2c1e09ac2f88fe128",
+                "sha256:90ed0e36455a81b25b7034038e40880189169c308a3df360861ad74da7b68c1a",
+                "sha256:95e67224815ef86924fbc2b71a9dbd1f7262384bca4bc4793645794ac4200717",
+                "sha256:afdb34b715daf814d1abea0317b6d672476b498472f1e5aacbadc34ebbc26e89",
+                "sha256:b4b2c63cc7963aedd08a5f5a454c9f67251b1ac9e22fd9d72836206c42dc2a72",
+                "sha256:d068f55bda3c2c3fcaec24bd083d9e2eede32c583faf084d6e4b9daaea77dde8",
+                "sha256:d5b3c4b7edd2e770375a01139be11307f04341ec709cf724e0f26ebb1eef12c3",
+                "sha256:deadf4df349d1dcd7b2853a2c8796593cc346600726eff680ed8ed11812382a7",
+                "sha256:df533af6f88080419c5a604d0d63b2c33b1c0c4409aba7d0cb6de305147ea8c8",
+                "sha256:e4aa948eb15018a657702fee0b9db47e908491c64d36b4a90f59a64741516e77",
+                "sha256:e5d842c73e4ef6ed8c1bd77806bf84a7cb535f9c0cf9b2c74d02ebda310070e1",
+                "sha256:ebec08091a22c2be870890913bdadd86fcd8e9f0f22bcb398abd3af914690c15",
+                "sha256:edc15fcfd77395e24543be48871c251f38132bb834d9fdfdad756adb6ea37679",
+                "sha256:f2b74784ed7e0bc2d02bd53e48ad6ba523c9b36c194260b7a5045071abbb1012",
+                "sha256:fa071559f14bd1e92077b1b5f6c22cf09756c6de7139370249eb372854ce51e6",
+                "sha256:fd52e796fee7171c4361d441796b64df1acfceb51f29e545e812f16d023c4bbc",
+                "sha256:fe976a0f1ef09b3638778024ab9fb8cde3118f203364212c198f71341c0715ca"
            ],
            "index": "pypi",
-            "version": "==4.4.1"
+            "version": "==4.5.0"
        },
        "multidict": {
            "hashes": [
-                "sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f",
-                "sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3",
-                "sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef",
-                "sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b",
-                "sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73",
-                "sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc",
-                "sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3",
-                "sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd",
-                "sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351",
-                "sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941",
-                "sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d",
-                "sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1",
-                "sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b",
-                "sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a",
-                "sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3",
-                "sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7",
-                "sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0",
-                "sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0",
-                "sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014",
-                "sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5",
-                "sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036",
-                "sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d",
-                "sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a",
-                "sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce",
-                "sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1",
-                "sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a",
-                "sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9",
-                "sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7",
-                "sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b"
+                "sha256:13f3ebdb5693944f52faa7b2065b751cb7e578b8dd0a5bb8e4ab05ad0188b85e",
+                "sha256:26502cefa86d79b86752e96639352c7247846515c864d7c2eb85d036752b643c",
+                "sha256:4fba5204d32d5c52439f88437d33ad14b5f228e25072a192453f658bddfe45a7",
+                "sha256:527124ef435f39a37b279653ad0238ff606b58328ca7989a6df372fd75d7fe26",
+                "sha256:5414f388ffd78c57e77bd253cf829373721f450613de53dc85a08e34d806e8eb",
+                "sha256:5eee66f882ab35674944dfa0d28b57fa51e160b4dce0ce19e47f495fdae70703",
+                "sha256:63810343ea07f5cd86ba66ab66706243a6f5af075eea50c01e39b4ad6bc3c57a",
+                "sha256:6bd10adf9f0d6a98ccc792ab6f83d18674775986ba9bacd376b643fe35633357",
+                "sha256:83c6ddf0add57c6b8a7de0bc7e2d656be3eefeff7c922af9a9aae7e49f225625",
+                "sha256:93166e0f5379cf6cd29746989f8a594fa7204dcae2e9335ddba39c870a287e1c",
+                "sha256:9a7b115ee0b9b92d10ebc246811d8f55d0c57e82dbb6a26b23c9a9a6ad40ce0c",
+                "sha256:a38baa3046cce174a07a59952c9f876ae8875ef3559709639c17fdf21f7b30dd",
+                "sha256:a6d219f49821f4b2c85c6d426346a5d84dab6daa6f85ca3da6c00ed05b54022d",
+                "sha256:a8ed33e8f9b67e3b592c56567135bb42e7e0e97417a4b6a771e60898dfd5182b",
+                "sha256:d7d428488c67b09b26928950a395e41cc72bb9c3d5abfe9f0521940ee4f796d4",
+                "sha256:dcfed56aa085b89d644af17442cdc2debaa73388feba4b8026446d168ca8dad7",
+                "sha256:f29b885e4903bd57a7789f09fe9d60b6475a6c1a4c0eca874d8558f00f9d4b51"
            ],
-            "version": "==4.5.2"
-        },
-        "odfpy": {
-            "hashes": [
-                "sha256:596021f0519623ca8717331951c95e3b8d7b21e86edc7efe8cb650a0d0f59a2b"
-            ],
-            "version": "==1.4.0"
+            "version": "==4.7.4"
        },
        "openpyxl": {
            "hashes": [
@@ -424,30 +264,6 @@
            ],
            "version": "==2.4.11"
        },
-        "pycares": {
-            "hashes": [
-                "sha256:2ca080db265ea238dc45f997f94effb62b979a617569889e265c26a839ed6305",
-                "sha256:6f79c6afb6ce603009db2042fddc2e348ad093ece9784cbe2daa809499871a23",
-                "sha256:70918d06eb0603016d37092a5f2c0228509eb4e6c5a3faacb4184f6ab7be7650",
-                "sha256:755187d28d24a9ea63aa2b4c0638be31d65fbf7f0ce16d41261b9f8cb55a1b99",
-                "sha256:7baa4b1f2146eb8423ff8303ebde3a20fb444a60db761fba0430d104fe35ddbf",
-                "sha256:90b27d4df86395f465a171386bc341098d6d47b65944df46518814ae298f6cc6",
-                "sha256:9e090dd6b2afa65cb51c133883b2bf2240fd0f717b130b0048714b33fb0f47ce",
-                "sha256:a11b7d63c3718775f6e805d6464cb10943780395ab042c7e5a0a7a9f612735dd",
-                "sha256:b253f5dcaa0ac7076b79388a3ac80dd8f3bd979108f813baade40d3a9b8bf0bd",
-                "sha256:c7f4f65e44ba35e35ad3febc844270665bba21cfb0fb7d749434e705b556e087",
-                "sha256:cdb342e6a254f035bd976d95807a2184038fc088d957a5104dcaab8be602c093",
-                "sha256:cf08e164f8bfb83b9fe633feb56f2754fae6baefcea663593794fa0518f8f98c",
-                "sha256:df9bc694cf03673878ea8ce674082c5acd134991d64d6c306d4bd61c0c1df98f"
-            ],
-            "version": "==3.0.0"
-        },
-        "pycparser": {
-            "hashes": [
-                "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3"
-            ],
-            "version": "==2.19"
-        },
        "pyjsparser": {
            "hashes": [
                "sha256:2b12842df98d83f65934e0772fa4a5d8b123b3bc79f1af1789172ac70265dd21",
@@ -455,13 +271,6 @@
            ],
            "version": "==2.7.1"
        },
-        "pyopenssl": {
-            "hashes": [
-                "sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200",
-                "sha256:c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6"
-            ],
-            "version": "==19.0.0"
-        },
        "pysocks": {
            "hashes": [
                "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299",
@@ -478,24 +287,6 @@
            ],
            "version": "==2019.3"
        },
-        "pyyaml": {
-            "hashes": [
-                "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9",
-                "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4",
-                "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8",
-                "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696",
-                "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34",
-                "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9",
-                "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73",
-                "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299",
-                "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b",
-                "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae",
-                "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681",
-                "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41",
-                "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8"
-            ],
-            "version": "==5.1.2"
-        },
        "records": {
            "hashes": [
                "sha256:47e4874096f4a8f4b5bcad8c7c7cf512be36186e6e263ff3dfd750b05ff0d3c4",
@@ -528,32 +319,32 @@
        },
        "six": {
            "hashes": [
-                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
-                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
+                "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
+                "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"
            ],
-            "version": "==1.12.0"
+            "version": "==1.14.0"
        },
        "soupsieve": {
            "hashes": [
-                "sha256:605f89ad5fdbfefe30cdc293303665eff2d188865d4dbe4eb510bba1edfbfce3",
-                "sha256:b91d676b330a0ebd5b21719cb6e9b57c57d433671f65b9c28dd3461d9a1ed0b6"
+                "sha256:bdb0d917b03a1369ce964056fc195cfdff8819c40de04695a80bc813c3cfa1f5",
+                "sha256:e2c1c5dee4a1c36bcb790e0fabd5492d874b8ebd4617622c4f6a731701060dda"
            ],
-            "version": "==1.9.4"
+            "version": "==1.9.5"
        },
        "sqlalchemy": {
            "hashes": [
-                "sha256:0f0768b5db594517e1f5e1572c73d14cf295140756431270d89496dc13d5e46c"
+                "sha256:64a7b71846db6423807e96820993fa12a03b89127d278290ca25c0b11ed7b4fb"
            ],
            "markers": "python_version >= '3.0'",
-            "version": "==1.3.10"
+            "version": "==1.3.13"
        },
        "tablib": {
            "hashes": [
-                "sha256:0f88a9cebdaa1a2cc29ae57387082ee81015d1149ecd34e48a8c8d3b4dd21670",
-                "sha256:5f33c079b07eb10cf9c4b4696add2ecf32c89db7729240546ecdcd5c92f67e13"
+                "sha256:4d1909aa3ff1c85ba97ad16176c0aeec33c8e894dc7ea6f10f2dd44701e99ba7",
+                "sha256:80f6c3453431cedf1125f23d16b3d96b92b426495714ebf0b4dede1fa75b447d"
            ],
            "index": "pypi",
-            "version": "==0.13.0"
+            "version": "==1.1.0"
        },
        "termcolor": {
            "hashes": [
@@ -571,11 +362,11 @@
        },
        "tqdm": {
            "hashes": [
-                "sha256:abc25d0ce2397d070ef07d8c7e706aede7920da163c64997585d42d3537ece3d",
-                "sha256:dd3fcca8488bb1d416aa7469d2f277902f26260c45aa86b667b074cd44b3b115"
+                "sha256:251ee8440dbda126b8dfa8a7c028eb3f13704898caaef7caa699b35e119301e2",
+                "sha256:fe231261cfcbc6f4a99165455f8f6b9ef4e1032a6e29bccf168b4bf42012f09c"
            ],
            "index": "pypi",
-            "version": "==4.36.1"
+            "version": "==4.42.1"
        },
        "tzlocal": {
            "hashes": [
@@ -586,10 +377,10 @@
        },
        "urllib3": {
            "hashes": [
-                "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
-                "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
+                "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
+                "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"
            ],
-            "version": "==1.25.6"
+            "version": "==1.25.8"
        },
        "win32-setctime": {
            "hashes": [
@@ -599,36 +390,105 @@
            "markers": "sys_platform == 'win32'",
            "version": "==1.0.1"
        },
-        "xlrd": {
-            "hashes": [
-                "sha256:546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2",
-                "sha256:e551fb498759fa3a5384a94ccd4c3c02eb7c00ea424426e212ac0c57be9dfbde"
-            ],
-            "version": "==1.2.0"
-        },
-        "xlwt": {
-            "hashes": [
-                "sha256:a082260524678ba48a297d922cc385f58278b8aa68741596a87de01a9c628b2e",
-                "sha256:c59912717a9b28f1a3c2a98fd60741014b06b043936dcecbc113eaaada156c88"
-            ],
-            "version": "==1.3.0"
-        },
        "yarl": {
            "hashes": [
-                "sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9",
-                "sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f",
-                "sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb",
-                "sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320",
-                "sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842",
-                "sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0",
-                "sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829",
-                "sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310",
-                "sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4",
-                "sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8",
-                "sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1"
+                "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce",
+                "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6",
+                "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce",
+                "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae",
+                "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d",
+                "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f",
+                "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b",
+                "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b",
+                "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb",
+                "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462",
+                "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea",
+                "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70",
+                "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1",
+                "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a",
+                "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b",
+                "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080",
+                "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2"
            ],
-            "version": "==1.3.0"
+            "version": "==1.4.2"
        }
    },
-    "develop": {}
+    "develop": {
+        "astroid": {
+            "hashes": [
+                "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a",
+                "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42"
+            ],
+            "version": "==2.3.3"
+        },
+        "colorama": {
+            "hashes": [
+                "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff",
+                "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"
+            ],
+            "markers": "sys_platform == 'win32'",
+            "version": "==0.4.3"
+        },
+        "isort": {
+            "hashes": [
+                "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
+                "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"
+            ],
+            "version": "==4.3.21"
+        },
+        "lazy-object-proxy": {
+            "hashes": [
+                "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d",
+                "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449",
+                "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08",
+                "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a",
+                "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50",
+                "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd",
+                "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239",
+                "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb",
+                "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea",
+                "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e",
+                "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156",
+                "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142",
+                "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442",
+                "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62",
+                "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db",
+                "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531",
+                "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383",
+                "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a",
+                "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357",
+                "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4",
+                "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0"
+            ],
+            "version": "==1.4.3"
+        },
+        "mccabe": {
+            "hashes": [
+                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
+                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
+            ],
+            "version": "==0.6.1"
+        },
+        "pylint": {
+            "hashes": [
+                "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd",
+                "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4"
+            ],
+            "index": "pypi",
+            "version": "==2.4.4"
+        },
+        "six": {
+            "hashes": [
+                "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
+                "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"
+            ],
+            "version": "==1.14.0"
+        },
+        "wrapt": {
+            "hashes": [
+                "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
+            ],
+            "version": "==1.11.2"
+        }
+    }
 }
@@ -4,8 +4,8 @@
 [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
-[![python](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.0.7-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/python-3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/release-v0.0.9-brightgreen)](https://github.com/shmilylty/OneForAll/releases)

 👊**OneForAll is a powerful subdomain collection tool**  📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)

@@ -35,17 +35,17 @@ At present, OneForAll is still under development, there must be a lot of problem
 * **Powerful collection capability**，For more information, please see [collection module description](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md).
  1. Collect subdomains using certificate transparency (there are currently 6 modules: `censys_api`，`spyse_api`，`certspotter`，`crtsh`，`entrust`，`google`）

-  2. General check collection subdomains (there are currently 4 modules: domain transfer vulnerability exploitation`cdx`, cross-domain policy file `cdx`, HTTPS certificate `cert`, content security policy `csp`, robots file `robots`, and sitemap file `sitemap`. Check NSEC record, NSEC3 record and other modules will be added later).
+  2. General check collection subdomains (there are currently 4 modules: domain transfer vulnerability exploitation`axfr`, cross-domain policy file `cdx`, HTTPS certificate `cert`, content security policy `csp`, robots file `robots`, and sitemap file `sitemap`. Check NSEC record, NSEC3 record and other modules will be added later).

  3.  Collect subdomains using web crawler files (there are currently two modules: `archirawl`, `commoncrawl`, which is still being debugged and needs to be added and improved). 

-  4.  Collect subdomains using DNS datasets (there are currently 19 modules: `CeBaidu`, `binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`,  `ptrarchive`, `sitedossier`,`threatcrowd`）
+  4.  Collect subdomains using DNS datasets (there are currently 22 modules: `ip138`, `ximcx`, `CeBaidu`, `binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`,  `ptrarchive`, `sitedossier`,`threatcrowd`）

  5.  Collect subdomains using DNS queries (there is currently a module to collect subdomains `srv` by enumerating common SRV records and making queries, which needs to be added and improved). 

-  6.  Collect subdomains using threat intelligence platform data (there are currently five modules: `riskiq_ api`, `threatbook_ api`, `threatkeeper `, `virustotal`, `virustotal_ api`, which need to be added and improved). 
+  6.  Collect subdomains using threat intelligence platform data (there are currently 6 modules: `alienvault`, `riskiq_ api`, `threatbook_ api`, `threatkeeper `, `virustotal`, `virustotal_ api`, which need to be added and improved).

-  7.  Use search engines to discover subdomains (there are currently 16 modules: `ask`, `bing_ api`, `fofa_ api`, `shodan_ api`, `yahoo`, `baidu`, `duckduckgo`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_ api`, `sogou`, `zoomeye_ api`), except for special search engines in the search module. General search engines support automatic exclusion of search, full search, recursive search. 
+  7.  Use search engines to discover subdomains (there are currently 17 modules: `ask`, `bing_ api`, `fofa_ api`, `shodan_ api`, `yahoo`, `baidu`, `duckduckgo`, `gitee`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_ api`, `sogou`, `zoomeye_ api`), except for special search engines in the search module. General search engines support automatic exclusion of search, full search, recursive search. 
 * **Support subdomain blasting**，This module has both conventional dictionary blasting and custom fuzz mode. It supports batch blasting and recursive blasting, and automatically judges pan-parsing and processing.
 * **Support subdmain verification**，default to enable subdomain verification, automatically resolve subdomain DNS, automatically request subdomain to obtain title and banner, and comprehensively determine subdomain survival.
 * **Support subdomain takeover**，By default, subdomain takeover risk checking is enabled. Automatic subdomain takeover is supported (only Github, remains to be improved at present), and batch inspection is supported.
@@ -55,22 +55,22 @@ At present, OneForAll is still under development, there must be a lot of problem

 ## 🚀Start Guide

-📢Currently, the project is **under development**, so it will continue to update iterations. It is best to **clone** the project when downloading and using OneForAll. Please take a moment to read this document to help you quickly get familiar with OneForAll!
+📢 Please take a moment to read this document to help you quickly get familiar with OneForAll!

 **🐍Installation requirements**

-OneForAll is based on CPython, so you need a Python environment to run. If your system doesn't already have a Python environment, you can refer to the [Python 3 Installation Guide](https://pythonguidecn.readthedocs.io/en/latest/starting/installation.html#python-3), theoretically Python 3.6, 3.7 and 3.8 can run OneForAll, **but** many tests are done on Python 3.7, so **recommended** you use **Python 3.7** version runs OneForAll. Run the following command to check the Python and pip3 versions:
+OneForAll is developed and tested based on [Python 3.8.0](https://www.python.org/downloads/release/python-380/). Please use a stable release higher than Python 3.8.0, other versions may have some problems. For more information on installing the Python environment, please see [Python 3 installation Guide](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3) . Run the following command to check the Python and pip3 versions:
 ```bash
 python -V
 pip3 -V
 ```
 If you see the following output, there is no problem with the Python environment:
 ```bash
-Python 3.7.4
-pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.7)
+Python 3.8.0
+pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.8)
 ```

-**✔Installation steps**
+**✔Installation steps (Git version)**

 1. **Download**
   
@@ -85,12 +85,14 @@ pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages
   ```

 2. **Installation**
+
+   Since the project is under development and will continue to be updated iteratively, `git clone` is used to clone the latest code repository during download, which is also convenient for subsequent updates. Downloading from Releases is not recommended because the update of the version in Releases is slow and inconvenient.
   
-   You can install OneForAll dependencies via pip3 (if you are familiar with [pipenv](https://docs.pipenv.org/en/latest/), then it is recommended that you use [pipenv install dependencies](https://github.com/shmilylty/OneForAll/tree/master/docs/Installation_dependency.md), the following is an example of using **pip3** to install dependencies under **Windows system**: (Note: If your Python3 is installed in the system Program Files In the directory, such as: `C:\Program Files\Python37`, then run the command prompt cmd as an administrator to execute the following command!)
+   You can install OneForAll dependencies via pip3 (if you are familiar with [pipenv](https://docs.pipenv.org/en/latest/), then it is recommended that you use [pipenv install dependencies](https://github.com/shmilylty/OneForAll/tree/master/docs/Installation_dependency.md), the following is an example of using **pip3** to install dependencies under **Windows system**: (Note: If your Python3 is installed in the system Program Files In the directory, such as: `C:\Program Files\Python38`, then run the command prompt cmd as an administrator to execute the following command!)
 ```bash
   cd OneForAll/
-   python -m pip install --user  -U pip setuptools wheel
-   pip3 install --user -r requirements.txt
+   python -m pip install -U pip setuptools wheel
+   pip3 install -r requirements.txt
   cd oneforall/
   python oneforall.py --help
 ```
@@ -106,6 +108,22 @@ pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages
   git pull
   ```

+**✔Installation steps (Docker version)**
+
+Method 1: directly pull the deployed image (not updated in time)
+
+```shell
+docker pull tardis07/oneforall
+docker run -it oneforall
+```
+
+Method 2: build from `Dockerfile` (same as git version)
+
+```shell
+docker build -t oneforall .
+docker run -it oneforall
+```
+
 **✨Demonstration**

 1. If you are installing dependencies through pip3, run the example using the following command: 
@@ -124,7 +142,7 @@ pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages

 **🤔Help**

-The command line parameters only provide some common parameters. For more detailed parameter configuration, please see [config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py) if you think Some parameters are frequently used in the command interface or missing parameters. Feedback is welcome. For well-known reasons, if you want to use some of the wall's collection interface, please go to [config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py) to configure the proxy, some collection Modules need to provide APIs (most of which are freely available for registered accounts). If you need to use them, please go to [config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py) to configure the API. Information, if not used, please ignore the error message. (For detailed modules, please read [collection module description](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md))
+The command line parameters only provide some common parameters. For more detailed parameter configuration, please see [api.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/api.py) if you think Some parameters are frequently used in the command interface or missing parameters. Feedback is welcome. For well-known reasons, if you want to use some of the wall's collection interface, please go to [api.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/api.py) to configure the proxy, some collection Modules need to provide APIs (most of which are freely available for registered accounts). If you need to use them, please go to [api.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/api.py) to configure the API. Information, if not used, please ignore the error message. (For detailed modules, please read [collection module description](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md))

 The OneForAll command line interface is based on [Fire](https://github.com/google/python-fire/). For more advanced usage of Fire, please refer to [using the Fire CLI](https://github.com/google/Python-fire/blob/master/docs/using-cli.md), if you have any doubts during the use, please feel free to give me feedback.

@@ -139,37 +157,38 @@ The OneForAll command line interface is based on [Fire](https://github.com/googl
   ```
   ```bash
   NAME
-       oneforall.py - OneForAll is a powerful subdomain collection tool
+       oneforall.py - OneForAll help Information
   
   SYNOPSIS
       oneforall.py --target=TARGET <flags>
   
   DESCRIPTION
-       Version: 0.0.6
-       Project: https://git.io/fjHT1
+       OneForAll is a powerful subdomain collection tool
   
       Example:
+           python3 oneforall.py version
           python3 oneforall.py --target example.com run
           python3 oneforall.py --target ./domains.txt run
-           python3 oneforall.py --target example.com --brute True run
-           python3 oneforall.py --target example.com --verify False run
           python3 oneforall.py --target example.com --valid None run
-           python3 oneforall.py --target example.com --port medium run
+           python3 oneforall.py --target example.com --brute True run
+           python3 oneforall.py --target example.com --port small run
           python3 oneforall.py --target example.com --format csv run
+           python3 oneforall.py --target example.com --dns False run
+           python3 oneforall.py --target example.com --req False run
+           python3 oneforall.py --target example.com --takeover False run
           python3 oneforall.py --target example.com --show True run
   
       Note:
           Parameter valid optional value 1, 0, none indicates that the export is 
           valid, invalid, and all subdomains, respectively.
-           Parameter verify for True attempts to resolve and request the subdomain 
-           and tag the validity of the subdomain based on the result.
-           Parameter port have optional values 'small', 'medium', 'large', 'xlarge', 
+           Parameter port have optional values 'default' 'small', 'large',
           See config.py configuration for details.
           Parameter format have optional values 'txt', 'rst', 'csv', 'tsv', 'json', 
           'yaml', 'html', 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'.
           If the parameter path is None, the appropriate file is generated in the 
           project result directory based on the format parameter and the domain 
           name.
+           Parameter path default None uses the OneForAll result directory generation path
   
   ARGUMENTS
       TARGET
@@ -178,16 +197,20 @@ The OneForAll command line interface is based on [Fire](https://github.com/googl
   FLAGS
       --brute=BRUTE
           Use blasting module (default False)
-       --verify=VERIFY
-           Verify the validity of subdomains (default True)
+       --dns=DNS
+           DNS resolve subdomain (default True)
+       --req=REQ
+           HTTP request subdomain (default True)
       --port=PORT
-           Port range for request authentication (default medium)
+           Port range for request authentication (default 80 port)
       --valid=VALID
-           Export validity of subdomains (default 1)
-       --path=PATH
-           Export path (default None)
+           Export validity of subdomains (default None)
       --format=FORMAT
           Export format (default xls)
+       --path=PATH
+           Export path (default None)
+       --takeover=TAKEOVER
+           Check subdomain takeover (default False)
       --show=SHOW
           Terminal display exported data (default False)
   ```
@@ -201,7 +224,7 @@ The OneForAll command line interface is based on [Fire](https://github.com/googl

   - Third, considering the blasting efficiency, there is no HTTP response volume similarity comparison and response volume content judgment,  this function has not been implemented yet, and will be implemented if necessary.

-   After not rigorous testing, in the 16-core CPU, using 16 processes 64 coroutines, 100M network bandwidth environment, set the task to be divided into 50000, run two million dictionaries about 10 minutes to run, about 3333 subdomains per second.
+   After not rigorous testing, in the 16-core CPU, using 16 processes 64 coroutines, 100M network bandwidth environment, run two million dictionaries about 10 minutes to run, about 3333 subdomains per second.

   ```bash
   python aiobrute.py --help
@@ -224,13 +247,6 @@ The OneForAll command line interface is based on [Fire](https://github.com/googl
           python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
   
       Note:
-           The setting of the parameter segment is affected by CPU performance, 
-           network bandwidth, and operator restrictions. By default, 500 subdomains 
-           are set as task groups. When you feel that your environment is not 
-           affected by the above factors, the current blasting speed is slow, so it 
-           is strongly recommended to use the dictionary. Size resizing: 100,000 
-           dictionary suggestions set to 5000, million dictionary set to 50000.
-           
           Parameter valid optional value 1, 0, none indicates that the export is 
           valid, invalid, and all subdomains, respectively.
           
@@ -248,11 +264,9 @@ The OneForAll command line interface is based on [Fire](https://github.com/googl
       --process=PROCESS
           Number of processes blasted (default CPU core count)
       --coroutine=COROUTINE
-           Number of coroutines per blasting process (default 64)
+           Number of coroutines per blasting process (default 1024)
       --wordlist=WORDLIST
           Specify the dictionary path used for blasting (config.py is used by default)
-       --segment=SEGMENT
-           Blasting task segmentation (default 500)
       --recursive=RECURSIVE
           Whether to use recursive blasting (default False)
       --depth=DEPTH
@@ -302,6 +316,7 @@ D:.
 +---images
 \---oneforall
    |   aiobrute.py   Asynchronous multi-process multi-correlation subdomain blasting module, can be run separately
+    |   api.py        API configuration of some collection modules
    |   collect.py    Upper layer call of each collection module
    |   config.py     Configuration file
    |   dbexport.py   Database export module, can be run separately
@@ -334,7 +349,7 @@ Very warmly welcome all ace to improve the project together!
 ## ⌛Follow-up plan

 - [ ] Continuous optimization and improvement of each module
- [ ] Subdomain monitoring (marking each newly discovered subdomain)
+- [x] Subdomain monitoring (marking each newly discovered subdomain)
 - [ ] Subdomain collection crawler implementation (including collection of subdomains from static resource files such as JS)
 - [ ] Implementation of front-end interface for powerful interaction (tentative: front-end: Element + back-end: Flask)

@@ -375,4 +390,4 @@ Thanks ace of [A-Team](https://github.com/QAX-A-Team) for their enthusiastic and
 This tool is limited to legally authorized enterprise security construction. In the process of using this tool, you should ensure that all your actions comply with local laws and regulations and have obtained sufficient authorization.
 If you have any illegal behavior in the process of using this tool, you are responsible for all consequences, and all authors and all contributors of this tool do not assume any legal and joint responsibility.
 Unless you have fully read, fully understood and accepted all the terms of this Agreement, please do not install and use this tool.
-Your use or any other express or implied representation of you to this Agreement is deemed to have been read and agreed to be bound by this Agreement.
+Your use or any other express or implied representation of you to this Agreement is deemed to have been read and agreed to be bound by this Agreement.
@@ -4,8 +4,8 @@
 [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
-[![python](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.0.7-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/python-3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/release-v0.0.9-brightgreen)](https://github.com/shmilylty/OneForAll/releases)

 👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/README.en.md)

@@ -26,9 +26,9 @@

 * **效率问题**，没有利用多进程，多线程以及异步协程技术，速度较慢。

-为了解决以上痛点，此项目应用而生，OneForAll一词是来自我喜欢的一部日漫《[我的英雄学院](https://manhua.fzdm.com/131/)》，它是一种通过一代代的传承不断变强的潜力无穷的顶级个性，目前[番剧](https://www.bilibili.com/bangumi/media/md7452/)也更新到了第三季了，欢迎大佬们入坑😄。正如其名，我希望OneForAll是一款集百家之长，功能强大的全面快速子域收集终极神器🔨。
+为了解决以上痛点，此项目应用而生，OneForAll一词是来自我喜欢的一部日漫《[我的英雄学院](https://manhua.fzdm.com/131/)》，它是一种通过一代代的传承不断变强的潜力无穷的顶级个性，目前[番剧](https://www.bilibili.com/bangumi/media/md7452/)也更新到了第四季了，欢迎大佬们入坑😄。正如其名，我希望OneForAll是一款集百家之长，功能强大的全面快速子域收集终极神器🔨。

-目前OneForAll还在开发中，肯定有不少问题和需要改进的地方，欢迎大佬们提交[Issues](https://github.com/shmilylty/OneForAll/issues)和[PR](https://github.com/shmilylty/OneForAll/pulls)，用着还行给个小星星✨吧，目前有一个专门用于OneForAll交流和反馈QQ群👨‍👨‍👦‍👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)，也可以给我发邮件📧[admin@hackfun.org]。
+目前OneForAll还在开发中，肯定有不少问题和需要改进的地方，欢迎大佬们提交[Issues](https://github.com/shmilylty/OneForAll/issues)和[PR](https://github.com/shmilylty/OneForAll/pulls)，用着还行给个小星星✨吧，目前有一个专门用于OneForAll交流和反馈QQ群👨‍👨‍👦‍👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)（加群验证：我的英雄学院），也可以给我发邮件📧[admin@hackfun.org]。

 ## 👍功能特性

@@ -39,13 +39,13 @@

  3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善）

-  4. 利用DNS数据集收集子域（目前有19个模块：`CeBaidu`, `binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`,  `ptrarchive`, `sitedossier`,`threatcrowd`）
+  4. 利用DNS数据集收集子域（目前有21个模块：`ip138`, `ximcx`, `CeBaidu`, `binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`,  `ptrarchive`, `sitedossier`,`threatcrowd`）

  5. 利用DNS查询收集子域（目前有1个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，该模块还有待添加和完善）

-  6. 利用威胁情报平台数据收集子域（目前有5个模块：`riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）
+  6. 利用威胁情报平台数据收集子域（目前有6个模块：`alienvault`, `riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）

-  7. 利用搜索引擎发现子域（目前有16个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
+  7. 利用搜索引擎发现子域（目前有17个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `gitee`,`github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
 * **支持子域爆破**，该模块有常规的字典爆破，也有自定义的fuzz模式，支持批量爆破和递归爆破，自动判断泛解析并处理。
 * **支持子域验证**，默认开启子域验证，自动解析子域DNS，自动请求子域获取title和banner，并综合判断子域存活情况。
 * **支持子域接管**，默认开启子域接管风险检查，支持子域自动接管（目前只有Github，有待完善），支持批量检查。
@@ -55,24 +55,26 @@

 ## 🚀上手指南

-📢由于该项目**处于开发中**，会不断进行更新迭代，下载使用最好**克隆**最新项目，请务必花一点时间阅读此文档，有助于你快速熟悉OneForAll！
+📢 请务必花一点时间阅读此文档，有助于你快速熟悉OneForAll！

 **🐍安装要求**

-OneForAll是基于CPython开发的，所以你需要Python环境才能运行，如果你的系统还没有Python环境你可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)，理论上Python 3.6，3.7和3.8都可以正常运行OneForAll，**但是**许多测试都是在Python 3.7上进行的，所以**推荐**你使用**Python 3.7**版本运行OneForAll。运行以下命令检查Python和pip3版本：
+OneForAll基于[Python 3.8.0]( https://www.python.org/downloads/release/python-380/ )开发和测试，请使用高于Python 3.8.0的稳定发行版本，其他版本可能会出现一些问题，安装Python环境可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)。运行以下命令检查Python和pip3版本：
 ```bash
 python -V
 pip3 -V
 ```
 如果你看到以下类似输出便说明Python环境没有问题：
 ```bash
-Python 3.7.4
-pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.7)
+Python 3.8.0
+pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python38\site-packages\pip (python 3.8)
 ```

-**✔安装步骤**
+**✔安装步骤（git 版）**

 1. **下载**
+
+   由于该项目**处于开发中**，会不断进行更新迭代，下载时使用`git clone`**克隆**最新代码仓库，也方便后续的更新，不推荐从Releases下载，因为Releases里版本更新缓慢，也不方便更新，
   本项目已经在[码云](https://gitee.com/shmilylty/OneForAll.git)(Gitee)镜像了一份，国内推荐使用码云进行克隆比较快：

   ```bash
@@ -84,18 +86,19 @@ pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages
   ```

 2. **安装**
-   你可以通过pip3安装OneForAll的依赖（如果你熟悉[pipenv](https://docs.pipenv.org/en/latest/)，那么推荐你使用[pipenv安装依赖]((https://github.com/shmilylty/OneForAll/tree/master/docs/Installation_dependency.md))），以下为**Windows系统**下使用**pip3**安装依赖的示例：（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python37`，那么请以管理员身份运行命令提示符cmd执行以下命令！）
-   
+
+   你可以通过pip3安装OneForAll的依赖（如果你熟悉[pipenv](https://docs.pipenv.org/en/latest/)，那么推荐你使用[pipenv安装依赖]((https://github.com/shmilylty/OneForAll/tree/master/docs/Installation_dependency.md))），以下为**Windows系统**下使用**pip3**安装依赖的示例：（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python38`，那么请以管理员身份运行命令提示符cmd执行以下命令！）
 ```bash
   cd OneForAll/
-   python -m pip install --user  -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-   pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+   python -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+   pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
   cd oneforall/
   python oneforall.py --help
-   ```
+```
   其他系统平台的请参考[依赖安装](https://github.com/shmilylty/OneForAll/tree/master/docs/installation_dependency.md)，如果在安装依赖过程中发现编译某个依赖库失败时可以参考[Q&A](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md)中解决方法，如果还没有解决欢迎加群反馈。

 3. **更新**
+
   ❗注意：如果你之前已经克隆了项目运行之前请**备份**自己修改过的文件到项目外的地方（如**config.py**），然后执行以下命令**更新**项目：

   ```bash
@@ -104,6 +107,22 @@ pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages
   git pull
   ```

+**✔安装步骤（docker 版）**
+
+方法一：直接拉取部署好的镜像（更新不及时）
+
+```shell
+docker pull tardis07/oneforall
+docker run -it oneforall
+```
+
+方法二：从 `Dockerfile` 中构建（同git版）
+
+```shell
+docker build -t oneforall .
+docker run -it oneforall
+```
+
 **✨使用演示**

 1. 如果你是通过pip3安装的依赖则使用以下命令运行示例：   
@@ -122,7 +141,7 @@ pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages

 **🤔使用帮助**

-命令行参数只提供了一些常用参数，更多详细的参数配置请见[config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py)，如果你认为有些参数是命令界面经常使用到的或缺少了什么参数等问题非常欢迎反馈。由于众所周知的原因，如果要使用一些被墙的收集接口请先到[config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py)配置代理，有些收集模块需要提供API（大多都是可以注册账号免费获取），如果需要使用请到[config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py)配置API信息，如果不使用请忽略有关报错提示。（详细模块请阅读[收集模块说明](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md)）
+命令行参数只提供了一些常用参数，更多详细的参数配置请见[api.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/api.py)，如果你认为有些参数是命令界面经常使用到的或缺少了什么参数等问题非常欢迎反馈。由于众所周知的原因，如果要使用一些被墙的收集接口请先到[api.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/api.py)配置代理，有些收集模块需要提供API（大多都是可以注册账号免费获取），如果需要使用请到[api.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/api.py)配置API信息，如果不使用请忽略有关报错提示。（详细模块请阅读[收集模块说明](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md)）

 OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实现，有关Fire更高级使用方法请参阅[使用Fire CLI](https://github.com/google/python-fire/blob/master/docs/using-cli.md)，有任何使用疑惑欢迎加群交流。

@@ -136,53 +155,58 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
   python oneforall.py --help
   ```
   ```bash
-   NAME
-       oneforall.py - OneForAll是一款功能强大的子域收集工具
-   
-   SYNOPSIS
-       oneforall.py --target=TARGET <flags>
-   
-   DESCRIPTION
-       Version: 0.0.6
-       Project: https://git.io/fjHT1
-   
-       Example:
-           python3 oneforall.py --target example.com run
-           python3 oneforall.py --target ./domains.txt run
-           python3 oneforall.py --target example.com --brute True run
-           python3 oneforall.py --target example.com --verify False run
-           python3 oneforall.py --target example.com --valid None run
-           python3 oneforall.py --target example.com --port medium run
-           python3 oneforall.py --target example.com --format csv run
-           python3 oneforall.py --target example.com --show True run
-   
-       Note:
-           参数valid可选值1，0，None分别表示导出有效，无效，全部子域
-           参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
-           参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
-           参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
-                             'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
-           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
-   
-   ARGUMENTS
-       TARGET
-           单个域名或者每行一个域名的文件路径(必需参数)
-   
-   FLAGS
-       --brute=BRUTE
-           使用爆破模块(默认False)
-       --verify=VERIFY
-           验证子域有效性(默认True)
-       --port=PORT
-           请求验证的端口范围(默认medium)
-       --valid=VALID
-           导出子域的有效性(默认1)
-       --path=PATH
-           导出路径(默认None)
-       --format=FORMAT
-           导出格式(默认xls)
-       --show=SHOW
-           终端显示导出数据(默认False)
+    NAME
+        oneforall.py - OneForAll帮助信息
+
+    SYNOPSIS
+        oneforall.py COMMAND | --target=TARGET <flags>
+    
+    DESCRIPTION
+        OneForAll是一款功能强大的子域收集工具
+
+        Example:
+            python3 oneforall.py version
+            python3 oneforall.py --target example.com run
+            python3 oneforall.py --target ./domains.txt run
+            python3 oneforall.py --target example.com --valid None run
+            python3 oneforall.py --target example.com --brute True run
+            python3 oneforall.py --target example.com --port small run
+            python3 oneforall.py --target example.com --format csv run
+            python3 oneforall.py --target example.com --dns False run
+            python3 oneforall.py --target example.com --req False run
+            python3 oneforall.py --target example.com --takeover False run
+            python3 oneforall.py --target example.com --show True run
+    
+        Note:
+            参数valid可选值1，0，None分别表示导出有效，无效，全部子域
+            参数port可选值有'default', 'small', 'large', 详见config.py配置
+            参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                              'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+            参数path默认None使用OneForAll结果目录生成路径
+
+    ARGUMENTS
+        TARGET
+            单个域名或者每行一个域名的文件路径(必需参数)
+    
+    FLAGS
+        --brute=BRUTE
+            使用爆破模块(默认False)
+        --dns=DNS
+            DNS解析子域(默认True)
+        --req=REQ
+            HTTP请求子域(默认True)
+        --port=PORT
+            请求验证子域的端口范围(默认只探测80端口)
+        --valid=VALID
+            导出子域的有效性(默认None)
+        --format=FORMAT
+            导出文件格式(默认csv)
+        --path=PATH
+            导出文件路径(默认None)
+        --takeover=TAKEOVER
+            检查子域接管(默认False)
+        --show=SHOW
+            终端显示导出数据(默认False)
   ```

 2. **aiobrute.py使用帮助**
@@ -191,7 +215,7 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
   - 一是主要是与泛解析的IP集合和TTL值做对比，可以参考[这篇文章](http://sh3ll.me/archives/201704041222.txt)。
   - 二是多次解析到同一IP集合次数（默认设置为10，可以在config.py设置大小）。
   - 三是考虑爆破效率问题目前还没有加上HTTP响应体相似度对比和响应体内容判断，如果有必要后续添加。
-   经过不严谨测试在16核心的CPU，使用16进程64协程，100M带宽的环境下，设置任务分割为50000，跑两百万字典大概10分钟左右跑完，大概3333个子域每秒。
+   经过不严谨测试在16核心的CPU，使用16进程64协程，100M带宽的环境下，跑两百万字典大概10分钟左右跑完，大概3333个子域每秒。

   ```bash
   python aiobrute.py --help
@@ -214,9 +238,6 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
           python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
   
       Note:
-           参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为任务组，
-           当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
-           十万字典建议设置为5000，百万字典设置为50000
           参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
           参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
                             'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
@@ -230,11 +251,9 @@ OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实
       --process=PROCESS
           爆破的进程数(默认CPU核心数)
       --coroutine=COROUTINE
-           每个爆破进程下的协程数(默认64)
+           每个爆破进程下的协程数(默认1024)
       --wordlist=WORDLIST
           指定爆破所使用的字典路径(默认使用config.py配置)
-       --segment=SEGMENT
-           爆破任务分割(默认500)
       --recursive=RECURSIVE
           是否使用递归爆破(默认False)
       --depth=DEPTH
@@ -284,6 +303,7 @@ D:.
 +---images
 \---oneforall
    |   aiobrute.py   异步多进程多协程子域爆破模块，可以单独运行
+    |   api.py        一些收集模块的API配置
    |   collect.py    各个收集模块上层调用
    |   config.py     配置文件
    |   dbexport.py   数据库导出模块，可以单独运行
@@ -316,7 +336,7 @@ D:.
 ## ⌛后续计划

 - [ ] 各模块持续优化和完善
- [ ] 子域监控（标记每次新发现的子域）
+- [x] 子域监控（标记每次新发现的子域）
 - [ ] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
 - [ ] 操作强大交互人性的前端界面实现（暂定：前端：Element + 后端：Flask）

@@ -331,17 +351,13 @@ D:.
 * **[Jing Ling](https://github.com/shmilylty)**
  * 核心开发

-* **[Black Star](https://github.com/blackstar24)**，**[Echocipher](https://github.com/Echocipher)**
-  * 模块贡献
-
-* **[iceMatcha](https://github.com/iceMatcha)**，**[mikuKeeper](https://github.com/mikuKeeper)**
-  * 工具测试
-
-* **Anyone**
-  * 工具反馈

 你可以在[CONTRIBUTORS.md](https://github.com/shmilylty/OneForAll/tree/master/CONTRIBUTORS.md)中参看所有参与该项目的开发者。

+## ☕赞赏
+如果你觉得这个项目帮助到了你，你可以帮作者买一杯咖啡表示鼓励:)
+![](https://raw.githubusercontent.com/shmilylty/OneForAll/master/images/Donate.png)
+
 ## 📄版权

 该项目签署了GPL-3.0授权许可，详情请参阅[LICENSE](https://github.com/shmilylty/OneForAll/LICENSE)。
@@ -10,10 +10,10 @@ A: 可以尝试以下方法:
 * [https://www.lfd.uci.edu/~gohlke/pythonlibs](https://www.lfd.uci.edu/~gohlke/pythonlibs)
 * [https://pythonwheels.com/](https://pythonwheels.com/)

-选择好对应版本执行以下命令手动安装。举个例子，当编译pycares时失败时，找到[https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares](https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares)，由于我的系统是Windows 10 64位，使用的Python 3.7便下载`pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl`（一般来说下载最新版本的），然后手动安装：
+选择好对应版本执行以下命令手动安装。举个例子，当编译 brotlipy 时失败时，找到[https://www.lfd.uci.edu/~gohlke/pythonlibs/#brotlipy](https://www.lfd.uci.edu/~gohlke/pythonlibs/#brotlipy)，由于我的系统是Windows 10 64位，使用的Python 3.8便下载`brotlipy‑0.7.0‑cp38‑cp38‑win_amd64.whl`（一般来说下载最新版本的），然后手动安装：

 ```bash
-pip3 install pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl
+pip3 install brotlipy‑0.7.0‑cp38‑cp38‑win_amd64.whl
 ```

 2. 到库的项目地址issues和wiki等找找有没有解决方法，如果没有就给他们提issues发邮件😜。
@@ -1,15 +1,15 @@
 #  收集模块说明 #

-如果要使用通过API收集子域的模块请先到[config.py](../oneforall/config.py)配置相关信息，大多平台的API都是可以注册账号免费获取的。
+如果要使用通过API收集子域的模块请先到[api.py](../oneforall/api.py)配置相关信息，大多平台的API都是可以注册账号免费获取的。

-如果你指定使用某些模块可以在[config.py](../oneforall/config.py)中设置：
+如果你指定使用某些模块可以在[api.py](../oneforall/api.py)中设置：

 ```python
 enable_all_module = False  # 不开启所有模块
 enable_partial_module = [('modules.search', 'ask')，('modules.search', 'baidu')]  # 只使用ask和baidu搜索引擎收集子域
 ```

-如果你指定使用某些模块使用代理可以在[config.py](../oneforall/config.py)中设置：
+如果你指定使用某些模块使用代理可以在[api.py](../oneforall/api.py)中设置：

 ```python
 enable_proxy = True  # 使用代理
@@ -23,12 +23,12 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
    
      | 模块名称    | 是否需要代理 | 是否需要API | 其他说明                                           |
      | ----------- | ------------ | ----------- | -------------------------------------------------- |
-      | censys_api  | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | censys_api  | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py) |
      | certspotter | 否           | 否          |                                                    |
      | crtsh       | 否           | 否          |                                                    |
      | entrust     | 否           | 否          |                                                    |
      | google      | 是           | 否          |                                                    |
-      | spyse_api   | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | spyse_api   | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py) |


  2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，后续会添加检查NSEC记录，NSEC3记录等模块）
@@ -48,62 +48,66 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
      | archivecrawl | 否           | 否          |          |
      | commoncrawl  | 否           | 否          |          |

-  4. 利用DNS数据集收集子域（目前有19个模块：`cebaidu`, `binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`,  `ptrarchive`, `sitedossier`,`threatcrowd`）
+  4. 利用DNS数据集收集子域（目前有22个模块：`cebaidu`, `binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`,  `ptrarchive`, `sitedossier`,`threatcrowd`）

      | 模块名称           | 是否需要代理 | 是否需要API | 其他说明                                           |
      | ------------------ | ------------ | ----------- | -------------------------------------------------- |
-      | binaryedge_api     | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | binaryedge_api     | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py) |
      | bufferover         | 否           | 否          |                                                    |
      | cebaidu            | 否           | 否          |                                                    |
      | chinaz             | 否           | 否          |                                                    |
-      | chinaz_api         | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
-      | circl_api          | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | chinaz_api         | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py) |
+      | circl_api          | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py) |
      | dnsdb              | 否           | 否          |                                                    |
-      | dnsdb_api          | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | dnsdb_api          | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py) |
      | dnsdumpster        | 否           | 否          |                                                    |
      | hackertarget       | 否           | 否          |                                                    |
-      | ipv4info           | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | ip138              | 否           | 否          |                                                    |
+      | ipv4info           | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py) |
      | netcraft           | 否           | 否          |                                                    |
-      | passivedns_api     | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
-      | ptrarchive         | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
-      | riddler            | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | passivedns_api     | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py) |
+      | ptrarchive         | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py) |
+      | riddler            | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py) |
      | robtex             | 否           | 否          |                                                    |
-      | securitytrails_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | securitytrails_api | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py) |
      | sitedossier        | 否           | 否          |                                                    |
      | threatcrowd        | 否           | 否          |                                                    |
+      | ximcx              | 否           | 否          |                                                    |
  5. 利用DNS查询收集子域（目前有1个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，该模块还有待添加和完善）

      | 模块名称 | 是否需要代理 | 是否需要API | 其他说明                      |
      | -------- | ------------ | ----------- | ----------------------------- |
      | srv      | 否           | 否          | 枚举域名常见的SRV记录发现子域 |
-  6. 利用威胁平台数据收集子域（目前有5个模块：`riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）
+  6. 利用威胁平台数据收集子域（目前有6个模块：`riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）

      | 模块名称       | 是否需要代理 | 是否需要API | 其他说明                                           |
-      | -------------- | ------------ | ----------- | -------------------------------------------------- |
-      | riskiq_api     | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
-      | threatbook_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | -------------- | ------------ | ----------- | ------------------------------------------------- |
+      | alienvault     | 否           | 否          |                                                    |
+      | riskiq_api     | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py)         |
+      | threatbook_api | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py)         |
      | threatminer    | 否           | 否          |                                                    |
      | virustotal     | 否           | 否          |                                                    |
-      | virustotal_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | virustotal_api | 否           | 是          | API使用和申请见[api.py](../oneforall/api.py)         |
  7. 利用搜索引擎发现子域（目前有16个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`）

     除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。

-     | 模块        | 是否需要代理           | 是否需要API | 其他说明                                                    |
-     | ----------- | ---------------------- | ----------- | ----------------------------------------------------------- |
+     | 模块         | 是否需要代理            | 是否需要API  | 其他说明                                                        |
+     | ----------- | ---------------------- | -----------| ----------------------------------------------------------- |
     | ask         | 是                     | 否          |                                                             |
     | baidu       | 否                     | 否          |                                                             |
     | bing        | 否                     | 否          |                                                             |
-     | bing_api    | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
+     | bing_api    | 否                     | 是          | API使用和申请见[api.py](../oneforall/api.py)                  |
     | duckduckgo  | 是                     | 否          |                                                             |
-     | exalead     | 否，最好使用国外代理。 | 否          |                                                             |
-     | fofa_api    | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
-     | github      | 否                     | 否          | 在[config.py](../oneforall/config.py)设置Github邮件名和密码 |
+     | exalead     | 否，最好使用国外代理。    | 否          |                                                             |
+     | fofa_api    | 否                     | 是          | API使用和申请见[api.py](../oneforall/api.py)                  |
+     | gitee       | 否                     | 否          |                                                             |
+     | github      | 否                     | 否          | 在[api.py](../oneforall/api.py)设置Github邮件名和密码。        |
     | google      | 是                     | 否          |                                                             |
-     | google_api  | 是                     | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
-     | shodan_api  | 否，最好使用国外代理。 | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
+     | google_api  | 是                     | 是          | API使用和申请见[api.py](../oneforall/api.py)                  |
+     | shodan_api  | 否，最好使用国外代理。    | 是          | API使用和申请见[api.py](../oneforall/api.py)                  |
     | so          | 否                     | 否          |                                                             |
     | sogou       | 否                     | 否          |                                                             |
     | yahoo       | 是                     | 否          |                                                             |
     | yandex      | 是                     | 否          |                                                             |
-     | zoomeye_api | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
+     | zoomeye_api | 否                     | 是          | API使用和申请见[api.py](../oneforall/api.py)                  |
@@ -1,64 +1,84 @@
 # 安装依赖

-你可以通过pip3和pipenv两种方法安装OneForAll的依赖（如果你熟悉[pipenv](https://docs.pipenv.org/en/latest/)，那么推荐使用你使用pipenv）：
+你可以通过pip3和pipenv两种方法安装OneForAll的依赖（熟悉哪种就用哪种）：

-* **Windows系统**（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python37`，那么请以管理员身份运行命令提示符cmd执行以下命令！）
+## Windows系统

-    1. 使用pipenv
+注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python38`，那么请以管理员身份运行命令提示符cmd执行以下命令！

-    ```bash
-    cd OneForAll/
-    python -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-    pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
-    pipenv install --user --python 3.7
-    cd oneforall
-    pipenv run python oneforall.py --help
-    ```
+1. 使用pipenv安装依赖
+```bash
+cd OneForAll/
+python -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+pip3 install pipenv -i https://mirrors.aliyun.com/pypi/simple/
+pipenv install --python 3.8
+cd oneforall
+pipenv run python oneforall.py --help
+```

-    2. 使用pip3
+2. 使用pip3安装依赖
+```bash
+cd OneForAll/
+python -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+cd oneforall/
+python oneforall.py --help
+```
+## Linux系统

-    ```bash
-    cd OneForAll/
-    python -m pip install --user  -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
-    cd oneforall/
-    python oneforall.py --help
-    ```
-* **Linux系统**
+* **Ubuntu/Debian系统(包括kali)**
+安装git
+```bash
+sudo apt update
+sudo apt install git
+```

-    1. 使用pipenv
-    ```bash
-    cd OneForAll/
-    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-    sudo pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
-    sudo pipenv install --user --python 3.7
-    cd oneforall
-    pipenv run python3 oneforall.py --help
-    ```
-    2. 使用pip3
-    ```bash
-    cd OneForAll/
-    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
-    cd oneforall/
-    python3 oneforall.py --help
-    ```
-* **Darwin系统**
+克隆OneForAll项目
+```bash
+git clone https://gitee.com/shmilylty/OneForAll.git
+```

-    1. 使用pipenv
-    ```bash
-    cd OneForAll/
-    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-    pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
-    pipenv install --user --python 3.7
-    cd oneforall
-    pipenv run python3 oneforall.py --help
-    ```
-    2. 使用pip3
-    ```bash
-    cd OneForAll/
-    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
-    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
-    cd oneforall/
-    python3 oneforall.py --help
-    ```
+安装python及开发依赖
+```bash
+sudo apt install python3.8 python3.8-dev python3-testresources
+```
+接下来你可以使用以下一种方式安装OneForAll的Python库依赖:
+1. 使用pipenv安装依赖
+```bash
+cd OneForAll/
+sudo python3.8 -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+sudo apt install pipenv
+sudo pipenv install --python 3.8
+cd oneforall
+pipenv run python3 oneforall.py --help
+```
+
+2. 使用pip3安装依赖
+```bash
+cd OneForAll/
+sudo apt install python3-pip
+sudo python3.8 -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+sudo pip3 install --ignore-installed -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+cd oneforall/
+python3.8 oneforall.py --help
+```
+## Darwin系统
+
+1. 使用pipenv安装依赖
+```bash
+cd OneForAll/
+python3 -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+pip3 install pipenv -i https://mirrors.aliyun.com/pypi/simple/
+pipenv install --python 3.8
+cd oneforall
+pipenv run python3 oneforall.py --help
+```
+
+2. 使用pip3安装依赖
+```bash
+cd OneForAll/
+python3 -m pip install -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+cd oneforall/
+python3 oneforall.py --help
+```
@@ -8,12 +8,13 @@ OneForAll多进程多协程异步子域爆破模块
 :license: GNU General Public License v3.0, see LICENSE for more details.
 """

-import asyncio
-import queue
-import secrets
-import signal
 import time
-
+import queue
+import signal
+import asyncio
+import secrets
+import functools
+from multiprocessing import Manager
 import aiomultiprocess as aiomp
 import exrex
 import fire
@@ -27,10 +28,6 @@ from common.database import Database
 from config import logger


-def init_worker():
-    signal.signal(signal.SIGINT, signal.SIG_IGN)
-
-
 def detect_wildcard(domain):
    """
    探测域名是否使用泛解析
@@ -39,25 +36,28 @@ def detect_wildcard(domain):
    :return: 如果没有使用泛解析返回False 反之返回泛解析的IP集合和ttl整型值
    """
    logger.log('INFOR', f'正在探测{domain}是否使用泛解析')
-    token = secrets.token_hex(16)
+    token = secrets.token_hex(4)
    random_subdomain = f'{token}.{domain}'
    try:
-        answers = resolve.dns_query_a(random_subdomain)
+        resolver = resolve.dns_resolver()
+        answers = resolver.query(random_subdomain, 'A')
    # 如果查询随机域名A记录出错 说明不存在随机子域的A记录 即没有开启泛解析
    except Exception as e:
        logger.log('DEBUG', e)
        logger.log('INFOR', f'{domain}没有使用泛解析')
        return False, None, None
    ttl = answers.ttl
+    name = answers.name
    ips = {item.address for item in answers}
    logger.log('ALERT', f'{domain}使用了泛解析')
-    logger.log('ALERT', f'{random_subdomain} 解析到IP: {ips} TTL: {ttl}')
+    logger.log('ALERT', f'{random_subdomain} 解析到域名: {name} '
+                        f'IP: {ips} TTL: {ttl}')
    return True, ips, ttl


 def wildcard_by_compare(ips, ttl, wildcard_ips, wildcard_ttl):
    """
-    通过与泛解析返回的IP集合和判TTL值进行对比判断发现的子域是否是泛解析子域
+    通过与泛解析返回的IP集合和返回的TTL值进行对比判断发现的子域是否是泛解析子域

    :param set ips: 子域A记录查询出的IP集合
    :param int ttl: 子域A记录查询出的TTL整型值
@@ -119,7 +119,7 @@ def gen_fuzz_domains(domain, rule):

 def gen_brute_domains(domain, path):
    domains = set()
-    with open(path) as file:
+    with open(path, encoding='utf-8', errors='ignore') as file:
        for line in file:
            brute_domain = line.strip() + '.' + domain
            domains.add(brute_domain)
@@ -146,13 +146,12 @@ class AIOBrute(Module):
        参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
-        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+        参数path默认None使用OneForAll结果目录生成路径

    :param str target:       单个域名或者每行一个域名的文件路径
    :param int process:      爆破的进程数(默认CPU核心数)
    :param int coroutine:    每个爆破进程下的协程数(默认64)
    :param str wordlist:     指定爆破所使用的字典路径(默认使用config.py配置)
-    :param int segment:      爆破任务分割(默认500)
    :param bool recursive:   是否使用递归爆破(默认False)
    :param int depth:        递归爆破的深度(默认2)
    :param str namelist:     指定递归爆破所使用的字典路径(默认使用config.py配置)
@@ -165,10 +164,10 @@ class AIOBrute(Module):
    :param bool show:        终端显示导出数据(默认False)
    """

-    def __init__(self, target, process=None, coroutine=64, wordlist=None,
-                 segment=500, recursive=False, depth=2, namelist=None,
-                 fuzz=False, rule=None, export=True, valid=None, format='csv',
-                 path=None, show=False):
+    def __init__(self, target, process=None, coroutine=None, wordlist=None,
+                 recursive=False, depth=None, namelist=None, fuzz=False,
+                 rule=None, export=True, valid=None, format='csv', path=None,
+                 show=False):
        Module.__init__(self)
        self.domains = set()
        self.domain = str()
@@ -178,7 +177,6 @@ class AIOBrute(Module):
        self.process = process or config.brute_process_num
        self.coroutine = coroutine or config.brute_coroutine_num
        self.wordlist = wordlist or config.brute_wordlist_path
-        self.segment = segment or config.brute_task_segment
        self.recursive_brute = recursive or config.enable_recursive_brute
        self.recursive_depth = depth or config.brute_recursive_depth
        self.recursive_namelist = namelist or config.recursive_namelist_path
@@ -192,6 +190,8 @@ class AIOBrute(Module):
        self.nameservers = config.resolver_nameservers
        self.ips_times = dict()  # IP集合出现次数
        self.enable_wildcard = False  # 当前域名是否使用泛解析
+        self.wildcard_check = config.enable_wildcard_check
+        self.wildcard_deal = config.enable_wildcard_deal
        self.wildcard_ips = set()  # 泛解析IP集合
        self.wildcard_ttl = int()  # 泛解析TTL整型值

@@ -207,63 +207,54 @@ class AIOBrute(Module):
            logger.log('INFOR', f'使用{self.wordlist}字典')
            domains = gen_brute_domains(domain, self.wordlist)
        domains = list(domains)
-        return utils.split_list(domains, self.segment)  # 分割任务组
+        return domains

    def deal_results(self, results):
        for result in results:
-            if result is None:
+            hostname, answer = result
+            if answer is None:
                continue
-            if isinstance(result, Exception):
+            if isinstance(answer, Exception):
                # logger.log('DEBUG', f'爆破{subdomain}时出错 {str(answers)}')
                continue
-            if isinstance(result, tuple):
-                subdomain, answers = result
-                if not answers:
+            name, alias, ips = answer
+            if name.endswith('.'):
+                name = name[0:-1]
+            # 取值 如果是首次出现的IP集合 出现次数先赋值0
+            value = self.ips_times.setdefault(str(ips), 0)
+            self.ips_times[str(ips)] = value + 1
+            # 目前域名开启了泛解析
+            if self.enable_wildcard and self.wildcard_deal:
+                # 通过对比查询的子域和响应的子域来判断真实子域
+                # 去掉解析到CDN的情况
+                if 'cdn' in name or 'waf' in name:
                    continue
-                ips = {record.host for record in answers}
-                # 取值 如果是首次出现的IP集合 出现次数先赋值0
-                value = self.ips_times.setdefault(str(ips), 0)
-                self.ips_times[str(ips)] = value + 1
-                ttl = answers[0].ttl
-                if self.enable_wildcard:
-                    if wildcard_by_compare(ips,
-                                           ttl,
-                                           self.wildcard_ips,
-                                           self.wildcard_ttl):
-                        continue
+                if not name.endswith(self.domain):
+                    continue
+                # 通过对比解析到的IP集合的次数来判断真实子域
                if wildcard_by_times(ips, self.ips_times):
                    continue
-                logger.log('INFOR', f'发现{self.domain}的子域: {subdomain} '
-                                    f'解析IP: {ips} TTL: {ttl}')
-                self.subdomains.add(subdomain)
-                self.records[subdomain] = str(ips)
+            # 只添加没有出现过的子域
+            if hostname not in self.subdomains:
+                logger.log('INFOR', f'发现{self.domain}的子域: {hostname} '
+                                    f'解析到: {name} IP: {ips}')
+                self.subdomains.add(hostname)
+                self.records[hostname] = str(ips)[1:-1]

    async def main(self, domain, rx_queue):
        if not self.fuzz:  # fuzz模式不探测域名是否使用泛解析
-            self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl \
-                = detect_wildcard(domain)
+            if self.wildcard_check:
+                self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl \
+                    = detect_wildcard(domain)
        tasks = self.gen_tasks(domain)
        logger.log('INFOR', f'正在爆破{domain}的域名')
-        for task in tqdm.tqdm(tasks,
-                              desc='Progress',
-                              smoothing=1.0,
-                              ncols=True):
-            async with aiomp.Pool(processes=self.process,
-                                  initializer=init_worker,
-                                  childconcurrency=self.coroutine) as pool:
-                try:
-                    results = await pool.map(resolve.aiodns_query_a, task)
-                except KeyboardInterrupt:
-                    logger.log('ALERT', '爆破终止正在退出')
-                    pool.terminate()  # 关闭pool，结束工作进程，不在处理未完成的任务。
-                    self.save_json()
-                    self.gen_result()
-                    rx_queue.put(self.results)
-                    return
-                self.deal_results(results)
-                self.save_json()
-                self.gen_result()
-                rx_queue.put(self.results)
+        # for task in tqdm.tqdm(tasks, total=len(tasks),
+        #                       desc='Progress'):
+        results = await resolve.aio_resolve(tasks, self.process, self.coroutine)
+        self.deal_results(results)
+        self.save_json()
+        self.gen_result()
+        rx_queue.put(self.results)

    def run(self, rx_queue=None):
        self.domains = utils.get_domains(self.target)
@@ -298,9 +289,9 @@ class AIOBrute(Module):
                                                              rx_queue))
            # 队列不空就一直取数据存数据库
            while not rx_queue.empty():
-                source, results = rx_queue.get()
+                results = rx_queue.get()
                # 将结果存入数据库中
-                db.save_db(self.domain, results, source)
+                db.save_db(self.domain, results, self.source)

            end = time.time()
            self.elapsed = round(end - start, 1)
@@ -310,14 +301,14 @@ class AIOBrute(Module):
                                f'发现{self.domain}的域名{length}个')
            logger.log('DEBUG', f'{self.source}模块发现{self.domain}的域名:\n'
                                f'{self.subdomains}')
+            if not self.path:
+                name = f'{self.domain}_brute_result.{self.format}'
+                self.path = config.result_save_path.joinpath(name)
            # 数据库导出
            if self.export:
-                if not self.path:
-                    name = f'{self.domain}_brute.{self.format}'
-                    self.path = config.result_save_path.joinpath(name)
                dbexport.export(self.domain,
                                valid=self.valid,
-                                dpath=self.path,
+                                path=self.path,
                                format=self.format,
                                show=self.show)

@@ -0,0 +1,74 @@
+# 模块API配置
+# Censys可以免费注册获取API：https://censys.io/api
+censys_api_id = ''
+censys_api_secret = ''
+
+# Binaryedge可以免费注册获取API：https://app.binaryedge.io/account/api
+# 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
+binaryedge_api = ''
+
+# Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
+chinaz_api = ''
+
+# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/
+# cognitive-services/bing-web-search-api/#web-json
+bing_api_id = ''
+bing_api_key = ''
+
+# SecurityTrails可以免费注册获取API：https://securitytrails.com/corp/api
+securitytrails_api = ''
+
+# https://fofa.so/api
+fofa_api_email = ''  # fofa用户邮箱
+fofa_api_key = ''  # fofa用户key
+
+# Google可以免费注册获取API:
+# https://developers.google.com/custom-search/v1/overview
+# 免费的API只能查询前100条结果
+google_api_key = ''  # Google API搜索key
+google_api_cx = ''  # Google API搜索cx
+
+# https://api.passivetotal.org/api/docs/
+riskiq_api_username = ''
+riskiq_api_key = ''
+
+# Shodan可以免费注册获取API: https://account.shodan.io/register
+# 免费的API限速1秒查询1次
+shodan_api_key = ''
+# ThreatBook API 查询子域名需要收费 https://x.threatbook.cn/nodev4/vb4/myAPI
+threatbook_api_key = ''
+
+# VirusTotal可以免费注册获取API: https://developers.virustotal.com/reference
+virustotal_api_key = ''
+
+# https://www.zoomeye.org/doc?channel=api
+zoomeye_api_usermail = ''
+zoomeye_api_password = ''
+
+# Spyse可以免费注册获取API: https://spyse.com/
+spyse_api_token = ''
+
+# https://www.circl.lu/services/passive-dns/
+circl_api_username = ''
+circl_api_password = ''
+
+# https://www.dnsdb.info/
+dnsdb_api_key = ''
+
+# ipv4info可以免费注册获取API: http://ipv4info.com/tools/api/
+# 免费的API有效期只有2天，到期之后可以再次生成，每天可以查询50次。
+ipv4info_api_key = ''
+
+# https://github.com/360netlab/flint
+# passivedns_api_addr默认空使用http://api.passivedns.cn
+# passivedns_api_token可为空
+passivedns_api_addr = ''
+passivedns_api_token = ''
+
+# Github Token可以访问https://github.com/settings/tokens生成,user为Github用户名
+# 用于子域接管
+github_api_user = ''
+github_api_token = ''
+# Github子域收集模块使用
+github_email = ''
+github_password = ''
@@ -14,7 +14,7 @@ class Collect(object):
        self.domain = domain
        self.elapsed = 0.0
        self.modules = []
-        self.collect_func = []
+        self.collect_funcs = []
        self.path = None
        self.export = export
        self.format = 'csv'
@@ -44,8 +44,9 @@ class Collect(object):
        导入脚本的do函数
        """
        for package, name in self.modules:
-            import_object = importlib.import_module('.'+name, package)
-            self.collect_func.append(getattr(import_object, 'do'))
+            import_object = importlib.import_module('.' + name, package)
+            func = getattr(import_object, 'do')
+            self.collect_funcs.append([func, name])

    def run(self):
        """
@@ -58,8 +59,10 @@ class Collect(object):

        threads = []
        # 创建多个子域收集线程
-        for collect_func in self.collect_func:
-            thread = threading.Thread(target=collect_func,
+        for collect in self.collect_funcs:
+            func_obj, func_name = collect
+            thread = threading.Thread(target=func_obj,
+                                      name=func_name,
                                      args=(self.domain,),
                                      daemon=True)
            threads.append(thread)
@@ -68,14 +71,20 @@ class Collect(object):
            thread.start()
        # 等待所有线程完成
        for thread in threads:
-            thread.join()
+            # 挨个线程判断超时 最坏情况主线程阻塞时间=线程数*module_thread_timeout
+            # 超时线程将脱离主线程 由于创建线程时已添加守护属于 所有超时线程会随着主线程结束
+            thread.join(config.module_thread_timeout)
+
+        for thread in threads:
+            if thread.is_alive():
+                logger.log('ALERT', f'{thread.name}模块线程发生超时')

        # 数据库导出
        if self.export:
            if not self.path:
                name = f'{self.domain}.{self.format}'
                self.path = config.result_save_path.joinpath(name)
-            dbexport.export(self.domain, dpath=self.path, format=self.format)
+            dbexport.export(self.domain, path=self.path, format=self.format)
        end = time.time()
        self.elapsed = round(end - start, 1)

@@ -23,7 +23,7 @@ class Database(object):
        :param db_path: 数据库连接或路径
        :return: SQLite数据库
        """
-        logger.log('DEBUG', f'正在获取数据库连接')
+        logger.log('TRACE', f'正在获取数据库连接')
        if isinstance(db_path, Connection):
            return db_path
        protocol = 'sqlite:///'
@@ -32,35 +32,46 @@ class Database(object):
        else:
            db_path = protocol + db_path
        db = records.Database(db_path)  # 不存在数据库时会新建一个数据库
-        logger.log('DEBUG', f'使用数据库: {db_path}')
+        logger.log('TRACE', f'使用数据库: {db_path}')
        return db.get_connection()

+    def query(self, sql):
+        try:
+            results = self.conn.query(sql)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+        else:
+            return results
+
    def create_table(self, table_name):
        """
-        初始化数据库
+        创建表结构

        :param str table_name: 要创建的表名
        """
        table_name = table_name.replace('.', '_')
-        logger.log('DEBUG', f'正在创建{table_name}表')
-        try:
-            self.conn.query(f'create table if not exists "{table_name}" ('
-                            f'id integer primary key,'
-                            f'url text,'
-                            f'subdomain text,'
-                            f'port int,'
-                            f'ips text,'
-                            f'status int,'
-                            f'reason text,'
-                            f'valid int,'
-                            f'title text,'
-                            f'banner text,'
-                            f'module text,'
-                            f'source text,'
-                            f'elapsed float,'
-                            f'count int)')
-        except Exception as e:
-            logger.log('ERROR', e)
+        if self.exist_table(table_name):
+            logger.log('TRACE', f'已经存在{table_name}表')
+            return
+        logger.log('TRACE', f'正在创建{table_name}表')
+        self.query(f'create table "{table_name}" ('
+                   f'id integer primary key,'
+                   f'url text,'
+                   f'subdomain text,'
+                   f'port int,'
+                   f'ips text,'
+                   f'status int,'
+                   f'reason text,'
+                   f'valid int,'
+                   f'new int,'
+                   f'title text,'
+                   f'banner text,'
+                   f'header text,'
+                   f'response text,'
+                   f'module text,'
+                   f'source text,'
+                   f'elapsed float,'
+                   f'count int)')

    def save_db(self, table_name, results, module_name=None):
        """
@@ -70,21 +81,39 @@ class Database(object):
        :param list results: 结果列表
        :param str module_name: 模块名
        """
-        logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域'
-                   '结果存入数据库')
+        logger.log('TRACE', f'正在将{module_name}模块发现{table_name}的子域'
+                            '结果存入数据库')
        table_name = table_name.replace('.', '_')
        if results:
            try:
                self.conn.bulk_query(
                    f'insert into "{table_name}" ('
                    f'id, url, subdomain, port, ips, status, reason, valid,'
-                    f'title, banner, module, source, elapsed, count)'
+                    f'new, title, banner, header, response, module, source, '
+                    f'elapsed, count)'
                    f'values (:id, :url, :subdomain, :port, :ips, :status,'
-                    f':reason, :valid, :title, :banner, :module, :source,'
-                    f':elapsed, :count)',
+                    f':reason, :valid, :new, :title, :banner, :header,'
+                    f':response, :module, :source,:elapsed, :count)',
                    results)
            except Exception as e:
-                logger.log('ERROR', e)
+                logger.log('ERROR', e.args)
+
+    def exist_table(self, table_name):
+        """
+        判断是否存在某表
+
+        :param str table_name: 表名
+        :return: 是否存在某表
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('TRACE', f'正在查询是否存在{table_name}表')
+        results = self.query(f'select count() from sqlite_master '
+                             f'where type = "table" and '
+                             f'name = "{table_name}"')
+        if results.scalar() == 0:
+            return False
+        else:
+            return True

    def copy_table(self, table_name, bak_table_name):
        """
@@ -95,13 +124,10 @@ class Database(object):
        """
        table_name = table_name.replace('.', '_')
        bak_table_name = bak_table_name.replace('.', '_')
-        logger.log('DEBUG', f'正在将{table_name}表复制到{bak_table_name}新表')
-        try:
-            self.conn.query(f'drop table if exists "{bak_table_name}"')
-            self.conn.query(f'create table "{bak_table_name}" '
-                            f'as select * from "{table_name}"')
-        except Exception as e:
-            logger.log('ERROR', e)
+        logger.log('TRACE', f'正在将{table_name}表复制到{bak_table_name}新表')
+        self.query(f'drop table if exists "{bak_table_name}"')
+        self.query(f'create table "{bak_table_name}" '
+                   f'as select * from "{table_name}"')

    def clear_table(self, table_name):
        """
@@ -110,11 +136,8 @@ class Database(object):
        :param str table_name: 表名
        """
        table_name = table_name.replace('.', '_')
-        logger.log('DEBUG', f'正在清空{table_name}表中的数据')
-        try:
-            self.conn.query(f'delete from "{table_name}"')
-        except Exception as e:
-            logger.log('ERROR', e)
+        logger.log('TRACE', f'正在清空{table_name}表中的数据')
+        self.query(f'delete from "{table_name}"')

    def drop_table(self, table_name):
        """
@@ -123,27 +146,21 @@ class Database(object):
        :param str table_name: 表名
        """
        table_name = table_name.replace('.', '_')
-        logger.log('DEBUG', f'正在删除{table_name}表')
-        try:
-            self.conn.query(f'drop table if exists "{table_name}"')
-        except Exception as e:
-            logger.log('ERROR', e)
+        logger.log('TRACE', f'正在删除{table_name}表')
+        self.query(f'drop table if exists "{table_name}"')

    def rename_table(self, table_name, new_table_name):
        """
-        复制表创建备份
+        重命名表名

        :param str table_name: 表名
        :param str new_table_name: 新表名
        """
        table_name = table_name.replace('.', '_')
        new_table_name = new_table_name.replace('.', '_')
-        logger.log('DEBUG', f'正在将{table_name}表重命名为{table_name}表')
-        try:
-            self.conn.query(f'alter table "{table_name}" '
-                            f'rename to "{new_table_name}"')
-        except Exception as e:
-            logger.log('ERROR', e)
+        logger.log('TRACE', f'正在将{table_name}表重命名为{table_name}表')
+        self.query(f'alter table "{table_name}" '
+                   f'rename to "{new_table_name}"')

    def deduplicate_subdomain(self, table_name):
        """
@@ -152,13 +169,10 @@ class Database(object):
        :param str table_name: 表名
        """
        table_name = table_name.replace('.', '_')
-        logger.log('DEBUG', f'正在去重{table_name}表中的子域')
-        try:
-            self.conn.query(
-                f'delete from "{table_name}" where id not in (select min(id) '
-                f'from "{table_name}" group by subdomain)')
-        except Exception as e:
-            logger.log('ERROR', e)
+        logger.log('TRACE', f'正在去重{table_name}表中的子域')
+        self.query(f'delete from "{table_name}" where '
+                   f'id not in (select min(id) '
+                   f'from "{table_name}" group by subdomain)')

    def remove_invalid(self, table_name):
        """
@@ -167,13 +181,9 @@ class Database(object):
        :param str table_name: 表名
        """
        table_name = table_name.replace('.', '_')
-        logger.log('DEBUG', f'正在去除{table_name}表中的无效子域')
-        try:
-            self.conn.query(
-                f'delete from "{table_name}" where '
-                f'subdomain is null or valid == 0')
-        except Exception as e:
-            logger.log('ERROR', e)
+        logger.log('TRACE', f'正在去除{table_name}表中的无效子域')
+        self.query(f'delete from "{table_name}" where '
+                   f'subdomain is null or valid == 0')

    def get_data(self, table_name):
        """
@@ -182,30 +192,24 @@ class Database(object):
        :param str table_name: 表名
        """
        table_name = table_name.replace('.', '_')
-        logger.log('DEBUG', f'获取{table_name}表中的所有数据')
-        try:
-            rows = self.conn.query(f'select * from "{table_name}"')
-        except Exception as e:
-            logger.log('ERROR', e)
-        else:
-            return rows
+        logger.log('TRACE', f'获取{table_name}表中的所有数据')
+        return self.query(f'select * from "{table_name}"')

-    def get_subdomain(self, table_name, valid):
+    def export_data(self, table_name, valid):
        """
-        获取表中的子域数据
+        获取表中的部分数据

        :param str table_name: 表名
-        :param int valid: 是否有效
+        :param any valid: 有效性
        """
        table_name = table_name.replace('.', '_')
-        logger.log('DEBUG', f'获取{table_name}表中的所有数据')
-        try:
-            rows = self.conn.query(
-                f'select * from "{table_name}" where valid = {valid}')
-        except Exception as e:
-            logger.log('ERROR', e)
-        else:
-            return rows
+        query = f'select id, url, subdomain, port, ips, status, reason,' \
+                f'valid, new, title, banner from "{table_name}"'
+        if valid == 0 or valid == 1:
+            where = f' where valid = {valid}'
+            query += where
+        logger.log('TRACE', f'获取{table_name}表中的所有数据')
+        return self.query(query)

    def close(self):
        self.conn.close()
@@ -15,7 +15,6 @@ from . import utils
 from .domain import Domain
 from common.database import Database

-
 lock = threading.Lock()


@@ -34,17 +33,18 @@ class Module(object):
        self.records = dict()  # 存放子域解析记录
        self.results = list()  # 存放模块结果
        self.start = time.time()  # 模块开始执行时间
-        self.end = None
+        self.end = None  # 模块结束执行时间
        self.elapsed = None  # 模块执行耗时

    def check(self, *apis):
        """
        简单检查是否配置了api信息
+
        :param apis: api信息元组
        :return: 检查结果
        """
        if not all(apis):
-            logger.log('ALERT', f'{self.source}模块API配置有误跳过执行')
+            logger.log('ALERT', f'{self.source}模块没有配置API跳过执行')
            return False
        return True

@@ -62,9 +62,37 @@ class Module(object):
        self.elapsed = round(self.end - self.start, 1)
        logger.log('DEBUG', f'结束执行{self.source}模块收集{self.domain}的子域')
        logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒发现子域'
-                   f'{len(self.subdomains)}个')
+                            f'{len(self.subdomains)}个')
        logger.log('DEBUG', f'{self.source}模块发现{self.domain}的子域\n'
-                   f'{self.subdomains}')
+                            f'{self.subdomains}')
+
+    def head(self, url, params=None, check=True, **kwargs):
+        """
+        自定义head请求
+
+        :param str url: 请求地址
+        :param dict params: 请求参数
+        :param bool check: 检查响应
+        :param kwargs: 其他参数
+        :return: requests响应对象
+        """
+        try:
+            resp = requests.head(url,
+                                 params=params,
+                                 cookies=self.cookie,
+                                 headers=self.header,
+                                 proxies=self.proxy,
+                                 timeout=self.timeout,
+                                 verify=self.verify,
+                                 **kwargs)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return None
+        if not check:
+            return resp
+        if utils.check_response('HEAD', resp):
+            return resp
+        return None

    def get(self, url, params=None, check=True, **kwargs):
        """
@@ -86,7 +114,7 @@ class Module(object):
                                verify=self.verify,
                                **kwargs)
        except Exception as e:
-            logger.log('ERROR', e)
+            logger.log('ERROR', e.args)
            return None
        if not check:
            return resp
@@ -114,11 +142,11 @@ class Module(object):
                                 verify=self.verify,
                                 **kwargs)
        except Exception as e:
-            logger.log('ERROR', e)
+            logger.log('ERROR', e.args)
            return None
        if not check:
            return resp
-        if utils.check_response('GET', resp):
+        if utils.check_response('POST', resp):
            return resp
        return None

@@ -142,16 +170,16 @@ class Module(object):
        :return: 代理字典
        """
        if not config.enable_proxy:
-            logger.log('DEBUG', f'所有模块不使用代理')
+            logger.log('TRACE', f'所有模块不使用代理')
            return self.proxy
        if config.proxy_all_module:
-            logger.log('DEBUG', f'{module}模块使用代理')
+            logger.log('TRACE', f'{module}模块使用代理')
            return utils.get_random_proxy()
        if module in config.proxy_partial_module:
-            logger.log('DEBUG', f'{module}模块使用代理')
+            logger.log('TRACE', f'{module}模块使用代理')
            return utils.get_random_proxy()
        else:
-            logger.log('DEBUG', f'{module}模块不使用代理')
+            logger.log('TRACE', f'{module}模块不使用代理')
            return self.proxy

    @staticmethod
@@ -165,7 +193,7 @@ class Module(object):
        :return: 匹配出的子域集合或列表
        :rtype: set or list
        """
-        logger.log('DEBUG', f'正则匹配响应体中的子域')
+        logger.log('TRACE', f'正则匹配响应体中的子域')
        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
                 r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
                 + domain.replace('.', r'\.')
@@ -192,26 +220,32 @@ class Module(object):
    def save_json(self):
        """
        将各模块结果保存为json文件
+
+        :return： 是否保存成功
        """
-        logger.log('DEBUG', f'将{self.source}模块发现的子域结果保存为json文件')
-        if config.save_module_result:
-            dpath = config.result_save_path.joinpath(self.domain, self.module)
-            dpath.mkdir(parents=True, exist_ok=True)
-            name = self.source + '.json'
-            path = dpath.joinpath(name)
-            with open(path, mode='w', encoding='utf-8') as file:
-                result = {'domain': self.domain,
-                          'name': self.module,
-                          'source': self.source,
-                          'elapsed': self.elapsed,
-                          'count': len(self.subdomains),
-                          'subdomains': list(self.subdomains),
-                          'records': self.records}
-                json.dump(result, file, ensure_ascii=False, indent=4)
+        if not config.save_module_result:
+            return False
+        logger.log('TRACE', f'将{self.source}模块发现的子域结果保存为json文件')
+        path = config.result_save_path.joinpath(self.domain, self.module)
+        path.mkdir(parents=True, exist_ok=True)
+        name = self.source + '.json'
+        path = path.joinpath(name)
+        with open(path, mode='w', encoding='utf-8', errors='ignore') as file:
+            result = {'domain': self.domain,
+                      'name': self.module,
+                      'source': self.source,
+                      'elapsed': self.elapsed,
+                      'count': len(self.subdomains),
+                      'subdomains': list(self.subdomains),
+                      'records': self.records}
+            json.dump(result, file, ensure_ascii=False, indent=4)
+        return True

    def gen_result(self):
-        results = list()
-        if not len(self.subdomains):  # 一个子域都没有发现的情况
+        """
+        生成结果
+        """
+        if not len(self.subdomains):  # 该模块一个子域都没有发现的情况
            result = {'id': None,
                      'url': None,
                      'subdomain': None,
@@ -220,14 +254,16 @@ class Module(object):
                      'status': None,
                      'reason': None,
                      'valid': None,
+                      'new': None,
                      'title': None,
                      'banner': None,
+                      'header': None,
+                      'response': None,
                      'module': self.module,
                      'source': self.source,
                      'elapsed': self.elapsed,
                      'count': 0}
-            results.append(result)
-            self.results = (self.source, results)
+            self.results.append(result)
        else:
            for subdomain in self.subdomains:
                url = 'http://' + subdomain
@@ -240,21 +276,25 @@ class Module(object):
                          'status': None,
                          'reason': None,
                          'valid': None,
+                          'new': None,
                          'title': None,
                          'banner': None,
                          'module': self.module,
+                          'header': None,
+                          'response': None,
                          'source': self.source,
                          'elapsed': self.elapsed,
                          'count': len(self.subdomains)}
-                results.append(result)
-            self.results = (self.source, results)
+                self.results.append(result)

    def save_db(self):
+        """
+        将模块结果存入数据库中
+
+        """
        lock.acquire()
        db = Database()
        db.create_table(self.domain)
-        source, results = self.results
-        # 将结果存入数据库中
-        db.save_db(self.domain, results, source)
+        db.save_db(self.domain, self.results, self.source)
        db.close()
        lock.release()
@@ -1,51 +1,59 @@
-# coding=utf-8
-
 import asyncio
 import functools

 import aiohttp
 import tqdm
 from aiohttp import ClientSession
-from aiohttp.resolver import AsyncResolver
 from bs4 import BeautifulSoup
 import config
 from common import utils
 from config import logger


+def get_limit_conn():
+    limit_open_conn = config.limit_open_conn
+    if limit_open_conn is None:  # 默认情况
+        limit_open_conn = utils.get_semaphore()
+    elif not isinstance(limit_open_conn, int):  # 如果传入不是数字的情况
+        limit_open_conn = utils.get_semaphore()
+    return limit_open_conn
+
+
 def get_ports(port):
-    logger.log('INFOR', f'正在获取请求端口范围')
+    logger.log('DEBUG', f'正在获取请求探测端口范围')
    ports = set()
-    if isinstance(port, set):
+    if isinstance(port, (set, list, tuple)):
        ports = port
-    elif isinstance(port, str):
-        if port not in {'small', 'medium', 'large', 'xlarge'}:
-            logger.log('ERROR', f'不存在{port}等端口范围')
-            port = 'medium'
+    elif isinstance(port, int):
+        if 0 <= port <= 65535:
+            ports = {port}
+    elif port in {'default', 'small', 'large'}:
+        logger.log('DEBUG', f'探测{port}等端口范围')
        ports = config.ports.get(port)
-        logger.log('INFOR', f'使用{port}等端口范围')
-    if not ports:  # 意外情况 ports_range为空使用使用中等端口范围
-        logger.log('ALERT', f'使用medium等端口范围')
-        ports = config.ports.get('medium')
-    return ports
+    if not ports:  # 意外情况
+        logger.log('ERROR', f'指定探测端口范围有误')
+        ports = {80}
+    logger.log('INFOR', f'探测端口范围：{ports}')
+    return set(ports)


 def gen_new_datas(datas, ports):
    logger.log('INFOR', f'正在生成请求地址')
    new_datas = []
-    protocols = ['http://']
    for data in datas:
        valid = data.get('valid')
        if valid is None:  # 子域有效性未知的才进行http请求探测
            subdomain = data.get('subdomain')
            for port in ports:
-                for protocol in protocols:
-                    if port == 443:
-                        url = f'https://{subdomain}:{port}'
-                    elif port == 8443:
-                        url = f'https://{subdomain}:{port}'
-                    else:
-                        url = f'{protocol}{subdomain}:{port}'
+                if str(port).endswith('443'):
+                    url = f'https://{subdomain}:{port}'
+                    data['id'] = None
+                    data['url'] = url
+                    data['port'] = port
+                    new_datas.append(data)
+                    data = dict(data)  # 需要生成一个新的字典对象
+                else:
+                    url = f'http://{subdomain}:{port}'
                    data['id'] = None
                    data['url'] = url
                    data['port'] = port
@@ -54,28 +62,49 @@ def gen_new_datas(datas, ports):
    return new_datas


-async def fetch(session, url, semaphore):
+async def fetch(session, url):
    """
    请求

    :param session: session对象
-    :param url: url地址
-    :param semaphore: 并发信号量
+    :param str url: url地址
    :return: 响应对象和响应文本
    """
-    timeout = aiohttp.ClientTimeout(total=config.get_timeout)
-    async with semaphore:
-        async with session.get(url,
-                               ssl=config.verify_ssl,
-                               allow_redirects=config.get_redirects,
-                               timeout=timeout,
-                               proxy=config.get_proxy) as resp:
+    method = config.request_method.upper()
+    timeout = aiohttp.ClientTimeout(total=None,
+                                    connect=None,
+                                    sock_read=config.sockread_timeout,
+                                    sock_connect=config.sockconn_timeout)
+    try:
+        if method == 'HEAD':
+            async with session.head(url,
+                                    ssl=config.verify_ssl,
+                                    allow_redirects=config.allow_redirects,
+                                    timeout=timeout,
+                                    proxy=config.aiohttp_proxy) as resp:
+                text = await resp.text()
+        else:
+            async with session.get(url,
+                                   ssl=config.verify_ssl,
+                                   allow_redirects=config.allow_redirects,
+                                   timeout=timeout,
+                                   proxy=config.aiohttp_proxy) as resp:

-            try:
-                text = await resp.text(encoding='gb2312')  # 先尝试用fb2312解码
-            except UnicodeDecodeError:
-                text = await resp.text(errors='ignore')
+                try:
+                    # 先尝试用utf-8解码
+                    text = await resp.text(encoding='utf-8', errors='strict')
+                except UnicodeError:
+                    try:
+                        # 再尝试用gb18030解码
+                        text = await resp.text(encoding='gb18030',
+                                               errors='strict')
+                    except UnicodeError:
+                        # 最后尝试自动解码
+                        text = await resp.text(encoding=None,
+                                               errors='ignore')
        return resp, text
+    except Exception as e:
+        return e


 def get_title(markup):
@@ -89,46 +118,47 @@ def get_title(markup):

    title = soup.title
    if title:
-        return title.text.strip()
+        return title.text

    h1 = soup.h1
    if h1:
-        return h1.text.strip()
+        return h1.text

    h2 = soup.h2
    if h2:
-        return h2.text.strip()
+        return h2.text
+
+    h3 = soup.h3
+    if h2:
+        return h3.text

    desc = soup.find('meta', attrs={'name': 'description'})
    if desc:
-        return desc['content'].strip()
+        return desc['content']

    word = soup.find('meta', attrs={'name': 'keywords'})
    if word:
-        return word['content'].strip()
-
-    if len(markup) <= 200:
-        return markup.strip()
+        return word['content']

    text = soup.text
    if len(text) <= 200:
-        return text.strip()
+        return text

-    return None
+    return 'None'


 def request_callback(future, index, datas):
-    try:
-        result = future.result()
-    except BaseException as e:
-        logger.log('DEBUG', e.args)
-        datas[index]['reason'] = str(e.args)
+    result = future.result()
+    if isinstance(result, BaseException):
+        logger.log('TRACE', result.args)
+        name = utils.get_classname(result)
+        datas[index]['reason'] = name + ' ' + str(result)
        datas[index]['valid'] = 0
-    else:
+    elif isinstance(result, tuple):
        resp, text = result
        datas[index]['reason'] = resp.reason
        datas[index]['status'] = resp.status
-        if resp.status >= 500:
+        if resp.status == 400 or resp.status >= 500:
            datas[index]['valid'] = 0
        else:
            datas[index]['valid'] = 1
@@ -136,52 +166,60 @@ def request_callback(future, index, datas):
            banner = str({'Server': headers.get('Server'),
                          'Via': headers.get('Via'),
                          'X-Powered-By': headers.get('X-Powered-By')})
-            datas[index]['banner'] = banner
-            datas[index]['title'] = get_title(text)
+            datas[index]['banner'] = banner[1:-1]
+            datas[index]['header'] = str(dict(headers))[1:-1]
+            if isinstance(text, str):
+                title = get_title(text).strip()
+                datas[index]['title'] = utils.remove_string(title)
+                datas[index]['response'] = utils.remove_string(text)


-async def bulk_get_request(datas, port):
-    ports = get_ports(port)
-    new_datas = gen_new_datas(datas, ports)
-    logger.log('INFOR', f'正在异步进行子域的GET请求')
-
-    limit_open_conn = config.limit_open_conn
-    if limit_open_conn is None:  # 默认情况
-        limit_open_conn = utils.get_semaphore()
-    elif not isinstance(limit_open_conn, int):  # 如果传入不是数字的情况
-        limit_open_conn = utils.get_semaphore()
-    # 使用异步域名解析器 自定义域名服务器
-    resolver = AsyncResolver(nameservers=config.resolver_nameservers)
-    conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
+def get_connector():
+    limit_open_conn = get_limit_conn()
+    return aiohttp.TCPConnector(ttl_dns_cache=300,
+                                ssl=config.verify_ssl,
                                limit=limit_open_conn,
-                                limit_per_host=config.limit_per_host,
-                                resolver=resolver)
+                                limit_per_host=config.limit_per_host)

-    semaphore = asyncio.Semaphore(limit_open_conn)
+
+def get_header():
    header = None
    if config.fake_header:
        header = utils.gen_fake_header()
-    async with ClientSession(connector=conn, headers=header) as session:
+    return header
+
+
+async def bulk_request(datas, port):
+    ports = get_ports(port)
+    new_datas = gen_new_datas(datas, ports)
+    method = config.request_method
+    logger.log('INFOR', f'使用{method}请求方法')
+    logger.log('INFOR', f'正在进行异步子域请求')
+    connector = get_connector()
+    header = get_header()
+    async with ClientSession(connector=connector, headers=header) as session:
        tasks = []
        for i, data in enumerate(new_datas):
            url = data.get('url')
-            task = asyncio.ensure_future(fetch(session, url, semaphore))
+            task = asyncio.ensure_future(fetch(session, url))
            task.add_done_callback(functools.partial(request_callback,
                                                     index=i,
                                                     datas=new_datas))
            tasks.append(task)
-        if tasks:  # 任务列表里有任务不空时才进行解析
+        # 任务列表里有任务不空时才进行解析
+        if tasks:
            # 等待所有task完成 错误聚合到结果列表里
            futures = asyncio.as_completed(tasks)
            for future in tqdm.tqdm(futures,
                                    total=len(tasks),
                                    desc='Progress',
-                                    smoothing=1.0,
-                                    ncols=True):
-                try:
-                    await future
-                except:
-                    pass
+                                    ncols=60):
+                await future

    logger.log('INFOR', f'完成异步进行子域的GET请求')
    return new_datas
+
+
+def run_bulk_query(datas, port):
+    new_datas = asyncio.run(bulk_request(datas, port))
+    return new_datas
@@ -1,13 +1,14 @@
-# coding=utf-8
+import signal
+import socket
 import asyncio
 import functools
+from multiprocessing import Manager

-import dns.resolver
-import aiodns
 import tqdm
+import aiomultiprocess as aiomp
+from dns.resolver import Resolver

 import config
-from common import utils
 from config import logger


@@ -15,110 +16,156 @@ def dns_resolver():
    """
    dns解析器
    """
-    resolver = dns.resolver.Resolver()
+    resolver = Resolver()
    resolver.nameservers = config.resolver_nameservers
    resolver.timeout = config.resolver_timeout
    resolver.lifetime = config.resolver_lifetime
    return resolver


-def dns_query_a(hostname):
-    """
-    查询A记录
-
-    :param str hostname: 主机名
-    :return: 查询结果
-    """
-    resolver = dns_resolver()
-    return resolver.query(hostname, 'A')
-
-
-def aiodns_resolver():
-    """
-    异步dns解析器
-    """
-    return aiodns.DNSResolver(nameservers=config.resolver_nameservers,
-                              timeout=config.resolver_timeout)
-
-
-async def aiodns_query_a(hostname, semaphore=None):
+async def aiodns_query_a(hostname):
    """
    异步查询A记录

    :param str hostname: 主机名
-    :param semaphore: 并发查询数量
-    :return: 主机名或查询结果或查询异常
-    """
-    if semaphore is None:
-        resolver = aiodns_resolver()
-        try:
-            answers = await resolver.query(hostname, 'A')
-        except BaseException as e:
-            logger.log('DEBUG', e.args)
-            answers = None
-        return hostname, answers
-    else:
-        async with semaphore:
-            resolver = aiodns_resolver()
-            try:
-                answers = await resolver.query(hostname, 'A')
-            except BaseException as e:
-                logger.log('DEBUG', e.args)
-                answers = None
-            return hostname, answers
-
-
-def resolve_callback(future, index, datas):
-    """
-    解析结果回调处理
-    :param future: future对象
-    :param index: 下标
-    :param datas: 结果集
+    :return: 查询结果
    """
    try:
-        result = future.result()
+        loop = asyncio.get_event_loop()
+        socket.setdefaulttimeout(20)
+        # answer = await loop.getaddrinfo(hostname, 80)
+        answer = await loop.run_in_executor(None,
+                                            socket.gethostbyname_ex,
+                                            hostname)
    except BaseException as e:
-        datas[index]['ips'] = str(e.args)
-        datas[index]['valid'] = 0
-    else:
-        if isinstance(result, tuple):
-            _, answers = result
-            if answers:
-                ips = {record.host for record in answers}
-                datas[index]['ips'] = str(ips)
-            else:
-                datas[index]['ips'] = 'No answers'
+        logger.log('TRACE', e.args)
+        answer = e
+    return hostname, answer


-async def bulk_query_a(datas):
+def convert_results(result_list):
    """
-    批量查询A记录
+    将结果列表类型转换为结果字典类型

-    :param datas: 待查的数据集
-    :return: 查询过得到的结果集
+    :param result_list: 待转换的结果列表
+    :return: 转换后的结果字典
    """
-    logger.log('INFOR', '正在异步查询子域的A记录')
-    tasks = []
-    semaphore = asyncio.Semaphore(config.limit_resolve_conn)
-    for i, data in enumerate(datas):
+    result_dict = {}
+    for result in result_list:
+        hostname, answer = result
+        value_dict = {'ips': None, 'reason': None, 'valid': None}
+        if isinstance(answer, tuple):
+            value_dict['ips'] = str(answer[2])[1:-1]
+            result_dict[hostname] = value_dict
+        elif isinstance(answer, Exception):
+            value_dict['reason'] = str(answer.args)
+            value_dict['valid'] = 0
+            result_dict[hostname] = value_dict
+        else:
+            value_dict['valid'] = 0
+            result_dict[hostname] = value_dict
+    return result_dict
+
+
+def filter_subdomain(data_list):
+    """
+    过滤出无IPS值的子域到新的子域列表
+
+    :param list data_list: 待过滤的数据列表
+    :return: 符合条件的子域列表
+    """
+    subdomains = []
+    for data in data_list:
        if not data.get('ips'):
            subdomain = data.get('subdomain')
-            task = asyncio.ensure_future(aiodns_query_a(subdomain, semaphore))
-            task.add_done_callback(functools.partial(resolve_callback,
-                                                     index=i,
-                                                     datas=datas))  # 回调
-            tasks.append(task)
-    if tasks:  # 任务列表里有任务不空时才进行解析
-        futures = asyncio.as_completed(tasks)
-        for future in tqdm.tqdm(futures,
-                                total=len(tasks),
-                                desc='Progress',
-                                smoothing=1.0,
-                                ncols=True):
-            try:
-                await future
-            except:
-                pass
-        # await asyncio.wait(tasks)  # 等待所有task完成
+            subdomains.append(subdomain)
+    return subdomains
+
+
+def update_data(data_list, results_dict):
+    """
+    更新解析结果
+
+    :param list data_list: 待更新的数据列表
+    :param dict results_dict: 解析结果字典
+    :return: 更新后的数据列表
+    """
+    for index, data in enumerate(data_list):
+        if not data.get('ips'):
+            subdomain = data.get('subdomain')
+            value_dict = results_dict.get(subdomain)
+            data.update(value_dict)
+            data_list[index] = data
+    return data_list
+
+
+def init_worker():
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+
+def query_progress(pr_queue, total):
+    bar = tqdm.tqdm()
+    bar.total = total
+    bar.desc = 'Resolve Progress'
+    bar.ncols = 80
+    bar.smoothing = 0
+    while True:
+        done = pr_queue.qsize()
+        bar.n = done
+        bar.update()
+        if done == total:
+            break
+    bar.close()
+
+
+async def aio_query(pr_queue, hostname):
+    """
+    异步查询主机名的A记录
+
+    :param pr_queue: 进度队列
+    :param str hostname: 主机名
+    :return: 查询结果
+    """
+    results = await aiodns_query_a(hostname)
+    pr_queue.put(1)
+    return results
+
+
+async def aio_resolve(subdomain_list, process_num, coroutine_num):
+    """
+    异步解析子域A记录
+
+    :param list subdomain_list: 待解析的子域列表
+    :param int process_num: 解析进程数
+    :param int coroutine_num: 每个解析进程下的协程数
+    :return: 解析结果
+    """
+    m = Manager()
+    pr_queue = m.Queue()
+    loop = asyncio.get_event_loop()
+    loop.run_in_executor(None, query_progress, pr_queue, len(subdomain_list))
+    wrapped_query = functools.partial(aio_query, pr_queue)
+    async with aiomp.Pool(processes=process_num,
+                          initializer=init_worker,
+                          childconcurrency=coroutine_num) as pool:
+        results = await pool.map(wrapped_query, subdomain_list)
+        return results
+
+
+async def bulk_resolve(data_list):
+    """
+    批量解析A记录并返回解析结果
+
+    :param list data_list: 待查的数据列表
+    :return: 查询过得到的结果列表
+    """
+    logger.log('INFOR', '正在异步查询子域的A记录')
+    # semaphore = asyncio.Semaphore(config.limit_resolve_conn)
+    query_subdomains = filter_subdomain(data_list)
+    process_num = config.brute_process_num
+    coroutine_num = config.brute_coroutine_num
+    results = await aio_resolve(query_subdomains, process_num, coroutine_num)
+    results_dict = convert_results(results)
+    data_list = update_data(data_list, results_dict)
    logger.log('INFOR', '完成异步查询子域的A记录')
-    return datas
+    return data_list
@@ -47,7 +47,10 @@ class Search(Module):
        :return: 匹配的子域
        :rtype set
        """
-        resp = requests.head(url, headers=self.header, proxies=self.proxy,
-                             timeout=self.timeout, allow_redirects=False)
+        resp = self.head(url, check=False, allow_redirects=False)
+        if not resp:
+            return set()
        location = resp.headers.get('location')
+        if not location:
+            return set()
        return set(utils.match_subdomain(domain, location))
@@ -1,10 +1,13 @@
 # coding=utf-8
 import re
+import sys
+import time
 import random
 import ipaddress
 import platform
 import config
 from pathlib import Path
+from records import Record, RecordCollection
 from common.domain import Domain
 from config import logger

@@ -118,7 +121,7 @@ def get_domains(target):
    elif isinstance(target, str):
        path = Path(target)
        if path.is_file():
-            with open(target) as file:
+            with open(target, encoding='utf-8', errors='ignore') as file:
                for line in file:
                    domain = Domain(line.strip()).match()
                    if domain:
@@ -137,63 +140,101 @@ def get_semaphore():
    """
    system = platform.system()
    if system == 'Windows':
-        return 300
+        return 800
    elif system == 'Linux':
        return 800
    elif system == 'Darwin':
        return 800


-def check_dpath(dpath):
+def check_path(path, name, format):
    """
-    检查目录路径
+    检查结果输出目录路径

-    :param dpath: 传入的目录路径
-    :return: 目录路径
+    :param path: 保存路径
+    :param name: 导出名字
+    :param format: 保存格式
+    :return: 保存路径
    """
-    if isinstance(dpath, str):
-        dpath = Path(dpath)
+    filename = f'{name}.{format}'
+    default_path = config.result_save_path.joinpath(filename)
+    if path is None:
+        path = default_path
+    try:
+        path = Path(path)
+    except Exception as e:
+        logger.log('ERROR', e.args)
+        path = default_path
    else:
-        dpath = config.result_save_path
-    if not dpath.is_dir():
-        logger.log('FATAL', f'{dpath}不是目录')
-    if not dpath.exists():
-        logger.log('ALERT', f'不存在{dpath}将会新建此目录')
-        dpath.mkdir(parents=True, exist_ok=True)
-    return dpath
+        if not path.exists():
+            logger.log('ALERT', f'不存在{path}目录将会新建')
+            path.mkdir(parents=True, exist_ok=True)
+            path = path.joinpath(filename)
+        if path.is_dir():
+            path = path.joinpath(filename)
+        if path.exists():
+            logger.log('ALERT', f'存在{path}文件将会覆盖')
+    # 意外情况
+    if not path:
+        path = default_path
+    logger.log('DEBUG', f'结果保存路径{path}')
+    return path


-def check_format(format):
+def check_format(format, count):
    """
    检查导出格式

    :param format: 传入的导出格式
+    :param count: 数量
    :return: 导出格式
    """
    formats = ['txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
               'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods']
+    if format == 'xls' and count > 65000:
+        logger.log('ALERT', 'xls文件限制为最多65000行')
+        logger.log('ALERT', '使用xlsx格式导出')
+        return 'xlsx'
    if format in formats:
        return format
    else:
        logger.log('ALERT', f'不支持{format}格式导出')
        logger.log('ALERT', '默认使用csv格式导出')
-        return 'xls'
+        return 'csv'


-def save_data(fpath, data):
+def save_data(path, data):
+    """
+    保存结果数据到文件
+
+    :param path: 保存路径
+    :param data: 待存数据
+    :return: 保存成功与否
+    """
    try:
-        with open(fpath, 'w', encoding="utf-8", newline='') as file:
+        with open(path, 'w', encoding="utf-8",
+                  errors='ignore', newline='') as file:
            file.write(data)
-            logger.log('ALERT', fpath)
+            logger.log('ALERT', f'结果输出{path}')
+            return True
    except TypeError:
-        with open(fpath, 'wb') as file:
+        with open(path, 'wb') as file:
            file.write(data)
-            logger.log('ALERT', fpath)
+            logger.log('ALERT', f'结果输出{path}')
+            return True
    except Exception as e:
-        logger.log('ERROR', e)
+        logger.log('ERROR', e.args)
+        return False


 def check_response(method, resp):
+    """
+    检查响应 输出非正常响应返回json的信息
+
+    :param method: 请求方法
+    :param resp: 响应体
+    :return: 是否正常响应
+    """
    if resp.status_code == 200 and resp.content:
        return True
    logger.log('ALERT', f'{method} {resp.url} {resp.status_code} - '
@@ -207,3 +248,90 @@ def check_response(method, resp):
        else:
            logger.log('ALERT', msg)
    return False
+
+
+def mark_subdomain(old_data, new_data):
+    """
+    标记新增子域并返回新的数据集
+
+    :param old_data: 之前数据集
+    :param new_data: 现在数据集
+    :return: 已标记的新的数据集
+    """
+    # 第一次收集子域的情况
+    if not old_data:
+        for index, item in enumerate(new_data):
+            item['new'] = 1
+            new_data[index] = item
+        return new_data
+    # 非第一次收集子域的情况
+    old_subdomains = {item.get('subdomain') for item in old_data}
+    for index, item in enumerate(new_data):
+        subdomain = item.get('subdomain')
+        if subdomain in old_subdomains:
+            item['new'] = 0
+        else:
+            item['new'] = 1
+        new_data[index] = item
+    return new_data
+
+
+def remove_string(string):
+    # Excel文件中单元格值不能直接存储以下非法字符
+    return re.sub(r'[\000-\010]|[\013-\014]|[\016-\037]', r'', string)
+
+
+def check_value(values):
+    for i, value in enumerate(values):
+        if value is None:
+            continue
+        if isinstance(value, str) and len(value) > 32767:
+            # Excel文件中单元格值长度不能超过32767
+            values[i] = value[:32767]
+    return values
+
+
+def export_all(format, path, datas):
+    """
+    将所有结果数据导出到一个文件
+
+    :param str format: 导出文件格式
+    :param str path: 导出文件路径
+    :param list datas: 待导出的结果数据
+    """
+    format = check_format(format, len(datas))
+    timestamp = get_timestamp()
+    name = f'all_subdomain_result_{timestamp}'
+    path = check_path(path, name, format)
+    row_list = list()
+    for row in datas:
+        row.pop('header')
+        row.pop('response')
+        row.pop('module')
+        row.pop('source')
+        row.pop('elapsed')
+        row.pop('count')
+        keys = row.keys()
+        values = row.values()
+        if format in {'xls', 'xlsx'}:
+            values = check_value(values)
+        row_list.append(Record(keys, values))
+    rows = RecordCollection(iter(row_list))
+    content = rows.export(format)
+    save_data(path, content)
+
+
+def get_timestamp():
+    return int(time.time())
+
+
+def get_classname(clsobj):
+    return clsobj.__class__.__name__
+
+
+def python_version():
+    return sys.version
+
+
+def count_valid(data):
+    return len(list(filter(lambda item: item.get('valid') == 1, data)))
@@ -1,222 +1,160 @@
-# coding=utf-8
-"""
-OneForAll配置
-"""
-import os
-import sys
-import pathlib
-
-import requests
-from loguru import logger
-
-# 路径设置
-oneforall_relpath = pathlib.Path(__file__).parent  # oneforall代码相对路径
-oneforall_abspath = oneforall_relpath.resolve()  # oneforall代码绝对路径
-oneforall_module_path = oneforall_relpath.joinpath('modules')  # oneforall模块目录
-data_storage_path = oneforall_relpath.joinpath('data')  # 数据存放目录
-result_save_path = oneforall_relpath.joinpath('results')  # 结果保存目录
-
-# 模块设置
-save_module_result = True  # 保存模块中各脚本结果(默认True)
-enable_all_module = True  # 启用所有模块(默认True)
-enable_partial_module = []  # 启用部分模块 必须禁用enable_all_module才能生效
-# 只使用ask和baidu搜索引擎收集子域
-# enable_partial_module = [('modules.search', 'ask')
-#                          ('modules.search', 'baidu')]
-
-
-# 爆破模块设置
-enable_brute_module = False  # 使用爆破模块(默认禁用)
-enable_verify_subdomain = True  # 验证子域有效性(默认True)
-enable_wildcard_check = True  # 开启泛解析检测 会去掉泛解析的子域
-# 爆破时使用的进程数(根据系统中CPU数量情况设置 不宜大于CPU数量 默认为系统中的CPU数量)
-brute_process_num = os.cpu_count()
-brute_coroutine_num = 128  # 爆破时每个进程下的协程数(不宜大于1000)
-# 爆破所使用的字典路径 默认data/subdomains.txt
-brute_wordlist_path = data_storage_path.joinpath('subnames.txt')
-brute_task_segment = 500
-# 参数segment的设置受CPU性能，网络带宽，运营商限制等限制，默认500个子域为一任务组，
-# 当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
-# 十万字典建议设置为5000，百万字典设置为50000
-enable_recursive_brute = False  # 是否使用递归爆破(默认禁用)
-brute_recursive_depth = 2  # 递归爆破深度(默认2层)
-# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt
-recursive_namelist_path = data_storage_path.joinpath('next_subnames.txt')
-enable_fuzz = False  # 是否使用fuzz模式枚举域名
-fuzz_rule = ''  # fuzz域名的正则 示例：[a-z][0-9] 第一位是字母 第二位是数字
-ips_appear_maximum = 10  # 同一IP集合出现次数超过10认为是泛解析
-
-# 代理设置
-enable_proxy = False  # 是否使用代理(全局开关)
-proxy_all_module = False  # 代理所有模块
-proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
-                        'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
-                        'YandexSearch', 'CrossDomainXml',
-                        'ContentSecurityPolicy']  # 代理自定义的模块
-proxy_pool = [{'http': 'http://127.0.0.1:1080',
-               'https': 'https://127.0.0.1:1080'}]  # 代理池
-# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
-#                'https': 'socks5h://127.0.0.1:10808'}]  # 代理池
-
-
-# 网络请求设置
-enable_fake_header = True  # 启用伪造请求头
-request_delay = 1  # 请求时延
-request_timeout = 30  # 请求超时
-request_verify = True  # 请求SSL验证
-requests.packages.urllib3.disable_warnings()  # 禁用安全警告信息
-
-# 搜索模块设置
-enable_recursive_search = False  # 递归搜索子域
-search_recursive_times = 2  # 递归搜索层数
-
-# DNS解析设置
-resolver_nameservers = [
-    '119.29.29.29', '182.254.116.116',  # DNSPod
-    '180.76.76.76',  # Baidu DNS
-    '223.5.5.5', '223.6.6.6',  # AliDNS
-    '114.114.114.114', '114.114.115.115'  # 114DNS
-    # '8.8.8.8', '8.8.4.4',  # Google DNS
-    # '1.0.0.1', '1.1.1.1'  # CloudFlare DNS
-    # '208.67.222.222', '208.67.220.220'  # OpenDNS
-]  # 指定查询的DNS域名服务器
-resolver_timeout = 5.0  # 解析超时时间
-resolver_lifetime = 30.0  # 解析存活时间
-limit_resolve_conn = 50  # 限制同一时间解析的数量(默认50)
-
-# http探测设置
-small_ports = {80, 443}
-medium_ports = {80, 443, 8000, 8080, 8443}  # 默认使用
-large_ports = {80, 81, 443, 591, 2082, 2087, 2095, 2096, 3000, 8000, 8001,
-               8008, 8080, 8083, 8443, 8834, 8888}
-xlarge_ports = {80, 81, 300, 443, 591, 593, 832, 981, 1010, 1311, 2082,
-                2087, 2095, 2096, 2480, 3000, 3128, 3333, 4243, 4567, 4711,
-                4712, 4993, 5000, 5104, 5108, 5800, 6543, 7000, 7396, 7474,
-                8000, 8001, 8008, 8014, 8042, 8069, 8080, 8081, 8088, 8090,
-                8091, 8016, 8118, 8123, 8172, 8222, 8243, 8280, 8281, 8333,
-                8443, 8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080,
-                9090, 9091, 9200, 9443, 9800, 9981, 12443, 16080, 18091, 18092,
-                20720, 28017}
-ports = {'small': small_ports, 'medium': medium_ports,
-         'large': large_ports, 'xlarge': xlarge_ports}
-verify_ssl = False
-# aiohttp 支持 HTTP/HTTPS形式的代理
-get_proxy = None  # proxy="http://user:pass@some.proxy.com"
-get_timeout = 120  # http请求探测总超时时间 None或者0则表示不检测超时
-get_redirects = True  # 允许请求跳转
-fake_header = True  # 使用伪造请求头
-# 限制同一时间打开的连接数(默认None，根据系统不同设置，Windows系统400 其他系统800)
-limit_open_conn = None
-# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
-limit_per_host = 0  # 默认0表示不限制
-
-
-# 模块API配置
-# Censys可以免费注册获取API：https://censys.io/api
-censys_api_id = ''
-censys_api_secret = ''
-
-# Binaryedge可以免费注册获取API：https://app.binaryedge.io/account/api
-# 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
-binaryedge_api = ''
-
-# Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
-chinaz_api = ''
-
-# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/
-# cognitive-services/bing-web-search-api/#web-json
-bing_api_id = ''
-bing_api_key = ''
-
-# SecurityTrails可以免费注册获取API：https://securitytrails.com/corp/api
-securitytrails_api = ''
-
-# https://fofa.so/api
-fofa_api_email = ''  # fofa用户邮箱
-fofa_api_key = ''  # fofa用户key
-
-# Google可以免费注册获取API:
-# https://developers.google.com/custom-search/v1/overview
-# 免费的API只能查询前100条结果
-google_api_key = ''  # Google API搜索key
-google_api_cx = ''  # Google API搜索cx
-
-# https://api.passivetotal.org/api/docs/
-riskiq_api_username = ''
-riskiq_api_key = ''
-
-# Shodan可以免费注册获取API: https://account.shodan.io/register
-# 免费的API限速1秒查询1次
-shodan_api_key = ''
-# ThreatBook API 查询子域名需要收费 https://x.threatbook.cn/nodev4/vb4/myAPI
-threatbook_api_key = ''
-
-# VirusTotal可以免费注册获取API: https://developers.virustotal.com/reference
-virustotal_api_key = ''
-
-# https://www.zoomeye.org/doc?channel=api
-zoomeye_api_username = ''
-zoomeye_api_password = ''
-
-# Spyse可以免费注册获取API: https://spyse.com/
-spyse_api_token = ''
-
-# https://www.circl.lu/services/passive-dns/
-circl_api_username = ''
-circl_api_password = ''
-
-# https://www.dnsdb.info/
-dnsdb_api_key = ''
-
-# ipv4info可以免费注册获取API: http://ipv4info.com/tools/api/
-# 免费的API有效期只有2天，到期之后可以再次生成，每天可以查询50次。
-ipv4info_api_key = ''
-
-# https://github.com/360netlab/flint
-# passivedns_api_addr默认空使用http://api.passivedns.cn
-# passivedns_api_token可为空
-passivedns_api_addr = ''
-passivedns_api_token = ''
-
-# Github Token可以访问https://github.com/settings/tokens生成,user为Github用户名
-github_api_user = ''
-github_api_token = ''
-# github子域收集模块使用
-github_email = ''
-github_password = ''
-
-subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
-                     'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
-                     'help', 'live', 'mall', 'blogs', 'files', 'forum',
-                     'store', 'mobile'}
-
-# 日志配置
-# 终端日志输出格式
-stdout_fmt = '<cyan>{time:HH:mm:ss,SSS}</cyan> ' \
-          '[<level>{level: <5}</level>] ' \
-          '<blue>{module}</blue>:<cyan>{line}</cyan> - ' \
-          '<level>{message}</level>'
-# 日志文件记录格式
-logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
-          '[<level>{level: <5}</level>] ' \
-          '<cyan>{process.name}({process.id})</cyan>:' \
-          '<cyan>{thread.name: <10}({thread.id: <5})</cyan> | ' \
-          '<blue>{module}</blue>.<blue>{function}</blue>:' \
-          '<blue>{line}</blue> - <level>{message}</level>'
-
-log_path = result_save_path.joinpath('oneforall.log')
-
-logger.remove()
-logger.level(name='TRACE', no=5, color='<cyan><bold>', icon='✏️')
-logger.level(name='DEBUG', no=10, color='<blue><bold>', icon='🐞 ')
-logger.level(name='INFOR', no=20, color='<green><bold>', icon='ℹ️')
-logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️')
-logger.level(name='ERROR', no=40, color='<red><bold>', icon='❌️')
-logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')
-
-if not os.environ.get('PYTHONIOENCODING'):  # 设置编码
-    os.environ['PYTHONIOENCODING'] = 'utf-8'
-
-logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True)
-logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True,
-           encoding='utf-8')
+# coding=utf-8
+"""
+OneForAll配置
+"""
+import os
+import sys
+import pathlib
+
+import urllib3
+from loguru import logger
+
+# 路径设置
+oneforall_relpath = pathlib.Path(__file__).parent  # oneforall代码相对路径
+oneforall_abspath = oneforall_relpath.resolve()  # oneforall代码绝对路径
+oneforall_module_path = oneforall_relpath.joinpath('modules')  # oneforall模块目录
+data_storage_path = oneforall_relpath.joinpath('data')  # 数据存放目录
+result_save_path = oneforall_relpath.joinpath('results')  # 结果保存目录
+
+# 模块设置
+save_module_result = False  # 保存各模块发现结果为json文件(默认False)
+enable_all_module = True  # 启用所有模块(默认True)
+enable_partial_module = []  # 启用部分模块 必须禁用enable_all_module才能生效
+# 只使用ask和baidu搜索引擎收集子域
+# enable_partial_module = [('modules.search', 'ask')
+#                          ('modules.search', 'baidu')]
+module_thread_timeout = 360.0  # 每个收集模块线程超时时间(默认6分钟)
+
+# 爆破模块设置
+enable_brute_module = False  # 使用爆破模块(默认False)
+enable_dns_resolve = True  # DNS解析子域(默认True)
+enable_http_request = True  # HTTP请求子域(默认True)
+enable_wildcard_check = True  # 开启泛解析检测(默认True)
+enable_wildcard_deal = True  # 开启泛解析处理(默认True)
+# 爆破时使用的进程数(根据系统中CPU数量情况设置 不宜大于CPU数量 默认为系统中的CPU数量)
+brute_process_num = os.cpu_count()
+brute_coroutine_num = 1024  # 爆破时每个进程下的协程数
+# 爆破所使用的字典路径 默认data/subdomains.txt
+brute_wordlist_path = data_storage_path.joinpath('subnames.txt')
+enable_recursive_brute = False  # 是否使用递归爆破(默认禁用)
+brute_recursive_depth = 2  # 递归爆破深度(默认2层)
+# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt
+recursive_namelist_path = data_storage_path.joinpath('next_subnames.txt')
+enable_fuzz = False  # 是否使用fuzz模式枚举域名
+fuzz_rule = ''  # fuzz域名的正则 示例：[a-z][0-9] 第一位是字母 第二位是数字
+ips_appear_maximum = 10  # 同一IP集合出现次数超过10认为是泛解析
+
+# 代理设置
+enable_proxy = False  # 是否使用代理(全局开关)
+proxy_all_module = False  # 代理所有模块
+proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
+                        'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
+                        'YandexSearch', 'CrossDomainXml',
+                        'ContentSecurityPolicy']  # 代理自定义的模块
+proxy_pool = [{'http': 'http://127.0.0.1:1080',
+               'https': 'https://127.0.0.1:1080'}]  # 代理池
+# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
+#                'https': 'socks5h://127.0.0.1:10808'}]  # 代理池
+
+
+# 网络请求设置
+enable_fake_header = True  # 启用伪造请求头
+request_delay = 1  # 请求时延
+request_timeout = 30  # 请求超时
+request_verify = False  # 请求SSL验证
+# 禁用安全警告信息
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+# 搜索模块设置
+enable_recursive_search = False  # 递归搜索子域
+search_recursive_times = 2  # 递归搜索层数
+
+# DNS解析设置
+resolver_nameservers = [
+    '119.29.29.29', '182.254.116.116',  # DNSPod
+    '180.76.76.76',  # Baidu DNS
+    '223.5.5.5', '223.6.6.6',  # AliDNS
+    '114.114.114.114', '114.114.115.115'  # 114DNS
+    # '8.8.8.8', '8.8.4.4',  # Google DNS
+    # '1.0.0.1', '1.1.1.1'  # CloudFlare DNS
+    # '208.67.222.222', '208.67.220.220'  # OpenDNS
+]  # 指定查询的DNS域名服务器
+resolver_timeout = 5.0  # 解析超时时间
+resolver_lifetime = 30.0  # 解析存活时间
+limit_resolve_conn = 500  # 限制同一时间解析的数量(默认500)
+
+# 请求端口探测设置
+# 你可以在端口列表添加自定义端口
+default_ports = [80]  # 默认使用
+small_ports = [80, 443, 8000, 8080, 8443]
+# 注意：建议大厂的域名尽量不使用大端口范围，因为大厂的子域太多，加上使用大端口范围会导致生成的
+# 请求上十万，百万，千万级，可能会导致内存不足程序奔溃，另外这样级别的请求量等待时间也是漫长的。
+# OneForAll不是一个端口扫描工具，如果要扫端口建议使用nmap,zmap之类的工具。
+large_ports = [80, 81, 280, 300, 443, 591, 593, 832, 888, 901, 981, 1010, 1080,
+               1100, 1241, 1311, 1352, 1434, 1521, 1527, 1582, 1583, 1944, 2082,
+               2082, 2086, 2087, 2095, 2096, 2222, 2301, 2480, 3000, 3128, 3333,
+               4000, 4001, 4002, 4100, 4125, 4243, 4443, 4444, 4567, 4711, 4712,
+               4848, 4849, 4993, 5000, 5104, 5108, 5432, 5555, 5800, 5801, 5802,
+               5984, 5985, 5986, 6082, 6225, 6346, 6347, 6443, 6480, 6543, 6789,
+               7000, 7001, 7002, 7396, 7474, 7674, 7675, 7777, 7778, 8000, 8001,
+               8002, 8003, 8004, 8005, 8006, 8008, 8009, 8010, 8014, 8042, 8069,
+               8075, 8080, 8081, 8083, 8088, 8090, 8091, 8092, 8093, 8016, 8118,
+               8123, 8172, 8181, 8200, 8222, 8243, 8280, 8281, 8333, 8384, 8403,
+               8443, 8500, 8530, 8531, 8800, 8806, 8834, 8880, 8887, 8888, 8910,
+               8983, 8989, 8990, 8991, 9000, 9043, 9060, 9080, 9090, 9091, 9200,
+               9294, 9295, 9443, 9444, 9800, 9981, 9988, 9990, 9999, 10000,
+               10880, 11371, 12043, 12046, 12443, 15672, 16225, 16080, 18091,
+               18092, 20000, 20720, 24465, 28017, 28080, 30821, 43110, 61600]
+ports = {'default': default_ports, 'small': small_ports, 'large': large_ports}
+
+# aiohttp有关配置
+verify_ssl = False
+# aiohttp 支持 HTTP/HTTPS形式的代理
+aiohttp_proxy = None  # proxy="http://user:pass@some.proxy.com"
+allow_redirects = True  # 允许请求跳转
+fake_header = True  # 使用伪造请求头
+# 为了保证请求质量 请谨慎更改以下设置
+# request_method只能是HEAD或GET,HEAD请求方法更快，但是不能获取响应体并提取从中提取
+request_method = 'GET'  # 使用请求方法，默认GET
+sockread_timeout = 5  # 每个请求socket读取超时时间，默认5秒
+sockconn_timeout = 5  # 每个请求socket连接超时时间，默认5秒
+# 限制同一时间打开的连接总数
+limit_open_conn = 100  # 默认100
+# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
+limit_per_host = 10  # 0表示不限制,默认10
+
+subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
+                     'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
+                     'help', 'live', 'mall', 'blogs', 'files', 'forum',
+                     'store', 'mobile'}
+
+# 日志配置
+# 终端日志输出格式
+stdout_fmt = '<cyan>{time:HH:mm:ss,SSS}</cyan> ' \
+             '[<level>{level: <5}</level>] ' \
+             '<blue>{module}</blue>:<cyan>{line}</cyan> - ' \
+             '<level>{message}</level>'
+# 日志文件记录格式
+logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
+              '[<level>{level: <5}</level>] ' \
+              '<cyan>{process.name}({process.id})</cyan>:' \
+              '<cyan>{thread.name: <18}({thread.id: <5})</cyan> | ' \
+              '<blue>{module}</blue>.<blue>{function}</blue>:' \
+              '<blue>{line}</blue> - <level>{message}</level>'
+
+log_path = result_save_path.joinpath('oneforall.log')
+
+logger.remove()
+logger.level(name='TRACE', no=5, color='<cyan><bold>', icon='✏️')
+logger.level(name='DEBUG', no=10, color='<blue><bold>', icon='🐞 ')
+logger.level(name='INFOR', no=20, color='<green><bold>', icon='ℹ️')
+logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️')
+logger.level(name='ERROR', no=40, color='<red><bold>', icon='❌️')
+logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')
+
+if not os.environ.get('PYTHONIOENCODING'):  # 设置编码
+    os.environ['PYTHONIOENCODING'] = 'utf-8'
+
+logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True)
+logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True,
+           encoding='utf-8')
@@ -1,102 +1,102 @@
-[
-"_afpovertcp._tcp.",
-"_aix._tcp.",
-"_autodiscover._tcp.",
-"_caldav._tcp.",
-"_certificates._tcp.",
-"_client._smtp.",
-"_cmp._tcp.",
-"_crls._tcp.",
-"_crl._tcp.",
-"_finger._tcp.",
-"_ftp._tcp.",
-"_gc._tcp.",
-"_h323be._tcp.",
-"_h323be._udp.",
-"_h323cs._tcp.",
-"_h323cs._udp.",
-"_h323ls._tcp.",
-"_h323ls._udp.",
-"_h323rs._tcp.",
-"_hkps._tcp.",
-"_hkp._tcp.",
-"_http._tcp.",
-"_iax.udp.",
-"_imaps._tcp.",
-"_imap._tcp.",
-"_jabber-client._tcp.",
-"_jabber-client._udp.",
-"_jabber._tcp.",
-"_jabber._udp.",
-"_kerberos-adm._tcp.",
-"_kerberos._tcp.",
-"_kerberos._tcp.dc._msdcs.",
-"_kerberos._udp.",
-"_kpasswd._tcp.",
-"_kpasswd._udp.",
-"_ldap._tcp.",
-"_ldap._tcp.dc._msdcs.",
-"_ldap._tcp.gc._msdcs.",
-"_ldap._tcp.pdc._msdcs.",
-"_msdcs.",
-"_mysqlsrv._tcp.",
-"_nntp._tcp.",
-"_ntp._udp.",
-"_ocsp._tcp.",
-"_pgpkeys._tcp.",
-"_pgprevokations._tcp.",
-"_PKIXREP._tcp.",
-"_pop3s._tcp.",
-"_pop3._tcp.",
-"_sipfederationtls._tcp.",
-"_sipinternal._tcp.",
-"_sipinternaltls._tcp.",
-"_sips._tcp.",
-"_sip._tcp.",
-"_sip._tls.",
-"_sip._udp.",
-"_smtp._tcp.",
-"_ssh._tcp.",
-"_stun._tcp.",
-"_stun._udp.",
-"_svcp._tcp.",
-"_tcp.",
-"_telnet._tcp.",
-"_test._tcp.",
-"_tls.",
-"_udp.",
-"_vlmcs._tcp.",
-"_vlmcs._udp.",
-"_whois._tcp.",
-"_wpad._tcp.",
-"_xmpp-client._tcp.",
-"_xmpp-client._udp.",
-"_xmpp-server._tcp.",
-"_xmpp-server._udp.",
-"_https._tcp.",
-"_imap.tcp.",
-"_kerberos.tcp.dc._msdcs.",
-"_ldap._tcp.ForestDNSZones.",
-"_submission._tcp.",
-"_caldavs._tcp.",
-"_carddav._tcp.",
-"_carddavs._tcp.",
-"_x-puppet._tcp.",
-"_x-puppet-ca._tcp.",
-"_domainkey.",
-"_pkixrep._tcp.",
-"_cisco-phone-http.",
-"_cisco-phone-tftp.",
-"_cisco-uds._tcp.",
-"_ciscowtp._tcp.",
-"_collab-edge._tls.",
-"_cuplogin._tcp.",
-"_client._smtp._tcp.",
-"_sftp._tcp.",
-"_h323rs._udp.",
-"_sql._tcp.",
-"_sip._tcp.internal.",
-"_snmp._udp.",
-"_rdp._tcp.",
-"_xmpp-server._udp."
-]
+[
+"_afpovertcp._tcp.",
+"_aix._tcp.",
+"_autodiscover._tcp.",
+"_caldav._tcp.",
+"_certificates._tcp.",
+"_client._smtp.",
+"_cmp._tcp.",
+"_crls._tcp.",
+"_crl._tcp.",
+"_finger._tcp.",
+"_ftp._tcp.",
+"_gc._tcp.",
+"_h323be._tcp.",
+"_h323be._udp.",
+"_h323cs._tcp.",
+"_h323cs._udp.",
+"_h323ls._tcp.",
+"_h323ls._udp.",
+"_h323rs._tcp.",
+"_hkps._tcp.",
+"_hkp._tcp.",
+"_http._tcp.",
+"_iax.udp.",
+"_imaps._tcp.",
+"_imap._tcp.",
+"_jabber-client._tcp.",
+"_jabber-client._udp.",
+"_jabber._tcp.",
+"_jabber._udp.",
+"_kerberos-adm._tcp.",
+"_kerberos._tcp.",
+"_kerberos._tcp.dc._msdcs.",
+"_kerberos._udp.",
+"_kpasswd._tcp.",
+"_kpasswd._udp.",
+"_ldap._tcp.",
+"_ldap._tcp.dc._msdcs.",
+"_ldap._tcp.gc._msdcs.",
+"_ldap._tcp.pdc._msdcs.",
+"_msdcs.",
+"_mysqlsrv._tcp.",
+"_nntp._tcp.",
+"_ntp._udp.",
+"_ocsp._tcp.",
+"_pgpkeys._tcp.",
+"_pgprevokations._tcp.",
+"_PKIXREP._tcp.",
+"_pop3s._tcp.",
+"_pop3._tcp.",
+"_sipfederationtls._tcp.",
+"_sipinternal._tcp.",
+"_sipinternaltls._tcp.",
+"_sips._tcp.",
+"_sip._tcp.",
+"_sip._tls.",
+"_sip._udp.",
+"_smtp._tcp.",
+"_ssh._tcp.",
+"_stun._tcp.",
+"_stun._udp.",
+"_svcp._tcp.",
+"_tcp.",
+"_telnet._tcp.",
+"_test._tcp.",
+"_tls.",
+"_udp.",
+"_vlmcs._tcp.",
+"_vlmcs._udp.",
+"_whois._tcp.",
+"_wpad._tcp.",
+"_xmpp-client._tcp.",
+"_xmpp-client._udp.",
+"_xmpp-server._tcp.",
+"_xmpp-server._udp.",
+"_https._tcp.",
+"_imap.tcp.",
+"_kerberos.tcp.dc._msdcs.",
+"_ldap._tcp.ForestDNSZones.",
+"_submission._tcp.",
+"_caldavs._tcp.",
+"_carddav._tcp.",
+"_carddavs._tcp.",
+"_x-puppet._tcp.",
+"_x-puppet-ca._tcp.",
+"_domainkey.",
+"_pkixrep._tcp.",
+"_cisco-phone-http.",
+"_cisco-phone-tftp.",
+"_cisco-uds._tcp.",
+"_ciscowtp._tcp.",
+"_collab-edge._tls.",
+"_cuplogin._tcp.",
+"_client._smtp._tcp.",
+"_sftp._tcp.",
+"_h323rs._udp.",
+"_sql._tcp.",
+"_sip._tcp.internal.",
+"_snmp._udp.",
+"_rdp._tcp.",
+"_xmpp-server._udp."
+]
@@ -13,7 +13,7 @@ from common import utils
 from common.database import Database


-def export(table, db=None, valid=None, dpath=None, format='csv', show=False):
+def export(table, db=None, valid=None, path=None, format='csv', show=False):
    """
    OneForAll数据库导出模块

@@ -25,24 +25,20 @@ def export(table, db=None, valid=None, dpath=None, format='csv', show=False):
        参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
-        参数dpath为None默认使用OneForAll结果目录
+        参数path默认None使用OneForAll结果目录生成路径

    :param str table:   要导出的表
    :param str db:      要导出的数据库路径(默认为results/result.sqlite3)
    :param int valid:   导出子域的有效性(默认None)
-    :param str format:  导出格式(默认xls)
-    :param str dpath:    导出目录(默认None)
+    :param str format:  导出文件格式(默认csv)
+    :param str path:    导出文件路径(默认None)
    :param bool show:   终端显示导出数据(默认False)
    """
-    format = utils.check_format(format)
-    dpath = utils.check_dpath(dpath)
+
    database = Database(db)
-    if valid is None:
-        rows = database.get_data(table)
-    elif isinstance(valid, int):
-        rows = database.get_subdomain(table, valid)
-    else:
-        rows = database.get_data(table)  # 意外情况导出全部子域
+    rows = database.export_data(table, valid)
+    format = utils.check_format(format, len(rows))
+    path = utils.check_path(path, table, format)
    if show:
        print(rows.dataset)
    if format == 'txt':
@@ -50,8 +46,7 @@ def export(table, db=None, valid=None, dpath=None, format='csv', show=False):
    else:
        data = rows.export(format)
    database.close()
-    fpath = dpath.joinpath(f'{table}.{format}')
-    utils.save_data(fpath, data)
+    utils.save_data(path, data)


 if __name__ == '__main__':
@@ -7,5 +7,8 @@

 import oneforall

-test = oneforall.OneForAll(target='example.com')
-test.run()
+if __name__ == '__main__':
+    test = oneforall.OneForAll(target='example.com')
+    test.brute = True
+    test.takeover = True
+    test.run()
@@ -8,7 +8,7 @@ github自动接管
 import json
 import base64
 import requests
-import config
+import api

 HEADERS = {
    "Accept": "application/json, text/javascript, */*; q=0.01",
@@ -21,8 +21,8 @@ def github_takeover(url):
    # 读取config配置文件
    repo_name = url
    print('[*]正在读取配置文件...')
-    user = config.github_api_user
-    token = config.github_api_token
+    user = api.github_api_user
+    token = api.github_api_token
    CHECK_HEADERS = {
        "Authorization": 'token ' + token,
        "Accept": "application/vnd.github.switcheroo-preview+json"
@@ -1,5 +1,4 @@
-import time
-import config
+import api
 from common.query import Query
 from config import logger

@@ -11,8 +10,8 @@ class CensysAPI(Query):
        self.module = 'Certificate'
        self.source = "CensysAPIQuery"
        self.addr = 'https://www.censys.io/api/v1/search/certificates'
-        self.id = config.censys_api_id
-        self.secret = config.censys_api_secret
+        self.id = api.censys_api_id
+        self.secret = api.censys_api_secret
        self.delay = 3.0  # Censys 接口查询速率限制 最快2.5秒查1次

    def query(self):
@@ -38,7 +37,6 @@ class CensysAPI(Query):
        self.subdomains = self.subdomains.union(subdomains)
        pages = json.get('metadata').get('pages')
        for page in range(2, pages + 1):
-            time.sleep(self.delay)
            data['page'] = page
            resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
            if not resp:
@@ -1,4 +1,3 @@
-import time
 from common import utils
 from common.query import Query

@@ -15,7 +14,6 @@ class CertSpotter(Query):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        params = {'domain': self.domain,
@@ -1,4 +1,3 @@
-import time
 from common import utils
 from common.query import Query

@@ -15,7 +14,6 @@ class Crtsh(Query):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        params = {'q': f'%.{self.domain}', 'output': 'json'}
@@ -1,4 +1,3 @@
-import time
 from common import utils
 from common.query import Query

@@ -15,7 +14,6 @@ class Entrust(Query):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        params = {'fields': 'subjectDN',
@@ -1,4 +1,3 @@
-import time
 from common import utils
 from common.query import Query

@@ -16,7 +15,6 @@ class Google(Query):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        params = {'include_expired': 'true',
@@ -1,5 +1,4 @@
-import time
-import config
+import api
 from common import utils
 from common.query import Query

@@ -11,7 +10,7 @@ class SpyseAPI(Query):
        self.module = 'Certificate'
        self.source = 'CertDBAPIQuery'
        self.addr = 'https://api.spyse.com/v1/subdomains'
-        self.token = config.spyse_api_token
+        self.token = api.spyse_api_token

    def query(self):
        """
@@ -19,7 +18,6 @@ class SpyseAPI(Query):
        """
        page_num = 1
        while True:
-            time.sleep(self.delay)
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            params = {'domain': self.domain,
@@ -34,10 +34,11 @@ class CheckAXFR(Module):
        """
        logger.log('DEBUG', f'尝试对{self.domain}的域名服务器{server}进行域传送')
        try:
-            xfr = dns.query.xfr(server, self.domain, timeout=30.0)
+            xfr = dns.query.xfr(where=server, zone=self.domain,
+                                timeout=5.0, lifetime=10.0)
            zone = dns.zone.from_xfr(xfr)
        except Exception as e:
-            logger.log('DEBUG', str(e))
+            logger.log('DEBUG', e.args)
            logger.log('DEBUG', f'对{self.domain}的域名服务器{server}进行域传送失败')
            return
        names = zone.nodes.keys()
@@ -60,7 +61,7 @@ class CheckAXFR(Module):
        try:
            answers = resolver.query(self.domain, "NS")
        except Exception as e:
-            logger.log('ERROR', e)
+            logger.log('ERROR', e.args)
            return
        nsservers = [str(answer) for answer in answers]
        if not len(nsservers):
@@ -93,4 +94,4 @@ def do(domain):  # 统一入口名字 方便多线程调用

 if __name__ == '__main__':
    do('ZoneTransfer.me')
-    do('example.com')
+    # do('example.com')
@@ -24,16 +24,15 @@ class CheckCDX(Module):
                f'https://{self.domain}/crossdomain.xml',
                f'http://www.{self.domain}/crossdomain.xml',
                f'https://www.{self.domain}/crossdomain.xml']
-        response = None
        for url in urls:
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
-            response = self.get(url)
-            if response:
-                break
-        if not response:
-            return
-        self.subdomains = utils.match_subdomain(self.domain, response.text)
+            response = self.get(url, check=False)
+            if not response:
+                return
+            if response and len(response.content):
+                self.subdomains = utils.match_subdomain(self.domain,
+                                                        response.text)

    def run(self):
        """
@@ -23,13 +23,14 @@ class CheckCert(Module):
        """
        获取域名证书并匹配证书中的子域名
        """
-        ctx = ssl.create_default_context()
-        sock = ctx.wrap_socket(socket.socket(), server_hostname=self.domain)
        try:
+            ctx = ssl.create_default_context()
+            sock = ctx.wrap_socket(socket.socket(),
+                                   server_hostname=self.domain)
            sock.connect((self.domain, self.port))
            cert_dict = sock.getpeercert()
        except Exception as e:
-            logger.log('ERROR', e)
+            logger.log('ERROR', e.args)
            return
        subdomains = utils.match_subdomain(self.domain, str(cert_dict))
        self.subdomains = self.subdomains.union(subdomains)
@@ -12,38 +12,46 @@ class CheckCSP(Module):
    """
    检查内容安全策略收集子域名
    """
-
    def __init__(self, domain, header):
        Module.__init__(self)
        self.domain = self.register(domain)
        self.module = 'Check'
        self.source = 'ContentSecurityPolicy'
-        self.header = header
+        self.csp_header = header
+
+    def grab_header(self):
+        """
+        抓取请求头
+
+        :return: 请求头
+        """
+        csp_header = dict()
+        urls = [f'http://{self.domain}',
+                f'https://{self.domain}',
+                f'http://www.{self.domain}',
+                f'https://www.{self.domain}']
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            response = self.get(url, check=False)
+            if response:
+                csp_header = response.headers
+                break
+        return csp_header

    def check(self):
        """
        正则匹配响应头中的内容安全策略字段以发现子域名
        """
-        if not self.header:
-            urls = [f'http://{self.domain}',
-                    f'https://{self.domain}',
-                    f'http://www.{self.domain}',
-                    f'https://www.{self.domain}']
-            response = None
-            for url in urls:
-                self.header = self.get_header()
-                self.proxy = self.get_proxy(self.source)
-                response = self.get(url)
-                if response:
-                    break
-            if not response:
-                return
-            self.header = response.headers
+        if not self.csp_header:
+            self.csp_header = self.grab_header()
        csp = self.header.get('Content-Security-Policy')
+        if not self.csp_header:
+            logger.log('DEBUG', f'获取{self.domain}域的请求头失败')
+            return
        if not csp:
            logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段')
            return
-        logger.log('DEBUG', f'{self.domain}域的响应头存在内容安全策略字段')
        self.subdomains = utils.match_subdomain(self.domain, csp)

    def run(self):
@@ -1,18 +1,14 @@
 """
 检查内容安全策略收集子域名收集子域名
 """
-import requests
-
 from common.module import Module
 from common import utils
-from config import logger


 class CheckRobots(Module):
    """
    检查robots.txt收集子域名
    """
-
    def __init__(self, domain):
        Module.__init__(self)
        self.domain = self.register(domain)
@@ -27,16 +23,15 @@ class CheckRobots(Module):
                f'https://{self.domain}/robots.txt',
                f'http://www.{self.domain}/robots.txt',
                f'https://www.{self.domain}/robots.txt']
-        response = None
        for url in urls:
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
-            response = self.get(url, allow_redirects=False)
-            if response:
-                break
-        if not response:
-            return
-        self.subdomains = utils.match_subdomain(self.domain, response.text)
+            response = self.get(url, check=False, allow_redirects=False)
+            if not response:
+                return
+            if response and len(response.content):
+                self.subdomains = utils.match_subdomain(self.domain,
+                                                        response.text)

    def run(self):
        """
@@ -1,18 +1,14 @@
 """
 检查内容安全策略收集子域名收集子域名
 """
-import requests
-
 from common.module import Module
 from common import utils
-from config import logger


 class CheckRobots(Module):
    """
    检查sitemap收集子域名
    """
-
    def __init__(self, domain):
        Module.__init__(self)
        self.domain = self.register(domain)
@@ -39,17 +35,17 @@ class CheckRobots(Module):
                f'https://{self.domain}/sitemap_index.xml',
                f'http://www.{self.domain}/sitemap_index.xml',
                f'https://www.{self.domain}/sitemap_index.xml']
-        response = None
        for url in urls:
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            self.timeout = 10
-            response = self.get(url, allow_redirects=False)
-            if response:
-                break
-        if not response:
-            return
-        self.subdomains = utils.match_subdomain(self.domain, response.text)
+            response = self.get(url, check=False, allow_redirects=False)
+            if not response:
+                return
+            if response and len(response.content):
+                self.subdomains = utils.match_subdomain(self.domain,
+                                                        response.text)
+

    def run(self):
        """
@@ -1,5 +1,4 @@
-import time
-import config
+import api
 from common.query import Query


@@ -10,13 +9,12 @@ class BinaryEdgeAPI(Query):
        self.module = 'Dataset'
        self.source = 'BinaryEdgeAPIQuery'
        self.addr = 'https://api.binaryedge.io/v2/query/domains/subdomain/'
-        self.api = config.binaryedge_api
+        self.api = api.binaryedge_api

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.header.update({'X-Key': self.api})
        self.proxy = self.get_proxy(self.source)
@@ -1,4 +1,3 @@
-import time
 import cloudscraper
 from common.query import Query
 from config import logger
@@ -16,7 +15,6 @@ class BufferOver(Query):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        # 绕过cloudFlare验证
        scraper = cloudscraper.create_scraper()
        scraper.interpreter = 'js2py'
@@ -27,7 +25,7 @@ class BufferOver(Query):
        except Exception as e:
            logger.log('ERROR', e.args)
            return
-        if not resp:
+        if resp.status_code != 200:
            return
        subdomains = self.match(self.domain, str(resp.json()))
        # 合并搜索子域名搜索结果
@@ -1,4 +1,3 @@
-import time
 from common.query import Query


@@ -8,13 +7,12 @@ class CeBaidu(Query):
        self.domain = self.register(domain)
        self.module = 'Dataset'
        self.source = 'CeBaiduQuery'
-        self.addr = 'http://ce.baidu.com/index/getRelatedSites'
+        self.addr = 'https://ce.baidu.com/index/getRelatedSites'

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        params = {'site_address': self.domain}
@@ -24,7 +22,6 @@ class CeBaidu(Query):
        subdomains = self.match(self.domain, str(resp.json()))
        # 合并搜索子域名搜索结果
        self.subdomains = self.subdomains.union(subdomains)
-        print(self.subdomains)

    def run(self):
        """
@@ -1,4 +1,3 @@
-import time
 from common.query import Query


@@ -14,7 +13,6 @@ class Chinaz(Query):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        self.addr = self.addr + self.domain
@@ -1,5 +1,4 @@
-import time
-import config
+import api
 from common.query import Query


@@ -10,13 +9,12 @@ class ChinazAPI(Query):
        self.module = 'Dataset'
        self.source = 'ChinazAPIQuery'
        self.addr = 'https://apidata.chinaz.com/CallAPI/Alexa'
-        self.api = config.chinaz_api
+        self.api = api.chinaz_api

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        params = {'key': self.api, 'domainName': self.domain}
@@ -1,5 +1,4 @@
-import time
-import config
+import api
 from common.query import Query


@@ -10,14 +9,13 @@ class CirclAPI(Query):
        self.module = 'Dataset'
        self.source = 'CirclAPIQuery'
        self.addr = 'https://www.circl.lu/pdns/query/'
-        self.user = config.circl_api_username
-        self.pwd = config.circl_api_password
+        self.user = api.circl_api_username
+        self.pwd = api.circl_api_password

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
@@ -1,92 +0,0 @@
-import time
-import random
-import cloudscraper
-from bs4 import BeautifulSoup
-from common.query import Query
-from config import logger
-
-
-class DNSdb(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = self.register(domain)
-        self.module = 'Dataset'
-        self.source = 'DNSdbQuery'
-        self.addr = 'http://www.dnsdb.org/'
-        self.url = f'{self.addr}{self.domain}/'
-
-    def get_tokens(self):
-        """
-        绕过cloudFlare验证并获取taken
-
-        :return: 绕过失败返回None 成功返回tokens
-        """
-        scraper = cloudscraper.create_scraper()
-        scraper.interpreter = 'js2py'
-        scraper.proxies = self.get_proxy(self.source)
-        scraper.timeout = 10
-        try:
-            tokens = scraper.get_tokens(self.url)
-        except Exception as e:
-            logger.log('ERROR', e.args)
-            return None
-        if len(tokens) != 2:
-            return None
-        return tokens
-
-    def query(self):
-        """
-        向接口查询子域并做子域匹配
-        """
-        tokens = self.get_tokens()
-        if not tokens:
-            logger.log('ALERT', f'{self.source}模块绕过cloudFlare检查失败')
-            return False
-        self.cookie = tokens[0]
-        self.header = {'User-Agent': tokens[1]}
-        self.timeout = 10
-        resp = self.get(self.url)
-        if not resp:
-            return
-        if 'index' in resp.text:
-            soup = BeautifulSoup(resp.text, features='lxml')
-            base = self.addr+self.domain
-            urls = list(map(lambda a: base + '/' + a.get('href'),
-                            soup.find_all('a')))
-            urls = urls[:-1]  # idn域名暂时不考虑
-            for url in urls:
-                resp = self.get(url)
-                if not resp:
-                    return
-                subdomains = self.match(self.domain, resp.text)
-                # 合并搜索子域名搜索结果
-                self.subdomains = self.subdomains.union(subdomains)
-        else:
-            subdomains = self.match(self.domain, resp.text)
-            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains)
-
-    def run(self):
-        """
-        类执行入口
-        """
-        self.begin()
-        self.query()
-        self.finish()
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-
-
-def do(domain):  # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    """
-    query = DNSdb(domain)
-    query.run()
-
-
-if __name__ == '__main__':
-    do('example.com')
@@ -1,5 +1,4 @@
-import time
-import config
+import api
 from common import utils
 from common.query import Query

@@ -11,13 +10,12 @@ class DNSdbAPI(Query):
        self.module = 'Dataset'
        self.source = 'DNSdbAPIQuery'
        self.addr = 'https://api.dnsdb.info/lookup/rrset/name/'
-        self.api = config.dnsdb_api_key
+        self.api = api.dnsdb_api_key

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.header.update({'X-API-Key': self.api})
        self.proxy = self.get_proxy(self.source)
@@ -1,4 +1,3 @@
-import time
 from common import utils
 from common.query import Query

@@ -15,7 +14,6 @@ class DNSdumpster(Query):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.header.update({'Referer': 'https://dnsdumpster.com'})
        self.proxy = self.get_proxy(self.source)
@@ -0,0 +1,49 @@
+from common.query import Query
+
+
+class IP138(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'IP138Query'
+        self.addr = 'https://site.ip138.com/{domain}/domain.htm'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        self.addr = self.addr.format(domain=self.domain)
+        resp = self.get(self.addr)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = IP138(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,73 +1,73 @@
-import config
-from common.query import Query
-from config import logger
-
-
-class IPv4InfoAPI(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = self.register(domain)
-        self.module = 'Dataset'
-        self.source = 'IPv4InfoAPIQuery'
-        self.addr = ' http://ipv4info.com/api_v1/'
-        self.api = config.ipv4info_api_key
-
-    def query(self):
-        """
-        向接口查询子域并做子域匹配
-        """
-        page = 0
-        while True:
-            self.header = self.get_header()
-            self.proxy = self.get_proxy(self.source)
-            params = {'type': 'SUBDOMAINS', 'key': self.api,
-                      'value': self.domain, 'page': page}
-            resp = self.get(self.addr, params)
-            if not resp:
-                return
-            if resp.status_code != 200:
-                break  # 请求不正常通常网络是有问题，不再继续请求下去
-            try:
-                json = resp.json()
-            except Exception as e:
-                logger.log('DEBUG', e.args)
-                break
-            subdomains = self.match(self.domain, str(json))
-            if not subdomains:
-                break
-            # 合并搜索子域名搜索结果
-            self.subdomains = self.subdomains.union(subdomains)
-            # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
-            subdomains = json.get('Subdomains')
-            if subdomains:
-                # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
-                if len(subdomains) < 300:
-                    break
-            page += 1
-            if page >= 50:  # ipv4info子域查询接口最多允许查询50页
-                break
-
-    def run(self):
-        """
-        类执行入口
-        """
-        self.begin()
-        self.query()
-        self.finish()
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-
-
-def do(domain):  # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    """
-    query = IPv4InfoAPI(domain)
-    query.run()
-
-
-if __name__ == '__main__':
-    do('example.com')
+import api
+from common.query import Query
+from config import logger
+
+
+class IPv4InfoAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'IPv4InfoAPIQuery'
+        self.addr = ' http://ipv4info.com/api_v1/'
+        self.api = api.ipv4info_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        page = 0
+        while True:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            params = {'type': 'SUBDOMAINS', 'key': self.api,
+                      'value': self.domain, 'page': page}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            if resp.status_code != 200:
+                break  # 请求不正常通常网络是有问题，不再继续请求下去
+            try:
+                json = resp.json()
+            except Exception as e:
+                logger.log('DEBUG', e.args)
+                break
+            subdomains = self.match(self.domain, str(json))
+            if not subdomains:
+                break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
+            subdomains = json.get('Subdomains')
+            if subdomains:
+                # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
+                if len(subdomains) < 300:
+                    break
+            page += 1
+            if page >= 50:  # ipv4info子域查询接口最多允许查询50页
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = IPv4InfoAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,5 +1,4 @@
-import time
-import config
+import api
 from common.query import Query


@@ -9,14 +8,13 @@ class PassiveDnsAPI(Query):
        self.domain = self.register(domain)
        self.module = 'Dataset'
        self.source = 'PassiveDnsQuery'
-        self.addr = config.passivedns_api_addr or 'http://api.passivedns.cn'
-        self.token = config.passivedns_api_token
+        self.addr = api.passivedns_api_addr or 'http://api.passivedns.cn'
+        self.token = api.passivedns_api_token

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.header.update({'X-AuthToken': self.token})
        self.proxy = self.get_proxy(self.source)
@@ -1,4 +1,3 @@
-import time
 from common.query import Query


@@ -14,7 +13,6 @@ class Riddler(Query):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        params = {'q': 'pld:' + self.domain}
@@ -1,5 +1,4 @@
-import time
-import config
+import api
 from common.query import Query


@@ -8,16 +7,15 @@ class SecurityTrailsAPI(Query):
        Query.__init__(self)
        self.domain = self.register(domain)
        self.module = 'Dataset'
-        self.source = 'SecurityTrailsQuery'
+        self.source = 'SecurityTrailsAPIQuery'
        self.addr = 'https://api.securitytrails.com/v1/domain/'
-        self.api = config.securitytrails_api
+        self.api = api.securitytrails_api
        self.delay = 2  # SecurityTrails查询时延至少2秒

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        params = {'apikey': self.api}
@@ -1,4 +1,3 @@
-import time

 from common.query import Query

@@ -19,7 +18,6 @@ class SiteDossier(Query):
        向接口查询子域并做子域匹配
        """
        while True:
-            time.sleep(self.delay)
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            url = f'{self.addr}{self.domain}/{self.page_num}'
@@ -23,7 +23,7 @@ class ThreatCrowd(Query):
        except Exception as e:
            logger.log('ERROR', e.args)
            return
-        if not resp:
+        if resp.status_code != 200:
            return
        subdomains = self.match(self.domain, str(resp.json()))
        # 合并搜索子域名搜索结果
@@ -52,4 +52,4 @@ def do(domain):  # 统一入口名字 方便多线程调用


 if __name__ == '__main__':
-    do('example.com')
+    do('mi.com')
@@ -0,0 +1,50 @@
+from common.query import Query
+
+
+class Ximcx(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'XimcxQuery'
+        self.addr = 'http://sbd.ximcx.cn/DomainServlet'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        data = {'domain': self.domain}
+        resp = self.post(self.addr, data=data)
+        if not resp:
+            return
+        json = resp.json()
+        subdomains = self.match(self.domain, str(json))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Ximcx(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -4,69 +4,54 @@
 通过枚举域名常见的SRV记录并做查询来发现子域
 """

-import asyncio
 import json
-
-import aiodns
+import queue
+import threading

 from common import utils
+from common import resolve
 from common.module import Module
-from config import data_storage_path, logger, resolver_nameservers
+from config import data_storage_path, logger


 class BruteSRV(Module):
-    def __init__(self, domain: str):
+    def __init__(self, domain):
        Module.__init__(self)
        self.domain = self.register(domain)
        self.module = 'dnsquery'
        self.source = "BruteSRV"
-        self.loop = asyncio.new_event_loop()
-        self.nameservers = resolver_nameservers
-        self.resolver = aiodns.DNSResolver(self.nameservers, self.loop)
+        self.thread_num = 10
+        self.names_que = queue.Queue()
+        self.answers_que = queue.Queue()

-    async def query(self, name):
-        """
-        查询域名的SRV记录
-        :param str name: SRV记录
-        :return: 查询结果
-        """
-        logger.log('TRACE', f'尝试查询{name}的SRV记录')
-        try:
-            answers = await self.resolver.query(name, 'SRV')
-        except Exception as e:
-            logger.log('TRACE', e)
-            logger.log('TRACE', f'查询{name}的SRV记录失败')
-            return None
-        else:
-            logger.log('TRACE', f'查询{name}的SRV记录成功')
-            return answers
+    def gen_names(self):
+        path = data_storage_path.joinpath('srv_prefixes.json')
+        with open(path, encoding='utf-8', errors='ignore') as file:
+            prefixes = json.load(file)
+        names = map(lambda prefix: prefix + self.domain, prefixes)
+
+        for name in names:
+            self.names_que.put(name)

    def brute(self):
        """
        枚举域名的SRV记录
        """
-        names_path = data_storage_path.joinpath('srv_names.json')
-        with open(names_path) as fp:
-            names_dict = json.load(fp)
-        query_map = map(lambda name: name + self.domain, names_dict)
+        self.gen_names()

-        tasks = []
-        for query in query_map:
-            tasks.append(self.query(query))
-        task_group = asyncio.gather(*tasks, loop=self.loop)
-        self.loop.run_until_complete(asyncio.gather(task_group))
-        self.loop.close()
-        results = task_group.result()
-        for result in results:
-            if result:
-                for answer in result:
-                    subdomains = utils.match_subdomain(self.domain, answer.host)
-                    if subdomains:
-                        self.subdomains = self.subdomains.union(subdomains)
-                    else:
-                        logger.log('DEBUG', f'{answer.host}不是{self.domain}的子域')
-        if not len(self.subdomains):
-            logger.log('DEBUG', f'没有找到{self.domain}的SRV记录')
+        for i in range(self.thread_num):
+            thread = BruteThread(self.names_que, self.answers_que)
+            thread.daemon = True
+            thread.start()
+
+        self.names_que.join()
+
+        while not self.answers_que.empty():
+            answer = self.answers_que.get()
+            if answer is not None:
+                for item in answer:
+                    subdomains = utils.match_subdomain(self.domain, str(item))
+                    self.subdomains = self.subdomains.union(subdomains)

    def run(self):
        """
@@ -80,6 +65,38 @@ class BruteSRV(Module):
        self.save_db()


+class BruteThread(threading.Thread):
+    def __init__(self, names_que, answers_que):
+        threading.Thread.__init__(self)
+        self.names_que = names_que
+        self.answers_que = answers_que
+        self.resolver = resolve.dns_resolver()
+
+    def query(self, name):
+        """
+        查询域名的SRV记录
+        :param str name: SRV记录
+        :return: 查询结果
+        """
+        logger.log('TRACE', f'尝试查询{name}的SRV记录')
+        try:
+            answer = self.resolver.query(name, 'SRV')
+        except Exception as e:
+            logger.log('TRACE', e.args)
+            logger.log('TRACE', f'查询{name}的SRV记录失败')
+            return None
+        else:
+            logger.log('TRACE', f'查询{name}的SRV记录成功')
+            return answer
+
+    def run(self):
+        while True:
+            name = self.names_que.get()
+            answer = self.query(name)
+            self.answers_que.put(answer)
+            self.names_que.task_done()
+
+
 def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口
@@ -91,5 +108,5 @@ def do(domain):  # 统一入口名字 方便多线程调用


 if __name__ == '__main__':
-
-    do('example.com')
+    do('zonetransfer.me')
+    # do('example.com')
@@ -0,0 +1,58 @@
+from common.query import Query
+
+
+class AlienVault(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Intelligence'
+        self.source = 'AlienVaultQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+
+        base = 'https://otx.alienvault.com/api/v1/indicators/domain'
+        dns = f'{base}/{self.domain}/passive_dns'
+        resp = self.get(dns)
+        if not resp:
+            return
+        json = resp.json()
+        subdomains = self.match(self.domain, str(json))
+        self.subdomains = self.subdomains.union(subdomains)
+
+        url = f'{base}/{self.domain}/url_list'
+        resp = self.get(url)
+        if not resp:
+            return
+        json = resp.json()
+        subdomains = self.match(self.domain, str(json))
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = AlienVault(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query


@@ -9,8 +9,8 @@ class RiskIQ(Query):
        self.module = 'Intelligence'
        self.source = 'RiskIQAPIQuery'
        self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
-        self.user = config.riskiq_api_username
-        self.key = config.riskiq_api_key
+        self.user = api.riskiq_api_username
+        self.key = api.riskiq_api_key

    def query(self):
        """
@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query


@@ -9,7 +9,7 @@ class ThreatBookAPI(Query):
        self.module = 'Intelligence'
        self.source = 'ThreatBookAPIQuery'
        self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
-        self.key = config.threatbook_api_key
+        self.key = api.threatbook_api_key

    def query(self):
        """
@@ -1,5 +1,3 @@
-import time
-
 from common.query import Query


@@ -15,7 +13,6 @@ class ThreatMiner(Query):
        """
        向接口查询子域并做子域匹配
        """
-        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        params = {'e': 'subdomains_container',
@@ -1,5 +1,3 @@
-import time
-
 from common.query import Query

 '''
@@ -21,7 +19,6 @@ class VirusTotal(Query):
        """
        next_cursor = ''
        while True:
-            time.sleep(self.delay)
            self.header = self.get_header()
            self.header.update({'Referer': 'https://www.virustotal.com/',
                                'TE': 'Trailers'})
@@ -1,4 +1,4 @@
-import config
+import api
 from common.query import Query


@@ -9,7 +9,7 @@ class VirusTotalAPI(Query):
        self.module = 'Intelligence'
        self.source = 'VirusTotalAPIQuery'
        self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'
-        self.key = config.virustotal_api_key
+        self.key = api.virustotal_api_key

    def query(self):
        """
@@ -15,18 +15,19 @@ class Baidu(Search):

    def redirect_match(self, domain, html):
        """
+        获取跳转地址并传递地址进行跳转head请求

-        :param domain:
-        :param html:
-        :return:
+        :param domain: 域名
+        :param html: 响应体
+        :return: 子域
        """
        bs = BeautifulSoup(html, features='lxml')
        subdomains_all = set()
        # 获取搜索结果中所有的跳转URL地址
        for find_res in bs.find_all('a', {'class': 'c-showurl'}):
            url = find_res.get('href')
-            subdomain = self.match_location(domain, url)
-            subdomains_all = subdomains_all.union(subdomain)
+            subdomains = self.match_location(domain, url)
+            subdomains_all = subdomains_all.union(subdomains)
        return subdomains_all

    def search(self, domain, filtered_subdomain='', full_search=False):
@@ -43,7 +44,8 @@ class Baidu(Search):
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'wd': query, 'pn': self.page_num,
+            params = {'wd': query,
+                      'pn': self.page_num,
                      'rn': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
@@ -73,9 +75,7 @@ class Baidu(Search):
        类执行入口
        """
        self.begin()
-
        self.search(self.domain, full_search=True)
-
        # 排除同一子域搜索结果过多的子域以发现新的子域
        for statement in self.filter(self.domain, self.subdomains):
            self.search(self.domain, filtered_subdomain=statement)
@@ -106,4 +106,4 @@ def do(domain):  # 统一入口名字 方便多线程调用


 if __name__ == '__main__':
-    do('example.com')
+    do('huayunshuzi.com')
@@ -1,5 +1,5 @@
 import time
-import config
+import api
 from common.search import Search


@@ -11,8 +11,8 @@ class BingAPI(Search):
        self.source = 'BingAPISearch'
        self.addr = 'https://api.cognitive.microsoft.com/' \
                    'bingcustomsearch/v7.0/search'
-        self.id = config.bing_api_id
-        self.key = config.bing_api_key
+        self.id = api.bing_api_id
+        self.key = api.bing_api_key
        self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数
        self.delay = 1  # 必应自定义搜索限制时延1秒

@@ -87,4 +87,4 @@ def do(domain):  # 统一入口名字 方便多线程调用

 if __name__ == '__main__':

-    do('example.com')
+    do('huawei.com')
@@ -1,7 +1,7 @@
 import base64
 import time

-import config
+import api
 from common.search import Search
 from config import logger

@@ -14,8 +14,8 @@ class FoFa(Search):
        self.source = 'FoFaAPISearch'
        self.addr = 'https://fofa.so/api/v1/search/all'
        self.delay = 1
-        self.email = config.fofa_api_email
-        self.key = config.fofa_api_key
+        self.email = api.fofa_api_email
+        self.key = api.fofa_api_key

    def search(self):
        """
@@ -0,0 +1,72 @@
+import time
+from bs4 import BeautifulSoup
+from common.search import Search
+from config import logger
+
+
+class Gitee(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.source = 'GiteeSearch'
+        self.module = 'Search'
+        self.addr = 'https://search.gitee.com/'
+        self.domain = self.register(domain)
+        self.header = self.get_header()
+
+    def search(self, full_search=False):
+        """
+        向接口查询子域并做子域匹配
+        """
+        page_num = 1
+        while True:
+            time.sleep(self.delay)
+            params = {'pageno': page_num, 'q': self.domain, 'type': 'code'}
+            try:
+                resp = self.get(self.addr, params=params)
+            except Exception as e:
+                logger.log('ERROR', e.args)
+                break
+            if resp.status_code != 200:
+                logger.log('ERROR', f'{self.source}模块搜索出错')
+                break
+            if 'class="empty-box"' in resp.text:
+                break
+            soup = BeautifulSoup(resp.text, 'lxml')
+            subdomains = self.match(self.domain, soup.text)
+            self.subdomains = self.subdomains.union(subdomains)
+            if not subdomains:
+                break
+            if not full_search:
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
+                    break
+            if '<li class="disabled"><a href="###">' in resp.text:
+                break
+            if page_num > 100:
+                break
+            page_num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Gitee(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,7 +1,7 @@
 import re
 import time
 import requests
-import config
+import api
 from bs4 import BeautifulSoup
 from common.search import Search
 from config import logger
@@ -18,8 +18,8 @@ class Github(Search):
        self.session = requests.Session()
        self.login_url = 'https://github.com/login'
        self.post_url = 'https://github.com/session'
-        self.email = config.github_email
-        self.password = config.github_password
+        self.email = api.github_email
+        self.password = api.github_password

    def login_github(self):
        """
@@ -38,11 +38,15 @@ class Github(Search):
            'login': self.email,
            'password': self.password
        }
-        resp = self.session.post(self.post_url, data=post_data)
+        try:
+            resp = self.session.post(self.post_url, data=post_data)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return False
        if resp.status_code != 200:
            return False
-        match = re.search(r'"user-login" content="(.*?)"', resp.text)
-        if match:
+        result = re.search(r'"user-login" content="(.*?)"', resp.text)
+        if result:
            return True

    def get_token(self):
@@ -51,16 +55,21 @@ class Github(Search):

        :return: 获取失败返回None，成功返回token
        """
-        resp = self.session.get(self.login_url)
+        try:
+
+            resp = self.session.get(self.login_url)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return None
        if resp.status_code != 200:
            return None
-        match = re.search(
-            r'name="authenticity_token" value="(.*?)"', resp.text)
-        if not match:
+        pattern = r'name="authenticity_token" value="(.*?)"'
+        result = re.findall(pattern, resp.text)
+        if not result:
            return None
-        return match.group(1)
+        return result[0]

-    def search(self, full_search=True):
+    def search(self, full_search=False):
        """
        向接口查询子域并做子域匹配
        """
@@ -68,15 +77,21 @@ class Github(Search):
        self.session.proxies = self.get_proxy(self.source)
        self.session.verify = self.verify
        if not self.login_github():
-            logger.log('ERROR', f'{self.session}模块登录失败')
+            logger.log('ERROR', f'{self.source}模块登录失败')
            return
        page_num = 1
        while True:
            time.sleep(self.delay)
            params = {'p': page_num, 'q': f'"{self.domain}"', 'type': 'Code'}
-            resp = self.session.get(self.addr, params=params)
+            try:
+                resp = self.session.get(self.addr, params=params)
+            except Exception as e:
+                logger.log('ERROR', e.args)
+                break
            if resp.status_code != 200:
-                logger.log('ERROR', f'{self.session}模块搜索出错')
+                logger.log('ERROR', f'{self.source}模块搜索出错')
+                break
+            if 'couldn’t find any code' in resp.text:
                break
            soup = BeautifulSoup(resp.text, 'lxml')
            subdomains = self.match(self.domain, soup.text)
@@ -87,7 +102,7 @@ class Github(Search):
                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
                if subdomains.issubset(self.subdomains):
                    break
-            if 'class="next_page disabled"' in resp.text:
+            if 'class="next_page"' not in resp.text:
                break
            if page_num > 100:
                break
@@ -118,4 +133,4 @@ def do(domain):  # 统一入口名字 方便多线程调用


 if __name__ == '__main__':
-    do('mi.com')
+    do('example.com')
@@ -1,5 +1,5 @@
 import time
-import config
+import api
 from common.search import Search


@@ -11,8 +11,8 @@ class GoogleAPI(Search):
        self.source = 'GoogleAPISearch'
        self.addr = 'https://www.googleapis.com/customsearch/v1'
        self.delay = 1
-        self.key = config.google_api_key
-        self.cx = config.google_api_cx
+        self.key = api.google_api_key
+        self.cx = api.google_api_cx
        self.per_page_num = 10  # 每次只能请求10个结果

    def search(self, domain, filtered_subdomain='', full_search=False):
@@ -1,4 +1,4 @@
-import config
+import api
 from common.search import Search


@@ -9,7 +9,7 @@ class ShodanAPI(Search):
        self.module = 'Search'
        self.source = 'ShodanAPISearch'
        self.addr = 'https://api.shodan.io/shodan/host/search'
-        self.key = config.shodan_api_key
+        self.key = api.shodan_api_key

    def search(self):
        """
@@ -9,7 +9,7 @@ class So(Search):
        self.domain = domain
        self.module = 'Search'
        self.source = 'SoSearch'
-        self.addr = 'http://www.so.com/s'
+        self.addr = 'https://www.so.com/s'
        self.limit_num = 640  # 限制搜索条数
        self.per_page_num = 10  # 默认每页显示10页

@@ -1,5 +1,5 @@
 import time
-import config
+import api
 from common.search import Search
 from config import logger

@@ -12,8 +12,8 @@ class ZoomEyeAPI(Search):
        self.source = 'ZoomEyeAPISearch'
        self.addr = 'https://api.zoomeye.org/web/search'
        self.delay = 2
-        self.user = config.zoomeye_api_username
-        self.pwd = config.zoomeye_api_password
+        self.user = api.zoomeye_api_usermail
+        self.pwd = api.zoomeye_api_password

    def login(self):
        """
@@ -24,7 +24,7 @@ class ZoomEyeAPI(Search):
        resp = self.post(url=url, json=data)
        if not resp:
            logger.log('FATAL', f'登录失败无法获取{self.source}的访问token')
-            return
+            exit(1)
        data = resp.json()
        if resp.status_code == 200:
            logger.log('DEBUG', f'{self.source}模块登录成功')
@@ -83,4 +83,4 @@ def do(domain):  # 统一入口名字 方便多线程调用


 if __name__ == '__main__':
-    do('example.com')
+    do('mi.com')
@@ -28,69 +28,84 @@ blue = '\033[01;34m'
 red = '\033[1;31m'
 end = '\033[0m'

-version = white + '{' + red + 'v0.0.7#dev' + white + '}'
+version = 'v0.0.9#dev'
+message = white + '{' + red + version + white + '}'

-banner = f"""{yellow}
+banner = f"""
+OneForAll是一款功能强大的子域收集工具{yellow}
             ___             _ _ 
- ___ ___ ___|  _|___ ___ ___| | | {version}{green}
+ ___ ___ ___|  _|___ ___ ___| | | {message}{green}
 | . |   | -_|  _| . |  _| .'| | | {blue}
-|___|_|_|___|_| |___|_| |__,|_|_| {white}git.io/fjHT1{end}
+|___|_|_|___|_| |___|_| |__,|_|_| {white}git.io/fjHT1
+
+{red}OneForAll处于开发中，会进行版本快速迭代，请每次在使用前进行更新！{end}
 """


 class OneForAll(object):
    """
+    OneForAll帮助信息
+
    OneForAll是一款功能强大的子域收集工具

-    Version: 0.0.7
-    Project: https://git.io/fjHT1
-
    Example:
+        python3 oneforall.py version
        python3 oneforall.py --target example.com run
-        python3 oneforall.py --target ./subdomains.txt run
+        python3 oneforall.py --target ./domains.txt run
        python3 oneforall.py --target example.com --valid None run
        python3 oneforall.py --target example.com --brute True run
        python3 oneforall.py --target example.com --port medium run
        python3 oneforall.py --target example.com --format csv run
-        python3 oneforall.py --target example.com --verify False run
+        python3 oneforall.py --target example.com --dns False run
+        python3 oneforall.py --target example.com --req False run
        python3 oneforall.py --target example.com --takeover False run
        python3 oneforall.py --target example.com --show True run

    Note:
        参数valid可选值1，0，None分别表示导出有效，无效，全部子域
-        参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
-        参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+        参数port可选值有'default', 'small', 'large', 详见config.py配置
        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+        参数path默认None使用OneForAll结果目录生成路径

-    :param str target:  单个域名或者每行一个域名的文件路径(必需参数)
-    :param bool brute:  使用爆破模块(默认False)
-    :param bool verify: 验证子域有效性(默认True)
-    :param str port:    请求验证的端口范围(默认medium)
-    :param int valid:   导出子域的有效性(默认1)
-    :param str format:  导出格式(默认csv)
-    :param bool show:   终端显示导出数据(默认False)
+    :param str target:     单个域名或者每行一个域名的文件路径(必需参数)
+    :param bool brute:     使用爆破模块(默认False)
+    :param bool dns:       DNS解析子域(默认True)
+    :param bool req:       HTTP请求子域(默认True)
+    :param str port:       请求验证子域的端口范围(默认只探测80端口)
+    :param int valid:      导出子域的有效性(默认None)
+    :param str format:     导出文件格式(默认csv)
+    :param str path:       导出文件路径(默认None)
+    :param bool takeover:  检查子域接管(默认False)
+    :param bool show:      终端显示导出数据(默认False)
    """
-    def __init__(self, target, brute=None, verify=None, port='medium', valid=1,
-                 format='csv', takeover=True, show=False):
+    def __init__(self, target, brute=None, dns=None, req=None,
+                 port='default', valid=None, format='csv', path=None,
+                 takeover=False, show=False):
        self.target = target
        self.port = port
        self.domains = set()
        self.domain = str()
+        self.data = list()
        self.datas = list()
        self.brute = brute
-        self.verify = verify
+        self.dns = dns
+        self.req = req
        self.takeover = takeover
        self.valid = valid
        self.format = format
+        self.path = path
        self.show = show

    def main(self):
        if self.brute is None:
            self.brute = config.enable_brute_module
-        if self.verify is None:
-            self.verify = config.enable_verify_subdomain
-        rename_table = self.domain + '_last'
+        if self.dns is None:
+            self.dns = config.enable_dns_resolve
+        if self.req is None:
+            self.req = config.enable_http_request
+        old_table = self.domain + '_last_result'
+        new_table = self.domain + '_now_result'
        collect = Collect(self.domain, export=False)
        collect.run()
        if self.brute:
@@ -99,52 +114,80 @@ class OneForAll(object):
            brute.run()

        db = Database()
-        db.copy_table(self.domain, self.domain+'_ori')
+        original_table = self.domain + '_original_result'
+        db.copy_table(self.domain, original_table)
        db.remove_invalid(self.domain)
        db.deduplicate_subdomain(self.domain)
-        # 不验证子域的情况
-        if not self.verify:
+
+        old_data = []
+        # 非第一次收集子域的情况时数据库预处理
+        if db.exist_table(new_table):
+            db.drop_table(old_table)  # 如果存在上次收集结果表就先删除
+            db.rename_table(new_table, old_table)  # 新表重命名为旧表
+            old_data = db.get_data(old_table).as_dict()
+
+        # 不解析子域直接导出结果
+        if not self.dns:
            # 数据库导出
-            self.valid = None
-            dbexport.export(self.domain, valid=self.valid, format=self.format,
-                            show=self.show)
-            db.drop_table(rename_table)
-            db.rename_table(self.domain, rename_table)
+            dbexport.export(self.domain, valid=self.valid,
+                            format=self.format, show=self.show)
+            db.drop_table(new_table)
+            db.rename_table(self.domain, new_table)
+            db.close()
            return
-        # 开始验证子域工作
-        self.datas = db.get_data(self.domain).as_dict()
+
+        self.data = db.get_data(self.domain).as_dict()
+
+        # 标记新发现子域
+        self.data = utils.mark_subdomain(old_data, self.data)
+
+        # 获取事件循环
        loop = asyncio.get_event_loop()
        asyncio.set_event_loop(loop)

-        # 解析域名地址
-        task = resolve.bulk_query_a(self.datas)
-        self.datas = loop.run_until_complete(task)
+        # 解析子域
+        task = resolve.bulk_resolve(self.data)
+        self.data = loop.run_until_complete(task)

        # 保存解析结果
-        resolve_table = self.domain + '_res'
+        resolve_table = self.domain + '_resolve_result'
        db.drop_table(resolve_table)
        db.create_table(resolve_table)
-        db.save_db(resolve_table, self.datas, 'resolve')
+        db.save_db(resolve_table, self.data, 'resolve')

-        # 请求域名地址
-        task = request.bulk_get_request(self.datas, self.port)
-        self.datas = loop.run_until_complete(task)
+        # 不请求子域直接导出结果
+        if not self.req:
+            # 数据库导出
+            dbexport.export(resolve_table, valid=self.valid,
+                            format=self.format, show=self.show)
+            db.drop_table(new_table)
+            db.rename_table(self.domain, new_table)
+            db.close()
+            return
+
+        # 请求子域
+        task = request.bulk_request(self.data, self.port)
+        self.data = loop.run_until_complete(task)
+        self.datas.extend(self.data)
        # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
        loop.run_until_complete(asyncio.sleep(0.25))
+        count = utils.count_valid(self.data)
+        logger.log('INFOR', f'经验证{self.domain}有效子域{count}个')

+        # 保存请求结果
        db.clear_table(self.domain)
-        db.save_db(self.domain, self.datas)
+        db.save_db(self.domain, self.data, 'request')

        # 数据库导出
-        dbexport.export(self.domain, valid=self.valid, format=self.format,
-                        show=self.show)
-        db.drop_table(rename_table)
-        db.rename_table(self.domain, rename_table)
+        dbexport.export(self.domain, valid=self.valid,
+                        format=self.format, show=self.show)
+        db.drop_table(new_table)
+        db.rename_table(self.domain, new_table)
        db.close()
-        # 子域接管检查

+        # 子域接管检查
        if self.takeover:
-            subdomains = set(map(lambda x: x.get('subdomain'), self.datas))
+            subdomains = set(map(lambda x: x.get('subdomain'), self.data))
            takeover = Takeover(subdomains)
            takeover.run()

@@ -152,15 +195,23 @@ class OneForAll(object):
        print(banner)
        dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        print(f'[*] Starting OneForAll @ {dt}\n')
+        logger.log('DEBUG', 'Python ' + utils.python_version())
+        logger.log('DEBUG', 'OneForAll ' + version)
        logger.log('INFOR', f'开始运行OneForAll')
        self.domains = utils.get_domains(self.target)
        if self.domains:
            for self.domain in self.domains:
                self.main()
+            utils.export_all(self.format, self.path, self.datas)
        else:
            logger.log('FATAL', f'获取域名失败')
        logger.log('INFOR', f'结束运行OneForAll')

+    @staticmethod
+    def version():
+        print(banner)
+        exit(0)
+

 if __name__ == '__main__':
    fire.Fire(OneForAll)
@@ -25,7 +25,7 @@ from common.domain import Domain

 def get_fingerprint():
    path = config.data_storage_path.joinpath('fingerprints.json')
-    with open(path) as file:
+    with open(path, encoding='utf-8', errors='ignore') as file:
        fingerprints = json.load(file)
    return fingerprints

@@ -35,7 +35,7 @@ def get_cname(subdomain):
    try:
        answers = resolver.query(subdomain, 'CNAME')
    except Exception as e:
-        logger.log('DEBUG', e.args)
+        logger.log('TRACE', e.args)
        return None
    for answer in answers:
        return answer.to_text()  # 一个子域只有一个CNAME记录
@@ -56,21 +56,21 @@ class Takeover(Module):
    Note:
        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
-        参数dpath为None默认使用OneForAll结果目录
+        参数path默认None使用OneForAll结果目录生成路径

-    :param str target:  单个子域或者每行一个子域的文件路径(必需参数)
+    :param any target:  单个子域或者每行一个子域的文件路径(必需参数)
    :param int thread:  线程数(默认100)
    :param str format:  导出格式(默认csv)
-    :param str dpath:   导出目录(默认None)
+    :param str path:    导出路径(默认None)
    """
-    def __init__(self, target, thread=100, dpath=None, format='csv'):
+    def __init__(self, target, thread=100, path=None, format='csv'):
        Module.__init__(self)
        self.subdomains = set()
        self.module = 'Check'
        self.source = 'Takeover'
        self.target = target
        self.thread = thread
-        self.dpath = dpath
+        self.path = path
        self.format = format
        self.fingerprints = None
        self.subdomainq = Queue()
@@ -78,17 +78,16 @@ class Takeover(Module):
        self.results = Dataset()

    def save(self):
-        logger.log('INFOR', '正在保存检查结果')
+        logger.log('DEBUG', '正在保存检查结果')
        if self.format == 'txt':
            data = str(self.results)
        else:
            data = self.results.export(self.format)
-        fpath = self.dpath.joinpath(f'takeover.{self.format}')
-        utils.save_data(fpath, data)
+        utils.save_data(self.path, data)

    def compare(self, subdomain, cname, responses):
        domain_resp = self.get('http://' + subdomain, check=False)
-        cname_resp = self.get('http://'+cname, check=False)
+        cname_resp = self.get('http://' + cname, check=False)
        if domain_resp is None or cname_resp is None:
            return

@@ -121,21 +120,23 @@ class Takeover(Module):
        bar = tqdm()
        bar.total = len(self.subdomains)
        bar.desc = 'Progress'
-        bar.ncols = True
+        bar.ncols = 60
        while True:
            done = bar.total - self.subdomainq.qsize()
            bar.n = done
            bar.update()
            if done == bar.total:  # 完成队列中所有子域的检查退出
                break
-        bar.close()
+        # bar.close()

    def run(self):
        start = time.time()
        logger.log('INFOR', f'开始执行{self.source}模块')
-        self.format = utils.check_format(self.format)
-        self.dpath = utils.check_dpath(self.dpath)
        self.subdomains = utils.get_domains(self.target)
+        self.format = utils.check_format(self.format, len(self.subdomains))
+        timestamp = utils.get_timestamp()
+        name = f'all_subdomain_{timestamp}'
+        self.path = utils.check_path(self.path, name, self.format)
        if self.subdomains:
            logger.log('INFOR', f'正在检查子域接管风险')
            self.fingerprints = get_fingerprint()
@@ -1,22 +1,14 @@
-aiodns==2.0.0
 aiohttp==3.6.2
-aiomultiprocess==0.6.1
-asn1crypto==1.2.0
+aiomultiprocess==0.7.0
 async-timeout==3.0.1
 attrs==19.3.0
-backports.csv==1.0.7
-beautifulsoup4==4.8.1
+beautifulsoup4==4.8.2
 Brotli==1.0.7
-brotlipy==0.7.0
 bs4==0.0.1
-cchardet==2.1.4
-certifi==2019.9.11
-cffi==1.13.0
+certifi==2019.11.28
 chardet==3.0.4
-cloudscraper==1.2.2
-colorama==0.4.1
-cryptography==2.7
-defusedxml==0.6.0
+cloudscraper==1.2.23
+colorama==0.4.3
 dnspython==1.16.0
 docopt==0.6.2
 et-xmlfile==1.0.1
@@ -24,33 +16,26 @@ exrex==0.10.5
 fire==0.2.1
 idna==2.8
 jdcal==1.4.1
-Js2Py==0.66
-loguru==0.3.2
-lxml==4.4.1
-multidict==4.5.2
-odfpy==1.4.0
+Js2Py==0.67
+loguru==0.4.1
+lxml==4.5.0
+multidict==4.7.4
 openpyxl==2.4.11
-pycares==3.0.0
-pycparser==2.19
 pyjsparser==2.7.1
-pyOpenSSL==19.0.0
 PySocks==1.7.1
 pytz==2019.3
-PyYAML==5.1.2
 records==0.5.3
 requests==2.22.0
 requests-file==1.4.3
 requests-toolbelt==0.9.1
-six==1.12.0
-soupsieve==1.9.4
-SQLAlchemy==1.3.10
-tablib==0.13.0
+six==1.14.0
+soupsieve==1.9.5
+SQLAlchemy==1.3.13
+tablib==1.1.0
 termcolor==1.1.0
 tldextract==2.2.2
-tqdm==4.36.1
+tqdm==4.42.1
 tzlocal==2.0.0
-urllib3==1.25.6
+urllib3==1.25.8
 win32-setctime==1.0.1
-xlrd==1.2.0
-xlwt==1.3.0
-yarl==1.3.0
+yarl==1.4.2
Author	SHA1	Message	Date
Jing Ling	5ff1598d82	v0.0.9	2020-02-20 15:05:46 +08:00
Jing Ling	57ddd501b4	v0.0.9	2020-02-20 15:04:17 +08:00
Jing Ling	46ebb3479c	更新文档	2020-02-20 14:49:55 +08:00
Jing Ling	488e9cfce7	typo	2020-02-20 14:37:05 +08:00
Jing Ling	f75836d58d	版本信息移到oneforall.py	2020-02-20 14:27:38 +08:00
Jing Ling	4c3e4bb203	更新文档	2020-02-20 14:19:51 +08:00
Jing Ling	56f6a35cd6	添加兼容的路径处理	2020-02-19 16:06:17 +08:00
Jing Ling	247cb65212	为OneForAll入口添加path参数	2020-02-18 16:00:11 +08:00
Jing Ling	51307ae19c	不使用encode函数	2020-02-17 13:02:05 +08:00
Jing Ling	6b831c376a	Update bug_report_zh.md	2020-02-17 12:57:47 +08:00
Jing Ling	8b2c13b22b	Update bug_report_zh.md	2020-02-17 12:56:40 +08:00
Jing Ling	c638acdfc7	Update and rename --bug.md to bug_report_zh.md	2020-02-17 12:53:19 +08:00
Jing Ling	ece8001f00	Update .travis.yml	2020-02-17 12:42:19 +08:00
Jing Ling	8353acb404	Update .travis.yml	2020-02-17 12:38:16 +08:00
Jing Ling	8c2f10c7d7	添加编码	2020-02-17 12:17:23 +08:00
Jing Ling	b60838ebee	Update .travis.yml	2020-02-17 11:09:45 +08:00
Jing Ling	e43c510a5f	在Window上指定shell使用UTF-8编码	2020-02-17 10:45:49 +08:00
Jing Ling	f9b706db4e	解决解码问题	2020-02-17 04:05:12 +08:00
Jing Ling	f2ff9c166a	调宽进度条	2020-02-17 03:51:47 +08:00
Jing Ling	6c902544b5	重构解析模块	2020-02-17 03:39:37 +08:00
Jing Ling	246cad2511	重构	2020-02-16 19:53:00 +08:00
Jing Ling	581122ac47	修复IP138请求问题	2020-02-16 15:34:01 +08:00
Jing Ling	80cae7c2d2	更新依赖	2020-02-16 02:47:20 +08:00
Jing Ling	41166ce1ff	为搜索模块线程设置超时	2020-02-16 01:51:57 +08:00
Jing Ling	38afa8807b	解决RuntimeError	2020-02-15 01:50:28 +08:00
Jing Ling	e8bc8e2941	GitHub登录页面发生了变化	2020-02-14 16:09:47 +08:00
Jing Ling	d06f8c2a40	增加HEAD请求功能	2020-02-12 18:21:52 +08:00
Jing Ling	cd88e34dd9	增加测试功能	2020-02-12 15:01:26 +08:00
Jing Ling	29e849ef1a	更改api名称	2020-02-12 15:00:59 +08:00
Jing Ling	5695b36c77	更改api配置说明信息	2020-02-12 10:53:22 +08:00
Jing Ling	2f6e8206d8	添加gitee模块说明	2020-02-12 10:46:55 +08:00
Jing Ling	a7b0def3c2	修复路径问题	2020-02-12 10:44:15 +08:00
Jing Ling	d527536192	重构请求解决大量超时导致子域无效问题	2020-02-11 03:05:16 +08:00
Jing Ling	8ba01b4527	调整日志等级	2020-02-11 01:55:33 +08:00
Jing Ling	58301560d0	提前路径判断是否为None	2020-02-10 22:04:01 +08:00
Jing Ling	135fce0993	修复路径问题和解包错误问题	2020-02-10 21:59:47 +08:00
Jing Ling	38c4de6896	api配置分离	2020-02-10 20:11:15 +08:00
Jing Ling	3f8aa8b74d	pep8	2020-02-10 18:48:42 +08:00
Jing Ling	463325e694	解决Github登录出现两个authenticity_token问题	2020-02-10 18:34:38 +08:00
Jing Ling	46521a8170	正确地判断某表是否存在	2020-02-09 21:22:24 +08:00
Jing Ling	ab2932642f	用列表存放端口	2020-02-09 19:23:40 +08:00
Jing Ling	1cd2082e71	添加函数说明	2020-02-09 19:00:04 +08:00
Jing Ling	a5f51aaefc	添加获取版本信息命令	2020-02-09 18:58:47 +08:00
Jing Ling	c4eb6cdcfa	更友好的提示	2020-02-09 16:10:34 +08:00
Jing Ling	e0ca9dbea0	添加返回值说明	2020-02-02 18:18:50 +08:00
Jing Ling	02ab78d835	调整顺序	2020-02-02 18:17:20 +08:00
Jing Ling	e66edb9717	添加Gitee码云搜索模块	2020-02-02 18:06:37 +08:00
Jing Ling	e3cb914901	模块提示更正	2020-02-02 18:04:59 +08:00
Jing Ling	d2e350b851	增加常见Web应用端口	2020-02-01 22:39:58 +08:00
Jing Ling	6cc6341e3a	Merge remote-tracking branch 'origin/master'	2020-02-01 17:45:02 +08:00
Jing Ling	2c7a7fa663	重构	2020-02-01 17:44:21 +08:00
Jing Ling	562dcefc7a	Merge pull request #44 from azimut/patch-1 Fix axfr module name in README.en.md	2020-01-30 21:20:04 +08:00
NCM	a21947fc5c	Fix axfr module name in README.en.md	2020-01-27 23:14:59 -03:00
Jing Ling	65a64e3929	解码兼容处理	2020-01-23 15:57:32 +08:00
Jing Ling	67937a6313	更加兼容的解码方式	2020-01-23 00:29:54 +08:00
Jing Ling	6b3636ba3b	Merge remote-tracking branch 'origin/master'	2020-01-10 15:59:38 +08:00
Jing Ling	bc4d4853ec	整理字典	2020-01-10 15:57:49 +08:00
Jing Ling	2ff7d45348	增加open容错处理	2020-01-10 15:26:41 +08:00
Jing Ling	d99cf64ae8	移除segment参数	2020-01-10 15:21:29 +08:00
Jing Ling	e1d224f8c7	更新文档	2020-01-10 15:20:47 +08:00
Jing Ling	501159f6eb	参数调优	2020-01-09 22:53:33 +08:00
Jing Ling	5553bd10f4	实在没有获取到标题就返回None	2020-01-09 22:06:22 +08:00
Jing Ling	2e003137de	修改请求默认不验证SSL	2020-01-09 20:54:29 +08:00
Jing Ling	9386643c5d	修复reason记录问题	2020-01-08 23:00:39 +08:00
Jing Ling	3220b363bf	正确计算有效子域数量	2020-01-07 01:35:32 +08:00
Jing Ling	ae8a7a65d0	重构	2020-01-07 01:34:43 +08:00
Jing Ling	c16bcada89	重构	2020-01-05 09:51:25 +08:00
Jing Ling	55b9188a13	添加大字典	2020-01-03 10:49:54 +08:00
Jing Ling	d6a85ccd53	Merge pull request #37 from MaxSecurity/master 解决在win下面无法读取字典小Bug	2020-01-03 10:19:58 +08:00
Jing Ling	fefdeef55a	参数优化	2020-01-03 00:38:07 +08:00
Jing Ling	8408d538de	重构	2020-01-02 23:12:58 +08:00
Jing Ling	4cb3c05b15	更新依赖	2020-01-02 22:20:12 +08:00
Jing Ling	175e8a47f0	默认全导出	2020-01-02 22:07:18 +08:00
MAX丶	b8ea1eb226	Rename oneforall/aiobrute.py to oneforall/aiobrute.py	2020-01-01 12:50:40 +08:00
Jing Ling	16f2bf4eda	增加编码错误忽略	2020-01-01 12:32:09 +08:00
MaxSecurity	eda4aa13f1	解决在win下面无法读取字典小Bug	2020-01-01 12:17:55 +08:00
Jing Ling	5cd71ca6e7	修复从config.py传入参数无效问题	2019-12-29 17:17:33 +08:00
Jing Ling	aab9bcd3b4	修复导出问题	2019-12-29 12:48:30 +08:00
Jing Ling	e47df26436	修改入口	2019-12-28 14:24:40 +08:00
Jing Ling	212e548486	模块优化	2019-12-28 13:56:22 +08:00
Jing Ling	9daefc9eee	修复进度条问题	2019-12-28 13:53:19 +08:00
Jing Ling	d5f478654b	添加当查询多个主域时结果自动合并导出功能	2019-12-27 18:54:10 +08:00
Jing Ling	7c8bd44c37	Update installation_dependency.md	2019-12-25 18:00:11 +08:00
Jing Ling	dd952759e6	Update installation_dependency.md	2019-12-25 17:56:30 +08:00
Jing Ling	93c31fd826	默认关闭保存各个模块结果为json文件	2019-12-23 23:09:31 +08:00
Jing Ling	d013fc2042	打印有效子域个数	2019-12-23 23:06:31 +08:00
Jing Ling	ac15eb567b	降低代码重复率	2019-12-23 23:06:07 +08:00
Jing Ling	9752e49e2a	Merge remote-tracking branch 'origin/master'	2019-12-23 21:56:37 +08:00
Jing Ling	d03429900b	更新依赖	2019-12-23 21:35:50 +08:00
shmilylty	6f4fd16cdb	移除没有必要的请求时延	2019-12-17 18:50:36 +08:00
shmilylty	05141046bd	不进行响应检查	2019-12-17 18:49:53 +08:00
shmilylty	6a22a3aed9	优化模块	2019-12-17 18:47:15 +08:00
shmilylty	05d281521d	完善模块	2019-12-17 17:43:17 +08:00
shmilylty	ce43bd75de	默认不启用子域接管检查	2019-12-17 11:55:38 +08:00
Jing Ling	30bc74a9a9	注释pytest	2019-12-13 01:46:54 +08:00
Jing Ling	70d3267803	创建测试	2019-12-13 01:43:14 +08:00
shmilylty	1fb30647df	不使用全搜索	2019-12-13 01:22:33 +08:00
shmilylty	6ff55b3693	Python38	2019-12-13 01:12:50 +08:00
shmilylty	207064e975	更新依赖	2019-12-10 22:25:00 +08:00
shmilylty	d8aa7487d2	修改为响应状态码判断	2019-12-10 22:06:21 +08:00
shmilylty	00928c367a	Merge remote-tracking branch 'origin/master'	2019-12-10 14:10:35 +08:00
shmilylty	f5d02d74ef	统一异常处理	2019-12-10 12:12:44 +08:00
shmilylty	0cafe696a6	添加Docker	2019-12-10 12:10:57 +08:00
Jing Ling	28c7305d52	Merge pull request #31 from Tardis07/master Add Docker supports	2019-12-09 22:52:33 +08:00
shmilylty	0093cd0522	使用https	2019-12-09 17:13:51 +08:00
milktea	7e22e2c7cd	Update README.md	2019-12-04 20:25:38 +08:00
milktea	abb10159b9	Add Docker supports Docker版基于alpine构建，轻量化。已知缺点：中文显示不全。推荐将结果导出后浏览后期维护考虑改为Ubuntu作为基础镜像	2019-12-04 20:24:46 +08:00
shmilylty	8c9e69ab9d	添加AlienVault接口说明	2019-11-30 23:28:19 +08:00
shmilylty	b6c4724dcb	添加超时	2019-11-30 21:45:34 +08:00
shmilylty	9421e16826	添加AlienVault接口	2019-11-30 21:00:04 +08:00
shmilylty	a94001e7e2	修复github搜索时间长问题	2019-11-30 20:59:24 +08:00
shmilylty	81520c43db	更新Linux下依赖安装帮助	2019-11-29 01:27:58 +08:00
shmilylty	59b2e4e59b	修复github搜索时间长问题	2019-11-25 18:00:54 +08:00
shmilylty	c4f77f16a9	重构异步DNS解析方法修复DNS解析进度条问题	2019-11-18 22:51:18 +08:00
shmilylty	85b6c28d5d	新添2个查询接口	2019-11-18 15:46:10 +08:00
shmilylty	971b5fef1e	添加ip138查询接口	2019-11-18 15:41:18 +08:00
shmilylty	2840583bc1	说明只支持Python 3.8	2019-11-15 15:08:09 +08:00
shmilylty	f64a9712f5	添加 http://sbd.ximcx.cn/查询接口	2019-11-14 23:08:17 +08:00
shmilylty	a6a9c6d3fa	更新依赖	2019-11-14 22:33:13 +08:00
shmilylty	f323e1364c	增加请求异常处理	2019-11-06 01:17:11 +08:00
shmilylty	3e19062670	增加HEAD请求异常处理	2019-11-06 01:09:30 +08:00
shmilylty	a987b9d96c	登录失败线程退出	2019-11-06 00:26:37 +08:00
shmilylty	d8a0c55fe5	添加takeover参数说明	2019-10-31 00:02:08 +08:00
shmilylty	9119e051b7	修复dns和req参数传入无效的问题	2019-10-30 23:59:28 +08:00
shmilylty	9f23e580e4	添加dns和req参数说明，移除verity参数说明。	2019-10-30 23:01:49 +08:00
shmilylty	3b24db2d89	添加dns和req参数，移除verity参数。	2019-10-30 22:51:45 +08:00
shmilylty	3f56cda716	调整大小	2019-10-30 20:40:16 +08:00
shmilylty	1690683f41	添加赞赏	2019-10-30 19:10:53 +08:00
shmilylty	e662f28407	v0.0.8	2019-10-30 00:46:06 +08:00
shmilylty	26d32d3cd9	实现新子域标记	2019-10-29 22:47:44 +08:00
shmilylty	55c4cf734a	去掉中括号	2019-10-29 16:00:25 +08:00
shmilylty	c568f73b51	去掉中括号	2019-10-28 20:06:16 +08:00
shmilylty	9c25b77aa6	新增new,header,response字段	2019-10-28 18:45:52 +08:00
shmilylty	9fbaf19f35	修正安装命令	2019-10-28 11:33:58 +08:00
shmilylty	e3c3965020	优化字典	2019-10-28 11:20:05 +08:00
shmilylty	40000472a5	由于dnsdb.org域名不在暂时删除此模块	2019-10-28 10:19:22 +08:00
shmilylty	cb8ac909d7	更改cname查询错误日志级别	2019-10-28 10:18:00 +08:00
shmilylty	7627b53650	使用多线程重写	2019-10-27 18:21:14 +08:00
shmilylty	f30f48aea2	修复解析没有结果的问题	2019-10-26 17:58:31 +08:00
shmilylty	e66c6d7e21	更新依赖	2019-10-25 21:40:24 +08:00
shmilylty	8ede7d9943	更新依赖	2019-10-25 21:40:04 +08:00
shmilylty	0d72aefb62	更改日志级别	2019-10-25 21:39:28 +08:00
shmilylty	893e6628e6	优化标题获取	2019-10-25 15:46:27 +08:00
shmilylty	baf394fc0b	参数优化	2019-10-24 22:30:49 +08:00
shmilylty	035963758d	修复端口重复问题	2019-10-24 21:56:07 +08:00
shmilylty	d760e02289	请求优化	2019-10-24 19:06:21 +08:00
shmilylty	e6b145d563	修正入口	2019-10-24 18:46:19 +08:00
shmilylty	7afb359918	调整参数	2019-10-24 18:44:58 +08:00
shmilylty	707a5eda93	请求优化	2019-10-24 18:44:35 +08:00
shmilylty	6d3c0ae873	请求优化	2019-10-24 16:08:55 +08:00
shmilylty	476e75869c	修改提示类型	2019-10-24 15:57:39 +08:00
shmilylty	e7110c24c5	默认请求只探测80端口	2019-10-23 01:31:51 +08:00
shmilylty	0e4c7961fe	请求子域返回状态码为400的暂时标记为无效子域	2019-10-22 23:49:29 +08:00
shmilylty	6f0cb70e02	推荐使用Python 3.8	2019-10-21 21:58:00 +08:00
shmilylty	4dec25d187	优化真实子域判断	2019-10-21 21:30:32 +08:00
shmilylty	e5e24804f6	更新番剧季数	2019-10-21 09:39:33 +08:00
shmilylty	f1b24e5674	不使用aiodns库并更新依赖库	2019-10-20 23:53:33 +08:00
shmilylty	d0edcb9a0b	不使用aiodns	2019-10-20 23:44:11 +08:00
shmilylty	2dbb5a1097	不使用aiodns	2019-10-20 23:42:24 +08:00
shmilylty	451992f07c	不使用aiodns	2019-10-20 23:24:16 +08:00
shmilylty	5cfbf885ca	不使用aiodns	2019-10-20 16:22:11 +08:00
shmilylty	ecc1cb53dc	简化命名	2019-10-19 21:26:48 +08:00
shmilylty	39d9adb6bc	正确判断查询结果	2019-10-19 21:15:09 +08:00
shmilylty	8675dc0202	移除aiodns依赖	2019-10-19 21:10:17 +08:00
shmilylty	de81a73097	目前加上超时会查不出结果，暂时去掉超时参数。	2019-10-19 18:38:03 +08:00
shmilylty	9d6f122254	删除无用输出	2019-10-19 16:48:19 +08:00
shmilylty	be11d5c3a2	更新Python 3.8在Win10上的配置	2019-10-18 15:29:36 +08:00