v0.0.7

更新依赖
添加百度云观测接口查询
2026-08-26 04:47:48 +08:00 · 2019-10-18 01:06:05 +08:00 · 2019-10-18 00:34:21 +08:00 · 2019-10-18 00:33:31 +08:00 · 2019-10-14 22:21:37 +08:00 · 2019-10-14 22:09:37 +08:00
101 changed files with 28073 additions and 124680 deletions
@@ -0,0 +1,36 @@
+---
+name: 提交Bug
+about: "请务必按照模板提交Bug\U0001F64F"
+title: 请填写BUG标题
+labels: bug
+assignees: shmilylty
+
+---
+
+**是否使用了最新代码**
+是或否（如果不是的话尝试克隆最新的代码再跑一下）
+
+**Bug描述**
+清晰而简洁的Bug描述
+
+**如何复现**
+复现步骤（可不写）
+复现命令
+
+**预期结果**
+清晰而简洁的预期结果描述（可不写）
+
+**实际结果**
+清晰而简洁的实际结果描述（如出现什么错误）
+
+**屏幕截图**
+
+
+**运行环境**
+ - 系统：[例如Windows 10 x64]
+ - Python版本：[例如3.7.1]
+- OneForAll版本：[例如0.0.6]
+
+
+**报错文本**
+复制完整的报错文本
@@ -0,0 +1,238 @@
+
+# Created by https://www.gitignore.io/api/python,windows,pycharm
+# Edit at https://www.gitignore.io/?templates=python,windows,pycharm
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+.idea/sonarlint
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# End of https://www.gitignore.io/api/python,windows,pycharm
@@ -0,0 +1,39 @@
+sudo: true
+
+notifications:
+  email: false
+
+language: python
+
+matrix:
+  include:
+    - name: "Python 3.6 on Linux"
+      python: 3.6           # this works for Linux but is ignored on macOS or Windows
+    - name: "Python 3.7 on Xenial Linux"
+      python: 3.7           # this works for Linux but is ignored on macOS or Windows
+      dist: xenial          # required for Python >= 3.7
+    - name: "Python 3.8 on Xenial Linux"
+      python: 3.8-dev           # this works for Linux but is ignored on macOS or Windows
+      dist: xenial          # required for Python >= 3.7
+    - name: "Python 3.7 on macOS"
+      os: osx
+      osx_image: xcode10.2  # Python 3.7 running on macOS 10.14.3
+      language: shell       # 'language: python' is an error on Travis CI macOS
+    - name: "Python 3.7 on Windows"
+      os: windows           # Windows 10.0.17134 N/A Build 17134
+      language: shell
+      before_install:
+        - choco install python
+        - python -m pip install --upgrade pip
+      env: PATH=/c/Python37:/c/Python37/Scripts:$PATH
+
+install:
+  - pip3 install -U pip
+  - pip3 install codecov
+  - pip3 install -r requirements.txt
+
+script:
+ - coverage run oneforall/example.py
+
+after_success:
+ - codecov
@@ -1,18 +1,60 @@
-# Changelog
+# 更新日志
 OneForAll的所有值得注意的更改都将记录在此文件中。

 OneForAll的更新日志格式基于[Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/)。

 OneForAll遵守[语义化版本格式](https://semver.org/)。

-## [Unreleased](https://github.com/shmilylty/onefroall/compare/v1.0.0...HEAD)
+## Unreleased
+## [0.0.7](https://github.com/shmilylty/oneforall/releases/tag/v0.0.7) - 2019-10-18
+- 修复一些已知问题
+- 添加百度云观测接口
+- 添加添加英文Readme文档
+- 更新有关文档
+- 优化标题获取
+- 更新依赖

-## [0.0.1](https://github.com/shmilylty/onefroall/releases/tag/v0.0.1) - 2019-08-01
-### Added
- 添加检查crossdomain.xml收集子域功能
- 添加检查域名证书收集子域功能
- 添加检查内容安全策略头收集子域功能
- 添加域传送利用功能
- 添加子域收集功能（搜索引擎，DNS数据集，证书透明度，网上爬虫档案）
- 添加子域爆破功能
- 添加数据库导出功能
+## [0.0.6](https://github.com/shmilylty/oneforall/releases/tag/v0.0.6) - 2019-08-27
+- 修复一些已知问题
+- 添加PassiveDNS查询和Github子域搜索模块
+- 优化FoFa和BufferOver收集模块
+- 更新有关文档
+- 更新依赖
+
+## [0.0.5](https://github.com/shmilylty/oneforall/releases/tag/v0.0.5) - 2019-08-19
+- 修复一些已知Bugs
+- 优化各子域收集接口并添加新的子域收集接口
+- 添加子域DNS解析和子域HTTP探测进度条
+- 添加子域接管风险检查模块及其使用说明
+- 更新OneForAll依赖
+
+## [0.0.4](https://github.com/shmilylty/oneforall/releases/tag/v0.0.4) - 2019-08-11
+### 修复
+- 修复一些已知Bugs
+
+## [0.0.3](https://github.com/shmilylty/oneforall/releases/tag/v0.0.3) - 2019-08-08
+### 修改
+- 代码PEP8格式化
+### 修改
+- 修改一些已知Bugs
+
+## [0.0.2](https://github.com/shmilylty/oneforall/releases/tag/v0.0.2) - 2019-08-04
+### 新增
+- 新增有关文档
+### 修改
+- 修改有关日志输出格式和信息
+### 修复
+- 升级fire库版本解决运行报错问题
+### 移除
+- 移除brotlipy依赖
+
+
+## [0.0.1](https://github.com/shmilylty/oneforall/releases/tag/v0.0.1) - 2019-08-02
+### 新增
+- 新增检查crossdomain.xml收集子域功能
+- 新增检查域名证书收集子域功能
+- 新增检查内容安全策略头收集子域功能
+- 新增域传送利用功能
+- 新增子域收集功能（搜索引擎，DNS数据集，证书透明度，网上爬虫档案）
+- 新增子域爆破功能
+- 新增数据库导出功能
@@ -3,5 +3,12 @@
 * **[Jing Ling](https://github.com/shmilylty)**
  * 核心开发

-* **[Black Star](https://github.com/shmilylty)**
-  * 模块贡献
+* **[Black Star](https://github.com/blackstar24)**，**[Echocipher](https://github.com/Echocipher)**
+  * 模块贡献
+
+* **[iceMatcha](https://github.com/iceMatcha)**，**[mikuKeeper](https://github.com/mikuKeeper)**
+  * 工具测试
+
+* **Anyone**
+  * 工具反馈
+
@@ -1,6 +1,6 @@
 [[source]]
 name = "pypi"
-url = "https://mirrors.aliyun.com/pypi/simple/"
+url = "https://pypi.python.org/simple/"
 verify_ssl = true

 [dev-packages]
@@ -17,12 +17,14 @@ tldextract = "*"
 exrex = "*"
 aiohttp = "*"
 fire = "*"
-fake-useragent = "*"
 bs4 = "*"
 cchardet = "*"
 lxml = "*"
-brotlipy = "*"
 pysocks = "*"
+cloudscraper = "*"
+js2py = "*"
+tablib = "*"
+brotlipy = "*"

 [requires]
-python_version = "3.7"
+python_version = "3.8"
@@ -1,11 +1,11 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "c2d9fe3a7303466070733472b7b06b491d35bbbcf3db8863152e721d797e9195"
+            "sha256": "656e9f0f133ba443049ac692b2551b8a0daa69542d471dc3dff965f7cdb3a308"
        },
        "pipfile-spec": 6,
        "requires": {
-            "python_version": "3.7"
+            "python_version": "3.8"
        },
        "sources": [
            {
@@ -26,38 +26,35 @@
        },
        "aiohttp": {
            "hashes": [
-                "sha256:00d198585474299c9c3b4f1d5de1a576cc230d562abc5e4a0e81d71a20a6ca55",
-                "sha256:0155af66de8c21b8dba4992aaeeabf55503caefae00067a3b1139f86d0ec50ed",
-                "sha256:09654a9eca62d1bd6d64aa44db2498f60a5c1e0ac4750953fdd79d5c88955e10",
-                "sha256:199f1d106e2b44b6dacdf6f9245493c7d716b01d0b7fbe1959318ba4dc64d1f5",
-                "sha256:296f30dedc9f4b9e7a301e5cc963012264112d78a1d3094cd83ef148fdf33ca1",
-                "sha256:368ed312550bd663ce84dc4b032a962fcb3c7cae099dbbd48663afc305e3b939",
-                "sha256:40d7ea570b88db017c51392349cf99b7aefaaddd19d2c78368aeb0bddde9d390",
-                "sha256:629102a193162e37102c50713e2e31dc9a2fe7ac5e481da83e5bb3c0cee700aa",
-                "sha256:6d5ec9b8948c3d957e75ea14d41e9330e1ac3fed24ec53766c780f82805140dc",
-                "sha256:87331d1d6810214085a50749160196391a712a13336cd02ce1c3ea3d05bcf8d5",
-                "sha256:9a02a04bbe581c8605ac423ba3a74999ec9d8bce7ae37977a3d38680f5780b6d",
-                "sha256:9c4c83f4fa1938377da32bc2d59379025ceeee8e24b89f72fcbccd8ca22dc9bf",
-                "sha256:9cddaff94c0135ee627213ac6ca6d05724bfe6e7a356e5e09ec57bd3249510f6",
-                "sha256:a25237abf327530d9561ef751eef9511ab56fd9431023ca6f4803f1994104d72",
-                "sha256:a5cbd7157b0e383738b8e29d6e556fde8726823dae0e348952a61742b21aeb12",
-                "sha256:a97a516e02b726e089cffcde2eea0d3258450389bbac48cbe89e0f0b6e7b0366",
-                "sha256:acc89b29b5f4e2332d65cd1b7d10c609a75b88ef8925d487a611ca788432dfa4",
-                "sha256:b05bd85cc99b06740aad3629c2585bda7b83bd86e080b44ba47faf905fdf1300",
-                "sha256:c2bec436a2b5dafe5eaeb297c03711074d46b6eb236d002c13c42f25c4a8ce9d",
-                "sha256:cc619d974c8c11fe84527e4b5e1c07238799a8c29ea1c1285149170524ba9303",
-                "sha256:d4392defd4648badaa42b3e101080ae3313e8f4787cb517efd3f5b8157eaefd6",
-                "sha256:e1c3c582ee11af7f63a34a46f0448fca58e59889396ffdae1f482085061a2889"
+                "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e",
+                "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326",
+                "sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a",
+                "sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654",
+                "sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a",
+                "sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4",
+                "sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17",
+                "sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec",
+                "sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd",
+                "sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48",
+                "sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59",
+                "sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965"
            ],
            "index": "pypi",
-            "version": "==3.5.4"
+            "version": "==3.6.2"
        },
        "aiomultiprocess": {
            "hashes": [
-                "sha256:fd9b616d5145ac2b01f315725277231ffad7e56eb6675885598f6fe861ef9fa3"
+                "sha256:c704383be74ccb806e2617cb3bb6a5faa1774b1c6643d62785c332eb6da9e742"
            ],
            "index": "pypi",
-            "version": "==0.6.0"
+            "version": "==0.6.1"
+        },
+        "asn1crypto": {
+            "hashes": [
+                "sha256:7bb1cc02a5620b3d72da4ba070bda2f44f0e61b44dee910a302eddff802b6fb5",
+                "sha256:87620880a477123e01177a1f73d0f327210b43a3cdbd714efcd2fa49a8d7b384"
+            ],
+            "version": "==1.2.0"
        },
        "async-timeout": {
            "hashes": [
@@ -68,10 +65,10 @@
        },
        "attrs": {
            "hashes": [
-                "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
-                "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
+                "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
+                "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
            ],
-            "version": "==19.1.0"
+            "version": "==19.3.0"
        },
        "backports.csv": {
            "hashes": [
@@ -82,11 +79,44 @@
        },
        "beautifulsoup4": {
            "hashes": [
-                "sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612",
-                "sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b",
-                "sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469"
+                "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
+                "sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931",
+                "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
            ],
-            "version": "==4.8.0"
+            "version": "==4.8.1"
+        },
+        "brotli": {
+            "hashes": [
+                "sha256:0538dc1744fd17c314d2adc409ea7d1b779783b89fd95bcfb0c2acc93a6ea5a7",
+                "sha256:0970a47f471782912d7705160b2b0a9306e68e6fadf9cffcaeb42d8f0951e26c",
+                "sha256:113f51658e6fe548dce4b3749f6ef6c24de4184ba9c10a909cbee4261c2a5da0",
+                "sha256:1e1aa9c4d1558889f42749c8baf846007953bfd32c8209230cf1cd1f5ef33495",
+                "sha256:2f2f4f78f29ac4a45d15b3d9fc3fd9705e0ad313a44b129f6e1d0c6916bad0e2",
+                "sha256:3269f6de1dd150fd0cce1c158b61ff5ac06d627fd3ae9c6ea03aed26fbbff7ea",
+                "sha256:50dd9ad2a2bb12da4e9002a438672d182f98e546e99952de80280a1e1729664f",
+                "sha256:5519a4b01b1a4f965083cbfa2ef2b9774c5a5f352341c47b50776ad109423d72",
+                "sha256:5eb27722d320370315971c427eb8aa7cc0791f2a458840d357ac653bd0ad3a14",
+                "sha256:5f06b4d5b6f58e5b5c220c2f23cad034dc5efa51b01fde2351ced1605bd980e2",
+                "sha256:72848d25a5f9e736db4af4512e0c3feecc094d57d241f8f1ae959115a2c39756",
+                "sha256:743001bca75f4a6b4454be3510feca46f9d61a0c782a9bc2bc684bdb245e279e",
+                "sha256:9d1c2dd27a1083fefd05b1b2f8df4a6bc2aaa6c21dd82cd41c8ae5e7c23a87f8",
+                "sha256:a13ce9b419fe9f277c63f700efb0e444331509d1881b5610d2ba7e9080606967",
+                "sha256:a19ef0952b9d2803df88dff07f45a6c92d5676afb9b8d69cf32232d684036d11",
+                "sha256:ad766ca8b8c1419b71a22756b45264f45725c86133dc80a7cbe30b6b78c75620",
+                "sha256:ad7963f261988ee0883816b6b9f206f11461c9b3cb5cfbca0c9ab5adc406d395",
+                "sha256:c16201060c5a3f8742e3deae759014251ac92f382f82bc2a41dc079ff18c3f24",
+                "sha256:c43b202f65891861a9a336984a103de25de235f756de69e32db893156f767013",
+                "sha256:c675c6cce4295cb1a692f3de7416aacace7314e064b94bc86e93aceefce7fd3e",
+                "sha256:d17cec0b992b1434f5f9df9986563605a4d1b1acd5574c87fc2ac014bcbd3316",
+                "sha256:dc91f6129953861a73d9a65c52a8dd682b561a9ebaf65283541645cab6489917",
+                "sha256:e2f4cbd1760d2bf2f30e396c2301999aab0191aec031a6a8a04950b2f575a536",
+                "sha256:f192e6d3556714105c10486bbd6d045e38a0c04d9da3cef21e0a8dfd8e162df4",
+                "sha256:f775b07026af2b1b0b5a8b05e41571cdcf3a315a67df265d60af301656a5425b",
+                "sha256:f969ec7f56ba9636679e69ca07fba548312ccaca37412ee823c7f413541ad7e0",
+                "sha256:f9dc52cd70907aafb99a773b66b156f2f995c7a0d284397c487c8b71ddbef2f9",
+                "sha256:fc7212e36ebeb81aebf7949c92897b622490d7c0e333a479c0395591e7994600"
+            ],
+            "version": "==1.0.7"
        },
        "brotlipy": {
            "hashes": [
@@ -95,6 +125,7 @@
                "sha256:09ec3e125d16749b31c74f021aba809541b3564e5359f8c265cbae442810b41a",
                "sha256:0be698678a114addcf87a4b9496c552c68a2c99bf93cf8e08f5738b392e82057",
                "sha256:0fa6088a9a87645d43d7e21e32b4a6bf8f7c3939015a50158c10972aa7f425b7",
+                "sha256:1379347337dc3d20b2d61456d44ccce13e0625db2611c368023b4194d5e2477f",
                "sha256:1ea4e578241504b58f2456a6c69952c88866c794648bdc74baee74839da61d44",
                "sha256:2699945a0a992c04fc7dc7fa2f1d0575a2c8b4b769f2874a08e8eae46bef36ae",
                "sha256:2a80319ae13ea8dd60ecdc4f5ccf6da3ae64787765923256b62c598c5bba4121",
@@ -102,6 +133,7 @@
                "sha256:36def0b859beaf21910157b4c33eb3b06d8ce459c942102f16988cca6ea164df",
                "sha256:3a3e56ced8b15fbbd363380344f70f3b438e0fd1fcf27b7526b6172ea950e867",
                "sha256:3c1d5e2cf945a46975bdb11a19257fa057b67591eb232f393d260e7246d9e571",
+                "sha256:4e4638b49835d567d447a2cfacec109f9a777f219f071312268b351b6839436d",
                "sha256:50ca336374131cfad20612f26cc43c637ac0bfd2be3361495e99270883b52962",
                "sha256:5de6f7d010b7558f72f4b061a07395c5c3fd57f0285c5af7f126a677b976a868",
                "sha256:637847560d671657f993313ecc6c6c6666a936b7a925779fd044065c7bc035b9",
@@ -161,43 +193,47 @@
        },
        "certifi": {
            "hashes": [
-                "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939",
-                "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695"
+                "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50",
+                "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"
            ],
-            "version": "==2019.6.16"
+            "version": "==2019.9.11"
        },
        "cffi": {
            "hashes": [
-                "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774",
-                "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d",
-                "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90",
-                "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b",
-                "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63",
-                "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45",
-                "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25",
-                "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3",
-                "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b",
-                "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647",
-                "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016",
-                "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4",
-                "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb",
-                "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753",
-                "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7",
-                "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9",
-                "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f",
-                "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8",
-                "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f",
-                "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc",
-                "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42",
-                "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3",
-                "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909",
-                "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45",
-                "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d",
-                "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512",
-                "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff",
-                "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201"
+                "sha256:08f99e8b38d5134d504aa7e486af8e4fde66a2f388bbecc270cdd1e00fa09ff8",
+                "sha256:1112d2fc92a867a6103bce6740a549e74b1d320cf28875609f6e93857eee4f2d",
+                "sha256:1b9ab50c74e075bd2ae489853c5f7f592160b379df53b7f72befcbe145475a36",
+                "sha256:24eff2997436b6156c2f30bed215c782b1d8fd8c6a704206053c79af95962e45",
+                "sha256:2eff642fbc9877a6449026ad66bf37c73bf4232505fb557168ba5c502f95999b",
+                "sha256:362e896cea1249ed5c2a81cf6477fabd9e1a5088aa7ea08358a4c6b0998294d2",
+                "sha256:40eddb3589f382cb950f2dcf1c39c9b8d7bd5af20665ce273815b0d24635008b",
+                "sha256:5ed40760976f6b8613d4a0db5e423673ca162d4ed6c9ed92d1f4e58a47ee01b5",
+                "sha256:632c6112c1e914c486f06cfe3f0cc507f44aa1e00ebf732cedb5719e6aa0466a",
+                "sha256:64d84f0145e181f4e6cc942088603c8db3ae23485c37eeda71cb3900b5e67cb4",
+                "sha256:6cb4edcf87d0e7f5bdc7e5c1a0756fbb37081b2181293c5fdf203347df1cd2a2",
+                "sha256:6f19c9df4785305669335b934c852133faed913c0faa63056248168966f7a7d5",
+                "sha256:719537b4c5cd5218f0f47826dd705fb7a21d83824920088c4214794457113f3f",
+                "sha256:7b0e337a70e58f1a36fb483fd63880c9e74f1db5c532b4082bceac83df1523fa",
+                "sha256:853376efeeb8a4ae49a737d5d30f5db8cdf01d9319695719c4af126488df5a6a",
+                "sha256:85bbf77ffd12985d76a69d2feb449e35ecdcb4fc54a5f087d2bd54158ae5bb0c",
+                "sha256:8978115c6f0b0ce5880bc21c967c65058be8a15f1b81aa5fdbdcbea0e03952d1",
+                "sha256:8f7eec920bc83692231d7306b3e311586c2e340db2dc734c43c37fbf9c981d24",
+                "sha256:8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226",
+                "sha256:92068ebc494b5f9826b822cec6569f1f47b9a446a3fef477e1d11d7fac9ea895",
+                "sha256:b57e1c8bcdd7340e9c9d09613b5e7fdd0c600be142f04e2cc1cc8cb7c0b43529",
+                "sha256:ba956c9b44646bc1852db715b4a252e52a8f5a4009b57f1dac48ba3203a7bde1",
+                "sha256:ca42034c11eb447497ea0e7b855d87ccc2aebc1e253c22e7d276b8599c112a27",
+                "sha256:dc9b2003e9a62bbe0c84a04c61b0329e86fccd85134a78d7aca373bbbf788165",
+                "sha256:dd308802beb4b2961af8f037becbdf01a1e85009fdfc14088614c1b3c383fae5",
+                "sha256:e77cd105b19b8cd721d101687fcf665fd1553eb7b57556a1ef0d453b6fc42faa",
+                "sha256:f56dff1bd81022f1c980754ec721fb8da56192b026f17f0f99b965da5ab4fbd2",
+                "sha256:fa4cc13c03ea1d0d37ce8528e0ecc988d2365e8ac64d8d86cafab4038cb4ce89",
+                "sha256:fa8cf1cb974a9f5911d2a0303f6adc40625c05578d8e7ff5d313e1e27850bd59",
+                "sha256:fb003019f06d5fc0aa4738492ad8df1fa343b8a37cbcf634018ad78575d185df",
+                "sha256:fd409b7778167c3bcc836484a8f49c0e0b93d3e745d975749f83aa5d18a5822f",
+                "sha256:fe5d65a3ee38122003245a82303d11ac05ff36531a8f5ce4bc7d4bbc012797e1"
            ],
-            "version": "==1.12.3"
+            "version": "==1.13.0"
        },
        "chardet": {
            "hashes": [
@@ -206,6 +242,14 @@
            ],
            "version": "==3.0.4"
        },
+        "cloudscraper": {
+            "hashes": [
+                "sha256:2316ddc0c00905536a3f1801917e68fb78610776b1b65f1b126627a7793c397b",
+                "sha256:a27d2452edbe3d77d089c71f74783edfd24802e4f100aef0fad0de4505a2b840"
+            ],
+            "index": "pypi",
+            "version": "==1.2.2"
+        },
        "colorama": {
            "hashes": [
                "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d",
@@ -214,6 +258,27 @@
            "markers": "sys_platform == 'win32'",
            "version": "==0.4.1"
        },
+        "cryptography": {
+            "hashes": [
+                "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c",
+                "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643",
+                "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216",
+                "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799",
+                "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a",
+                "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9",
+                "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc",
+                "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8",
+                "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53",
+                "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1",
+                "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609",
+                "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292",
+                "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e",
+                "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6",
+                "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed",
+                "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d"
+            ],
+            "version": "==2.7"
+        },
        "defusedxml": {
            "hashes": [
                "sha256:6687150770438374ab581bb7a1b327a847dd9c5749e396102de3fad4e8a3ef93",
@@ -248,19 +313,12 @@
            "index": "pypi",
            "version": "==0.10.5"
        },
-        "fake-useragent": {
-            "hashes": [
-                "sha256:c104998b750eb097eefc28ae28e92d66397598d2cf41a31aa45d5559ef1adf35"
-            ],
-            "index": "pypi",
-            "version": "==0.1.11"
-        },
        "fire": {
            "hashes": [
-                "sha256:cdae28ea728d6cad487c900d534426d2cd09ddd2993287a885ff7c8accb097c8"
+                "sha256:6865fefc6981a713d2ce56a2a2c92c56c729269f74a6cddd6f4b94d16ae084c9"
            ],
            "index": "pypi",
-            "version": "==0.2.0"
+            "version": "==0.2.1"
        },
        "idna": {
            "hashes": [
@@ -276,6 +334,14 @@
            ],
            "version": "==1.4.1"
        },
+        "js2py": {
+            "hashes": [
+                "sha256:6e5628abfff2fb4051e8e77a353e44831f474e2ceb865278271897f7f326aeb6",
+                "sha256:bf87cb4432944470f11fed9c1cb8d0312dd505e7b867362f55102f24379ab94f"
+            ],
+            "index": "pypi",
+            "version": "==0.66"
+        },
        "loguru": {
            "hashes": [
                "sha256:b6fad0d7aed357b5c147edcc6982606b933754338950b72d8123f48c150c5a4f",
@@ -286,31 +352,31 @@
        },
        "lxml": {
            "hashes": [
-                "sha256:06e5599b9c54f797a3c0f384c67705a0d621031007aa2400a6c7d17300fdb995",
-                "sha256:092237cfe4ece074401b75001a2e525fa6e1fb9d40fee8b7b132b1947d3bd2f8",
-                "sha256:0b6d49d0a26fe8207df8dd27c40b75be4deb2277173903aa76ec3e82df77cbe7",
-                "sha256:0f77061c20b4f32b1cf39e8f661c74e966344084c996e7b23c3a94e472461df0",
-                "sha256:0fef86edfa2f146b4b0ae2c6c05c3e4a8f3388b3655eafbc4aab3247f4dabb24",
-                "sha256:2f163c8844db4ed06a230ef092e2461ad01830972a896b8f3cf8b5bac70ae85d",
-                "sha256:350333190052bbfbc3222b1805b59b7979d7276e57af2257367e15a2db27082d",
-                "sha256:3b57dc5ed7b6a7d852c961f2389ca99404c2b59fd2088baec6fbaca02f688be4",
-                "sha256:3e86e5df4a8edd6f725f3c76f1d45e046d4f3aa40478092e4f5f373ad1f526e2",
-                "sha256:43dac60d10341d3e56be089cd0798b70e70d45ce32279f4c3190d8cbd71350e4",
-                "sha256:4665ee84ac8ba11d58f1ed517e29ea8536b4ae4e0c6fb6c7d3dce70abcd279f0",
-                "sha256:5033cf606a7cb559db967689b1b2e743994000f783607ba4c484e90917395ad7",
-                "sha256:75d731af05bf40f808d7716e0d26b4b02913402f861c032ce8c36efca350ae72",
-                "sha256:7720174604c7647e357566ac9e4d135c137caed5e7b01223551a4c81c8dc8b9a",
-                "sha256:b33ec641309bcea40c76c1b105f988e4e8f9a2f1ee1486aa5c0eeef33956c9bb",
-                "sha256:d1135dc0ac197242028ede085b693ba1f2bff7f0f9b91080e2540348312bfa53",
-                "sha256:d5a61e9c2322b45f259909a02b76bc98c4641214e22a37191d00c151aa9cdb9a",
-                "sha256:da22c4b17bc17dad9c8faf6d94c8fe568ac71c867a56631ab874da418fc7f8f7",
-                "sha256:da5c48ec9f8d8b5df42d328b6d1fb8d9413cd664a2367ef4f6f7cc48ee5b82c0",
-                "sha256:db2794bad21b7b30b6849b4e1537171cae8a7087711d958d69c233470dc612e7",
-                "sha256:f1c2f67df727034f94ccb590142d1d110f3dd38f638a4f1567fdd9f39892ba05",
-                "sha256:f840dddded8b046edc774c88ed8d2442cdb231a68894c42c74e3a809450fae76"
+                "sha256:02ca7bf899da57084041bb0f6095333e4d239948ad3169443f454add9f4e9cb4",
+                "sha256:096b82c5e0ea27ce9138bcbb205313343ee66a6e132f25c5ed67e2c8d960a1bc",
+                "sha256:0a920ff98cf1aac310470c644bc23b326402d3ef667ddafecb024e1713d485f1",
+                "sha256:17cae1730a782858a6e2758fd20dd0ef7567916c47757b694a06ffafdec20046",
+                "sha256:17e3950add54c882e032527795c625929613adbd2ce5162b94667334458b5a36",
+                "sha256:1f4f214337f6ee5825bf90a65d04d70aab05526c08191ab888cb5149501923c5",
+                "sha256:2e8f77db25b0a96af679e64ff9bf9dddb27d379c9900c3272f3041c4d1327c9d",
+                "sha256:4dffd405390a45ecb95ab5ab1c1b847553c18b0ef8ed01e10c1c8b1a76452916",
+                "sha256:6b899931a5648862c7b88c795eddff7588fb585e81cecce20f8d9da16eff96e0",
+                "sha256:726c17f3e0d7a7200718c9a890ccfeab391c9133e363a577a44717c85c71db27",
+                "sha256:760c12276fee05c36f95f8040180abc7fbebb9e5011447a97cdc289b5d6ab6fc",
+                "sha256:796685d3969815a633827c818863ee199440696b0961e200b011d79b9394bbe7",
+                "sha256:891fe897b49abb7db470c55664b198b1095e4943b9f82b7dcab317a19116cd38",
+                "sha256:a471628e20f03dcdfde00770eeaf9c77811f0c331c8805219ca7b87ac17576c5",
+                "sha256:a63b4fd3e2cabdcc9d918ed280bdde3e8e9641e04f3c59a2a3109644a07b9832",
+                "sha256:b0b84408d4eabc6de9dd1e1e0bc63e7731e890c0b378a62443e5741cfd0ae90a",
+                "sha256:be78485e5d5f3684e875dab60f40cddace2f5b2a8f7fede412358ab3214c3a6f",
+                "sha256:c27eaed872185f047bb7f7da2d21a7d8913457678c9a100a50db6da890bc28b9",
+                "sha256:c81cb40bff373ab7a7446d6bbca0190bccc5be3448b47b51d729e37799bb5692",
+                "sha256:d11874b3c33ee441059464711cd365b89fa1a9cf19ae75b0c189b01fbf735b84",
+                "sha256:e9c028b5897901361d81a4718d1db217b716424a0283afe9d6735fe0caf70f79",
+                "sha256:fe489d486cd00b739be826e8c1be188ddb74c7a1ca784d93d06fda882a6a1681"
            ],
            "index": "pypi",
-            "version": "==4.4.0"
+            "version": "==4.4.1"
        },
        "multidict": {
            "hashes": [
@@ -382,14 +448,35 @@
            ],
            "version": "==2.19"
        },
+        "pyjsparser": {
+            "hashes": [
+                "sha256:2b12842df98d83f65934e0772fa4a5d8b123b3bc79f1af1789172ac70265dd21",
+                "sha256:be60da6b778cc5a5296a69d8e7d614f1f870faf94e1b1b6ac591f2ad5d729579"
+            ],
+            "version": "==2.7.1"
+        },
+        "pyopenssl": {
+            "hashes": [
+                "sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200",
+                "sha256:c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6"
+            ],
+            "version": "==19.0.0"
+        },
        "pysocks": {
            "hashes": [
-                "sha256:15d38914b60dbcb231d276f64882a20435c049450160e953ca7d313d1405f16f",
-                "sha256:32238918ac0f19e9fd870a8692ac9bd14f5e8752b3c62624cda5851424642210",
-                "sha256:d9031ea45fdfacbe59a99273e9f0448ddb33c1580fe3831c1b09557c5718977c"
+                "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299",
+                "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5",
+                "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0"
            ],
            "index": "pypi",
-            "version": "==1.7.0"
+            "version": "==1.7.1"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
+                "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
+            ],
+            "version": "==2019.3"
        },
        "pyyaml": {
            "hashes": [
@@ -432,6 +519,13 @@
            ],
            "version": "==1.4.3"
        },
+        "requests-toolbelt": {
+            "hashes": [
+                "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f",
+                "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"
+            ],
+            "version": "==0.9.1"
+        },
        "six": {
            "hashes": [
                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
@@ -441,23 +535,24 @@
        },
        "soupsieve": {
            "hashes": [
-                "sha256:72b5f1aea9101cf720a36bb2327ede866fd6f1a07b1e87c92a1cc18113cbc946",
-                "sha256:e4e9c053d59795e440163733a7fec6c5972210e1790c507e4c7b051d6c5259de"
+                "sha256:605f89ad5fdbfefe30cdc293303665eff2d188865d4dbe4eb510bba1edfbfce3",
+                "sha256:b91d676b330a0ebd5b21719cb6e9b57c57d433671f65b9c28dd3461d9a1ed0b6"
            ],
-            "version": "==1.9.2"
+            "version": "==1.9.4"
        },
        "sqlalchemy": {
            "hashes": [
-                "sha256:217e7fc52199a05851eee9b6a0883190743c4fb9c8ac4313ccfceaffd852b0ff"
+                "sha256:0f0768b5db594517e1f5e1572c73d14cf295140756431270d89496dc13d5e46c"
            ],
            "markers": "python_version >= '3.0'",
-            "version": "==1.3.6"
+            "version": "==1.3.10"
        },
        "tablib": {
            "hashes": [
                "sha256:0f88a9cebdaa1a2cc29ae57387082ee81015d1149ecd34e48a8c8d3b4dd21670",
                "sha256:5f33c079b07eb10cf9c4b4696add2ecf32c89db7729240546ecdcd5c92f67e13"
            ],
+            "index": "pypi",
            "version": "==0.13.0"
        },
        "termcolor": {
@@ -468,26 +563,33 @@
        },
        "tldextract": {
            "hashes": [
-                "sha256:2c1c5d9d454f79734b4f3da0d603856dd9f820753410a3e9abf0a0c9fde33e97",
-                "sha256:b72bef6013de67c7fa181250bc2c2e089a994d259c09ca95a9771f2f97e29ed1"
+                "sha256:16b2f7e81d89c2a5a914d25bdbddd3932c31a6b510db886c3ce0764a195c0ee7",
+                "sha256:9aa21a1f7827df4209e242ec4fc2293af5940ec730cde46ea80f66ed97bfc808"
            ],
            "index": "pypi",
-            "version": "==2.2.1"
+            "version": "==2.2.2"
        },
        "tqdm": {
            "hashes": [
-                "sha256:14a285392c32b6f8222ecfbcd217838f88e11630affe9006cd0e94c7eff3cb61",
-                "sha256:25d4c0ea02a305a688e7e9c2cdc8f862f989ef2a4701ab28ee963295f5b109ab"
+                "sha256:abc25d0ce2397d070ef07d8c7e706aede7920da163c64997585d42d3537ece3d",
+                "sha256:dd3fcca8488bb1d416aa7469d2f277902f26260c45aa86b667b074cd44b3b115"
            ],
            "index": "pypi",
-            "version": "==4.32.2"
+            "version": "==4.36.1"
+        },
+        "tzlocal": {
+            "hashes": [
+                "sha256:11c9f16e0a633b4b60e1eede97d8a46340d042e67b670b290ca526576e039048",
+                "sha256:949b9dd5ba4be17190a80c0268167d7e6c92c62b30026cf9764caf3e308e5590"
+            ],
+            "version": "==2.0.0"
        },
        "urllib3": {
            "hashes": [
-                "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
-                "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
+                "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
+                "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
            ],
-            "version": "==1.25.3"
+            "version": "==1.25.6"
        },
        "win32-setctime": {
            "hashes": [
@@ -0,0 +1,378 @@
+# OneForAll
+
+[![Build Status](https://travis-ci.org/shmilylty/OneForAll.svg?branch=master)](https://travis-ci.org/shmilylty/OneForAll)
+[![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
+[![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
+[![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
+[![python](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/release-v0.0.7-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+
+👊**OneForAll is a powerful subdomain collection tool**  📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
+
+![](https://raw.githubusercontent.com/shmilylty/OneForAll/master/images/All_Might.jpg)
+
+## 🎉Project profile
+
+Project home：[https://shmilylty.github.io/OneForAll/](https://shmilylty.github.io/OneForAll/)
+
+Project address ：[https://github.com/shmilylty/OneForAll](https://github.com/shmilylty/OneForAll)
+
+The importance of information collection in penetration testing is self-evident. Subdomain collection is an essential and very important part of information collection. At present, there are many open source tools for subdomain collection on the Internet, but there are always some of the following problems:
+
+* **Not powerful enough**，there are not enough interfaces to collect subdomains automatically, and there are no functions such as automatic subdomain resolve, verification, FUZZ and information expansion.
+* **Not friendly enough**，although the command line module is more convenient, but when there are a lot of optional parameters and the operation to be implemented is complex, using command line mode is a bit unfriendly. If there is a good interaction, With a highly operable front end, the experience will be much better.
+
+* **Lack of maintenance**，Many tools have not been updated once in years, what issues and PR are, do not exist.
+
+* **Efficiency issues**，do not take advantage of multi-process, multi-threading and asynchronous cooperation technology, the speed is slow.
+
+In order to solve the above pain points, the term OneForAll comes from one of my favorite Japanese cartoons,《[My Hero Academia](https://manhua.fzdm.com/131/)》.It's a top personality that has grown from generation to generation, and now [TV series](https://www.bilibili.com/bangumi/media/md7452/)has been updated to season 4. As its name suggests, I want OneForAll to be a collection of 100 strong, powerful and fast subdomains that collect the ultimate artifact 🔨.
+
+At present, OneForAll is still under development, there must be a lot of problems and areas for improvement. Welcome to submit [Issues](https://github.com/shmilylty/OneForAll/issues) or [PR](https://github.com/shmilylty/OneForAll/pulls)，If you like, give it to a little star ✨，At present, there is a special QQ group for OneForAll communication and feedback: 👨‍👨‍👦‍👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)，You can also 📧[email](mailto://admin@hackfun.org) or [tweet](https://twitter.com/jinglingbiaodi) to me .
+
+## 👍Features
+
+* **Powerful collection capability**，For more information, please see [collection module description](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md).
+  1. Collect subdomains using certificate transparency (there are currently 6 modules: `censys_api`，`spyse_api`，`certspotter`，`crtsh`，`entrust`，`google`）
+
+  2. General check collection subdomains (there are currently 4 modules: domain transfer vulnerability exploitation`cdx`, cross-domain policy file `cdx`, HTTPS certificate `cert`, content security policy `csp`, robots file `robots`, and sitemap file `sitemap`. Check NSEC record, NSEC3 record and other modules will be added later).
+
+  3.  Collect subdomains using web crawler files (there are currently two modules: `archirawl`, `commoncrawl`, which is still being debugged and needs to be added and improved). 
+
+  4.  Collect subdomains using DNS datasets (there are currently 19 modules: `CeBaidu`, `binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`,  `ptrarchive`, `sitedossier`,`threatcrowd`）
+
+  5.  Collect subdomains using DNS queries (there is currently a module to collect subdomains `srv` by enumerating common SRV records and making queries, which needs to be added and improved). 
+
+  6.  Collect subdomains using threat intelligence platform data (there are currently five modules: `riskiq_ api`, `threatbook_ api`, `threatkeeper `, `virustotal`, `virustotal_ api`, which need to be added and improved). 
+
+  7.  Use search engines to discover subdomains (there are currently 16 modules: `ask`, `bing_ api`, `fofa_ api`, `shodan_ api`, `yahoo`, `baidu`, `duckduckgo`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_ api`, `sogou`, `zoomeye_ api`), except for special search engines in the search module. General search engines support automatic exclusion of search, full search, recursive search. 
+* **Support subdomain blasting**，This module has both conventional dictionary blasting and custom fuzz mode. It supports batch blasting and recursive blasting, and automatically judges pan-parsing and processing.
+* **Support subdmain verification**，default to enable subdomain verification, automatically resolve subdomain DNS, automatically request subdomain to obtain title and banner, and comprehensively determine subdomain survival.
+* **Support subdomain takeover**，By default, subdomain takeover risk checking is enabled. Automatic subdomain takeover is supported (only Github, remains to be improved at present), and batch inspection is supported.
+* **Powerful processing feature**，The found subdomain results support automatic removal, automatic DNS parsing, HTTP request detection, automatic filtering of valid subdomains, and expansion of Banner information for subdomains. The final supported export formats are `txt`, `rst`, `csv`, `tsv`, `json`, `yaml`, `html`, `xls`, `xlsx`, `dbf`, `latex`, `ods`.
+* **Very fast**，[collection module](https://github.com/shmilylty/OneForAll/tree/master/oneforall//collect.py) uses multithreaded calls, [blasting module](https://github.com/shmilylty/OneForAll/tree/master/oneforall/aiobrute.py) uses asynchronous multiprocess and multiprogramming, and DNS parsing and HTTP requests use asynchronous multiprogramming in subdomain verification. Multithreaded check [subdomain takeover](https://github.com/shmilylty/OneForAll/tree/master/oneforall/takeover.py) risk.
+* **Good experience**，Each module has a progress bar, and the results of each module are saved asynchronously.
+
+## 🚀Start Guide
+
+📢Currently, the project is **under development**, so it will continue to update iterations. It is best to **clone** the project when downloading and using OneForAll. Please take a moment to read this document to help you quickly get familiar with OneForAll!
+
+**🐍Installation requirements**
+
+OneForAll is based on CPython, so you need a Python environment to run. If your system doesn't already have a Python environment, you can refer to the [Python 3 Installation Guide](https://pythonguidecn.readthedocs.io/en/latest/starting/installation.html#python-3), theoretically Python 3.6, 3.7 and 3.8 can run OneForAll, **but** many tests are done on Python 3.7, so **recommended** you use **Python 3.7** version runs OneForAll. Run the following command to check the Python and pip3 versions:
+```bash
+python -V
+pip3 -V
+```
+If you see the following output, there is no problem with the Python environment:
+```bash
+Python 3.7.4
+pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.7)
+```
+
+**✔Installation steps**
+
+1. **Download**
+   
+   This project has been mirrored in [Gitee](https://gitee.com/shmilylty/OneForAll.git). If you are in China, it is recommended that you use Gitee for cloning, which is faster:
+
+   ```bash
+   git clone https://gitee.com/shmilylty/OneForAll.git
+   ```
+   or：
+   ```bash
+   git clone https://github.com/shmilylty/OneForAll.git
+   ```
+
+2. **Installation**
+   
+   You can install OneForAll dependencies via pip3 (if you are familiar with [pipenv](https://docs.pipenv.org/en/latest/), then it is recommended that you use [pipenv install dependencies](https://github.com/shmilylty/OneForAll/tree/master/docs/Installation_dependency.md), the following is an example of using **pip3** to install dependencies under **Windows system**: (Note: If your Python3 is installed in the system Program Files In the directory, such as: `C:\Program Files\Python37`, then run the command prompt cmd as an administrator to execute the following command!)
+```bash
+   cd OneForAll/
+   python -m pip install --user  -U pip setuptools wheel
+   pip3 install --user -r requirements.txt
+   cd oneforall/
+   python oneforall.py --help
+```
+      For other system platforms, please refer to [dependency installation](https://github.com/shmilylty/OneForAll/tree/master/docs/installation_dependency.md). If you find that compiling a dependent library fails during the installation dependencies, Refer to the solution in the [Q&A](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md) documentation, if not resolved, welcome feedback.
+
+3. **Update**
+
+   ❗Note: If you have cloned the project before, please **back** up your own modified files (such as **config.py**) to the location outside the project before updating, then execute the following command **update** project:
+
+   ```bash
+   git fetch --all
+   git reset --hard origin/master
+   git pull
+   ```
+
+**✨Demonstration**
+
+1. If you are installing dependencies through pip3, run the example using the following command: 
+    ```bash
+    cd oneforall/
+    python3 oneforall.py --target example.com run
+    ```
+
+    ![Example](./docs/usage_example.svg)
+
+2. If you install dependencies through pipenv, run the example using the following command: 
+   ```bash
+   cd oneforall/
+   pipenv run python oneforall.py --target example.com run
+   ```
+
+**🤔Help**
+
+The command line parameters only provide some common parameters. For more detailed parameter configuration, please see [config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py) if you think Some parameters are frequently used in the command interface or missing parameters. Feedback is welcome. For well-known reasons, if you want to use some of the wall's collection interface, please go to [config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py) to configure the proxy, some collection Modules need to provide APIs (most of which are freely available for registered accounts). If you need to use them, please go to [config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py) to configure the API. Information, if not used, please ignore the error message. (For detailed modules, please read [collection module description](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md))
+
+The OneForAll command line interface is based on [Fire](https://github.com/google/python-fire/). For more advanced usage of Fire, please refer to [using the Fire CLI](https://github.com/google/Python-fire/blob/master/docs/using-cli.md), if you have any doubts during the use, please feel free to give me feedback.
+
+[oneforall.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/oneforall.py) is the main program entry, and oneforall.py can call [aiobrute.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/aiobrute.py), [takerover.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/takerover.py) and [dbexport.py ](https://github.com/shmilylty/OneForAll/tree/master/oneforall/dbexport.py) and other modules, in order to facilitate the sub-field blasting, aiobrute.py is isolated independently, in order to facilitate the subdomain takeover risk check independently takeover.py, in order to facilitate the database export independently dbexport.py, these modules can be run separately, and the parameters accepted are more abundant.
+
+❗ Note: When you encounter some problems or doubts during use, please use [Issues](https://github.com/shmilylty/OneForAll/issues) to search for answers. Also see [Q&A](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md).
+
+1. **oneforall.py help**
+
+   ```bash
+   python oneforall.py --help
+   ```
+   ```bash
+   NAME
+       oneforall.py - OneForAll is a powerful subdomain collection tool
+   
+   SYNOPSIS
+       oneforall.py --target=TARGET <flags>
+   
+   DESCRIPTION
+       Version: 0.0.6
+       Project: https://git.io/fjHT1
+   
+       Example:
+           python3 oneforall.py --target example.com run
+           python3 oneforall.py --target ./domains.txt run
+           python3 oneforall.py --target example.com --brute True run
+           python3 oneforall.py --target example.com --verify False run
+           python3 oneforall.py --target example.com --valid None run
+           python3 oneforall.py --target example.com --port medium run
+           python3 oneforall.py --target example.com --format csv run
+           python3 oneforall.py --target example.com --show True run
+   
+       Note:
+           Parameter valid optional value 1, 0, none indicates that the export is 
+           valid, invalid, and all subdomains, respectively.
+           Parameter verify for True attempts to resolve and request the subdomain 
+           and tag the validity of the subdomain based on the result.
+           Parameter port have optional values 'small', 'medium', 'large', 'xlarge', 
+           See config.py configuration for details.
+           Parameter format have optional values 'txt', 'rst', 'csv', 'tsv', 'json', 
+           'yaml', 'html', 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'.
+           If the parameter path is None, the appropriate file is generated in the 
+           project result directory based on the format parameter and the domain 
+           name.
+   
+   ARGUMENTS
+       TARGET
+           Single domain name or file path for one domain name per line (required)
+   
+   FLAGS
+       --brute=BRUTE
+           Use blasting module (default False)
+       --verify=VERIFY
+           Verify the validity of subdomains (default True)
+       --port=PORT
+           Port range for request authentication (default medium)
+       --valid=VALID
+           Export validity of subdomains (default 1)
+       --path=PATH
+           Export path (default None)
+       --format=FORMAT
+           Export format (default xls)
+       --show=SHOW
+           Terminal display exported data (default False)
+   ```
+   
+2. **aiobrute.py help**
+
+   With regard to the handling of the universal parsing problem, first of all, OneForAll accesses a random subdomain to determine whether universal parsing is used, and if universal parsing is used, it is handled by the following judgment: 
+   - First, it is mainly compared with the pan-parsed IP set and TTL values, see [this article](http://sh3ll.me/archives/201704041222.txt).
+
+   - Second, the number of times to resolve to the same IP collection multiple times (the default is 10, which can be set to size in config.py).
+
+   - Third, considering the blasting efficiency, there is no HTTP response volume similarity comparison and response volume content judgment,  this function has not been implemented yet, and will be implemented if necessary.
+
+   After not rigorous testing, in the 16-core CPU, using 16 processes 64 coroutines, 100M network bandwidth environment, set the task to be divided into 50000, run two million dictionaries about 10 minutes to run, about 3333 subdomains per second.
+
+   ```bash
+   python aiobrute.py --help
+   ```
+
+   ```bash
+   NAME
+       aiobrute.py - OneForAll multi-process multi-correlation asynchronous subdomain blasting module
+   
+   SYNOPSIS
+       aiobrute.py --target=TARGET <flags>
+   
+   DESCRIPTION
+       Example：
+           python3 aiobrute.py --target example.com run
+           python3 aiobrute.py --target ./domains.txt run
+           python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
+           python3 aiobrute.py --target example.com --wordlist subdomains.txt run
+           python3 aiobrute.py --target example.com --recursive True --depth 2 run
+           python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
+   
+       Note:
+           The setting of the parameter segment is affected by CPU performance, 
+           network bandwidth, and operator restrictions. By default, 500 subdomains 
+           are set as task groups. When you feel that your environment is not 
+           affected by the above factors, the current blasting speed is slow, so it 
+           is strongly recommended to use the dictionary. Size resizing: 100,000 
+           dictionary suggestions set to 5000, million dictionary set to 50000.
+           
+           Parameter valid optional value 1, 0, none indicates that the export is 
+           valid, invalid, and all subdomains, respectively.
+           
+           Parameter format have optional values 'txt', 'rst', 'csv', 'tsv', 'json', 
+           'yaml', 'html', 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'.
+           If the parameter path is None, the appropriate file is generated in the 
+           project result directory based on the format parameter and the domain 
+           name.
+   
+   ARGUMENTS
+       TARGET
+           Single domain name or file path for one domain name per line (required)
+   
+   FLAGS
+       --process=PROCESS
+           Number of processes blasted (default CPU core count)
+       --coroutine=COROUTINE
+           Number of coroutines per blasting process (default 64)
+       --wordlist=WORDLIST
+           Specify the dictionary path used for blasting (config.py is used by default)
+       --segment=SEGMENT
+           Blasting task segmentation (default 500)
+       --recursive=RECURSIVE
+           Whether to use recursive blasting (default False)
+       --depth=DEPTH
+           Depth of recursive blasting (default 2)
+       --namelist=NAMELIST
+           Specifies the dictionary path used by recursive blasting (configured by default using config.py)
+       --fuzz=FUZZ
+           Whether to use the fuzz mode for blasting (default False, you must specify the fuzz regular rule)
+       --rule=RULE
+           Regular rules used by fuzz mode (configured by default using config.py)
+       --export=EXPORT
+           Whether to export the blast result (default True)
+       --valid=VALID
+           Export validity of subdomains (default None)
+       --format=FORMAT
+           Export format (default xls)
+       --path=PATH
+           Export path (default None)
+       --show=SHOW
+           Terminal display exported data (default False)
+   ```
+
+3. For other module usage, please refer to [usage help](https://github.com/shmilylty/OneForAll/tree/master/docs/using_help.md).
+
+## 👏Framework used
+
+* [aiodns](https://github.com/saghul/aiodns) - aiodns provides a simple way for doing asynchronous DNS resolutions using [pycares](https://github.com/saghul/pycares). 
+* [aiohttp](https://github.com/aio-libs/aiohttp) -  Asynchronous HTTP client/server framework for asyncio and Python 
+* [aiomultiprocess](https://github.com/jreese/aiomultiprocess) -  Take a modern Python codebase to the next level of performance. (Multiprocessing and asyncio combine to implement asynchronous multi-process multi-coroutine)
+* [beautifulsoup4](https://pypi.org/project/beautifulsoup4/) -  Beautiful Soup is a library that makes it easy to scrape information from web pages. 
+* [fire](https://github.com/google/python-fire) -  Python Fire is a library for automatically generating command line interfaces (CLIs) from absolutely any Python object. 
+* [loguru](https://github.com/Delgan/loguru) -  Loguru is a library which aims to bring enjoyable logging in Python. 
+* [records](https://github.com/kennethreitz/records) -  Records is a very simple, but powerful, library for making raw SQL queries to most relational databases.
+* [requests](https://github.com/psf/requests) -  A simple, yet elegant HTTP library. 
+* [tqdm](https://github.com/tqdm/tqdm) -  A Fast, Extensible Progress Bar for Python and CLI 
+
+Thanks to these great excellent Python libraries!
+
+## 🌲Directory structure
+
+```bash
+D:.
+|
+---.github
+---docs
+|       collection_modules.md collection module description
+---images
+\---oneforall
+    |   aiobrute.py   Asynchronous multi-process multi-correlation subdomain blasting module, can be run separately
+    |   collect.py    Upper layer call of each collection module
+    |   config.py     Configuration file
+    |   dbexport.py   Database export module, can be run separately
+    |   domains.txt   List of domain names to be blasted
+    |   oneforall.py  OneForAll main entrance, can be run separately
+    |   __init__.py
+    |
+    +---common Common call module
+    +---data   Store some of the required data
+    |       next_subdomains.txt     Next level subdomain dictionary
+    |       public_suffix_list.dat  Top-level domain name suffix
+    |       srv_names.json          Common SRV record prefix name
+    |       subdomains.txt          Common Dictionary of subdomain blasting
+    |
+    \---modules 
+        +---certificates     Using certificate transparency to collect subdomain
+        +---check            Using conventional methods to collect subdomain
+        +---crawl            Using web crawler files to collect subdomain
+        +---datasets         Using DNS datasets to collect subdomain
+        +---dnsquery         Using DNS query to collect subdomain
+        +---intelligence     Using threat intelligence platform to collect subdomain
+        \---search           Using search engine to collect subdomain
+
+```
+
+## 🙏Contribution
+
+Very warmly welcome all ace to improve the project together!
+
+## ⌛Follow-up plan
+
+- [ ] Continuous optimization and improvement of each module
+- [ ] Subdomain monitoring (marking each newly discovered subdomain)
+- [ ] Subdomain collection crawler implementation (including collection of subdomains from static resource files such as JS)
+- [ ] Implementation of front-end interface for powerful interaction (tentative: front-end: Element + back-end: Flask)
+
+For more details, see [TODO.md](https://github.com/shmilylty/OneForAll/tree/master/TODO.md).
+
+## 🔖Version control
+
+The project uses [SemVer](https://semver.org/) language version format for version management), and you can view the available version in [Releases](https://github.com/shmilylty/OneForAll/releases).
+
+## 👨‍💻Contributors
+
+* **[Jing Ling](https://github.com/shmilylty)**
+  * Core development
+
+* **[Black Star](https://github.com/blackstar24)**，**[Echocipher](https://github.com/Echocipher)**
+  * Module contribution
+
+* **[iceMatcha](https://github.com/iceMatcha)**，**[mikuKeeper](https://github.com/mikuKeeper)**
+  * Tool test
+
+* **Anyone**
+  * Tool feedback
+
+You can see all the developers involved in the project in [CONTRIBUTORS.md](https://github.com/shmilylty/OneForAll/tree/master/CONTRIBUTORS.md).
+
+## 📄License
+
+The project has signed a GPL-3.0 license, for more information, please see [LICENSE](https://github.com/shmilylty/OneForAll/LICENSE).
+
+## 😘Acknowledgement
+
+Thanks to the various subdomain collection projects of online open source!
+
+Thanks ace of [A-Team](https://github.com/QAX-A-Team) for their enthusiastic and unselfish answers!
+
+## 📜Disclaimer ##
+
+This tool is limited to legally authorized enterprise security construction. In the process of using this tool, you should ensure that all your actions comply with local laws and regulations and have obtained sufficient authorization.
+If you have any illegal behavior in the process of using this tool, you are responsible for all consequences, and all authors and all contributors of this tool do not assume any legal and joint responsibility.
+Unless you have fully read, fully understood and accepted all the terms of this Agreement, please do not install and use this tool.
+Your use or any other express or implied representation of you to this Agreement is deemed to have been read and agreed to be bound by this Agreement.
@@ -1,5 +1,360 @@
 # OneForAll
-Subdomain Collection One For All：Full Cowl-100% Smash

-# Developing
-OneForAll is under development, please look forward to it!
+[![Build Status](https://travis-ci.org/shmilylty/OneForAll.svg?branch=master)](https://travis-ci.org/shmilylty/OneForAll)
+[![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
+[![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
+[![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
+[![python](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/release-v0.0.7-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+
+👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/README.en.md)
+
+![](https://raw.githubusercontent.com/shmilylty/OneForAll/master/images/All_Might.jpg)
+
+## 🎉项目简介
+
+项目主页：[https://shmilylty.github.io/OneForAll/](https://shmilylty.github.io/OneForAll/)
+
+项目地址：[https://github.com/shmilylty/OneForAll](https://github.com/shmilylty/OneForAll)
+
+在渗透测试中信息收集的重要性不言而喻，子域收集是信息收集中必不可少且非常重要的一环，目前网上也开源了许多子域收集的工具，但是总是存在以下部分问题：
+
+* **不够强大**，子域收集的接口不够多，不能做到对批量子域自动收集，没有自动子域解析，验证，FUZZ以及信息拓展等功能。
+* **不够友好**，固然命令行模块比较方便，但是当可选的参数很多，要实现的操作复杂，用命令行模式就有点不够友好，如果有交互良好，高可操作的前端那么使用体验就会好很多。
+
+* **缺少维护**，很多工具几年没有更新过一次，issues和PR是啥，不存在的。
+
+* **效率问题**，没有利用多进程，多线程以及异步协程技术，速度较慢。
+
+为了解决以上痛点，此项目应用而生，OneForAll一词是来自我喜欢的一部日漫《[我的英雄学院](https://manhua.fzdm.com/131/)》，它是一种通过一代代的传承不断变强的潜力无穷的顶级个性，目前[番剧](https://www.bilibili.com/bangumi/media/md7452/)也更新到了第三季了，欢迎大佬们入坑😄。正如其名，我希望OneForAll是一款集百家之长，功能强大的全面快速子域收集终极神器🔨。
+
+目前OneForAll还在开发中，肯定有不少问题和需要改进的地方，欢迎大佬们提交[Issues](https://github.com/shmilylty/OneForAll/issues)和[PR](https://github.com/shmilylty/OneForAll/pulls)，用着还行给个小星星✨吧，目前有一个专门用于OneForAll交流和反馈QQ群👨‍👨‍👦‍👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)，也可以给我发邮件📧[admin@hackfun.org]。
+
+## 👍功能特性
+
+* **收集能力强大**，详细模块请阅读[收集模块说明](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md)。
+  1. 利用证书透明度收集子域（目前有6个模块：`censys_api`，`spyse_api`，`certspotter`，`crtsh`，`entrust`，`google`）
+
+  2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，检查robots文件`robots`，检查sitemap文件`sitemap`，后续会添加检查NSEC记录，NSEC3记录等模块）
+
+  3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善）
+
+  4. 利用DNS数据集收集子域（目前有19个模块：`CeBaidu`, `binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`,  `ptrarchive`, `sitedossier`,`threatcrowd`）
+
+  5. 利用DNS查询收集子域（目前有1个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，该模块还有待添加和完善）
+
+  6. 利用威胁情报平台数据收集子域（目前有5个模块：`riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）
+
+  7. 利用搜索引擎发现子域（目前有16个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
+* **支持子域爆破**，该模块有常规的字典爆破，也有自定义的fuzz模式，支持批量爆破和递归爆破，自动判断泛解析并处理。
+* **支持子域验证**，默认开启子域验证，自动解析子域DNS，自动请求子域获取title和banner，并综合判断子域存活情况。
+* **支持子域接管**，默认开启子域接管风险检查，支持子域自动接管（目前只有Github，有待完善），支持批量检查。
+* **处理功能强大**，发现的子域结果支持自动去除，自动DNS解析，HTTP请求探测，自动筛选出有效子域，拓展子域的Banner信息，最终支持的导出格式有`txt`, `rst`, `csv`, `tsv`, `json`, `yaml`, `html`, `xls`, `xlsx`, `dbf`, `latex`, `ods`。
+* **速度极快**，[收集模块](https://github.com/shmilylty/OneForAll/tree/master/oneforall//collect.py)使用多线程调用，[爆破模块](https://github.com/shmilylty/OneForAll/tree/master/oneforall/aiobrute.py)使用异步多进程多协程，子域验证中DNS解析和HTTP请求使用异步多协程，多线程检查[子域接管](https://github.com/shmilylty/OneForAll/tree/master/oneforall/takeover.py)风险。
+* **体验良好**，日志和终端输出全使用中文，各模块都有进度条，异步保存各模块结果。
+
+## 🚀上手指南
+
+📢由于该项目**处于开发中**，会不断进行更新迭代，下载使用最好**克隆**最新项目，请务必花一点时间阅读此文档，有助于你快速熟悉OneForAll！
+
+**🐍安装要求**
+
+OneForAll是基于CPython开发的，所以你需要Python环境才能运行，如果你的系统还没有Python环境你可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)，理论上Python 3.6，3.7和3.8都可以正常运行OneForAll，**但是**许多测试都是在Python 3.7上进行的，所以**推荐**你使用**Python 3.7**版本运行OneForAll。运行以下命令检查Python和pip3版本：
+```bash
+python -V
+pip3 -V
+```
+如果你看到以下类似输出便说明Python环境没有问题：
+```bash
+Python 3.7.4
+pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.7)
+```
+
+**✔安装步骤**
+
+1. **下载**
+   本项目已经在[码云](https://gitee.com/shmilylty/OneForAll.git)(Gitee)镜像了一份，国内推荐使用码云进行克隆比较快：
+
+   ```bash
+   git clone https://gitee.com/shmilylty/OneForAll.git
+   ```
+   或者：
+   ```bash
+   git clone https://github.com/shmilylty/OneForAll.git
+   ```
+
+2. **安装**
+   你可以通过pip3安装OneForAll的依赖（如果你熟悉[pipenv](https://docs.pipenv.org/en/latest/)，那么推荐你使用[pipenv安装依赖]((https://github.com/shmilylty/OneForAll/tree/master/docs/Installation_dependency.md))），以下为**Windows系统**下使用**pip3**安装依赖的示例：（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python37`，那么请以管理员身份运行命令提示符cmd执行以下命令！）
+   
+```bash
+   cd OneForAll/
+   python -m pip install --user  -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+   pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+   cd oneforall/
+   python oneforall.py --help
+   ```
+   其他系统平台的请参考[依赖安装](https://github.com/shmilylty/OneForAll/tree/master/docs/installation_dependency.md)，如果在安装依赖过程中发现编译某个依赖库失败时可以参考[Q&A](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md)中解决方法，如果还没有解决欢迎加群反馈。
+
+3. **更新**
+   ❗注意：如果你之前已经克隆了项目运行之前请**备份**自己修改过的文件到项目外的地方（如**config.py**），然后执行以下命令**更新**项目：
+
+   ```bash
+   git fetch --all
+   git reset --hard origin/master
+   git pull
+   ```
+
+**✨使用演示**
+
+1. 如果你是通过pip3安装的依赖则使用以下命令运行示例：   
+    ```bash
+    cd oneforall/
+    python3 oneforall.py --target example.com run
+    ```
+
+    ![Example](./docs/usage_example.svg)
+
+2. 如果你通过pipenv安装的依赖则使用以下命令运行示例：
+   ```bash
+   cd oneforall/
+   pipenv run python oneforall.py --target example.com run
+   ```
+
+**🤔使用帮助**
+
+命令行参数只提供了一些常用参数，更多详细的参数配置请见[config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py)，如果你认为有些参数是命令界面经常使用到的或缺少了什么参数等问题非常欢迎反馈。由于众所周知的原因，如果要使用一些被墙的收集接口请先到[config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py)配置代理，有些收集模块需要提供API（大多都是可以注册账号免费获取），如果需要使用请到[config.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/config.py)配置API信息，如果不使用请忽略有关报错提示。（详细模块请阅读[收集模块说明](https://github.com/shmilylty/OneForAll/tree/master/docs/collection_modules.md)）
+
+OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实现，有关Fire更高级使用方法请参阅[使用Fire CLI](https://github.com/google/python-fire/blob/master/docs/using-cli.md)，有任何使用疑惑欢迎加群交流。
+
+[oneforall.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/oneforall.py)是主程序入口，oneforall.py可以调用[aiobrute.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/aiobrute.py)，[takerover.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/takerover.py)及[dbexport.py](https://github.com/shmilylty/OneForAll/tree/master/oneforall/dbexport.py)等模块，为了方便进行子域爆破独立出了aiobrute.py，为了方便进行子域接管风险检查独立出了takerover.py，为了方便数据库导出独立出了dbexport.py，这些模块都可以单独运行，并且所接受参数要更丰富一点。
+
+❗注意：当你在使用过程中遇到一些问题或者疑惑时，请先到[Issues](https://github.com/shmilylty/OneForAll/issues)里使用搜索找找答案，还可以参阅[常见问题与回答](https://github.com/shmilylty/OneForAll/tree/master/docs/Q&A.md)。
+
+1. **oneforall.py使用帮助**
+
+   ```bash
+   python oneforall.py --help
+   ```
+   ```bash
+   NAME
+       oneforall.py - OneForAll是一款功能强大的子域收集工具
+   
+   SYNOPSIS
+       oneforall.py --target=TARGET <flags>
+   
+   DESCRIPTION
+       Version: 0.0.6
+       Project: https://git.io/fjHT1
+   
+       Example:
+           python3 oneforall.py --target example.com run
+           python3 oneforall.py --target ./domains.txt run
+           python3 oneforall.py --target example.com --brute True run
+           python3 oneforall.py --target example.com --verify False run
+           python3 oneforall.py --target example.com --valid None run
+           python3 oneforall.py --target example.com --port medium run
+           python3 oneforall.py --target example.com --format csv run
+           python3 oneforall.py --target example.com --show True run
+   
+       Note:
+           参数valid可选值1，0，None分别表示导出有效，无效，全部子域
+           参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
+           参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+           参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                             'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   ARGUMENTS
+       TARGET
+           单个域名或者每行一个域名的文件路径(必需参数)
+   
+   FLAGS
+       --brute=BRUTE
+           使用爆破模块(默认False)
+       --verify=VERIFY
+           验证子域有效性(默认True)
+       --port=PORT
+           请求验证的端口范围(默认medium)
+       --valid=VALID
+           导出子域的有效性(默认1)
+       --path=PATH
+           导出路径(默认None)
+       --format=FORMAT
+           导出格式(默认xls)
+       --show=SHOW
+           终端显示导出数据(默认False)
+   ```
+
+2. **aiobrute.py使用帮助**
+
+   关于泛解析问题处理程序首先会访问一个随机的子域判断是否泛解析，如果使用了泛解析则是通过以下判断处理：
+   - 一是主要是与泛解析的IP集合和TTL值做对比，可以参考[这篇文章](http://sh3ll.me/archives/201704041222.txt)。
+   - 二是多次解析到同一IP集合次数（默认设置为10，可以在config.py设置大小）。
+   - 三是考虑爆破效率问题目前还没有加上HTTP响应体相似度对比和响应体内容判断，如果有必要后续添加。
+   经过不严谨测试在16核心的CPU，使用16进程64协程，100M带宽的环境下，设置任务分割为50000，跑两百万字典大概10分钟左右跑完，大概3333个子域每秒。
+
+   ```bash
+   python aiobrute.py --help
+   ```
+
+   ```bash
+   NAME
+       aiobrute.py - OneForAll多进程多协程异步子域爆破模块
+   
+   SYNOPSIS
+       aiobrute.py --target=TARGET <flags>
+   
+   DESCRIPTION
+       Example：
+           python3 aiobrute.py --target example.com run
+           python3 aiobrute.py --target ./domains.txt run
+           python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
+           python3 aiobrute.py --target example.com --wordlist subdomains.txt run
+           python3 aiobrute.py --target example.com --recursive True --depth 2 run
+           python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
+   
+       Note:
+           参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为任务组，
+           当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+           十万字典建议设置为5000，百万字典设置为50000
+           参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+           参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                             'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   ARGUMENTS
+       TARGET
+           单个域名或者每行一个域名的文件路径
+   
+   FLAGS
+       --process=PROCESS
+           爆破的进程数(默认CPU核心数)
+       --coroutine=COROUTINE
+           每个爆破进程下的协程数(默认64)
+       --wordlist=WORDLIST
+           指定爆破所使用的字典路径(默认使用config.py配置)
+       --segment=SEGMENT
+           爆破任务分割(默认500)
+       --recursive=RECURSIVE
+           是否使用递归爆破(默认False)
+       --depth=DEPTH
+           递归爆破的深度(默认2)
+       --namelist=NAMELIST
+           指定递归爆破所使用的字典路径(默认使用config.py配置)
+       --fuzz=FUZZ
+           是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
+       --rule=RULE
+           fuzz模式使用的正则规则(默认使用config.py配置)
+       --export=EXPORT
+           是否导出爆破结果(默认True)
+       --valid=VALID
+           导出子域的有效性(默认None)
+       --format=FORMAT
+           导出格式(默认xls)
+       --path=PATH
+           导出路径(默认None)
+       --show=SHOW
+           终端显示导出数据(默认False)
+   ```
+
+3. 其他模块使用请参考[使用帮助](https://github.com/shmilylty/OneForAll/tree/master/docs/using_help.md)
+
+## 👏主要框架
+
+* [aiodns](https://github.com/saghul/aiodns) - 简单DNS异步解析库。
+* [aiohttp](https://github.com/aio-libs/aiohttp) - 异步http客户端/服务器框架
+* [aiomultiprocess](https://github.com/jreese/aiomultiprocess) - 将Python代码提升到更高的性能水平(multiprocessing和asyncio结合，实现异步多进程多协程)
+* [beautifulsoup4](https://pypi.org/project/beautifulsoup4/) - 可以轻松从HTML或XML文件中提取数据的Python库
+* [fire](https://github.com/google/python-fire) - Python Fire是一个纯粹根据任何Python对象自动生成命令行界面（CLI）的库
+* [loguru](https://github.com/Delgan/loguru) - 旨在带来愉快的日志记录Python库
+* [records](https://github.com/kennethreitz/records) - Records是一个非常简单但功能强大的库，用于对大多数关系数据库进行最原始SQL查询。
+* [requests](https://github.com/psf/requests) - Requests 唯一的一个非转基因的 Python HTTP 库，人类可以安全享用。
+* [tqdm](https://github.com/tqdm/tqdm) - 适用于Python和CLI的快速，可扩展的进度条库
+
+感谢这些伟大优秀的Python库！
+
+## 🌲目录结构
+
+```bash
+D:.
+|
+---.github
+---docs
+|       collection_modules.md 收集模块说明
+---images
+\---oneforall
+    |   aiobrute.py   异步多进程多协程子域爆破模块，可以单独运行
+    |   collect.py    各个收集模块上层调用
+    |   config.py     配置文件
+    |   dbexport.py   数据库导出模块，可以单独运行
+    |   domains.txt   要批量爆破的域名列表
+    |   oneforall.py  OneForAll主入口，可以单独运行
+    |   __init__.py
+    |
+    +---common 公共调用模块
+    +---data   存放一些所需数据
+    |       next_subdomains.txt     下一层子域字典
+    |       public_suffix_list.dat  顶级域名后缀 
+    |       srv_names.json          常见SRV记录前缀名
+    |       subdomains.txt          子域爆破常见字典
+    |
+    \---modules 
+        +---certificates     利用证书透明度收集子域模块
+        +---check            常规检查收集子域模块
+        +---crawl            利用网上爬虫档案收集子域模块
+        +---datasets         利用DNS数据集收集子域模块
+        +---dnsquery         利用DNS查询收集子域模块
+        +---intelligence     利用威胁情报平台数据收集子域模块
+        \---search           利用搜索引擎发现子域模块
+
+```
+
+## 🙏贡献
+
+非常热烈欢迎各位大佬一起完善本项目！
+
+## ⌛后续计划
+
+- [ ] 各模块持续优化和完善
+- [ ] 子域监控（标记每次新发现的子域）
+- [ ] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
+- [ ] 操作强大交互人性的前端界面实现（暂定：前端：Element + 后端：Flask）
+
+更多详细信息请阅读[TODO.md](https://github.com/shmilylty/OneForAll/tree/master/TODO.md)。
+
+## 🔖版本控制
+
+该项目使用[SemVer](https://semver.org/)语言化版本格式进行版本管理，你可以在[Releases](https://github.com/shmilylty/OneForAll/releases)查看可用版本。
+
+## 👨‍💻贡献者
+
+* **[Jing Ling](https://github.com/shmilylty)**
+  * 核心开发
+
+* **[Black Star](https://github.com/blackstar24)**，**[Echocipher](https://github.com/Echocipher)**
+  * 模块贡献
+
+* **[iceMatcha](https://github.com/iceMatcha)**，**[mikuKeeper](https://github.com/mikuKeeper)**
+  * 工具测试
+
+* **Anyone**
+  * 工具反馈
+
+你可以在[CONTRIBUTORS.md](https://github.com/shmilylty/OneForAll/tree/master/CONTRIBUTORS.md)中参看所有参与该项目的开发者。
+
+## 📄版权
+
+该项目签署了GPL-3.0授权许可，详情请参阅[LICENSE](https://github.com/shmilylty/OneForAll/LICENSE)。
+
+## 😘鸣谢
+
+感谢网上开源的各个子域收集项目！
+
+感谢[A-Team](https://github.com/QAX-A-Team)大哥们热情无私的问题解答！
+
+## 📜免责声明 ##
+
+本工具仅限于合法授权的企业安全建设，在使用本工具过程中，您应确保自己所有行为符合当地的法律法规，并且已经取得了足够的授权。
+如您在使用本工具的过程中存在任何非法行为，您需自行承担所有后果，本工具所有作者和所有贡献者不承担任何法律及连带责任。
+除非您已充分阅读、完全理解并接受本协议所有条款，否则，请您不要安装并使用本工具。
+您的使用行为或者您以其他任何明示或者默示方式表示接受本协议的，即视为您已阅读并同意本协议的约束。
@@ -2,8 +2,8 @@

 ## 下一步计划

- [ ] 子域收集模块优化
- [ ] 子域接管功能实现
- [ ] 子域收集爬虫实现
- [ ] 操作强大交互人性的前端界面实现
+- [ ] 各模块支持优化和完善
+- [ ] 子域监控（标记每次新发现的子域）
+- [ ] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
+- [ ] 操作强大交互人性的前端界面实现（暂定：Element+Flask）

@@ -0,0 +1 @@
+theme: jekyll-theme-architect
@@ -0,0 +1,40 @@
+# 常见问题与回答
+
+## 依赖问题
+
+**Q: 在安装依赖过程遇到编译某个依赖库失败，怎么解决？**
+
+A: 可以尝试以下方法:
+
+1. 到提供编译好的whl文件的第三方平台，找到对应库手动下载安装。第三方平台平台有： 
+ * [https://www.lfd.uci.edu/~gohlke/pythonlibs](https://www.lfd.uci.edu/~gohlke/pythonlibs)
+ * [https://pythonwheels.com/](https://pythonwheels.com/)
+
+选择好对应版本执行以下命令手动安装。举个例子，当编译pycares时失败时，找到[https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares](https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares)，由于我的系统是Windows 10 64位，使用的Python 3.7便下载`pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl`（一般来说下载最新版本的），然后手动安装：
+
+```bash
+pip3 install pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl
+```
+
+2. 到库的项目地址issues和wiki等找找有没有解决方法，如果没有就给他们提issues发邮件😜。
+
+## 使用问题
+
+**Q: 为什么运行OneForAll之后最终结果为空？**
+
+*A: 有几种可能性：第一可能目标域名没有子域。第二由于OneForAll默认会自动验证子域，在导出是只会有效子域，所以存在导出时没有有效子域的情况，你可以在运行OneForAll使用--valid=None指定导出所有发现的子域，你也可以使用--verify=False指定不验证子域的有效性。*
+
+**Q: 安装依赖时出现以下类似报错，怎么解决？**
+   Cannot uninstall 'PyYAML'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall.
+
+*A: 安装依赖时尝试加上--ignore-installed参数*
+
+**Q:使用Excel打开结果的csv中有中文乱码，怎么解决？**
+
+*A:由于Excel打开文件需要带BOM头识别编码，而默认生成的结果csv文件是UTF-8编码不是UTF-8-BOM，所以会使用Excel打开会出现中文乱码*，有以下解决办法：
+
+1.在一开始运行OneForAll，设置format参数为xls或者xlsx格式。
+
+2.使用NotePad++类似工具转化编码格式为UTF-8-BOM。
+
+3.使用Excel软件导入结果csv文件中的数据。
@@ -0,0 +1,109 @@
+#  收集模块说明 #
+
+如果要使用通过API收集子域的模块请先到[config.py](../oneforall/config.py)配置相关信息，大多平台的API都是可以注册账号免费获取的。
+
+如果你指定使用某些模块可以在[config.py](../oneforall/config.py)中设置：
+
+```python
+enable_all_module = False  # 不开启所有模块
+enable_partial_module = [('modules.search', 'ask')，('modules.search', 'baidu')]  # 只使用ask和baidu搜索引擎收集子域
+```
+
+如果你指定使用某些模块使用代理可以在[config.py](../oneforall/config.py)中设置：
+
+```python
+enable_proxy = True  # 使用代理
+proxy_all_module = False  # 不代理所有模块
+proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和AskSearch模块（各个模块的source属性值）
+```
+
+以下是各个模块说明：
+
+ 1. 利用证书透明度收集子域（目前有6个模块：`censys_api`，`certdb_api`，`certspotter`，`crtsh`，`entrust`，`google`）
+    
+      | 模块名称    | 是否需要代理 | 是否需要API | 其他说明                                           |
+      | ----------- | ------------ | ----------- | -------------------------------------------------- |
+      | censys_api  | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | certspotter | 否           | 否          |                                                    |
+      | crtsh       | 否           | 否          |                                                    |
+      | entrust     | 否           | 否          |                                                    |
+      | google      | 是           | 否          |                                                    |
+      | spyse_api   | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+
+
+  2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，后续会添加检查NSEC记录，NSEC3记录等模块）
+
+      | 模块名称 | 是否需要代理           | 是否需要API | 其他说明           |
+      | -------- | ---------------------- | ----------- | ------------------ |
+      | axfr     | 否                     | 否          | 域传送漏洞利用     |
+      | cdx      | 手动设置（默认不使用） | 否          | 检查跨域策略文件   |
+      | cert     | 否                     | 否          | 检查HTTPS证书      |
+      | csp      | 手动设置（默认不使用） | 否          | 检查内容安全策略   |
+      | robots   | 手动设置（默认不使用） | 否          | 检查robots.txt文件 |
+      | sitemap  | 手动设置（默认不使用） | 否          | 检查sitemap文件    |
+  3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善）
+
+      | 模块名称     | 是否需要代理 | 是否需要API | 其他说明 |
+      | ------------ | ------------ | ----------- | -------- |
+      | archivecrawl | 否           | 否          |          |
+      | commoncrawl  | 否           | 否          |          |
+
+  4. 利用DNS数据集收集子域（目前有19个模块：`cebaidu`, `binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `passivedns_api`,  `ptrarchive`, `sitedossier`,`threatcrowd`）
+
+      | 模块名称           | 是否需要代理 | 是否需要API | 其他说明                                           |
+      | ------------------ | ------------ | ----------- | -------------------------------------------------- |
+      | binaryedge_api     | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | bufferover         | 否           | 否          |                                                    |
+      | cebaidu            | 否           | 否          |                                                    |
+      | chinaz             | 否           | 否          |                                                    |
+      | chinaz_api         | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | circl_api          | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | dnsdb              | 否           | 否          |                                                    |
+      | dnsdb_api          | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | dnsdumpster        | 否           | 否          |                                                    |
+      | hackertarget       | 否           | 否          |                                                    |
+      | ipv4info           | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | netcraft           | 否           | 否          |                                                    |
+      | passivedns_api     | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | ptrarchive         | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | riddler            | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | robtex             | 否           | 否          |                                                    |
+      | securitytrails_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | sitedossier        | 否           | 否          |                                                    |
+      | threatcrowd        | 否           | 否          |                                                    |
+  5. 利用DNS查询收集子域（目前有1个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，该模块还有待添加和完善）
+
+      | 模块名称 | 是否需要代理 | 是否需要API | 其他说明                      |
+      | -------- | ------------ | ----------- | ----------------------------- |
+      | srv      | 否           | 否          | 枚举域名常见的SRV记录发现子域 |
+  6. 利用威胁平台数据收集子域（目前有5个模块：`riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）
+
+      | 模块名称       | 是否需要代理 | 是否需要API | 其他说明                                           |
+      | -------------- | ------------ | ----------- | -------------------------------------------------- |
+      | riskiq_api     | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | threatbook_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | threatminer    | 否           | 否          |                                                    |
+      | virustotal     | 否           | 否          |                                                    |
+      | virustotal_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+  7. 利用搜索引擎发现子域（目前有16个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`）
+
+     除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
+
+     | 模块        | 是否需要代理           | 是否需要API | 其他说明                                                    |
+     | ----------- | ---------------------- | ----------- | ----------------------------------------------------------- |
+     | ask         | 是                     | 否          |                                                             |
+     | baidu       | 否                     | 否          |                                                             |
+     | bing        | 否                     | 否          |                                                             |
+     | bing_api    | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
+     | duckduckgo  | 是                     | 否          |                                                             |
+     | exalead     | 否，最好使用国外代理。 | 否          |                                                             |
+     | fofa_api    | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
+     | github      | 否                     | 否          | 在[config.py](../oneforall/config.py)设置Github邮件名和密码 |
+     | google      | 是                     | 否          |                                                             |
+     | google_api  | 是                     | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
+     | shodan_api  | 否，最好使用国外代理。 | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
+     | so          | 否                     | 否          |                                                             |
+     | sogou       | 否                     | 否          |                                                             |
+     | yahoo       | 是                     | 否          |                                                             |
+     | yandex      | 是                     | 否          |                                                             |
+     | zoomeye_api | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py)          |
@@ -0,0 +1,64 @@
+# 安装依赖
+
+你可以通过pip3和pipenv两种方法安装OneForAll的依赖（如果你熟悉[pipenv](https://docs.pipenv.org/en/latest/)，那么推荐使用你使用pipenv）：
+
+* **Windows系统**（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python37`，那么请以管理员身份运行命令提示符cmd执行以下命令！）
+
+    1. 使用pipenv
+
+    ```bash
+    cd OneForAll/
+    python -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
+    pipenv install --user --python 3.7
+    cd oneforall
+    pipenv run python oneforall.py --help
+    ```
+
+    2. 使用pip3
+
+    ```bash
+    cd OneForAll/
+    python -m pip install --user  -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+    cd oneforall/
+    python oneforall.py --help
+    ```
+* **Linux系统**
+
+    1. 使用pipenv
+    ```bash
+    cd OneForAll/
+    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    sudo pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
+    sudo pipenv install --user --python 3.7
+    cd oneforall
+    pipenv run python3 oneforall.py --help
+    ```
+    2. 使用pip3
+    ```bash
+    cd OneForAll/
+    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+    cd oneforall/
+    python3 oneforall.py --help
+    ```
+* **Darwin系统**
+
+    1. 使用pipenv
+    ```bash
+    cd OneForAll/
+    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
+    pipenv install --user --python 3.7
+    cd oneforall
+    pipenv run python3 oneforall.py --help
+    ```
+    2. 使用pip3
+    ```bash
+    cd OneForAll/
+    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+    cd oneforall/
+    python3 oneforall.py --help
+    ```
@@ -0,0 +1,212 @@
+# 使用帮助
+
+oneforall.py是主程序入口，oneforall.py里有调用aiobrute.py和dbexport.py，为了方便进行子域爆破和数据库导出独立出了aiobrute.py和dbexport.py，这两个文件可以单独运行，并且所接受参数要更丰富一点。
+
+1. oneforall.py使用帮助
+
+   ```bash
+   python oneforall.py --help
+   ```
+   ```bash
+   NAME
+       oneforall.py - OneForAll是一款功能强大的子域收集工具
+   
+   SYNOPSIS
+       oneforall.py --target=TARGET <flags>
+   
+   DESCRIPTION
+       Version: 0.0.4
+       Project: https://git.io/fjHT1
+   
+       Example:
+           python3 oneforall.py --target example.com run
+           python3 oneforall.py --target ./domains.txt run
+           python3 oneforall.py --target example.com --brute True run
+           python3 oneforall.py --target example.com --verify False run
+           python3 oneforall.py --target example.com --valid None run
+           python3 oneforall.py --target example.com --port medium run
+           python3 oneforall.py --target example.com --format csv run
+           python3 oneforall.py --target example.com --show True run
+   
+       Note:
+           参数valid可选值1，0，None分别表示导出有效，无效，全部子域
+           参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
+           参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+           参数format可选格式有'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
+                             'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   ARGUMENTS
+       TARGET
+           单个域名或者每行一个域名的文件路径(必需参数)
+   
+   FLAGS
+       --brute=BRUTE
+           使用爆破模块(默认False)
+       --verify=VERIFY
+           验证子域有效性(默认True)
+       --port=PORT
+           请求验证的端口范围(默认medium)
+       --valid=VALID
+           导出子域的有效性(默认1)
+       --path=PATH
+           导出路径(默认None)
+       --format=FORMAT
+           导出格式(默认xlsx)
+       --show=SHOW
+           终端显示导出数据(默认False)
+   ```
+
+2. aiobrute.py使用帮助
+
+   关于泛解析问题处理程序首先会访问一个随机的子域判断是否泛解析，如果使用了泛解析则是通过以下判断处理：
+   - 一是主要是与泛解析的IP集合和TTL值做对比，可以参考[这篇文章](http://sh3ll.me/archives/201704041222.txt)。
+   - 二是多次解析到同一IP集合次数（默认设置为10，可以在config.py设置大小）
+   - 考虑爆破效率问题目前还没有加上HTTP响应体相似度对比和响应体内容判断
+   经过测试在16核心的CPU，使用16进程64协程，100M带宽的环境下，设置任务分割为50000，跑两百万字典大概10分钟左右跑完，大概3333个子域每秒。
+
+   ```bash
+   python aiobrute.py --help
+   ```
+
+   ```bash
+   NAME
+       aiobrute.py - OneForAll多进程多协程异步子域爆破模块
+   
+   SYNOPSIS
+       aiobrute.py --target=TARGET <flags>
+   
+   DESCRIPTION
+       Example：
+           python3 aiobrute.py --target example.com run
+           python3 aiobrute.py --target ./domains.txt run
+           python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
+           python3 aiobrute.py --target example.com --wordlist subdomains.txt run
+           python3 aiobrute.py --target example.com --recursive True --depth 2 run
+           python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
+   
+       Note:
+           参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为任务组，
+           当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+           十万字典建议设置为5000，百万字典设置为50000
+           参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+           参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
+                             'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   ARGUMENTS
+       TARGET
+           单个域名或者每行一个域名的文件路径
+   
+   FLAGS
+       --process=PROCESS
+           爆破的进程数(默认CPU核心数)
+       --coroutine=COROUTINE
+           每个爆破进程下的协程数(默认64)
+       --wordlist=WORDLIST
+           指定爆破所使用的字典路径(默认使用config.py配置)
+       --segment=SEGMENT
+           爆破任务分割(默认500)
+       --recursive=RECURSIVE
+           是否使用递归爆破(默认False)
+       --depth=DEPTH
+           递归爆破的深度(默认2)
+       --namelist=NAMELIST
+           指定递归爆破所使用的字典路径(默认使用config.py配置)
+       --fuzz=FUZZ
+           是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
+       --rule=RULE
+           fuzz模式使用的正则规则(默认使用config.py配置)
+       --export=EXPORT
+           是否导出爆破结果(默认True)
+       --valid=VALID
+           导出子域的有效性(默认None)
+       --format=FORMAT
+           导出格式(默认xlsx)
+       --path=PATH
+           导出路径(默认None)
+       --show=SHOW
+           终端显示导出数据(默认False)
+   
+   ```
+
+
+3. takeover.py使用帮助
+
+   ```bash
+   python takeover.py --help
+   ```
+   
+   ```bash
+   NAME
+    takeover.py - OneForAll多线程子域接管风险检查模块
+   
+   
+    SYNOPSIS
+        takeover.py COMMAND | --target=TARGET <flags>
+   
+    DESCRIPTION
+        Example:
+            python3 takeover.py --target www.example.com  --format csv run
+            python3 takeover.py --target ./subdomains.txt --thread 10 run
+   
+        Note:
+            参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                              'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+            参数dpath为None默认使用OneForAll结果目录
+   
+    ARGUMENTS
+        TARGET
+            单个子域或者每行一个子域的文件路径(必需参数)
+   
+    FLAGS
+        --thread=THREAD
+            线程数(默认100)
+        --dpath=DPATH
+            导出目录(默认None)
+        --format=FORMAT
+            导出格式(默认xls)
+   
+   ```
+
+
+4. dbexport.py使用帮助
+
+   ```bash
+   python dbexport.py --help
+   ```
+
+   ```bash
+   NAME
+       dbexport.py - OneForAll数据库导出模块
+   
+   SYNOPSIS
+       dbexport.py TABLE <flags>
+   
+   DESCRIPTION
+       Example:
+           python3 dbexport.py --table name --format csv --path= ./result.csv
+           python3 dbexport.py --db result.db --table name --show False
+   
+       Note:
+           参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+           参数format可选格式有'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
+                             'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   POSITIONAL ARGUMENTS
+       TABLE
+           要导出的表
+   
+   FLAGS
+       --db=DB
+           要导出的数据库路径(默认为results/result.sqlite3)
+       --valid=VALID
+           导出子域的有效性(默认None)
+       --path=PATH
+           导出路径(默认None)
+       --format=FORMAT
+           导出格式(默认xlsx)
+       --show=SHOW
+           终端显示导出数据(默认False)
+   ```
@@ -1 +0,0 @@
-# coding=utf-8
@@ -1,42 +1,50 @@
+#!/usr/bin/python3
 # coding=utf-8
-import os
-import time
-import queue
-import signal
-import pathlib
+
+"""
+OneForAll多进程多协程异步子域爆破模块
+
+:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
+:license: GNU General Public License v3.0, see LICENSE for more details.
+"""
+
 import asyncio
+import queue
+import secrets
+import signal
+import time
+
+import aiomultiprocess as aiomp
+import exrex
 import fire
 import tqdm
-import exrex
-import secrets
-import aiomultiprocess
+
 import config
-from config import logger
-from common import utils, database, resolve
+import dbexport
+from common import resolve, utils
 from common.module import Module
+from common.database import Database
+from config import logger


 def init_worker():
    signal.signal(signal.SIGINT, signal.SIG_IGN)


-def get_wordlist(name):
-    return config.data_storage_path.joinpath(name)
-
-
 def detect_wildcard(domain):
    """
    探测域名是否使用泛解析

    :param str domain: 域名
-    :return: 如果没有使用泛解析返回False 使用返回泛解析的IP集合和ttl整型值
+    :return: 如果没有使用泛解析返回False 反之返回泛解析的IP集合和ttl整型值
    """
    logger.log('INFOR', f'正在探测{domain}是否使用泛解析')
    token = secrets.token_hex(16)
    random_subdomain = f'{token}.{domain}'
    try:
        answers = resolve.dns_query_a(random_subdomain)
-    except Exception as e:  # 如果查询随机域名A记录出错 说明不存在随机子域的A记录 即没有开启泛解析
+    # 如果查询随机域名A记录出错 说明不存在随机子域的A记录 即没有开启泛解析
+    except Exception as e:
        logger.log('DEBUG', e)
        logger.log('INFOR', f'{domain}没有使用泛解析')
        return False, None, None
@@ -121,42 +129,66 @@ def gen_brute_domains(domain, path):

 class AIOBrute(Module):
    """
-    多进程多协程异步子域爆破
-
-    :param str target:       单个域名或者每行一个域名的文件路径
-    :param int processes:    爆破的进程数（默认CPU核心数）
-    :param int coroutine:    每个爆破进程下的协程数（默认16）
-    :param str wordlist:     指定爆破所使用的字典路径（默认使用config.py配置）
-    :param bool recursive:   是否使用递归爆破（默认禁用）
-    :param int depth:        递归爆破的深度（默认2）
-    :param str namelist:     指定递归爆破所使用的字典路径（默认使用config.py配置）
-    :param bool fuzz:        是否使用fuzz模式进行爆破（默认禁用，开启必须指定fuzz正则规则）
-    :param str rule:         fuzz模式使用的正则规则（默认使用config.py配置）
+    OneForAll多进程多协程异步子域爆破模块

    Example：
-    python aiobrute.py --target example.com run
-    python aiobrute.py --target ./domains.txt run
-    python aiobrute.py --target example.com --processes 4 --coroutine 64 --wordlist data/subdomains.txt run
-    python aiobrute.py --target example.com --recursive True --depth 2 --namelist data/next_subdomains.txt run
-    python aiobrute.py --target www.{fuzz}.example.com --fuzz True --rule [a-z][0-9] run
+        python3 aiobrute.py --target subdomain.com run
+        python3 aiobrute.py --target ./subdomains.txt run
+        python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
+        python3 aiobrute.py --target example.com --wordlist subnames.txt run
+        python3 aiobrute.py --target example.com --recursive True --depth 2 run
+        python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
+
+    Note:
+        参数segment的设置受CPU性能，网络带宽，运营商限制等限制，默认500个子域为任务组，
+        当你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+        十万字典建议设置为5000，百万字典设置为50000
+        参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+
+    :param str target:       单个域名或者每行一个域名的文件路径
+    :param int process:      爆破的进程数(默认CPU核心数)
+    :param int coroutine:    每个爆破进程下的协程数(默认64)
+    :param str wordlist:     指定爆破所使用的字典路径(默认使用config.py配置)
+    :param int segment:      爆破任务分割(默认500)
+    :param bool recursive:   是否使用递归爆破(默认False)
+    :param int depth:        递归爆破的深度(默认2)
+    :param str namelist:     指定递归爆破所使用的字典路径(默认使用config.py配置)
+    :param bool fuzz:        是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
+    :param str rule:         fuzz模式使用的正则规则(默认使用config.py配置)
+    :param bool export:      是否导出爆破结果(默认True)
+    :param int valid:        导出子域的有效性(默认None)
+    :param str format:       导出格式(默认csv)
+    :param str path:         导出路径(默认None)
+    :param bool show:        终端显示导出数据(默认False)
    """

-    def __init__(self, target, processes=None, coroutine=64, wordlist=None,
-                 recursive=False, depth=2, namelist=None, fuzz=False, rule=None):
+    def __init__(self, target, process=None, coroutine=64, wordlist=None,
+                 segment=500, recursive=False, depth=2, namelist=None,
+                 fuzz=False, rule=None, export=True, valid=None, format='csv',
+                 path=None, show=False):
        Module.__init__(self)
        self.domains = set()
        self.domain = str()
        self.module = 'Brute'
        self.source = 'AIOBrute'
        self.target = target
-        self.processes = processes or config.brute_processes_num or os.cpu_count()
+        self.process = process or config.brute_process_num
        self.coroutine = coroutine or config.brute_coroutine_num
-        self.wordlist = wordlist or config.brute_wordlist_path or get_wordlist('subdomains.txt')
+        self.wordlist = wordlist or config.brute_wordlist_path
+        self.segment = segment or config.brute_task_segment
        self.recursive_brute = recursive or config.enable_recursive_brute
        self.recursive_depth = depth or config.brute_recursive_depth
-        self.recursive_namelist = namelist or config.recursive_namelist_path or get_wordlist('next_subdomains.txt')
+        self.recursive_namelist = namelist or config.recursive_namelist_path
        self.fuzz = fuzz or config.enable_fuzz
        self.rule = rule or config.fuzz_rule
+        self.export = export
+        self.valid = valid
+        self.format = format
+        self.path = path
+        self.show = show
        self.nameservers = config.resolver_nameservers
        self.ips_times = dict()  # IP集合出现次数
        self.enable_wildcard = False  # 当前域名是否使用泛解析
@@ -164,44 +196,61 @@ class AIOBrute(Module):
        self.wildcard_ttl = int()  # 泛解析TTL整型值

    def gen_tasks(self, domain):
-        logger.log('INFOR', f'正在生成{domain}的字典')
-        if self.domain != domain:  # 如果domain不是self.domain，而是self.domain的子域 生成递归爆破字典
+        # 如果domain不是self.subdomain，而是self.domain的子域 生成递归爆破字典
+        if self.domain != domain:
+            logger.log('INFOR', f'使用{self.recursive_namelist}字典')
            domains = gen_brute_domains(domain, self.recursive_namelist)
        elif self.fuzz and self.rule:  # 开启fuzz模式并指定了fuzz正则规则
+            logger.log('INFOR', f'正在生成{domain}的fuzz字典')
            domains = gen_fuzz_domains(domain, self.rule)
        else:
+            logger.log('INFOR', f'使用{self.wordlist}字典')
            domains = gen_brute_domains(domain, self.wordlist)
        domains = list(domains)
-        return utils.split_list(domains, 500)  # 分割任务组 500个子域为一组任务
+        return utils.split_list(domains, self.segment)  # 分割任务组

    def deal_results(self, results):
        for result in results:
+            if result is None:
+                continue
            if isinstance(result, Exception):
                # logger.log('DEBUG', f'爆破{subdomain}时出错 {str(answers)}')
                continue
            if isinstance(result, tuple):
                subdomain, answers = result
+                if not answers:
+                    continue
                ips = {record.host for record in answers}
-                value = self.ips_times.setdefault(str(ips), 0)  # 取值 如果是首次出现的IP集合 出现次数先赋值0
+                # 取值 如果是首次出现的IP集合 出现次数先赋值0
+                value = self.ips_times.setdefault(str(ips), 0)
                self.ips_times[str(ips)] = value + 1
                ttl = answers[0].ttl
                if self.enable_wildcard:
-                    if wildcard_by_compare(ips, ttl, self.wildcard_ips, self.wildcard_ttl):
+                    if wildcard_by_compare(ips,
+                                           ttl,
+                                           self.wildcard_ips,
+                                           self.wildcard_ttl):
                        continue
                if wildcard_by_times(ips, self.ips_times):
                    continue
-                logger.log('INFOR', f'发现{self.domain}的子域: {subdomain} 解析IP: {ips} TTL: {ttl}')
+                logger.log('INFOR', f'发现{self.domain}的子域: {subdomain} '
+                                    f'解析IP: {ips} TTL: {ttl}')
                self.subdomains.add(subdomain)
                self.records[subdomain] = str(ips)

    async def main(self, domain, rx_queue):
        if not self.fuzz:  # fuzz模式不探测域名是否使用泛解析
-            self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl = detect_wildcard(domain)
+            self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl \
+                = detect_wildcard(domain)
        tasks = self.gen_tasks(domain)
        logger.log('INFOR', f'正在爆破{domain}的域名')
-        for task in tqdm.tqdm(tasks, desc='Progress', smoothing=1.0, ncols=True):
-            async with aiomultiprocess.Pool(processes=self.processes, initializer=init_worker,
-                                            childconcurrency=self.coroutine) as pool:
+        for task in tqdm.tqdm(tasks,
+                              desc='Progress',
+                              smoothing=1.0,
+                              ncols=True):
+            async with aiomp.Pool(processes=self.process,
+                                  initializer=init_worker,
+                                  childconcurrency=self.coroutine) as pool:
                try:
                    results = await pool.map(resolve.aiodns_query_a, task)
                except KeyboardInterrupt:
@@ -211,51 +260,66 @@ class AIOBrute(Module):
                    self.gen_result()
                    rx_queue.put(self.results)
                    return
-                else:
-                    self.deal_results(results)
-            self.save_json()
-            self.gen_result()
-            rx_queue.put(self.results)
+                self.deal_results(results)
+                self.save_json()
+                self.gen_result()
+                rx_queue.put(self.results)

    def run(self, rx_queue=None):
        self.domains = utils.get_domains(self.target)
        while self.domains:
            self.domain = self.domains.pop()
            start = time.time()
-            db_conn = database.connect_db()
-            table_name = self.domain.replace('.', '_')
-            database.create_table(db_conn, table_name)
+            db = Database()
+            db.create_table(self.domain)
            if not rx_queue:
                rx_queue = queue.Queue()
            logger.log('INFOR', f'开始执行{self.source}模块爆破域名{self.domain}')
-            logger.log('INFOR', f'{self.source}模块使用{self.processes}个进程乘{self.coroutine}个协程')
-            # logger.log('INFOR', f'{self.source}模块使用个进程乘{self.coroutine}个协程')
-            if self.recursive_brute and not self.fuzz:  # fuzz模式不使用递归爆破
+            logger.log('INFOR', f'使用{self.process}进程乘{self.coroutine}协程')
+            # fuzz模式不使用递归爆破
+            if self.recursive_brute and not self.fuzz:
                logger.log('INFOR', f'开始递归爆破{self.domain}的第1层子域')
            loop = asyncio.get_event_loop()
            asyncio.set_event_loop(loop)
            loop.run_until_complete(self.main(self.domain, rx_queue))

            # 递归爆破下一层的子域
-            if self.recursive_brute and not self.fuzz:  # fuzz模式不使用递归爆破
-                for layer_num in range(1, self.recursive_depth):  # 之前已经做过1层子域爆破 当前实际递归层数是layer+1
-                    logger.log('INFOR', f'开始递归爆破{self.domain}的第{layer_num+1}层子域')
+            # fuzz模式不使用递归爆破
+            if self.recursive_brute and not self.fuzz:
+                for layer_num in range(1, self.recursive_depth):
+                    # 之前已经做过1层子域爆破 当前实际递归层数是layer+1
+                    logger.log('INFOR', f'开始递归爆破{self.domain}的'
+                                        f'第{layer_num + 1}层子域')
                    for subdomain in self.subdomains.copy():
-                        if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域爆破的限制条件
-                            loop.run_until_complete(self.main(subdomain, rx_queue))
-
-
-            while not rx_queue.empty():  # 队列不空就一直取数据存数据库
-                database.save_db(db_conn, table_name, rx_queue.get())  # 将结果存入数据库中
-            database.copy_table(db_conn, table_name)
-            database.deduplicate_subdomain(db_conn, table_name)
-            database.remove_invalid(db_conn, table_name)
+                        # 进行下一层子域爆破的限制条件
+                        if subdomain.count('.') - self.domain.count('.') \
+                                == layer_num:
+                            loop.run_until_complete(self.main(subdomain,
+                                                              rx_queue))
+            # 队列不空就一直取数据存数据库
+            while not rx_queue.empty():
+                source, results = rx_queue.get()
+                # 将结果存入数据库中
+                db.save_db(self.domain, results, source)

            end = time.time()
            self.elapsed = round(end - start, 1)
            logger.log('INFOR', f'结束执行{self.source}模块爆破域名{self.domain}')
-            logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒发现{self.domain}的域名{len(self.subdomains)}个')
-            logger.log('DEBUG', f'{self.source}模块发现{self.domain}的的域名 {self.subdomains}')
+            length = len(self.subdomains)
+            logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒'
+                                f'发现{self.domain}的域名{length}个')
+            logger.log('DEBUG', f'{self.source}模块发现{self.domain}的域名:\n'
+                                f'{self.subdomains}')
+            # 数据库导出
+            if self.export:
+                if not self.path:
+                    name = f'{self.domain}_brute.{self.format}'
+                    self.path = config.result_save_path.joinpath(name)
+                dbexport.export(self.domain,
+                                valid=self.valid,
+                                dpath=self.path,
+                                format=self.format,
+                                show=self.show)


 def do(domain, result):  # 统一入口名字 方便多线程调用
@@ -270,6 +334,6 @@ def do(domain, result):  # 统一入口名字 方便多线程调用


 if __name__ == '__main__':
-    # fire.Fire(AIOBrute)
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    fire.Fire(AIOBrute)
+    # result_queue = queue.Queue()
+    # do('example.com', result_queue)
@@ -1,14 +1,9 @@
-# coding=utf-8
-"""
-被动收集类
-"""
 import time
-import queue
 import threading
 import importlib
 import config
 import dbexport
-from common import database
+from config import logger


 class Collect(object):
@@ -22,21 +17,24 @@ class Collect(object):
        self.collect_func = []
        self.path = None
        self.export = export
-        self.format = 'xlsx'
+        self.format = 'csv'

    def get_mod(self):
        """
        获取要运行的模块
-        :return: None
        """
        if config.enable_all_module:
-            # modules = ['brute', 'certificates', 'crawl', 'datasets', 'intelligence', 'search']
-            modules = ['certificates', 'check', 'datasets', 'dnsquery', 'intelligence', 'search']  # crawl模块还有点问题
+            # modules = ['brute', 'certificates', 'crawl',
+            # 'datasets', 'intelligence', 'search']
+            # crawl模块还有点问题
+            modules = ['certificates', 'check', 'datasets',
+                       'dnsquery', 'intelligence', 'search']
            # modules = ['intelligence']  # crawl模块还有点问题
            for module in modules:
                module_path = config.oneforall_module_path.joinpath(module)
                for path in module_path.rglob('*.py'):
-                    import_module = ('modules.' + module, path.stem)  # 需要导入的类
+                    # 需要导入的类
+                    import_module = ('modules.' + module, path.stem)
                    self.modules.append(import_module)
        else:
            self.modules = config.enable_partial_module
@@ -49,20 +47,21 @@ class Collect(object):
            import_object = importlib.import_module('.'+name, package)
            self.collect_func.append(getattr(import_object, 'do'))

-    def run(self, rx_queue=None):
+    def run(self):
        """
        类运行入口
        """
        start = time.time()
+        logger.log('INFOR', f'开始收集{self.domain}的子域')
        self.get_mod()
        self.import_func()

-        if not rx_queue:
-            rx_queue = queue.Queue(maxsize=len(self.collect_func))  # 结果集队列
        threads = []
        # 创建多个子域收集线程
        for collect_func in self.collect_func:
-            thread = threading.Thread(target=collect_func, args=(self.domain, rx_queue), daemon=True)
+            thread = threading.Thread(target=collect_func,
+                                      args=(self.domain,),
+                                      daemon=True)
            threads.append(thread)
        # 启动所有线程
        for thread in threads:
@@ -71,18 +70,12 @@ class Collect(object):
        for thread in threads:
            thread.join()

-        db_conn = database.connect_db()
-        table_name = self.domain.replace('.', '_')
-        database.create_table(db_conn, table_name)
-        database.copy_table(db_conn, table_name)
-        database.deduplicate_subdomain(db_conn, table_name)
-        database.remove_invalid(db_conn, table_name)
-        db_conn.close()
        # 数据库导出
        if self.export:
            if not self.path:
-                self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
-            dbexport.export(table_name, path=self.path, format=self.format)
+                name = f'{self.domain}.{self.format}'
+                self.path = config.result_save_path.joinpath(name)
+            dbexport.export(self.domain, dpath=self.path, format=self.format)
        end = time.time()
        self.elapsed = round(end - start, 1)

@@ -11,160 +11,201 @@ from records import Connection
 from config import logger


-def connect_db(db_path=None):
-    """
-    获取数据库对象
+class Database(object):
+    def __init__(self, db_path=None):
+        self.conn = self.get_conn(db_path)

-    :param db_path: 数据库连接或路径
-    :return: SQLite数据库
-    """
-    logger.log('DEBUG', f'正在获取数据库连接')
-    if isinstance(db_path, Connection):
-        return db_path
-    protocol = 'sqlite:///'
-    if not db_path:  # 数据库路径为空连接默认数据库
-        db_path = f'{protocol}{config.result_save_path}/result.sqlite3'
-    else:
-        db_path = protocol + db_path
-    db = records.Database(db_path)  # 不存在数据库时会新建一个数据库
-    logger.log('DEBUG', f'使用数据库: {db_path}')
-    return db.get_connection()
+    @staticmethod
+    def get_conn(db_path):
+        """
+        获取数据库对象

+        :param db_path: 数据库连接或路径
+        :return: SQLite数据库
+        """
+        logger.log('DEBUG', f'正在获取数据库连接')
+        if isinstance(db_path, Connection):
+            return db_path
+        protocol = 'sqlite:///'
+        if not db_path:  # 数据库路径为空连接默认数据库
+            db_path = f'{protocol}{config.result_save_path}/result.sqlite3'
+        else:
+            db_path = protocol + db_path
+        db = records.Database(db_path)  # 不存在数据库时会新建一个数据库
+        logger.log('DEBUG', f'使用数据库: {db_path}')
+        return db.get_connection()

-def create_table(db_conn, table_name):
-    """
-    初始化数据库
+    def create_table(self, table_name):
+        """
+        初始化数据库

-    :param db_conn: 数据库连接
-    :param str table_name: 要创建的表名
-    """
-    logger.log('DEBUG', f'正在创建{table_name}表')
-    try:
-        db_conn.query(f'create table if not exists {table_name} ('
-                      f'id integer primary key,'
-                      f'url text,'
-                      f'subdomain text,'
-                      f'port int,'
-                      f'ips text,'
-                      f'status int,'
-                      f'reason text,'
-                      f'valid int,'
-                      f'title text,'
-                      f'banner text,'
-                      f'module text,'
-                      f'source text,'
-                      f'elapsed float,'
-                      f'count int)')
-    except Exception as e:
-        logger.log('ERROR', e)
-
-
-def save_db(db_conn, table_name, results, module_name=None):
-    """
-    将各模块结果存入数据库
-
-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    :param list results: 结果列表
-    :param str module_name: 模块名
-    """
-    logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域结果存入数据库')
-    if results:
+        :param str table_name: 要创建的表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在创建{table_name}表')
        try:
-            db_conn.bulk_query(f'insert into {table_name} (id, url, subdomain, port, ips, status,'
-                               f'reason, valid, title, banner, module, source, elapsed, count)'
-                               f'values (:id, :url, :subdomain, :port, :ips, :status, :reason, :valid,'
-                               f':title, :banner, :module, :source, :elapsed, :count)', results)
+            self.conn.query(f'create table if not exists "{table_name}" ('
+                            f'id integer primary key,'
+                            f'url text,'
+                            f'subdomain text,'
+                            f'port int,'
+                            f'ips text,'
+                            f'status int,'
+                            f'reason text,'
+                            f'valid int,'
+                            f'title text,'
+                            f'banner text,'
+                            f'module text,'
+                            f'source text,'
+                            f'elapsed float,'
+                            f'count int)')
        except Exception as e:
            logger.log('ERROR', e)

+    def save_db(self, table_name, results, module_name=None):
+        """
+        将各模块结果存入数据库

-def copy_table(db_conn, table_name):
-    """
-    复制表创建备份
+        :param str table_name: 表名
+        :param list results: 结果列表
+        :param str module_name: 模块名
+        """
+        logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域'
+                   '结果存入数据库')
+        table_name = table_name.replace('.', '_')
+        if results:
+            try:
+                self.conn.bulk_query(
+                    f'insert into "{table_name}" ('
+                    f'id, url, subdomain, port, ips, status, reason, valid,'
+                    f'title, banner, module, source, elapsed, count)'
+                    f'values (:id, :url, :subdomain, :port, :ips, :status,'
+                    f':reason, :valid, :title, :banner, :module, :source,'
+                    f':elapsed, :count)',
+                    results)
+            except Exception as e:
+                logger.log('ERROR', e)

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    new_table_name = table_name + '_bak'
-    logger.log('DEBUG', f'正在将{table_name}表复制到{new_table_name}新表')
-    try:
-        db_conn.query(f'drop table if exists {new_table_name}')
-        db_conn.query(f'create table {new_table_name} as select * from {table_name}')
-    except Exception as e:
-        logger.log('ERROR', e)
+    def copy_table(self, table_name, bak_table_name):
+        """
+        复制表创建备份

+        :param str table_name: 表名
+        :param str bak_table_name: 新表名
+        """
+        table_name = table_name.replace('.', '_')
+        bak_table_name = bak_table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在将{table_name}表复制到{bak_table_name}新表')
+        try:
+            self.conn.query(f'drop table if exists "{bak_table_name}"')
+            self.conn.query(f'create table "{bak_table_name}" '
+                            f'as select * from "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

-def clear_table(db_conn, table_name):
-    """
-    清空表中数据
+    def clear_table(self, table_name):
+        """
+        清空表中数据

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    logger.log('DEBUG', f'正在清空{table_name}表中的数据')
-    try:
-        db_conn.query(f'delete from {table_name}')
-    except Exception as e:
-        logger.log('ERROR', e)
-
-
-def deduplicate_subdomain(db_conn, table_name):
-    """
-    去重表中的子域并删除空值和无效值
-
-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    logger.log('DEBUG', f'正在去重{table_name}表中的子域')
-    try:
-        db_conn.query(f'delete from {table_name} where id not in (select min(id) from {table_name} group by subdomain)')
-    except Exception as e:
-        logger.log('ERROR', e)
-
-
-def remove_invalid(db_conn, table_name):
-    """
-        去除表中的空值或无效子域
-
-        :param db_conn: 数据库连接
        :param str table_name: 表名
        """
-    logger.log('DEBUG', f'正在去除{table_name}表中的无效子域')
-    try:
-        db_conn.query(f'delete from {table_name} where subdomain is null or valid == 0')
-    except Exception as e:
-        logger.log('ERROR', e)
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在清空{table_name}表中的数据')
+        try:
+            self.conn.query(f'delete from "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

+    def drop_table(self, table_name):
+        """
+        删除表

-def get_data(db_conn, table_name):
-    """
-    获取表中的所有数据
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在删除{table_name}表')
+        try:
+            self.conn.query(f'drop table if exists "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    logger.log('DEBUG', f'获取{table_name}表中的所有数据')
-    try:
-        rows = db_conn.query(f'select * from {table_name}')
-    except Exception as e:
-        logger.log('ERROR', e)
-    else:
-        return rows
+    def rename_table(self, table_name, new_table_name):
+        """
+        复制表创建备份

+        :param str table_name: 表名
+        :param str new_table_name: 新表名
+        """
+        table_name = table_name.replace('.', '_')
+        new_table_name = new_table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在将{table_name}表重命名为{table_name}表')
+        try:
+            self.conn.query(f'alter table "{table_name}" '
+                            f'rename to "{new_table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

-def get_subdomain(db_conn, table_name, valid):
-    """
-    获取表中的子域数据
+    def deduplicate_subdomain(self, table_name):
+        """
+        去重表中的子域

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    :param int valid: 是否有效
-    """
-    logger.log('DEBUG', f'获取{table_name}表中的所有数据')
-    try:
-        rows = db_conn.query(f'select * from {table_name} where valid = {valid}')
-    except Exception as e:
-        logger.log('ERROR', e)
-    else:
-        return rows
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在去重{table_name}表中的子域')
+        try:
+            self.conn.query(
+                f'delete from "{table_name}" where id not in (select min(id) '
+                f'from "{table_name}" group by subdomain)')
+        except Exception as e:
+            logger.log('ERROR', e)
+
+    def remove_invalid(self, table_name):
+        """
+        去除表中的空值或无效子域
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在去除{table_name}表中的无效子域')
+        try:
+            self.conn.query(
+                f'delete from "{table_name}" where '
+                f'subdomain is null or valid == 0')
+        except Exception as e:
+            logger.log('ERROR', e)
+
+    def get_data(self, table_name):
+        """
+        获取表中的所有数据
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'获取{table_name}表中的所有数据')
+        try:
+            rows = self.conn.query(f'select * from "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)
+        else:
+            return rows
+
+    def get_subdomain(self, table_name, valid):
+        """
+        获取表中的子域数据
+
+        :param str table_name: 表名
+        :param int valid: 是否有效
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'获取{table_name}表中的所有数据')
+        try:
+            rows = self.conn.query(
+                f'select * from "{table_name}" where valid = {valid}')
+        except Exception as e:
+            logger.log('ERROR', e)
+        else:
+            return rows
+
+    def close(self):
+        self.conn.close()
@@ -31,9 +31,9 @@ class Domain(object):
        """
        域名导出

-        >>>d = Domain('www.example.com')
+        >>> d = Domain('www.example.com')
        <domain.Domain object>
-        >>>d.extract()
+        >>> d.extract()
        ExtractResult(subdomain='www', domain='example', suffix='com')

        :return: 导出结果
@@ -50,9 +50,9 @@ class Domain(object):
        """
        获取注册域名

-        >>>d = Domain('www.example.com')
+        >>> d = Domain('www.example.com')
        <domain.Domain object>
-        >>>d.registered()
+        >>> d.registered()
        example.com

        :return: 注册域名
@@ -6,13 +6,14 @@
 import json
 import re
 import threading
+import time

 import requests
 import config
 from config import logger
 from . import utils
 from .domain import Domain
-from common import database
+from common.database import Database


 lock = threading.Lock()
@@ -32,56 +33,94 @@ class Module(object):
        self.subdomains = set()  # 存放发现的子域
        self.records = dict()  # 存放子域解析记录
        self.results = list()  # 存放模块结果
-        self.elapsed = 0.0  # 模块执行耗时
+        self.start = time.time()  # 模块开始执行时间
+        self.end = None
+        self.elapsed = None  # 模块执行耗时

-    def get(self, url, params=None, **kwargs):
+    def check(self, *apis):
+        """
+        简单检查是否配置了api信息
+        :param apis: api信息元组
+        :return: 检查结果
+        """
+        if not all(apis):
+            logger.log('ALERT', f'{self.source}模块API配置有误跳过执行')
+            return False
+        return True
+
+    def begin(self):
+        """
+        输出模块开始信息
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块收集{self.domain}的子域')
+
+    def finish(self):
+        """
+        输出模块结束信息
+        """
+        self.end = time.time()
+        self.elapsed = round(self.end - self.start, 1)
+        logger.log('DEBUG', f'结束执行{self.source}模块收集{self.domain}的子域')
+        logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒发现子域'
+                   f'{len(self.subdomains)}个')
+        logger.log('DEBUG', f'{self.source}模块发现{self.domain}的子域\n'
+                   f'{self.subdomains}')
+
+    def get(self, url, params=None, check=True, **kwargs):
        """
        自定义get请求

        :param str url: 请求地址
        :param dict params: 请求参数
+        :param bool check: 检查响应
        :param kwargs: 其他参数
        :return: requests响应对象
        """
        try:
-            resp = requests.get(url, params=params, cookies=self.cookie, headers=self.header,
-                                proxies=self.proxy, timeout=self.timeout, verify=self.verify, **kwargs)
+            resp = requests.get(url,
+                                params=params,
+                                cookies=self.cookie,
+                                headers=self.header,
+                                proxies=self.proxy,
+                                timeout=self.timeout,
+                                verify=self.verify,
+                                **kwargs)
        except Exception as e:
            logger.log('ERROR', e)
            return None
-        if resp.status_code != 200:
-            logger.log('ALERT', f'GET {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
-            content_type = resp.headers.get('Content-Type')
-            if content_type:
-                if 'json' in content_type:
-                    logger.log('ALERT', resp.json())
-            return None
-        logger.log('DEBUG', f'GET {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
-        return resp
+        if not check:
+            return resp
+        if utils.check_response('GET', resp):
+            return resp
+        return None

-    def post(self, url, data=None, **kwargs):
+    def post(self, url, data=None, check=True, **kwargs):
        """
        自定义post请求

        :param str url: 请求地址
        :param dict data: 请求数据
+        :param bool check: 检查响应
        :param kwargs: 其他参数
        :return: requests响应对象
        """
        try:
-            resp = requests.post(url, data=data, cookies=self.cookie, headers=self.header,
-                                 proxies=self.proxy, timeout=self.timeout, verify=self.verify, **kwargs)
+            resp = requests.post(url,
+                                 data=data,
+                                 cookies=self.cookie,
+                                 headers=self.header,
+                                 proxies=self.proxy,
+                                 timeout=self.timeout,
+                                 verify=self.verify,
+                                 **kwargs)
        except Exception as e:
            logger.log('ERROR', e)
            return None
-        if resp.status_code != 200:
-            content_type = resp.headers.get('Content-Type')
-            if content_type:
-                if 'json' in content_type:
-                    logger.log('ALERT', resp.json())
-            return None
-        logger.log('DEBUG', f'POST {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
-        return resp
+        if not check:
+            return resp
+        if utils.check_response('GET', resp):
+            return resp
+        return None

    def get_header(self):
        """
@@ -127,12 +166,14 @@ class Module(object):
        :rtype: set or list
        """
        logger.log('DEBUG', f'正则匹配响应体中的子域')
-        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
+                 r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
                 + domain.replace('.', r'\.')
        result = re.findall(regexp, html, re.I)
        if not result:
            return set()
-        deal = map(lambda s: re.sub(r'(?:http://|https://)', '', s[1:].lower(), flags=re.I), result)
+        regexp = r'(?:http://|https://)'
+        deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
        if distinct:
            return set(deal)
        else:
@@ -154,38 +195,66 @@ class Module(object):
        """
        logger.log('DEBUG', f'将{self.source}模块发现的子域结果保存为json文件')
        if config.save_module_result:
-            dirpath = config.result_save_path.joinpath(self.domain, self.module)
-            dirpath.mkdir(parents=True, exist_ok=True)
+            dpath = config.result_save_path.joinpath(self.domain, self.module)
+            dpath.mkdir(parents=True, exist_ok=True)
            name = self.source + '.json'
-            path = dirpath.joinpath(name)
+            path = dpath.joinpath(name)
            with open(path, mode='w', encoding='utf-8') as file:
-                result = {'domain': self.domain, 'name': self.module, 'source': self.source, 'elapsed': self.elapsed,
-                          'count': len(self.subdomains), 'subdomains': list(self.subdomains), 'records': self.records}
+                result = {'domain': self.domain,
+                          'name': self.module,
+                          'source': self.source,
+                          'elapsed': self.elapsed,
+                          'count': len(self.subdomains),
+                          'subdomains': list(self.subdomains),
+                          'records': self.records}
                json.dump(result, file, ensure_ascii=False, indent=4)

    def gen_result(self):
        results = list()
        if not len(self.subdomains):  # 一个子域都没有发现的情况
-            result = {'id': None, 'url': None, 'subdomain': None, 'port': None, 'ips': None, 'status': None,
-                      'reason': None, 'valid': 1, 'title': None, 'banner': None, 'module': self.module,
-                      'source': self.source, 'elapsed': self.elapsed, 'count': 0}
+            result = {'id': None,
+                      'url': None,
+                      'subdomain': None,
+                      'port': None,
+                      'ips': None,
+                      'status': None,
+                      'reason': None,
+                      'valid': None,
+                      'title': None,
+                      'banner': None,
+                      'module': self.module,
+                      'source': self.source,
+                      'elapsed': self.elapsed,
+                      'count': 0}
            results.append(result)
            self.results = (self.source, results)
        else:
            for subdomain in self.subdomains:
                url = 'http://' + subdomain
                ips = self.records.get(subdomain)
-                result = {'id': None, 'url': url, 'subdomain': subdomain, 'port': None, 'ips': ips, 'status': None,
-                          'reason': None, 'valid': 1, 'title': None, 'banner': None, 'module': self.module,
-                          'source': self.source, 'elapsed': self.elapsed, 'count': len(self.subdomains)}
+                result = {'id': None,
+                          'url': url,
+                          'subdomain': subdomain,
+                          'port': None,
+                          'ips': ips,
+                          'status': None,
+                          'reason': None,
+                          'valid': None,
+                          'title': None,
+                          'banner': None,
+                          'module': self.module,
+                          'source': self.source,
+                          'elapsed': self.elapsed,
+                          'count': len(self.subdomains)}
                results.append(result)
            self.results = (self.source, results)

    def save_db(self):
        lock.acquire()
-        db_conn = database.connect_db()
-        table_name = self.domain.replace('.', '_')
-        database.create_table(db_conn, table_name)
+        db = Database()
+        db.create_table(self.domain)
        source, results = self.results
-        database.save_db(db_conn, table_name, results, source)  # 将结果存入数据库中
+        # 将结果存入数据库中
+        db.save_db(self.domain, results, source)
+        db.close()
        lock.release()
@@ -2,7 +2,9 @@

 import asyncio
 import functools
+
 import aiohttp
+import tqdm
 from aiohttp import ClientSession
 from aiohttp.resolver import AsyncResolver
 from bs4 import BeautifulSoup
@@ -31,13 +33,19 @@ def get_ports(port):
 def gen_new_datas(datas, ports):
    logger.log('INFOR', f'正在生成请求地址')
    new_datas = []
-    protocols = ['http://', 'https://']
+    protocols = ['http://']
    for data in datas:
-        if data.get('valid'):  # 有效的子域才进行http请求探测
+        valid = data.get('valid')
+        if valid is None:  # 子域有效性未知的才进行http请求探测
            subdomain = data.get('subdomain')
            for port in ports:
                for protocol in protocols:
-                    url = f'{protocol}{subdomain}:{port}'
+                    if port == 443:
+                        url = f'https://{subdomain}:{port}'
+                    elif port == 8443:
+                        url = f'https://{subdomain}:{port}'
+                    else:
+                        url = f'{protocol}{subdomain}:{port}'
                    data['id'] = None
                    data['url'] = url
                    data['port'] = port
@@ -52,64 +60,128 @@ async def fetch(session, url, semaphore):

    :param session: session对象
    :param url: url地址
-    :param semaphore: 同步对象（控制并发量）
+    :param semaphore: 并发信号量
    :return: 响应对象和响应文本
    """
    timeout = aiohttp.ClientTimeout(total=config.get_timeout)
    async with semaphore:
-        async with session.get(url, allow_redirects=config.get_redirects,
-                               timeout=timeout, proxy=config.get_proxy) as resp:
-            text = await resp.text()
-            return resp, text
+        async with session.get(url,
+                               ssl=config.verify_ssl,
+                               allow_redirects=config.get_redirects,
+                               timeout=timeout,
+                               proxy=config.get_proxy) as resp:
+
+            try:
+                text = await resp.text(encoding='gb2312')  # 先尝试用fb2312解码
+            except UnicodeDecodeError:
+                text = await resp.text(errors='ignore')
+        return resp, text
+
+
+def get_title(markup):
+    """
+    获取标题
+
+    :param markup: html标签
+    :return: 标题
+    """
+    soup = BeautifulSoup(markup, 'lxml')
+
+    title = soup.title
+    if title:
+        return title.text.strip()
+
+    h1 = soup.h1
+    if h1:
+        return h1.text.strip()
+
+    h2 = soup.h2
+    if h2:
+        return h2.text.strip()
+
+    desc = soup.find('meta', attrs={'name': 'description'})
+    if desc:
+        return desc['content'].strip()
+
+    word = soup.find('meta', attrs={'name': 'keywords'})
+    if word:
+        return word['content'].strip()
+
+    if len(markup) <= 200:
+        return markup.strip()
+
+    text = soup.text
+    if len(text) <= 200:
+        return text.strip()
+
+    return None


 def request_callback(future, index, datas):
    try:
-        resp, text = future.result()
-    except Exception as e:
+        result = future.result()
+    except BaseException as e:
        logger.log('DEBUG', e.args)
        datas[index]['reason'] = str(e.args)
        datas[index]['valid'] = 0
    else:
+        resp, text = result
        datas[index]['reason'] = resp.reason
        datas[index]['status'] = resp.status
-        if resp.status == 400 or resp.status >= 500:
+        if resp.status >= 500:
            datas[index]['valid'] = 0
        else:
+            datas[index]['valid'] = 1
            headers = resp.headers
-            banner = str({'Server': headers.get('Server'), 'Via': headers.get('Via'),
+            banner = str({'Server': headers.get('Server'),
+                          'Via': headers.get('Via'),
                          'X-Powered-By': headers.get('X-Powered-By')})
            datas[index]['banner'] = banner
-            soup = BeautifulSoup(text, 'lxml')
-            title = soup.title
-            head = soup.head
-            if title:
-                datas[index]['title'] = title.text
-            elif head:
-                datas[index]['title'] = head.text
-            else:
-                datas[index]['title'] = text
+            datas[index]['title'] = get_title(text)


 async def bulk_get_request(datas, port):
-    logger.log('INFOR', f'正在异步进行子域的GET请求')
    ports = get_ports(port)
    new_datas = gen_new_datas(datas, ports)
+    logger.log('INFOR', f'正在异步进行子域的GET请求')
+
+    limit_open_conn = config.limit_open_conn
+    if limit_open_conn is None:  # 默认情况
+        limit_open_conn = utils.get_semaphore()
+    elif not isinstance(limit_open_conn, int):  # 如果传入不是数字的情况
+        limit_open_conn = utils.get_semaphore()
+    # 使用异步域名解析器 自定义域名服务器
+    resolver = AsyncResolver(nameservers=config.resolver_nameservers)
+    conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
+                                limit=limit_open_conn,
+                                limit_per_host=config.limit_per_host,
+                                resolver=resolver)
+
+    semaphore = asyncio.Semaphore(limit_open_conn)
    header = None
    if config.fake_header:
        header = utils.gen_fake_header()
-    resolver = AsyncResolver(nameservers=config.resolver_nameservers)  # 使用异步域名解析器 自定义域名服务器
-    conn = aiohttp.TCPConnector(verify_ssl=config.verify_ssl, limit=config.limit_open_conn,
-                                limit_per_host=config.limit_per_host, resolver=resolver)
-    semaphore = asyncio.Semaphore(utils.get_semaphore())
    async with ClientSession(connector=conn, headers=header) as session:
        tasks = []
        for i, data in enumerate(new_datas):
            url = data.get('url')
            task = asyncio.ensure_future(fetch(session, url, semaphore))
-            task.add_done_callback(functools.partial(request_callback, index=i, datas=new_datas))
+            task.add_done_callback(functools.partial(request_callback,
+                                                     index=i,
+                                                     datas=new_datas))
            tasks.append(task)
        if tasks:  # 任务列表里有任务不空时才进行解析
-            await asyncio.wait(tasks)  # 等待所有task完成
+            # 等待所有task完成 错误聚合到结果列表里
+            futures = asyncio.as_completed(tasks)
+            for future in tqdm.tqdm(futures,
+                                    total=len(tasks),
+                                    desc='Progress',
+                                    smoothing=1.0,
+                                    ncols=True):
+                try:
+                    await future
+                except:
+                    pass
+
    logger.log('INFOR', f'完成异步进行子域的GET请求')
    return new_datas
@@ -4,6 +4,8 @@ import functools

 import dns.resolver
 import aiodns
+import tqdm
+
 import config
 from common import utils
 from config import logger
@@ -39,7 +41,7 @@ def aiodns_resolver():
                              timeout=config.resolver_timeout)


-async def aiodns_query_a(hostname, semaphore):
+async def aiodns_query_a(hostname, semaphore=None):
    """
    异步查询A记录

@@ -47,23 +49,45 @@ async def aiodns_query_a(hostname, semaphore):
    :param semaphore: 并发查询数量
    :return: 主机名或查询结果或查询异常
    """
-    async with semaphore:
+    if semaphore is None:
        resolver = aiodns_resolver()
-        answers = await resolver.query(hostname, 'A')
+        try:
+            answers = await resolver.query(hostname, 'A')
+        except BaseException as e:
+            logger.log('DEBUG', e.args)
+            answers = None
        return hostname, answers
+    else:
+        async with semaphore:
+            resolver = aiodns_resolver()
+            try:
+                answers = await resolver.query(hostname, 'A')
+            except BaseException as e:
+                logger.log('DEBUG', e.args)
+                answers = None
+            return hostname, answers


 def resolve_callback(future, index, datas):
+    """
+    解析结果回调处理
+    :param future: future对象
+    :param index: 下标
+    :param datas: 结果集
+    """
    try:
        result = future.result()
-    except aiodns.error.DNSError as e:
+    except BaseException as e:
        datas[index]['ips'] = str(e.args)
        datas[index]['valid'] = 0
    else:
        if isinstance(result, tuple):
            _, answers = result
-            ips = {record.host for record in answers}
-            datas[index]['ips'] = str(ips)
+            if answers:
+                ips = {record.host for record in answers}
+                datas[index]['ips'] = str(ips)
+            else:
+                datas[index]['ips'] = 'No answers'


 async def bulk_query_a(datas):
@@ -75,14 +99,26 @@ async def bulk_query_a(datas):
    """
    logger.log('INFOR', '正在异步查询子域的A记录')
    tasks = []
-    semaphore = asyncio.Semaphore(utils.get_semaphore())
+    semaphore = asyncio.Semaphore(config.limit_resolve_conn)
    for i, data in enumerate(datas):
        if not data.get('ips'):
            subdomain = data.get('subdomain')
            task = asyncio.ensure_future(aiodns_query_a(subdomain, semaphore))
-            task.add_done_callback(functools.partial(resolve_callback, index=i, datas=datas))  # 回调
+            task.add_done_callback(functools.partial(resolve_callback,
+                                                     index=i,
+                                                     datas=datas))  # 回调
            tasks.append(task)
    if tasks:  # 任务列表里有任务不空时才进行解析
-        await asyncio.wait(tasks)  # 等待所有task完成
+        futures = asyncio.as_completed(tasks)
+        for future in tqdm.tqdm(futures,
+                                total=len(tasks),
+                                desc='Progress',
+                                smoothing=1.0,
+                                ncols=True):
+            try:
+                await future
+            except:
+                pass
+        # await asyncio.wait(tasks)  # 等待所有task完成
    logger.log('INFOR', '完成异步查询子域的A记录')
    return datas
@@ -28,16 +28,18 @@ class Search(Module):
        :rtype: str
        """
        statements_list = []
-        subdomains_temp = set(map(lambda x: x + '.' + domain, config.subdomains_common))
+        subdomains_temp = set(map(lambda x: x + '.' + domain,
+                                  config.subdomains_common))
        subdomains_temp = list(subdomain.intersection(subdomains_temp))
        for i in range(0, len(subdomains_temp), 2):  # 同时排除2个子域
-            statements_list.append(''.join(set(map(lambda s: ' -site:' + s, subdomains_temp[i:i + 2]))))
+            statements_list.append(''.join(set(map(lambda s: ' -site:' + s,
+                                                   subdomains_temp[i:i + 2]))))
        return statements_list

    def match_location(self, domain, url):
        """
        匹配跳转之后的url
-        针对部分搜索引擎（如百度搜索）搜索展示url时有显示不全的情况
+        针对部分搜索引擎(如百度搜索)搜索展示url时有显示不全的情况
        此函数会向每条结果的链接发送head请求获取响应头的location值并做子域匹配

        :param str domain: 域名
@@ -45,6 +47,7 @@ class Search(Module):
        :return: 匹配的子域
        :rtype set
        """
-        resp = requests.head(url, headers=self.header, proxies=self.proxy, timeout=self.timeout, allow_redirects=False)
+        resp = requests.head(url, headers=self.header, proxies=self.proxy,
+                             timeout=self.timeout, allow_redirects=False)
        location = resp.headers.get('location')
        return set(utils.match_subdomain(domain, location))
@@ -1,14 +1,25 @@
 # coding=utf-8
 import re
-import pathlib
 import random
 import ipaddress
 import platform
 import config
-from fake_useragent import UserAgent
+from pathlib import Path
 from common.domain import Domain
 from config import logger

+user_agents = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/68.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:61.0) '
+    'Gecko/20100101 Firefox/68.0',
+    'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0']
+

 def match_subdomain(domain, text, distinct=True):
    """
@@ -20,7 +31,8 @@ def match_subdomain(domain, text, distinct=True):
    :return: 匹配结果
    :rtype: set or list
    """
-    regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' + domain.replace('.', r'\.')
+    regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+             + domain.replace('.', r'\.')
    result = re.findall(regexp, text, re.I)
    if not result:
        return set()
@@ -45,10 +57,11 @@ def gen_fake_header():
    """
    生成伪造请求头
    """
-    ua = UserAgent()
+    ua = random.choice(user_agents)
    ip = gen_random_ip()
    headers = {
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept': 'text/html,application/xhtml+xml,'
+                  'application/xml;q=0.9,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
        'Cache-Control': 'max-age=0',
@@ -56,7 +69,7 @@ def gen_fake_header():
        'DNT': '1',
        'Referer': 'https://www.google.com/',
        'Upgrade-Insecure-Requests': '1',
-        'User-Agent': ua.random,
+        'User-Agent': ua,
        'X-Forwarded-For': ip,
        'X-Real-IP': ip
    }
@@ -86,7 +99,7 @@ def split_list(ls, size):
    """
    if size == 0:
        return ls
-    return [ls[i:i+size] for i in range(0, len(ls), size)]
+    return [ls[i:i + size] for i in range(0, len(ls), size)]


 def get_domains(target):
@@ -96,20 +109,22 @@ def get_domains(target):
    :param set or str target:
    :return: 域名集合
    """
-    domains = set()
-    logger.log('INFOR', f'正在获取域名')
-    if isinstance(target, set):
+    domains = list()
+    logger.log('DEBUG', f'正在获取域名')
+    if isinstance(target, (set, tuple)):
+        domains = list(target)
+    elif isinstance(target, list):
        domains = target
    elif isinstance(target, str):
-        path = pathlib.Path(target)
+        path = Path(target)
        if path.is_file():
            with open(target) as file:
                for line in file:
                    domain = Domain(line.strip()).match()
                    if domain:
-                        domains.add(domain)
-        if Domain(target).match():
-            domains = {target}
+                        domains.append(domain)
+        elif Domain(target).match():
+            domains = [target]
    logger.log('INFOR', f'获取到{len(domains)}个域名')
    return domains

@@ -122,9 +137,73 @@ def get_semaphore():
    """
    system = platform.system()
    if system == 'Windows':
-        return 500
+        return 300
    elif system == 'Linux':
-        return 1000
+        return 800
    elif system == 'Darwin':
-        return 1000
+        return 800

+
+def check_dpath(dpath):
+    """
+    检查目录路径
+
+    :param dpath: 传入的目录路径
+    :return: 目录路径
+    """
+    if isinstance(dpath, str):
+        dpath = Path(dpath)
+    else:
+        dpath = config.result_save_path
+    if not dpath.is_dir():
+        logger.log('FATAL', f'{dpath}不是目录')
+    if not dpath.exists():
+        logger.log('ALERT', f'不存在{dpath}将会新建此目录')
+        dpath.mkdir(parents=True, exist_ok=True)
+    return dpath
+
+
+def check_format(format):
+    """
+    检查导出格式
+
+    :param format: 传入的导出格式
+    :return: 导出格式
+    """
+    formats = ['txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+               'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods']
+    if format in formats:
+        return format
+    else:
+        logger.log('ALERT', f'不支持{format}格式导出')
+        logger.log('ALERT', '默认使用csv格式导出')
+        return 'xls'
+
+
+def save_data(fpath, data):
+    try:
+        with open(fpath, 'w', encoding="utf-8", newline='') as file:
+            file.write(data)
+            logger.log('ALERT', fpath)
+    except TypeError:
+        with open(fpath, 'wb') as file:
+            file.write(data)
+            logger.log('ALERT', fpath)
+    except Exception as e:
+        logger.log('ERROR', e)
+
+
+def check_response(method, resp):
+    if resp.status_code == 200 and resp.content:
+        return True
+    logger.log('ALERT', f'{method} {resp.url} {resp.status_code} - '
+                        f'{resp.reason} {len(resp.content)}')
+    content_type = resp.headers.get('Content-Type')
+    if content_type and 'json' in content_type and resp.content:
+        try:
+            msg = resp.json()
+        except Exception as e:
+            logger.log('DEBUG', e.args)
+        else:
+            logger.log('ALERT', msg)
+    return False
@@ -2,9 +2,11 @@
 """
 OneForAll配置
 """
-
+import os
 import sys
 import pathlib
+
+import requests
 from loguru import logger

 # 路径设置
@@ -15,40 +17,54 @@ data_storage_path = oneforall_relpath.joinpath('data')  # 数据存放目录
 result_save_path = oneforall_relpath.joinpath('results')  # 结果保存目录

 # 模块设置
-save_module_result = True  # 保存模块中各脚本结果 默认保存
-enable_all_module = True  # 启用所有模块 默认启用
+save_module_result = True  # 保存模块中各脚本结果(默认True)
+enable_all_module = True  # 启用所有模块(默认True)
 enable_partial_module = []  # 启用部分模块 必须禁用enable_all_module才能生效
-# enable_partial_module = [('modules.search', 'google')]  # 启用部分模块 必须禁用enable_all_module才能生效
+# 只使用ask和baidu搜索引擎收集子域
+# enable_partial_module = [('modules.search', 'ask')
+#                          ('modules.search', 'baidu')]


 # 爆破模块设置
-enable_brute_module = False  # 使用爆破模块（默认禁用）
+enable_brute_module = False  # 使用爆破模块(默认禁用)
+enable_verify_subdomain = True  # 验证子域有效性(默认True)
 enable_wildcard_check = True  # 开启泛解析检测 会去掉泛解析的子域
-brute_processes_num = None  # 爆破时使用的进程数（根据系统中CPU数量情况设置 不宜大于CPU数量 默认None为系统中的CPU数量）
-brute_coroutine_num = 128  # 爆破时每个进程下的协程数（不宜大于1000）
-brute_wordlist_path = None  # 爆破所使用的字典路径 默认data/subdomains.dict
-use_general_wordlist = False  # 是否使用通用字典（默认禁用）
-enable_recursive_brute = False  # 是否使用递归爆破（默认禁用）
-brute_recursive_depth = 2  # 递归爆破深度（默认2层）
-recursive_namelist_path = None  # 爆破下一层子域所使用的字典路径 默认data/next_subdomains.dict
+# 爆破时使用的进程数(根据系统中CPU数量情况设置 不宜大于CPU数量 默认为系统中的CPU数量)
+brute_process_num = os.cpu_count()
+brute_coroutine_num = 128  # 爆破时每个进程下的协程数(不宜大于1000)
+# 爆破所使用的字典路径 默认data/subdomains.txt
+brute_wordlist_path = data_storage_path.joinpath('subnames.txt')
+brute_task_segment = 500
+# 参数segment的设置受CPU性能，网络带宽，运营商限制等限制，默认500个子域为一任务组，
+# 当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+# 十万字典建议设置为5000，百万字典设置为50000
+enable_recursive_brute = False  # 是否使用递归爆破(默认禁用)
+brute_recursive_depth = 2  # 递归爆破深度(默认2层)
+# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt
+recursive_namelist_path = data_storage_path.joinpath('next_subnames.txt')
 enable_fuzz = False  # 是否使用fuzz模式枚举域名
 fuzz_rule = ''  # fuzz域名的正则 示例：[a-z][0-9] 第一位是字母 第二位是数字
 ips_appear_maximum = 10  # 同一IP集合出现次数超过10认为是泛解析

 # 代理设置
-enable_proxy = True  # 是否使用代理 全局开关
+enable_proxy = False  # 是否使用代理(全局开关)
 proxy_all_module = False  # 代理所有模块
-proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch', 'GoogleAPISearch',
-                        'GoogleSearch', 'YahooSearch', 'YandexSearch']  # 代理自定义的模块
-proxy_pool = [{'http': 'http://127.0.0.1:1080', 'https': 'https://127.0.0.1:1080'}]  # 代理池
-# proxy_pool = [{'http': 'socks5://127.0.0.1:10808', 'https': 'socks5://127.0.0.1:10808'}]  # 代理池
+proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
+                        'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
+                        'YandexSearch', 'CrossDomainXml',
+                        'ContentSecurityPolicy']  # 代理自定义的模块
+proxy_pool = [{'http': 'http://127.0.0.1:1080',
+               'https': 'https://127.0.0.1:1080'}]  # 代理池
+# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
+#                'https': 'socks5h://127.0.0.1:10808'}]  # 代理池


 # 网络请求设置
 enable_fake_header = True  # 启用伪造请求头
 request_delay = 1  # 请求时延
-request_timeout = 30  # 请求超时(AskSearch和YahooSearch比较慢)
+request_timeout = 30  # 请求超时
 request_verify = True  # 请求SSL验证
+requests.packages.urllib3.disable_warnings()  # 禁用安全警告信息

 # 搜索模块设置
 enable_recursive_search = False  # 递归搜索子域
@@ -66,6 +82,7 @@ resolver_nameservers = [
 ]  # 指定查询的DNS域名服务器
 resolver_timeout = 5.0  # 解析超时时间
 resolver_lifetime = 30.0  # 解析存活时间
+limit_resolve_conn = 50  # 限制同一时间解析的数量(默认50)

 # http探测设置
 small_ports = {80, 443}
@@ -76,18 +93,22 @@ xlarge_ports = {80, 81, 300, 443, 591, 593, 832, 981, 1010, 1311, 2082,
                2087, 2095, 2096, 2480, 3000, 3128, 3333, 4243, 4567, 4711,
                4712, 4993, 5000, 5104, 5108, 5800, 6543, 7000, 7396, 7474,
                8000, 8001, 8008, 8014, 8042, 8069, 8080, 8081, 8088, 8090,
-                8091, 8118, 8123, 8172, 8222, 8243, 8280, 8281, 8333, 8443,
-                8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080, 9090,
-                9091, 9200, 9443, 9800, 9981, 12443, 16080, 18091, 18092,
+                8091, 8016, 8118, 8123, 8172, 8222, 8243, 8280, 8281, 8333,
+                8443, 8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080,
+                9090, 9091, 9200, 9443, 9800, 9981, 12443, 16080, 18091, 18092,
                20720, 28017}
-ports = {'small': small_ports, 'medium': medium_ports, 'large': large_ports, 'xlarge': xlarge_ports}
+ports = {'small': small_ports, 'medium': medium_ports,
+         'large': large_ports, 'xlarge': xlarge_ports}
 verify_ssl = False
-get_proxy = None  # aiohttp 支持 HTTP/HTTPS形式的代理 proxy="http://user:pass@some.proxy.com"
-get_timeout = 10  # http请求探测总超时时间 None或者0则表示不检测超时
+# aiohttp 支持 HTTP/HTTPS形式的代理
+get_proxy = None  # proxy="http://user:pass@some.proxy.com"
+get_timeout = 120  # http请求探测总超时时间 None或者0则表示不检测超时
 get_redirects = True  # 允许请求跳转
 fake_header = True  # 使用伪造请求头
-limit_open_conn = 100  # 限制同一时间打开的连接数（默认100），0表示不限制
-limit_per_host = 0  # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数（默认0表示不限制）
+# 限制同一时间打开的连接数(默认None，根据系统不同设置，Windows系统400 其他系统800)
+limit_open_conn = None
+# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
+limit_per_host = 0  # 默认0表示不限制


 # 模块API配置
@@ -99,10 +120,11 @@ censys_api_secret = ''
 # 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
 binaryedge_api = ''

-# Binaryedge可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
+# Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
 chinaz_api = ''

-# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/cognitive-services/bing-web-search-api/#web-json
+# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/
+# cognitive-services/bing-web-search-api/#web-json
 bing_api_id = ''
 bing_api_key = ''

@@ -113,7 +135,8 @@ securitytrails_api = ''
 fofa_api_email = ''  # fofa用户邮箱
 fofa_api_key = ''  # fofa用户key

-# Google可以免费注册获取API: https://developers.google.com/custom-search/v1/overview
+# Google可以免费注册获取API:
+# https://developers.google.com/custom-search/v1/overview
 # 免费的API只能查询前100条结果
 google_api_key = ''  # Google API搜索key
 google_api_cx = ''  # Google API搜索cx
@@ -135,8 +158,8 @@ virustotal_api_key = ''
 zoomeye_api_username = ''
 zoomeye_api_password = ''

-# Certdb可以免费注册获取API: https://spyse.com/
-certdb_api_token = ''
+# Spyse可以免费注册获取API: https://spyse.com/
+spyse_api_token = ''

 # https://www.circl.lu/services/passive-dns/
 circl_api_username = ''
@@ -149,15 +172,37 @@ dnsdb_api_key = ''
 # 免费的API有效期只有2天，到期之后可以再次生成，每天可以查询50次。
 ipv4info_api_key = ''

-subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs', 'web', 'www', 'job', 'docs', 'news', 'blog',
-                     'data', 'help', 'live', 'mall', 'blogs', 'files', 'forum', 'store', 'mobile'}
+# https://github.com/360netlab/flint
+# passivedns_api_addr默认空使用http://api.passivedns.cn
+# passivedns_api_token可为空
+passivedns_api_addr = ''
+passivedns_api_token = ''
+
+# Github Token可以访问https://github.com/settings/tokens生成,user为Github用户名
+github_api_user = ''
+github_api_token = ''
+# github子域收集模块使用
+github_email = ''
+github_password = ''
+
+subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
+                     'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
+                     'help', 'live', 'mall', 'blogs', 'files', 'forum',
+                     'store', 'mobile'}

 # 日志配置
-log_fmt = '<light-green>{time:HH:mm:ss,SSS}</light-green> ' \
+# 终端日志输出格式
+stdout_fmt = '<cyan>{time:HH:mm:ss,SSS}</cyan> ' \
          '[<level>{level: <5}</level>] ' \
-          '<cyan>{process.name}</cyan>:<cyan>{thread.name: <10}</cyan> | ' \
-          '<blue>{module}</blue>.<blue>{function}</blue>:<blue>{line}</blue> - ' \
+          '<blue>{module}</blue>:<cyan>{line}</cyan> - ' \
          '<level>{message}</level>'
+# 日志文件记录格式
+logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
+          '[<level>{level: <5}</level>] ' \
+          '<cyan>{process.name}({process.id})</cyan>:' \
+          '<cyan>{thread.name: <10}({thread.id: <5})</cyan> | ' \
+          '<blue>{module}</blue>.<blue>{function}</blue>:' \
+          '<blue>{line}</blue> - <level>{message}</level>'

 log_path = result_save_path.joinpath('oneforall.log')

@@ -169,13 +214,9 @@ logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️')
 logger.level(name='ERROR', no=40, color='<red><bold>', icon='❌️')
 logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')

-logger.add(sys.stdout, level='INFOR', format=log_fmt, enqueue=True)
-logger.add(log_path, level='TRACE', format=log_fmt, enqueue=True, encoding='utf-8')
+if not os.environ.get('PYTHONIOENCODING'):  # 设置编码
+    os.environ['PYTHONIOENCODING'] = 'utf-8'

-# 调试模式
-# import urllib3
-# urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-# request_proxy = [{'http': 'http://127.0.0.1:8080', 'https': 'https://127.0.0.1:8080'}]
-# request_verify = False
-# enable_all_module = False  # 启用所有模块 默认启用
-# enable_partial_module = [('modules.certificates', 'censys_api')]  # 启用部分模块 必须禁用enable_all_module才能生效
+logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True)
+logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True,
+           encoding='utf-8')
@@ -0,0 +1,249 @@
+[
+    {
+        "name":"github", 
+        "cname":["github.io", "github.map.fastly.net"], 
+        "response":["There isn't a GitHub Pages site here.", "For root URLs (like http://example.com/) you must provide an index.html file"]
+    },
+    {
+        "name":"heroku", 
+        "cname":["herokudns.com", "herokussl.com", "herokuapp.com"], 
+        "response":["There's nothing here, yet.", "herokucdn.com/error-pages/no-such-app.html", "<title>No such app</title>"]
+    },
+    {
+        "name":"unbounce",
+        "cname":["unbouncepages.com"],
+        "response":["Sorry, the page you were looking for doesn’t exist.", "The requested URL was not found on this server"]
+    },
+    {
+        "name":"tumblr",
+        "cname":["tumblr.com"],
+        "response":["There's nothing here.", "Whatever you were looking for doesn't currently exist at this address."]
+    },
+    {
+        "name":"shopify",
+        "cname":["myshopify.com"],
+        "response":["Sorry, this shop is currently unavailable.", "Only one step left!"]
+    },
+    {
+        "name":"instapage",
+        "cname":["pageserve.co", "secure.pageserve.co", "https://instapage.com/"],
+        "response":["Looks Like You're Lost","The page you're looking for is no longer available."]
+    },
+    {
+        "name":"desk",
+        "cname":["desk.com"],
+        "response":["Please try again or try Desk.com free for 14 days.", "Sorry, We Couldn't Find That Page"]
+    },
+    {
+        "name":"campaignmonitor",
+        "cname":["createsend.com", "name.createsend.com"],
+        "response":["Double check the URL", "<strong>Trying to access your account?</strong>"]
+    },
+    {
+        "name":"cargocollective",
+        "cname":["cargocollective.com"],
+        "response":["404 Not Found"]
+    },
+    {
+        "name":"statuspage",
+        "cname":["statuspage.io"],
+        "response":["Better Status Communication", "You are being <a href=\"https://www.statuspage.io\">redirected"]
+    },
+    {
+        "name":"amazonaws",
+        "cname":["amazonaws.com"],
+        "response":["NoSuchBucket", "The specified bucket does not exist"]
+    },
+    {
+        "name":"bitbucket",
+        "cname":["bitbucket.org"],  
+        "response":["The page you have requested does not exist","Repository not found"]
+    },
+    {
+        "name":"smartling",
+        "cname":["smartling.com"],
+        "response":["Domain is not configured"]
+    },
+    {
+        "name":"acquia",
+        "cname":["acquia.com"],
+        "response":["If you are an Acquia Cloud customer and expect to see your site at this address","The site you are looking for could not be found."]
+    },
+    {
+        "name":"fastly",
+        "cname":["fastly.net"],
+        "response":["Please check that this domain has been added to a service", "Fastly error: unknown domain"]
+    },
+    {
+        "name":"pantheon",
+        "cname":["pantheonsite.io"],
+        "response":["The gods are wise", "The gods are wise, but do not know of the site which you seek."]
+    },
+    {
+        "name":"zendesk",
+        "cname":["zendesk.com"],
+        "response":["Help Center Closed"]
+    },
+    {
+        "name":"uservoice",
+        "cname":["uservoice.com"],
+        "response":["This UserVoice subdomain is currently available!"]
+    },
+    {
+        "name":"ghost",
+        "cname":["ghost.io"],
+        "response":["The thing you were looking for is no longer here", "The thing you were looking for is no longer here, or never was"]
+    },
+    {
+        "name":"pingdom",
+        "cname":["stats.pingdom.com"],
+        "response":["pingdom"]
+    },
+    {
+        "name":"tilda",
+        "cname":["tilda.ws"],
+        "response":["Domain has been assigned"]
+    },
+    {
+        "name":"wordpress",
+        "cname":["wordpress.com"],  
+        "response":["Do you want to register"]
+    },
+    {
+        "name":"teamwork",
+        "cname":["teamwork.com"],
+        "response":["Oops - We didn't find your site."]
+    },
+    {
+        "name":"helpjuice",
+        "cname":["helpjuice.com"],
+        "response":["We could not find what you're looking for."]
+    },
+    {
+        "name":"helpscout",
+        "cname":["helpscoutdocs.com"],
+        "response":["No settings were found for this company:"]
+    },
+    {
+        "name":"cargo",
+        "cname":["cargocollective.com"],
+        "response":["If you're moving your domain away from Cargo you must make this configuration through your registrar's DNS control panel."]
+    },
+    {
+        "name":"feedpress",
+        "cname":["redirect.feedpress.me"],
+        "response":["The feed has not been found."]
+    },
+    {
+        "name":"surge",
+        "cname":["surge.sh"],
+        "response":["project not found"]
+    },
+    {
+        "name":"surveygizmo",
+        "cname":["privatedomain.sgizmo.com", "privatedomain.surveygizmo.eu", "privatedomain.sgizmoca.com"],
+        "response":["data-html-name"]
+    },
+    {
+        "name":"mashery",
+        "cname":["mashery.com"],
+        "response":["Unrecognized domain <strong>"]
+    },
+    {
+        "name":"intercom",
+        "cname":["custom.intercom.help"],
+        "response":["This page is reserved for artistic dogs.","<h1 class=\"headline\">Uh oh. That page doesn’t exist.</h1>"]
+    },
+    {
+        "name":"webflow",
+        "cname":["proxy.webflow.io"],
+        "response":["<p class=\"description\">The page you are looking for doesn't exist or has been moved.</p>"]
+    },
+    {
+        "name":"kajabi",
+        "cname":["endpoint.mykajabi.com"],
+        "response":["<h1>The page you were looking for doesn't exist.</h1>"]
+    },
+    {
+        "name":"thinkific",
+        "cname":["thinkific.com"],
+        "response":["You may have mistyped the address or the page may have moved."]
+    },
+    {
+        "name":"tave",
+        "cname":["clientaccess.tave.com"],
+        "response":["<h1>Error 404: Page Not Found</h1>"]
+    },
+    {
+        "name":"wishpond",
+        "cname":["wishpond.com"],
+        "response":["https://www.wishpond.com/404?campaign=true"]
+    },
+    {
+        "name":"aftership",
+        "cname":["aftership.com"],
+        "response":["Oops.</h2><p class=\"text-muted text-tight\">The page you're looking for doesn't exist."]
+    },
+    {
+        "name":"aha",
+        "cname":["ideas.aha.io"],
+        "response":["There is no portal here ... sending you back to Aha!"]
+    },
+    {
+        "name":"brightcove",
+        "cname":["brightcovegallery.com", "gallery.video", "bcvp0rtal.com"],
+        "response":["<p class=\"bc-gallery-error-code\">Error Code: 404</p>"]
+    },
+    {
+        "name":"bigcartel",
+        "cname":["bigcartel.com"],
+        "response":["<h1>Oops! We couldn&#8217;t find that page.</h1>"]
+    },
+    {
+        "name":"activecompaign",
+        "cname":["activehosted.com"],
+        "response":["alt=\"LIGHTTPD - fly light.\""]
+    },
+    {
+        "name":"compaignmonitor",
+        "cname":["createsend.com"],
+        "response":["Double check the URL or <a href=\"mailto:help@createsend.com"]
+    },
+    {
+        "name":"simplebooklet",
+        "cname":["simplebooklet.com"],
+        "response":["We can't find this <a href=\"https://simplebooklet.com"]
+    },
+    {
+        "name":"getresponse",
+        "cname":[".gr8.com"],
+        "response":["With GetResponse Landing Pages, lead generation has never been easier"]
+    },
+    {
+        "name":"vend",
+        "cname":["vendecommerce.com"],
+        "response":["Looks like you've traveled too far into cyberspace."]
+    },
+    {
+        "name":"jetbrains",
+        "cname":["myjetbrains.com"],
+        "response":["is not a registered InCloud YouTrack.","is not a registered InCloud YouTrack."]
+    },
+    {
+        "name":"azure",
+        "cname":["azurewebsites.net",
+            ".cloudapp.net",
+            ".cloudapp.azure.com",
+            ".trafficmanager.net",
+            ".blob.core.windows.net",
+            ".azure-api.net",
+            ".azurehdinsight.net",
+            ".azureedge.net"],
+        "response":["404 Web Site not found"]
+    },
+    {
+        "name":"readme",
+        "cname":["readme.io"],
+        "response":["Project doesnt exist... yet!"]
+    }
+]
@@ -1,163 +1,163 @@
-test
-test2
-t
-dev
-1
-2
-3
-s1
-s2
-s3
-admin
-adm
-a
-b
-c
-m
-ht
-adminht
-webht
-web
-gm
-sys
-system
-manage
-manager
-mgr
-passport
-bata
-wei
-weixin
-wechat
-wx
-wiki
-upload
-ftp
-pic
-jira
-zabbix
-nagios
-bug
-bugzilla
-sql
-mysql
-db
-stmp
-pop
-imap
-mail
-zimbra
-exchange
-forum
-bbs
-list
-count
-counter
-img
-img01
-img02
-img03
-img04
-api
-cache
-js
-css
-app
-apps
-wap
-sms
-zip
-monitor
-proxy
-update
-upgrade
-stat
-stats
-data
-portal
-blog
-autodiscover
-en
-search
-so
-oa
-database
-home
-sso
-help
-vip
-s
-w
-down
-download
-downloads
-dl
-svn
-git
-log
-staff
-vpn
-sslvpn
-ssh
-scanner
-sandbox
-ldap
-lab
-go
-demo
-console
-cms
-auth
-crm
-erp
-res
-static
-old
-new
-beta
-image
-service
-login
-3g
-docs
-it
-e
-live
-library
-files
-i
-d
-cp
-connect
-gateway
-lib
-preview
-backup
-share
-status
-assets
-user
-vote
-bugs
-cas
-feedback
-id
-edm
-survey
-union
-ceshi
-dev1
-updates
-phpmyadmin
-pma
-edit
-master
-xml
-control
-profile
-zhidao
-tool
-toolbox
-boss
-activity
-www
+test
+test2
+t
+dev
+1
+2
+3
+s1
+s2
+s3
+admin
+adm
+a
+b
+c
+m
+ht
+adminht
+webht
+web
+gm
+sys
+system
+manage
+manager
+mgr
+passport
+bata
+wei
+weixin
+wechat
+wx
+wiki
+upload
+ftp
+pic
+jira
+zabbix
+nagios
+bug
+bugzilla
+sql
+mysql
+db
+stmp
+pop
+imap
+mail
+zimbra
+exchange
+forum
+bbs
+list
+count
+counter
+img
+img01
+img02
+img03
+img04
+api
+cache
+js
+css
+app
+apps
+wap
+sms
+zip
+monitor
+proxy
+update
+upgrade
+stat
+stats
+data
+portal
+blog
+autodiscover
+en
+search
+so
+oa
+database
+home
+sso
+help
+vip
+s
+w
+down
+download
+downloads
+dl
+svn
+git
+log
+staff
+vpn
+sslvpn
+ssh
+scanner
+sandbox
+ldap
+lab
+go
+demo
+console
+cms
+auth
+crm
+erp
+res
+static
+old
+new
+beta
+image
+service
+login
+3g
+docs
+it
+e
+live
+library
+files
+i
+d
+cp
+connect
+gateway
+lib
+preview
+backup
+share
+status
+assets
+user
+vote
+bugs
+cas
+feedback
+id
+edm
+survey
+union
+ceshi
+dev1
+updates
+phpmyadmin
+pma
+edit
+master
+xml
+control
+profile
+zhidao
+tool
+toolbox
+boss
+activity
+www
@@ -1,53 +1,59 @@
+#!/usr/bin/python3
 # coding=utf-8

+"""
+OneForAll数据库导出模块
+
+:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
+:license: GNU General Public License v3.0, see LICENSE for more details.
+"""
+
 import fire
-from common import database
-from config import logger
+from common import utils
+from common.database import Database


-def export(table, db=None, valid=None, path=None, format='xlsx', output=False):
+def export(table, db=None, valid=None, dpath=None, format='csv', show=False):
    """
-    将数据库导出为指定格式文件
-
-    :param str table:   要导出的表
-    :param str db:      要导出的数据库路径（默认为results/result.sqlite3）
-    :param int valid:   导出子域的有效性（默认None）
-    :param str format:  导出格式（默认xlsx）
-    :param str path:    导出路径(默认None)
-    :param bool output: 是否将导出数据输出到终端（默认False）
-
-    Note:
-        参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
-        参数format可选格式：'csv','tsv','json','yaml','html','xls','xlsx','dbf','latex','ods'
-        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+    OneForAll数据库导出模块

    Example:
-        python dbexport.py --db result.db --table name --format csv --output False
-        python dbexport.py --db result.db --table name --format csv --path= ./result.csv
+        python3 dbexport.py --table name --format csv --dir= ./result.csv
+        python3 dbexport.py --db result.db --table name --show False
+
+    Note:
+        参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+        参数dpath为None默认使用OneForAll结果目录
+
+    :param str table:   要导出的表
+    :param str db:      要导出的数据库路径(默认为results/result.sqlite3)
+    :param int valid:   导出子域的有效性(默认None)
+    :param str format:  导出格式(默认xls)
+    :param str dpath:    导出目录(默认None)
+    :param bool show:   终端显示导出数据(默认False)
    """
-    db_conn = database.connect_db(db)
+    format = utils.check_format(format)
+    dpath = utils.check_dpath(dpath)
+    database = Database(db)
    if valid is None:
-        rows = database.get_data(db_conn, table)
+        rows = database.get_data(table)
    elif isinstance(valid, int):
-        rows = database.get_subdomain(db_conn, table, valid)
+        rows = database.get_subdomain(table, valid)
    else:
-        rows = database.get_data(db_conn, table)  # 意外情况导出全部子域
-    if output:
+        rows = database.get_data(table)  # 意外情况导出全部子域
+    if show:
        print(rows.dataset)
-    if not path:
-        path = 'export.' + format
-    logger.log('INFOR', f'正在将数据库中{table}表导出到{path}')
-    try:
-        with open(path, 'w') as file:
-            file.write(rows.export(format))
-            logger.log('INFOR', '成功完成导出')
-    except TypeError:
-        with open(path, 'wb') as file:
-            file.write(rows.export(format))
-            logger.log('INFOR', '成功完成导出')
-    except Exception as e:
-        logger.log('ERROR', e)
+    if format == 'txt':
+        data = str(rows.dataset)
+    else:
+        data = rows.export(format)
+    database.close()
+    fpath = dpath.joinpath(f'{table}.{format}')
+    utils.save_data(fpath, data)


 if __name__ == '__main__':
    fire.Fire(export)
+    # save('example_com_last', format='txt')
@@ -1 +1,2 @@
-example.com
+example.com
+hackfun.org
@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+"""
+示例
+"""
+
+import oneforall
+
+test = oneforall.OneForAll(target='example.com')
+test.run()
@@ -1 +0,0 @@
-# coding=utf-8
@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+"""
+github自动接管
+"""
+
+import json
+import base64
+import requests
+import config
+
+HEADERS = {
+    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "Accept-Language": "zh-CN,zh;q=0.9",
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
+}
+
+
+def github_takeover(url):
+    # 读取config配置文件
+    repo_name = url
+    print('[*]正在读取配置文件...')
+    user = config.github_api_user
+    token = config.github_api_token
+    CHECK_HEADERS = {
+        "Authorization": 'token ' + token,
+        "Accept": "application/vnd.github.switcheroo-preview+json"
+    }
+    repos_url = 'https://api.github.com/repos/' + user + '/' + repo_name
+    repos_r = requests.get(url=repos_url, headers=CHECK_HEADERS)
+    # 验证token是否正确
+    if 'message' in repos_r.json():
+        if repos_r.json()['message'] == 'Bad credentials':
+            print('[*]请检查Token是否正确')
+        elif repos_r.json()['message'] == 'Not Found':
+            print('[*]正在生成接管库...')  # 生成接管库
+            creat_repo_dict = {
+                "name": repo_name,
+                "description": "This is a subdomain takeover Repository",
+            }
+            creat_repo_url = 'https://api.github.com/user/repos'
+            creat_repo_r = requests.post(url=creat_repo_url,
+                                         headers=CHECK_HEADERS,
+                                         data=json.dumps(creat_repo_dict))
+            creat_repo_status = creat_repo_r.status_code
+            if creat_repo_status == 201:
+                print('[*]创建接管库' + repo_name + '成功，正在进行自动接管...')
+                # 接管文件生成
+                # index.html文件
+                html = b'''
+                <html>
+                    <p>Subdomain Takerover Test!</>
+                </html>
+                '''
+                html64 = base64.b64encode(html).decode('utf-8')
+                html_dict = {
+                    "message": "my commit message",
+                    "committer": {
+                        "name": "user",  # 提交id，非必改项
+                        "email": "user@163.com"  # 同上
+                    },
+                    "content": html64
+                }
+                # CNAME文件
+                cname_url = bytes(url, encoding='utf-8')
+                cname_url64 = base64.b64encode(cname_url).decode('utf-8')
+                url_dict = {
+                    "message": "my commit message",
+                    "committer": {
+                        "name": "user",
+                        "email": "user@163.com"
+                    },
+                    "content": cname_url64
+                }
+                html_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/index.html'
+                url_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/CNAME'
+                html_r = requests.put(url=html_url, data=json.dumps(html_dict),
+                                      headers=CHECK_HEADERS)  # 上传index.html
+                cname_r = requests.put(url=url_url, data=json.dumps(url_dict),
+                                       headers=CHECK_HEADERS)  # 上传CNAME
+                rs = cname_r.status_code
+                if rs == 201:
+                    print('[*]生成接管库成功，正在开启Github pages...')
+                    page_url = "https://api.github.com/repos/" + user + "/" + url + "/pages"
+                    page_dict = {
+                        "source": {
+                            "branch": "master"
+                        }
+                    }
+                    page_r = requests.post(url=page_url,
+                                           data=json.dumps(page_dict),
+                                           headers=CHECK_HEADERS)  # 开启page
+                    if page_r.status_code == 201:
+                        print('[+]自动接管成功，请稍后访问http://' + str(url) + '查看结果')
+                    else:
+                        print('[+]开启Github pages失败，请检查网络或稍后重试...')
+                else:
+                    print('[+]生成接管库失败，请检查网络或稍后重试...')
+    elif url in repos_r.json()['name']:
+        print('[*]生成接管库失败，请检查https://github.com/' + user +
+              '?tab=repositories是否存在同名接管库...')
@@ -1,6 +1,4 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
 from config import logger
@@ -24,64 +22,53 @@ class CensysAPI(Query):
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        data = {
-            'query': 'parsed.names: owasp.org',
+            'query': f'parsed.names: {self.domain}',
            'page': 1,
-            'fields': ['parsed.subject_dn'],
+            'fields': ['parsed.subject_dn', 'parsed.names'],
            'flatten': True}
-
        resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
        if not resp:
            return
-        resp_json = resp.json()
-        status = resp_json.get('status')
+        json = resp.json()
+        status = json.get('status')
        if status != 'ok':
            logger.log('ALERT', status)
            return
-        subdomains_find = self.match(self.domain, str(resp_json))
-        self.subdomains = self.subdomains.union(subdomains_find)
-        pages = resp_json.get('metadata').get('pages')
-        for page in range(2, pages+1):
+        subdomains = self.match(self.domain, str(json))
+        self.subdomains = self.subdomains.union(subdomains)
+        pages = json.get('metadata').get('pages')
+        for page in range(2, pages + 1):
            time.sleep(self.delay)
            data['page'] = page
            resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
            if not resp:
                return
-            subdomains_find = self.match(self.domain, str(resp.json()))
-            self.subdomains = self.subdomains.union(subdomains_find)
+            subdomains = self.match(self.domain, str(resp.json()))
+            self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not (self.id and self.secret):
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.id, self.secret):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = CensysAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,74 +0,0 @@
-# coding=utf-8
-import time
-import queue
-import config
-from common.query import Query
-from common import utils
-from config import logger
-
-
-class CertDBAPI(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = domain
-        self.module = 'Certificate'
-        self.source = 'CertDBQuery'
-        self.addr = 'https://api.spyse.com/v1/subdomains'
-        self.token = config.certdb_api_token
-
-    def query(self):
-        """
-        向接口查询子域并做子域匹配
-        """
-        page_num = 1
-        while True:
-            time.sleep(self.delay)
-            self.header = self.get_header()
-            self.proxy = self.get_proxy(self.source)
-            params = {'domain': self.domain, 'api_token': self.token, 'page': page_num}
-            resp = self.get(self.addr, params)
-            if not resp:
-                return
-            resp_json = resp.json()
-            subdomains_find = utils.match_subdomain(self.domain, str(resp_json))
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
-            page_num += 1
-            if resp_json.get('count') < 30:  # 默认每次查询最多返回30条 当前条数小于30条说明已经查完
-                break
-
-    def run(self, rx_queue):
-        """
-        类执行入口
-        """
-        if not self.token:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
-            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
-        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
-
-
-def do(domain, rx_queue):   # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    :param rx_queue: 结果集队列
-    """
-    query = CertDBAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
-
-
-if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class CertSpotter(Query):
@@ -21,42 +18,38 @@ class CertSpotter(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'domain': self.domain, 'include_subdomains': 'true', 'expand': 'dns_names'}
+        params = {'domain': self.domain,
+                  'include_subdomains': 'true',
+                  'expand': 'dns_names'}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = utils.match_subdomain(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
+
    """
    query = CertSpotter(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class Crtsh(Query):
@@ -25,38 +22,30 @@ class Crtsh(Query):
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = utils.match_subdomain(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Crtsh(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class Entrust(Query):
@@ -21,42 +18,36 @@ class Entrust(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'fields': 'subjectDN', 'domain': self.domain, 'includeExpired': 'true'}
+        params = {'fields': 'subjectDN',
+                  'domain': self.domain,
+                  'includeExpired': 'true'}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = utils.match_subdomain(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Entrust(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class Google(Query):
@@ -12,7 +9,8 @@ class Google(Query):
        self.domain = self.register(domain)
        self.module = 'Certificate'
        self.source = 'GoogleQuery'
-        self.addr = 'https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch'
+        self.addr = 'https://transparencyreport.google.com/' \
+                    'transparencyreport/api/v3/httpsreport/ct/certsearch'

    def query(self):
        """
@@ -21,42 +19,37 @@ class Google(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'include_expired': 'true', 'include_subdomains': 'true', 'domain': self.domain}
+        params = {'include_expired': 'true',
+                  'include_subdomains': 'true',
+                  'domain': self.domain}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = utils.match_subdomain(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Google(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -0,0 +1,67 @@
+import time
+import config
+from common import utils
+from common.query import Query
+
+
+class SpyseAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Certificate'
+        self.source = 'CertDBAPIQuery'
+        self.addr = 'https://api.spyse.com/v1/subdomains'
+        self.token = config.spyse_api_token
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        page_num = 1
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            params = {'domain': self.domain,
+                      'api_token': self.token,
+                      'page': page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            json = resp.json()
+            subdomains = utils.match_subdomain(self.domain, str(json))
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            page_num += 1
+            # 默认每次查询最多返回30条 当前条数小于30条说明已经查完
+            if json.get('count') < 30:
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.token):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = SpyseAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,19 +1,18 @@
-# coding=utf-8
 """
-查询域名的NS记录（域名服务器记录，记录该域名由哪台域名服务器解析），
-检查查出的域名服务器是否开启DNS域传送，如果开启且没做访问控制和身份验证便加以利用获取域名的所有记录
+查询域名的NS记录(域名服务器记录，记录该域名由哪台域名服务器解析)，检查查出的域名服务器是
+否开启DNS域传送，如果开启且没做访问控制和身份验证便加以利用获取域名的所有记录。

-DNS域传送（DNS zone transfer）指的是一台备用域名服务器使用来自主域名服务器的数据刷新自己的域数据库，
-目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
-当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获取某个域的所有记录。
+DNS域传送(DNS zone transfer)指的是一台备用域名服务器使用来自主域名服务器的数据刷新自己
+的域数据库，目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
+当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获
+取某个域的所有记录。
 """
-import time
-import queue
 import dns.resolver
 import dns.zone
-from config import logger
-from common import utils, resolve
+
+from common import resolve, utils
 from common.module import Module
+from config import logger


 class CheckAXFR(Module):
@@ -25,13 +24,37 @@ class CheckAXFR(Module):
        self.domain = self.register(domain)
        self.module = 'Check'
        self.source = 'AXFRCheck'
-        self.nsservers = []
        self.results = []

+    def axfr(self, server):
+        """
+        执行域传送
+
+        :param server: 域名服务器
+        """
+        logger.log('DEBUG', f'尝试对{self.domain}的域名服务器{server}进行域传送')
+        try:
+            xfr = dns.query.xfr(server, self.domain, timeout=30.0)
+            zone = dns.zone.from_xfr(xfr)
+        except Exception as e:
+            logger.log('DEBUG', str(e))
+            logger.log('DEBUG', f'对{self.domain}的域名服务器{server}进行域传送失败')
+            return
+        names = zone.nodes.keys()
+        for name in names:
+            full_domain = str(name) + '.' + self.domain
+            subdomain = utils.match_subdomain(self.domain, full_domain)
+            self.subdomains = self.subdomains.union(subdomain)
+            record = zone[name].to_text(name)
+            self.results.append(record)
+        if self.results:
+            logger.log('DEBUG', f'发现{self.domain}在{server}上的域传送记录')
+            logger.log('DEBUG', '\n'.join(self.results))
+            self.results = []
+
    def check(self):
        """
        正则匹配响应头中的内容安全策略字段以发现子域名
-        :return: None
        """
        resolver = resolve.dns_resolver()
        try:
@@ -39,60 +62,35 @@ class CheckAXFR(Module):
        except Exception as e:
            logger.log('ERROR', e)
            return
-        self.nsservers = [str(answer) for answer in answers]
-        if not len(self.nsservers):
+        nsservers = [str(answer) for answer in answers]
+        if not len(nsservers):
            logger.log('ALERT', f'没有找到{self.domain}的NS域名服务器记录')
            return
-        for nsserver in self.nsservers:
-            logger.log('DEBUG', f'正在尝试对{self.domain}的域名服务器{nsserver}进行域传送')
-            try:
-                xfr = dns.query.xfr(nsserver, self.domain)
-                zone = dns.zone.from_xfr(xfr)
-            except Exception as e:
-                logger.log('DEBUG', str(e))
-                logger.log('INFOR', f'对{self.domain}的域名服务器{nsserver}进行域传送失败')
-                continue
-            else:
-                names = zone.nodes.keys()
-                for name in names:
-                    subdomain = utils.match_subdomain(self.domain, str(name)+'.'+self.domain)
-                    self.subdomains = self.subdomains.union(subdomain)
-                    record = zone[name].to_text(name)
-                    self.results.append(record)
-            if self.results:
-                logger.log('INFOR', f'发现{self.domain}在{nsserver}上的域传送记录')
-                logger.log('DEBUG', '\n'.join(self.results))
-                self.results = []
+        for nsserver in nsservers:
+            self.axfr(nsserver)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}的域传送漏洞')
-        start = time.time()
+        self.begin()
        self.check()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}的域传送漏洞')
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口
+
    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    check = CheckAXFR(domain)
-    check.run(rx_queue)
-    logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
-    logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
+    check.run()


 if __name__ == '__main__':
-    # do('ZoneTransfer.me')
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('ZoneTransfer.me')
+    do('example.com')
@@ -1,12 +1,9 @@
-# coding=utf-8
 """
 检查crossdomain.xml文件收集子域名
 """
-import time
-import queue
-from config import logger
-from common.utils import match_subdomain
+
 from common.module import Module
+from common import utils


 class CheckCDX(Module):
@@ -22,43 +19,43 @@ class CheckCDX(Module):
    def check(self):
        """
        检查crossdomain.xml收集子域名
-        :return:
        """
-        url = f'http://{self.domain}/crossdomain.xml'
-        self.header = self.get_header()
-        self.proxy = self.get_proxy(self.source)
-        resp = self.get(url)
-        if not resp:
+        urls = [f'http://{self.domain}/crossdomain.xml',
+                f'https://{self.domain}/crossdomain.xml',
+                f'http://www.{self.domain}/crossdomain.xml',
+                f'https://www.{self.domain}/crossdomain.xml']
+        response = None
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            response = self.get(url)
+            if response:
+                break
+        if not response:
            return
-        self.subdomains = match_subdomain(self.domain, resp.text)
+        self.subdomains = utils.match_subdomain(self.domain, response.text)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的crossdomain.xml')
-        start = time.time()
+        self.begin()
        self.check()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的crossdomain.xml')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口
+
    :param domain: 域名
-    :param rx_queue: 结果集队列
    """
    check = CheckCDX(domain)
-    check.run(rx_queue)
-    logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
-    logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
+    check.run()


 if __name__ == '__main__':
-    do('163.com')
+    do('example.com')
@@ -1,16 +1,14 @@
 #!/usr/bin/env python3
-# coding=utf-8

 """
 检查域名证书收集子域名
 """
-import ssl
-import time
-import queue
 import socket
-from config import logger
+import ssl
+
 from common import utils
 from common.module import Module
+from config import logger


 class CheckCert(Module):
@@ -33,38 +31,30 @@ class CheckCert(Module):
        except Exception as e:
            logger.log('ERROR', e)
            return
-        subdomains_find = utils.match_subdomain(self.domain, str(cert_dict))
-        self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = utils.match_subdomain(self.domain, str(cert_dict))
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的证书中的子域')
-        start = time.time()
+        self.begin()
        self.check()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的证书中的子域')
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    check = CheckCert(domain)
-    check.run(rx_queue)
-    logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
-    logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
+    check.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,23 +1,23 @@
-# coding=utf-8
 """
 检查内容安全策略收集子域名收集子域名
 """
-import time
-import queue
-from config import logger
+import requests
+
 from common import utils
 from common.module import Module
+from config import logger


 class CheckCSP(Module):
    """
    检查内容安全策略收集子域名
    """
+
    def __init__(self, domain, header):
        Module.__init__(self)
        self.domain = self.register(domain)
        self.module = 'Check'
-        self.source = 'Content-Security-Policy'
+        self.source = 'ContentSecurityPolicy'
        self.header = header

    def check(self):
@@ -25,11 +25,20 @@ class CheckCSP(Module):
        正则匹配响应头中的内容安全策略字段以发现子域名
        """
        if not self.header:
-            url = f'http://www.{self.domain}'
-            resp = self.get(url)
-            if not resp:
+            urls = [f'http://{self.domain}',
+                    f'https://{self.domain}',
+                    f'http://www.{self.domain}',
+                    f'https://www.{self.domain}']
+            response = None
+            for url in urls:
+                self.header = self.get_header()
+                self.proxy = self.get_proxy(self.source)
+                response = self.get(url)
+                if response:
+                    break
+            if not response:
                return
-            self.header = resp.headers
+            self.header = response.headers
        csp = self.header.get('Content-Security-Policy')
        if not csp:
            logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段')
@@ -37,39 +46,29 @@ class CheckCSP(Module):
        logger.log('DEBUG', f'{self.domain}域的响应头存在内容安全策略字段')
        self.subdomains = utils.match_subdomain(self.domain, csp)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
-        start = time.time()
+        self.begin()
        self.check()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)


-def do(domain, rx_queue, header=None):  # 统一入口名字 方便多线程调用
+def do(domain, header=None):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    :param dict or None header: 响应头
    """
    check = CheckCSP(domain, header)
-    check.run(rx_queue)
-    logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
-    logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
+    check.run()


 if __name__ == '__main__':
-    import requests
-    # resp = requests.get('https://content-security-policy.com/')
-    result_queue = queue.Queue()
-    resp = requests.get('https://www.baidu.com/')
-    do('google-analytics.com', result_queue, resp.headers)
+    resp = requests.get('https://content-security-policy.com/')
+    do('google-analytics.com', resp.headers)
@@ -0,0 +1,64 @@
+"""
+检查内容安全策略收集子域名收集子域名
+"""
+import requests
+
+from common.module import Module
+from common import utils
+from config import logger
+
+
+class CheckRobots(Module):
+    """
+    检查robots.txt收集子域名
+    """
+
+    def __init__(self, domain):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Check'
+        self.source = 'Robots'
+
+    def check(self):
+        """
+        正则匹配域名的robots.txt文件中的子域
+        """
+        urls = [f'http://{self.domain}/robots.txt',
+                f'https://{self.domain}/robots.txt',
+                f'http://www.{self.domain}/robots.txt',
+                f'https://www.{self.domain}/robots.txt']
+        response = None
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            response = self.get(url, allow_redirects=False)
+            if response:
+                break
+        if not response:
+            return
+        self.subdomains = utils.match_subdomain(self.domain, response.text)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    check = CheckRobots(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    do('qq.com')
@@ -0,0 +1,77 @@
+"""
+检查内容安全策略收集子域名收集子域名
+"""
+import requests
+
+from common.module import Module
+from common import utils
+from config import logger
+
+
+class CheckRobots(Module):
+    """
+    检查sitemap收集子域名
+    """
+
+    def __init__(self, domain):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Check'
+        self.source = 'Sitemap'
+
+    def check(self):
+        """
+        正则匹配域名的sitemap文件中的子域
+        """
+        urls = [f'http://{self.domain}/sitemap.xml',
+                f'https://{self.domain}/sitemap.xml',
+                f'http://www.{self.domain}/sitemap.xml',
+                f'https://www.{self.domain}/sitemap.xml',
+                f'http://{self.domain}/sitemap.txt',
+                f'https://{self.domain}/sitemap.txt',
+                f'http://www.{self.domain}/sitemap.txt',
+                f'https://www.{self.domain}/sitemap.txt',
+                f'http://{self.domain}/sitemap.html',
+                f'https://{self.domain}/sitemap.html',
+                f'http://www.{self.domain}/sitemap.html',
+                f'https://www.{self.domain}/sitemap.html',
+                f'http://{self.domain}/sitemap_index.xml',
+                f'https://{self.domain}/sitemap_index.xml',
+                f'http://www.{self.domain}/sitemap_index.xml',
+                f'https://www.{self.domain}/sitemap_index.xml']
+        response = None
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            self.timeout = 10
+            response = self.get(url, allow_redirects=False)
+            if response:
+                break
+        if not response:
+            return
+        self.subdomains = utils.match_subdomain(self.domain, response.text)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    check = CheckRobots(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    do('qq.com')
@@ -1,6 +1,3 @@
-# coding=utf-8
-import time
-import queue
 import cdx_toolkit
 from common.crawl import Crawl
 from config import logger
@@ -29,43 +26,36 @@ class ArchiveCrawl(Crawl):
        for resp in cdx.iter(url, limit=limit):
            if resp.data.get('status') not in ['301', '302']:
                url = resp.data.get('url')
-                subdomains_find = self.match(self.register(domain), url + resp.text)
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                subdomains = self.match(self.register(domain),
+                                             url + resp.text)
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.crawl(self.domain, 50)
-
        # 爬取已发现的子域以发现新的子域
        for subdomain in self.subdomains:
            if subdomain != self.domain:
                self.crawl(subdomain, 10)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    crawl = ArchiveCrawl(domain)
-    crawl.run(result)
-    logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
-    logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
+    crawl.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,6 @@
-# coding=utf-8
-import time
-import queue
-from tqdm import tqdm
 import cdx_toolkit
+from tqdm import tqdm
+
 from common.crawl import Crawl
 from config import logger

@@ -29,43 +27,35 @@ class CommonCrawl(Crawl):

        for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
            if resp.data.get('status') not in ['301', '302']:
-                subdomains_find = self.match(self.register(domain), resp.text)
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                subdomains = self.match(self.register(domain), resp.text)
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.crawl(self.domain, 50)
-
        # 爬取已发现的子域以发现新的子域
        for subdomain in self.subdomains:
            if subdomain != self.domain:
                self.crawl(subdomain, 10)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    crawl = CommonCrawl(domain)
-    crawl.run(result)
-    logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
-    logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
+    crawl.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class BinaryEdgeAPI(Query):
@@ -27,42 +24,32 @@ class BinaryEdgeAPI(Query):
        resp = self.get(url)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = self.match(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.api:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.api):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = BinaryEdgeAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,6 +1,5 @@
-# coding=utf-8
 import time
-import queue
+import cloudscraper
 from common.query import Query
 from config import logger

@@ -11,51 +10,50 @@ class BufferOver(Query):
        self.domain = self.register(domain)
        self.module = 'Dataset'
        self.source = 'BufferOverQuery'
-        self.addr = 'https://dns.bufferover.run/dns'
+        self.addr = 'https://dns.bufferover.run/dns?q='

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
        time.sleep(self.delay)
-        self.header = self.get_header()
-        self.proxy = self.get_proxy(self.source)
-        params = {'q': self.domain}
-        resp = self.get(self.addr, params)
+        # 绕过cloudFlare验证
+        scraper = cloudscraper.create_scraper()
+        scraper.interpreter = 'js2py'
+        scraper.proxies = self.get_proxy(self.source)
+        url = self.addr + self.domain
+        try:
+            resp = scraper.get(url, timeout=self.timeout)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return
        if not resp:
            return
-        subdomains_find = self.match(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = BufferOver(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -0,0 +1,52 @@
+import time
+from common.query import Query
+
+
+class CeBaidu(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'CeBaiduQuery'
+        self.addr = 'http://ce.baidu.com/index/getRelatedSites'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'site_address': self.domain}
+        resp = self.get(self.addr, params)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+        print(self.subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = CeBaidu(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.query import Query
-from config import logger


 class Chinaz(Query):
@@ -24,38 +21,31 @@ class Chinaz(Query):
        resp = self.get(self.addr)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = self.match(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Chinaz(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class ChinazAPI(Query):
@@ -26,42 +23,33 @@ class ChinazAPI(Query):
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.api:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.api):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = ChinazAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class CirclAPI(Query):
@@ -26,39 +23,33 @@ class CirclAPI(Query):
        resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
        if not resp:
            return
-        subdomains_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        if self.user and self.pwd:
-            start = time.time()
-            self.query()
-            end = time.time()
-            self.elapsed = round(end - start, 1)
-            self.save_json()
-            self.gen_result()
-            self.save_db()
-            rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        if not self.check(self.user, self.pwd):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = CirclAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,7 +1,6 @@
-# coding=utf-8
-import random
 import time
-import queue
+import random
+import cloudscraper
 from bs4 import BeautifulSoup
 from common.query import Query
 from config import logger
@@ -13,64 +12,81 @@ class DNSdb(Query):
        self.domain = self.register(domain)
        self.module = 'Dataset'
        self.source = 'DNSdbQuery'
-        self.addr = 'https://www.dnsdb.org/'
+        self.addr = 'http://www.dnsdb.org/'
+        self.url = f'{self.addr}{self.domain}/'
+
+    def get_tokens(self):
+        """
+        绕过cloudFlare验证并获取taken
+
+        :return: 绕过失败返回None 成功返回tokens
+        """
+        scraper = cloudscraper.create_scraper()
+        scraper.interpreter = 'js2py'
+        scraper.proxies = self.get_proxy(self.source)
+        scraper.timeout = 10
+        try:
+            tokens = scraper.get_tokens(self.url)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return None
+        if len(tokens) != 2:
+            return None
+        return tokens

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        self.header = self.get_header()
-        self.header.update({'Referer': self.addr})
-        self.proxy = self.get_proxy(self.source)
-        url = self.addr + self.domain + '/'
-        resp = self.get(url)
+        tokens = self.get_tokens()
+        if not tokens:
+            logger.log('ALERT', f'{self.source}模块绕过cloudFlare检查失败')
+            return False
+        self.cookie = tokens[0]
+        self.header = {'User-Agent': tokens[1]}
+        self.timeout = 10
+        resp = self.get(self.url)
        if not resp:
            return
-        if resp.status_code == 200:
-            if 'index' in resp.text:
-                soup = BeautifulSoup(resp.text, features='lxml')
-                index_urls = set(map(lambda x: self.addr + self.domain + x.text, soup.find_all('a')))
-                for url in index_urls:
-                    self.delay = random.randint(2, 5)  # 休眠绕过CloudFlare的DDoS保护
-                    time.sleep(self.delay)
-                    resp = self.get(url)
-                    if not resp:
-                        return
-                    subdomains_find = self.match(self.domain, resp.text)
-                    self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
-            else:
-                subdomains_find = self.match(self.domain, resp.text)
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        if 'index' in resp.text:
+            soup = BeautifulSoup(resp.text, features='lxml')
+            base = self.addr+self.domain
+            urls = list(map(lambda a: base + '/' + a.get('href'),
+                            soup.find_all('a')))
+            urls = urls[:-1]  # idn域名暂时不考虑
+            for url in urls:
+                resp = self.get(url)
+                if not resp:
+                    return
+                subdomains = self.match(self.domain, resp.text)
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains)
+        else:
+            subdomains = self.match(self.domain, resp.text)
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = DNSdb(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,10 +1,7 @@
-# coding=utf-8
 import time
-import queue
 import config
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class DNSdbAPI(Query):
@@ -28,43 +25,34 @@ class DNSdbAPI(Query):
        resp = self.get(url)
        if not resp:
            return
-        if resp.status_code == 200:
-            subdomains_find = utils.match_subdomain(self.domain, resp.text)
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = utils.match_subdomain(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.api:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.api):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
+
    """
    query = DNSdbAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,15 +1,9 @@
-# coding=utf-8
 import time
-import queue
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class DNSdumpster(Query):
-    """
-
-    """
    def __init__(self, domain):
        Query.__init__(self)
        self.domain = self.register(domain)
@@ -29,43 +23,38 @@ class DNSdumpster(Query):
        if not resp:
            return
        self.cookie = resp.cookies
-        data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'), 'targetip': self.domain}
+        data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'),
+                'targetip': self.domain}
        resp = self.post(self.addr, data)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, resp.text)
-        if subdomains_find:
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = utils.match_subdomain(self.domain, resp.text)
+        if subdomains:
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = DNSdumpster(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+
+    do('example.com')
@@ -1,9 +1,5 @@
-# coding=utf-8
-import time
-import queue
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class HackerTarget(Query):
@@ -25,39 +21,32 @@ class HackerTarget(Query):
        if not resp:
            return
        if resp.status_code == 200:
-            subdomains_find = utils.match_subdomain(self.domain, resp.text)
-            if subdomains_find:
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            subdomains = utils.match_subdomain(self.domain, resp.text)
+            if subdomains:
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = HackerTarget(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,6 +1,3 @@
-# coding=utf-8
-import time
-import queue
 import config
 from common.query import Query
 from config import logger
@@ -23,53 +20,54 @@ class IPv4InfoAPI(Query):
        while True:
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
-            params = {'type': 'SUBDOMAINS', 'key': self.api, 'value': self.domain, 'page': page}
+            params = {'type': 'SUBDOMAINS', 'key': self.api,
+                      'value': self.domain, 'page': page}
            resp = self.get(self.addr, params)
            if not resp:
                return
            if resp.status_code != 200:
                break  # 请求不正常通常网络是有问题，不再继续请求下去
-            resp_json = resp.json()
-            subdomains_find = self.match(self.domain, str(resp_json))
-            if not subdomains_find:
+            try:
+                json = resp.json()
+            except Exception as e:
+                logger.log('DEBUG', e.args)
                break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
-            subdomains = resp_json.get('Subdomains')  # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
-            if len(subdomains) < 300:  # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
+            subdomains = self.match(self.domain, str(json))
+            if not subdomains:
                break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
+            subdomains = json.get('Subdomains')
+            if subdomains:
+                # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
+                if len(subdomains) < 300:
+                    break
            page += 1
            if page >= 50:  # ipv4info子域查询接口最多允许查询50页
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = IPv4InfoAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,11 +1,9 @@
-# coding=utf-8
+import hashlib
 import re
 import time
-import queue
-import hashlib
 from urllib import parse
+
 from common.query import Query
-from config import logger


 class NetCraft(Query):
@@ -24,16 +22,21 @@ class NetCraft(Query):
        绕过NetCraft的JS验证
        """
        self.header = self.get_header()  # Netcraft会检查User-Agent
-        self.cookie = self.get(self.init).cookies
+        resp = self.get(self.init)
+        if not resp:
+            return None
+        self.cookie = resp.cookies
        cookie_value = self.cookie['netcraft_js_verification_challenge']
-        verify_taken = hashlib.sha1(parse.unquote(cookie_value).encode('utf-8')).hexdigest()
+        cookie_encode = parse.unquote(cookie_value).encode('utf-8')
+        verify_taken = hashlib.sha1(cookie_encode).hexdigest()
        self.cookie['netcraft_js_verification_response'] = verify_taken

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        self.bypass_verification()
+        if not self.bypass_verification():
+            return
        last = ''
        while True:
            time.sleep(self.delay)
@@ -43,44 +46,37 @@ class NetCraft(Query):
            resp = self.get(self.addr + last, params)
            if not resp:
                return
-            subdomains_find = self.match(self.domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
            if 'Next page' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                break
            last = re.search(r'&last=.*' + self.domain, resp.text).group(0)
            self.page_num += self.per_page_num

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = NetCraft(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -0,0 +1,56 @@
+import time
+import config
+from common.query import Query
+
+
+class PassiveDnsAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Dataset'
+        self.source = 'PassiveDnsQuery'
+        self.addr = config.passivedns_api_addr or 'http://api.passivedns.cn'
+        self.token = config.passivedns_api_token
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        time.sleep(self.delay)
+        self.header = self.get_header()
+        self.header.update({'X-AuthToken': self.token})
+        self.proxy = self.get_proxy(self.source)
+        url = self.addr + '/flint/rrset/*.' + self.domain
+        resp = self.get(url)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.addr):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = PassiveDnsAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,10 +1,7 @@
-# coding=utf-8
-import time
-import queue
 import random
-from common.query import Query
+
 from common import utils
-from config import logger
+from common.query import Query


 class PTRArchive(Query):
@@ -13,7 +10,7 @@ class PTRArchive(Query):
        self.domain = self.register(domain)
        self.module = 'Dataset'
        self.source = "PTRArchiveQuery"
-        self.addr = 'http://ptrarchive.com/tools/search3.htm'
+        self.addr = 'http://ptrarchive.com/tools/search4.htm'

    def query(self):
        """
@@ -21,45 +18,39 @@ class PTRArchive(Query):
        """
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        self.cookie = {'pa_id': str(random.randint(0, 1000000000))}  # 绕过主页前端JS验证
+        # 绕过主页前端JS验证
+        self.cookie = {'pa_id': str(random.randint(0, 1000000000))}
        params = {'label': self.domain, 'date': 'ALL'}
        resp = self.get(self.addr, params)
        if not resp:
            return
        if resp.status_code == 200:
-            subdomains_find = utils.match_subdomain(self.domain, resp.text)
-            if subdomains_find:
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            subdomains = utils.match_subdomain(self.domain, resp.text)
+            if subdomains:
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = PTRArchive(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.query import Query
-from config import logger


 class Riddler(Query):
@@ -24,38 +21,31 @@ class Riddler(Query):
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = self.match(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Riddler(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,7 @@
-# coding=utf-8
-import time
-import queue
 import json
+import time
+
 from common.query import Query
-from config import logger


 class Robtex(Query):
@@ -34,39 +32,32 @@ class Robtex(Query):
                resp = self.get(url)
                if not resp:
                    return
-                subdomains_find = self.match(self.domain, resp.text)
-                if subdomains_find:
-                    self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                subdomains = self.match(self.domain, resp.text)
+                if subdomains:
+                    # 合并搜索子域名搜索结果
+                    self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Robtex(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class SecurityTrailsAPI(Query):
@@ -28,44 +25,35 @@ class SecurityTrailsAPI(Query):
        resp = self.get(url, params)
        if not resp:
            return
-        subdomains_prefix = resp.json()['subdomains']
-        subdomains_find = [f'{prefix}.{self.domain}' for prefix in subdomains_prefix]
-        if subdomains_find:
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        prefixs = resp.json()['subdomains']
+        subdomains = [f'{prefix}.{self.domain}' for prefix in prefixs]
+        if subdomains:
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.api:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.api):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = SecurityTrailsAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,6 @@
-# coding=utf-8
 import time
-import queue
+
 from common.query import Query
-from config import logger


 class SiteDossier(Query):
@@ -28,43 +26,37 @@ class SiteDossier(Query):
            resp = self.get(url)
            if not resp:
                return
-            subdomains_find = self.match(self.domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
-            if 'Show next 100 items' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            # 搜索页面没有出现下一页时停止搜索
+            if 'Show next 100 items' not in resp.text:
                break
            self.page_num += self.per_page_num

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = SiteDossier(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -0,0 +1,55 @@
+import cloudscraper
+
+from common.query import Query
+from config import logger
+
+
+class ThreatCrowd(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.source = 'ThreatCrowdQuery'
+        self.addr = 'https://www.threatcrowd.org/searchApi' \
+                    '/v2/domain/report?domain='
+
+    def query(self):
+        # 绕过cloudFlare验证
+        scraper = cloudscraper.create_scraper()
+        scraper.interpreter = 'js2py'
+        scraper.proxies = self.get_proxy(self.source)
+        url = self.addr + self.domain
+        try:
+            resp = scraper.get(url, timeout=self.timeout)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = ThreatCrowd(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,18 +1,17 @@
 #!/usr/bin/env python3
-# coding=utf-8

 """
 通过枚举域名常见的SRV记录并做查询来发现子域
 """

-import time
-import queue
-import json
 import asyncio
+import json
+
 import aiodns
-from common.module import Module
+
 from common import utils
-from config import logger, data_storage_path, resolver_nameservers
+from common.module import Module
+from config import data_storage_path, logger, resolver_nameservers


 class BruteSRV(Module):
@@ -61,42 +60,36 @@ class BruteSRV(Module):
        for result in results:
            if result:
                for answer in result:
-                    subdomain = utils.match_subdomain(self.domain, answer.host)
-                    if subdomain:
-                        self.subdomains = self.subdomains.union(subdomain)
+                    subdomains = utils.match_subdomain(self.domain, answer.host)
+                    if subdomains:
+                        self.subdomains = self.subdomains.union(subdomains)
                    else:
                        logger.log('DEBUG', f'{answer.host}不是{self.domain}的子域')
        if not len(self.subdomains):
            logger.log('DEBUG', f'没有找到{self.domain}的SRV记录')

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始枚举{self.domain}域的SRV记录')
-        start = time.time()
+        self.begin()
        self.brute()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束枚举{self.domain}域的SRV记录')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口
+
    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    brute = BruteSRV(domain)
-    brute.run(rx_queue)
-    logger.log('INFOR', f'{brute.source}模块耗时{brute.elapsed}秒发现{brute.domain}的子域{len(brute.subdomains)}个')
-    logger.log('DEBUG', f'{brute.source}模块发现{brute.domain}的子域 {brute.subdomains}')
+    brute.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+
+    do('example.com')
@@ -1,66 +0,0 @@
-# coding=utf-8
-import time
-import queue
-import config
-from common.query import Query
-from config import logger
-
-
-class RiskIQ(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = self.register(domain)
-        self.module = 'Intelligence'
-        self.source = 'RiskIQQuery'
-        self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
-        self.username = config.riskiq_api_username
-        self.key = config.riskiq_api_key
-
-    def query(self):
-        """
-        向接口查询子域并做子域匹配
-        """
-        self.header = self.get_header()
-        self.proxy = self.get_proxy(self.source)
-        params = {'query': self.domain}
-        resp = self.get(url=self.addr, params=params, auth=(self.username, self.key))
-        if not resp:
-            return
-        resp_json = resp.json()
-        subdomains_find = resp_json.get('subdomains')
-        if subdomains_find:
-            self.subdomains = set(map(lambda x: x + '.' + self.domain, subdomains_find))
-
-    def run(self, rx_queue):
-        """
-        类执行入口
-        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
-        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
-
-
-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    :param rx_queue: 结果集队列
-    """
-    return
-    query = RiskIQ(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
-
-
-if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
@@ -0,0 +1,56 @@
+import config
+from common.query import Query
+
+
+class RiskIQ(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Intelligence'
+        self.source = 'RiskIQAPIQuery'
+        self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
+        self.user = config.riskiq_api_username
+        self.key = config.riskiq_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'query': self.domain}
+        resp = self.get(url=self.addr,
+                        params=params,
+                        auth=(self.user, self.key))
+        if not resp:
+            return
+        data = resp.json()
+        names = data.get('subdomains')
+        self.subdomains = set(map(lambda sub: f'{sub}.{self.domain}', names))
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.user, self.key):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = RiskIQ(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,66 +0,0 @@
-# coding=utf-8
-import time
-import queue
-import config
-from common.query import Query
-from config import logger
-
-
-class ThreatBookAPI(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = self.register(domain)
-        self.module = 'Intelligence'
-        self.source = 'ThreatBookAPIQuery'
-        self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
-        self.key = config.threatbook_api_key
-
-    def query(self, domain):
-        """
-        向接口查询子域并做子域匹配
-        """
-        self.header = self.get_header()
-        self.proxy = self.get_proxy(self.source)
-        params = {'apikey': self.key, 'domain': domain, 'field': 'sub_domains'}
-        resp = self.post(self.addr, params)
-        if not resp:
-            return
-        subdomain_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomain_find)
-
-    def run(self, rx_queue):
-        """
-        类执行入口
-        """
-        if not self.key:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
-            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
-        self.query(self.domain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
-
-
-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    :param rx_queue: 结果集队列
-    """
-    query = ThreatBookAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
-
-
-if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
@@ -0,0 +1,54 @@
+import config
+from common.query import Query
+
+
+class ThreatBookAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Intelligence'
+        self.source = 'ThreatBookAPIQuery'
+        self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
+        self.key = config.threatbook_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'apikey': self.key,
+                  'domain': self.domain,
+                  'field': 'sub_domains'}
+        resp = self.post(self.addr, params)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.key):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = ThreatBookAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,8 +1,6 @@
-# coding=utf-8
 import time
-import queue
+
 from common.query import Query
-from config import logger


 class ThreatMiner(Query):
@@ -20,42 +18,36 @@ class ThreatMiner(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'e': 'subdomains_container', 'q': self.domain, 't': 0, 'rt': 10}
+        params = {'e': 'subdomains_container',
+                  'q': self.domain, 't': 0, 'rt': 10}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = self.match(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = ThreatMiner(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.query import Query
-from config import logger

+from common.query import Query

 '''
 最多查询100条
@@ -18,7 +15,7 @@ class VirusTotal(Query):
        self.addr = 'https://www.virustotal.com/ui/domains/{}/subdomains'
        self.domain = self.register(domain)

-    def query(self, domain):
+    def query(self):
        """
        向接口查询子域并做子域匹配
        """
@@ -30,56 +27,48 @@ class VirusTotal(Query):
                                'TE': 'Trailers'})
            self.proxy = self.get_proxy(self.source)
            params = {'limit': '40', 'cursor': next_cursor}
-            resp = self.get(url=self.addr.format(domain), params=params)
+            resp = self.get(url=self.addr.format(self.domain), params=params)
            if not resp:
                return
-            resp_json = resp.json()
-            subdomain_find = set()
-            datas = resp_json.get('data')
+            data = resp.json()
+            subdomains = set()
+            datas = data.get('data')

            if datas:
                for data in datas:
                    subdomain = data.get('id')
                    if subdomain:
-                        subdomain_find.add(subdomain)
+                        subdomains.add(subdomain)
            else:
                break
-            self.subdomains = self.subdomains.union(subdomain_find)
-            meta = resp_json.get('meta')
+            self.subdomains = self.subdomains.union(subdomains)
+            meta = data.get('meta')
            if meta:
                next_cursor = meta.get('cursor')
            else:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
-        self.query(self.domain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.begin()
+        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = VirusTotal(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,5 @@
-# coding=utf-8
-import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class VirusTotalAPI(Query):
@@ -15,54 +11,45 @@ class VirusTotalAPI(Query):
        self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'
        self.key = config.virustotal_api_key

-    def query(self, domain):
+    def query(self):
        """
        向接口查询子域并做子域匹配
        """
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'apikey': self.key, 'domain': domain}
+        params = {'apikey': self.key, 'domain': self.domain}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        resp_json = resp.json()
-        subdomain_find = set(resp_json.get('subdomains'))
-        self.subdomains = self.subdomains.union(subdomain_find)
+        json = resp.json()
+        data = json.get('subdomains')
+        if data:
+            subdomains = set(data)
+            self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.key:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.key):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
-        self.query(self.domain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.begin()
+        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
-    return
    query = VirusTotalAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.search import Search
-from config import logger


 class Ask(Search):
@@ -33,57 +30,52 @@ class Ask(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += 1
            if '>Next<' not in resp.text:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search(self.domain, full_search=True)
-
        # 排除同一子域搜索结果过多的子域以发现新的子域
        for statement in self.filter(self.domain, self.subdomains):
            self.search(self.domain, filtered_subdomain=statement)

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Ask(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.search import Search
 from bs4 import BeautifulSoup
-from config import logger
+from common.search import Search


 class Baidu(Search):
@@ -25,7 +22,8 @@ class Baidu(Search):
        """
        bs = BeautifulSoup(html, features='lxml')
        subdomains_all = set()
-        for find_res in bs.find_all('a', {'class': 'c-showurl'}):  # 获取搜索结果中所有的跳转URL地址
+        # 获取搜索结果中所有的跳转URL地址
+        for find_res in bs.find_all('a', {'class': 'c-showurl'}):
            url = find_res.get('href')
            subdomain = self.match_location(domain, url)
            subdomains_all = subdomains_all.union(subdomain)
@@ -45,32 +43,37 @@ class Baidu(Search):
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'wd': query, 'pn': self.page_num, 'rn': self.per_page_num}
+            params = {'wd': query, 'pn': self.page_num,
+                      'rn': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
            if len(domain) > 12:  # 解决百度搜索结果中域名过长会显示不全的问题
-                subdomains_find = self.redirect_match(domain, resp.text)  # 获取百度跳转URL响应头的Location字段获取直链
+                # 获取百度跳转URL响应头的Location字段获取直链
+                subdomains = self.redirect_match(domain, resp.text)
            else:
-                subdomains_find = self.match(domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+                subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += self.per_page_num
-            if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 搜索页面没有出现下一页时停止搜索
+            if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:
                break
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -79,32 +82,28 @@ class Baidu(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Baidu(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.search import Search
-from config import logger


 class Bing(Search):
@@ -34,29 +31,32 @@ class Bing(Search):
            time.sleep(self.delay)
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'q': query, 'first': self.page_num, 'count': self.per_page_num}
+            params = {'q': query, 'first': self.page_num,
+                      'count': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomains_find = self.match(domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
-            if '<div class="sw_next>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            # 搜索页面没有出现下一页时停止搜索
+            if '<div class="sw_next>' not in resp.text:
                break
            self.page_num += self.per_page_num
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -65,32 +65,28 @@ class Bing(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Bing(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.search import Search
-from config import logger


 class BingAPI(Search):
@@ -11,8 +8,9 @@ class BingAPI(Search):
        Search.__init__(self)
        self.domain = domain
        self.module = 'Search'
-        self.source = 'BingCustomSearch'
-        self.addr = 'https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search'
+        self.source = 'BingAPISearch'
+        self.addr = 'https://api.cognitive.microsoft.com/' \
+                    'bingcustomsearch/v7.0/search'
        self.id = config.bing_api_id
        self.key = config.bing_api_key
        self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数
@@ -38,27 +36,26 @@ class BingAPI(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomains_find = self.match(domain, str(resp.json()))
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(domain, str(resp.json()))
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += self.per_page_num
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not (self.id and self.key):
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.id, self.key):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -67,32 +64,28 @@ class BingAPI(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = BingAPI(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,7 +1,5 @@
-# coding=utf-8
 import re
 import time
-import queue
 from common.search import Search
 from config import logger

@@ -32,27 +30,29 @@ class DuckDuckGO(Search):
            resp = self.post(self.addr, data)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            try:
                s = re.findall(r'name="s" value="(\d.*)"', resp.text)[-1]
                dc = re.findall(r'name="dc" value="(\d.*)"', resp.text)
            except Exception as e:
                logger.error(e)
                break
-            data.update({'s': s, 'nextParams': '', 'o': 'json', 'dc': dc, 'api': '/d.js'})
+            data.update({'s': s, 'nextParams': '', 'o': 'json',
+                         'dc': dc, 'api': '/d.js'})

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -61,33 +61,30 @@ class DuckDuckGO(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    # return  # 暂时还有点问题
    search = DuckDuckGO(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
-import time
-import queue
 import random
+import time
 from common.search import Search
-from config import logger


 class Exalead(Search):
@@ -30,29 +27,30 @@ class Exalead(Search):
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'q': query, 'elements_per_page': '30', "start_index": self.page_num}
+            params = {'q': query, 'elements_per_page': '30',
+                      "start_index": self.page_num}
            resp = self.get(url=self.addr, params=params)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += self.per_page_num
            if self.page_num > 1999:
                break
            if 'title="Go to the next page"' not in resp.text:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -62,32 +60,28 @@ class Exalead(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Exalead(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,71 +0,0 @@
-# coding=utf-8
-import time
-import queue
-import json
-import base64
-import config
-from common.search import Search
-from config import logger
-
-
-class FoFa(Search):
-    def __init__(self, domain):
-        Search.__init__(self)
-        self.domain = domain
-        self.module = 'Search'
-        self.source = 'FoFaSearch'
-        self.addr = 'https://fofa.so/api/v1/search/all'
-        self.delay = 1
-        self.email = config.fofa_api_email
-        self.key = config.fofa_api_key
-
-    def search(self):
-        """
-        发送搜索请求并做子域匹配
-        """
-        self.page_num = 1
-        query_base64 = base64.b64encode(f'domain={self.domain}'.encode('utf-8'))
-        while True:
-            time.sleep(self.delay)
-            self.header = self.get_header()
-            self.proxy = self.get_proxy(self.source)
-            query = {'email': self.email, 'key': self.key, 'qbase64': query_base64, 'page': self.page_num}
-            resp = self.get(self.addr, query)
-            if not resp:
-                return
-            subdomain_find = self.match(self.domain, resp.text)
-            self.subdomains = self.subdomains.union(subdomain_find)
-            self.page_num += 1
-
-    def run(self, rx_queue):
-        """
-        类执行入口
-        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
-        self.search()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
-
-
-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    :param rx_queue: 结果集队列
-    """
-    search = FoFa(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
-
-
-if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
@@ -0,0 +1,74 @@
+import base64
+import time
+
+import config
+from common.search import Search
+from config import logger
+
+
+class FoFa(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'FoFaAPISearch'
+        self.addr = 'https://fofa.so/api/v1/search/all'
+        self.delay = 1
+        self.email = config.fofa_api_email
+        self.key = config.fofa_api_key
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        self.page_num = 1
+        subdomain_encode = f'domain={self.domain}'.encode('utf-8')
+        query_data = base64.b64encode(subdomain_encode)
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = {'email': self.email,
+                     'key': self.key,
+                     'qbase64': query_data,
+                     'page': self.page_num,
+                     'size': 10000}
+            resp = self.get(self.addr, query)
+            if not resp:
+                return
+            resp_json = resp.json()
+            subdomains = self.match(self.domain, str(resp_json))
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains = self.subdomains.union(subdomains)
+            size = resp_json.get('size')
+            if size < 10000:
+                break
+            self.page_num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.email, self.key):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = FoFa(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -0,0 +1,121 @@
+import re
+import time
+import requests
+import config
+from bs4 import BeautifulSoup
+from common.search import Search
+from config import logger
+
+
+class Github(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.source = 'GithubSearch'
+        self.module = 'Search'
+        self.addr = 'https://github.com/search'
+        self.domain = self.register(domain)
+        self.header = self.get_header()
+        self.session = requests.Session()
+        self.login_url = 'https://github.com/login'
+        self.post_url = 'https://github.com/session'
+        self.email = config.github_email
+        self.password = config.github_password
+
+    def login_github(self):
+        """
+        登录github
+
+        :return: 登录失败返回False 成功返回True
+        """
+        token = self.get_token()
+        if token is None:
+            logger.log('ERROR', f'{self.source}模块获取token失败')
+            return False
+        post_data = {
+            'commit': 'Sign in',
+            'utf8': '✓',
+            'authenticity_token': token,
+            'login': self.email,
+            'password': self.password
+        }
+        resp = self.session.post(self.post_url, data=post_data)
+        if resp.status_code != 200:
+            return False
+        match = re.search(r'"user-login" content="(.*?)"', resp.text)
+        if match:
+            return True
+
+    def get_token(self):
+        """
+        获取github登录token
+
+        :return: 获取失败返回None，成功返回token
+        """
+        resp = self.session.get(self.login_url)
+        if resp.status_code != 200:
+            return None
+        match = re.search(
+            r'name="authenticity_token" value="(.*?)"', resp.text)
+        if not match:
+            return None
+        return match.group(1)
+
+    def search(self, full_search=True):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.session.headers = self.get_header()
+        self.session.proxies = self.get_proxy(self.source)
+        self.session.verify = self.verify
+        if not self.login_github():
+            logger.log('ERROR', f'{self.session}模块登录失败')
+            return
+        page_num = 1
+        while True:
+            time.sleep(self.delay)
+            params = {'p': page_num, 'q': f'"{self.domain}"', 'type': 'Code'}
+            resp = self.session.get(self.addr, params=params)
+            if resp.status_code != 200:
+                logger.log('ERROR', f'{self.session}模块搜索出错')
+                break
+            soup = BeautifulSoup(resp.text, 'lxml')
+            subdomains = self.match(self.domain, soup.text)
+            self.subdomains = self.subdomains.union(subdomains)
+            if not subdomains:
+                break
+            if not full_search:
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
+                    break
+            if 'class="next_page disabled"' in resp.text:
+                break
+            if page_num > 100:
+                break
+            page_num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.email, self.password):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Github(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('mi.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
-import time
-import queue
 import random
+import time
 from common.search import Search
-from config import logger


 class Google(Search):
@@ -43,25 +40,25 @@ class Google(Search):
            resp = self.get(url=self.addr, params=payload)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            page_num += per_page_num
-            if 'start='+str(page_num) not in resp.text:
+            if 'start=' + str(page_num) not in resp.text:
                break
            if '302 Moved' in resp.text:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -70,32 +67,28 @@ class Google(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Google(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.search import Search
-from config import logger


 class GoogleAPI(Search):
@@ -32,32 +29,30 @@ class GoogleAPI(Search):
            time.sleep(self.delay)
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
-            params = {'key': self.key, 'cx': self.cx, 'q': word, 'fields': 'items/link',
+            params = {'key': self.key, 'cx': self.cx,
+                      'q': word, 'fields': 'items/link',
                      'start': self.page_num, 'num': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomain_find = self.match(domain, str(resp.json()))
-            if not subdomain_find:
+            subdomains = self.match(domain, str(resp.json()))
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += self.per_page_num
            if self.page_num > 100:  # 免费的API只能查询前100条结果
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not (self.cx and self.key):
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.cx, self.key):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -66,32 +61,28 @@ class GoogleAPI(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = GoogleAPI(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,11 +1,5 @@
-# coding=utf-8
-import time
-import queue
-import queue
 import config
-# from shodan import Shodan
 from common.search import Search
-from config import logger


 class ShodanAPI(Search):
@@ -13,7 +7,7 @@ class ShodanAPI(Search):
        Search.__init__(self)
        self.domain = self.register(domain)
        self.module = 'Search'
-        self.source = 'ShodanSearch'
+        self.source = 'ShodanAPISearch'
        self.addr = 'https://api.shodan.io/shodan/host/search'
        self.key = config.shodan_api_key

@@ -26,48 +20,41 @@ class ShodanAPI(Search):
        query = 'hostname:.' + self.domain
        page = 1
        while True:
-            params = {'key': self.key, 'page': page, 'query': query, 'minify': True, 'facets': {'hostnames'}}
+            params = {'key': self.key, 'page': page, 'query': query,
+                      'minify': True, 'facets': {'hostnames'}}
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomain_find = self.match(self.domain, resp.text)
-            if subdomain_find:
-                self.subdomains = self.subdomains.union(subdomain_find)
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            if subdomains:
+                self.subdomains = self.subdomains.union(subdomains)
            page += 1

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.key:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.key):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = ShodanAPI(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    results = queue.Queue()
-    do('qq.com', results)
+    do('example.com')
@@ -1,8 +1,6 @@
-# coding=utf-8
 import time
-import queue
+
 from common.search import Search
-from config import logger


 class So(Search):
@@ -33,25 +31,27 @@ class So(Search):
            resp = self.get(url=self.addr, params=payload)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            page_num += 1
-            if '<a id="snext"' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 搜索页面没有出现下一页时停止搜索
+            if '<a id="snext"' not in resp.text:
                break
-            if self.page_num * self.per_page_num >= self.limit_num:  # 搜索条数限制
+            # 搜索条数限制
+            if self.page_num * self.per_page_num >= self.limit_num:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -60,32 +60,28 @@ class So(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = So(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,4 @@
-# coding=utf-8
-import time
-import queue
 from common.search import Search
-from config import logger


 class Sogou(Search):
@@ -27,29 +23,33 @@ class Sogou(Search):
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            word = 'site:' + domain + filtered_subdomain
-            payload = {'query': word, 'page': self.page_num, "num": self.per_page_num}
+            payload = {'query': word, 'page': self.page_num,
+                       "num": self.per_page_num}
            resp = self.get(self.addr, payload)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += 1
-            if '<a id="sogou_next"' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 搜索页面没有出现下一页时停止搜索
+            if '<a id="sogou_next"' not in resp.text:
                break
-            if self.page_num * self.per_page_num >= self.limit_num:  # 搜索条数限制
+            # 搜索条数限制
+            if self.page_num * self.per_page_num >= self.limit_num:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -58,32 +58,28 @@ class Sogou(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Sogou(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.search import Search
-from config import logger


 class Yahoo(Search):
@@ -11,8 +8,8 @@ class Yahoo(Search):
        self.domain = domain
        self.module = 'Search'
        self.source = 'YahooSearch'
-        self.init = 'https://hk.search.yahoo.com/'
-        self.addr = 'https://hk.search.yahoo.com/search'
+        self.init = 'https://search.yahoo.com/'
+        self.addr = 'https://search.yahoo.com/search'
        self.limit_num = 1000  # 限制搜索条数
        self.delay = 5
        self.per_page_num = 40
@@ -39,25 +36,27 @@ class Yahoo(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomains_find = self.match(domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
            if '>Next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                break
            self.page_num += self.per_page_num
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -66,32 +65,28 @@ class Yahoo(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Yahoo(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.search import Search
-from config import logger


 class Yandex(Search):
@@ -25,35 +22,41 @@ class Yandex(Search):
        :param bool full_search: 全量搜索
        """
        self.page_num = 0  # 二次搜索重新置0
-        self.cookie = self.get(self.init).cookies  # 获取cookie bing在搜索时需要带上cookie
+        resp = self.get(self.init)
+        if not resp:
+            return
+        self.cookie = resp.cookies  # 获取cookie
        while True:
            time.sleep(self.delay)
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'text': query, 'p': self.page_num, 'numdoc': self.per_page_num}
+            params = {'text': query, 'p': self.page_num,
+                      'numdoc': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomains_find = self.match(domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
            if '>next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                break
            self.page_num += 1
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -62,32 +65,28 @@ class Yandex(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Yandex(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,6 +1,4 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.search import Search
 from config import logger
@@ -20,20 +18,19 @@ class ZoomEyeAPI(Search):
    def login(self):
        """
        登陆获取查询taken
-        :return:
        """
        url = 'https://api.zoomeye.org/user/login'
        data = {'username': self.user, 'password': self.pwd}
        resp = self.post(url=url, json=data)
        if not resp:
-            logger.log('FETAL', f'登录失败无法获取{self.source}的访问token')
+            logger.log('FATAL', f'登录失败无法获取{self.source}的访问token')
            return
-        resp_json = resp.json()
+        data = resp.json()
        if resp.status_code == 200:
-            # print('登陆成功')
-            return resp_json.get('access_token')
+            logger.log('DEBUG', f'{self.source}模块登录成功')
+            return data.get('access_token')
        else:
-            logger.log('ALERT', resp_json.get('message'))
+            logger.log('ALERT', data.get('message'))
            exit(1)

    def search(self):
@@ -51,47 +48,39 @@ class ZoomEyeAPI(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomain_find = self.match(self.domain, resp.text)
-            self.subdomains = self.subdomains.union(subdomain_find)
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains = self.subdomains.union(subdomains)
            page_num += 1
            if page_num > 500:
                break
            if resp.status_code == 403:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not (self.user and self.pwd):
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.user, self.pwd):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = ZoomEyeAPI(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -2,92 +2,161 @@
 # coding=utf-8

 """
-OneForAll是一款强大的子域收集神器
+OneForAll是一款功能强大的子域收集工具
+
+:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
+:license: GNU General Public License v3.0, see LICENSE for more details.
 """

 import asyncio
+
 import fire
 import config
 import dbexport
+from datetime import datetime
 from config import logger
 from collect import Collect
 from aiobrute import AIOBrute
-from common import utils, database, resolve, request
+from common import utils, resolve, request
+from common.database import Database
+from takeover import Takeover

-__author__ = 'Jing Ling, Black Star'
-__contact__ = 'admin@hackfun.org'
-__copyright__ = 'Copyright (c) 2019, Jing Ling. All rights reserved.'
-__license__ = 'GNU General Public License v3.0'
-__version__ = '0.0.1'
+yellow = '\033[01;33m'
+white = '\033[01;37m'
+green = '\033[01;32m'
+blue = '\033[01;34m'
+red = '\033[1;31m'
+end = '\033[0m'
+
+version = white + '{' + red + 'v0.0.7#dev' + white + '}'
+
+banner = f"""{yellow}
+             ___             _ _ 
+ ___ ___ ___|  _|___ ___ ___| | | {version}{green}
+| . |   | -_|  _| . |  _| .'| | | {blue}
+|___|_|_|___|_| |___|_| |__,|_|_| {white}git.io/fjHT1{end}
+"""


 class OneForAll(object):
    """
-    OneForAll是一款强大的子域收集神器
+    OneForAll是一款功能强大的子域收集工具

-    Version: 0.0.1
-    Project: https://github.com/shmilylty/OneForAll/
-
-    :param str target:  单个域名或者每行一个域名的文件路径
-    :param bool brute:  是否使用爆破模块（默认禁用）
-    :param str port:    HTTP请求验证的端口范围（默认medium）
-    :param int valid:   导出子域的有效性（默认1）
-    :param str format:  导出格式（默认xlsx）
-    :param str path:    导出路径(默认None)
-    :param bool output: 是否将导出数据输出到终端（默认False）
-
-    Note:
-        参数valid可选值有1，0，None，分别表示导出有效，无效，全部子域
-        参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
-        参数format可选格式有'csv','tsv','json','yaml','html','xls','xlsx','dbf','latex','ods'
-        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+    Version: 0.0.7
+    Project: https://git.io/fjHT1

    Example:
-    python oneforall.py --target example.com run
-    python oneforall.py --target example.com --brute True --port medium valid 1 run
-    python oneforall.py --target ./domains.txt --format csv --path= ./result.csv  --output True run
+        python3 oneforall.py --target example.com run
+        python3 oneforall.py --target ./subdomains.txt run
+        python3 oneforall.py --target example.com --valid None run
+        python3 oneforall.py --target example.com --brute True run
+        python3 oneforall.py --target example.com --port medium run
+        python3 oneforall.py --target example.com --format csv run
+        python3 oneforall.py --target example.com --verify False run
+        python3 oneforall.py --target example.com --takeover False run
+        python3 oneforall.py --target example.com --show True run
+
+    Note:
+        参数valid可选值1，0，None分别表示导出有效，无效，全部子域
+        参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
+        参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+
+    :param str target:  单个域名或者每行一个域名的文件路径(必需参数)
+    :param bool brute:  使用爆破模块(默认False)
+    :param bool verify: 验证子域有效性(默认True)
+    :param str port:    请求验证的端口范围(默认medium)
+    :param int valid:   导出子域的有效性(默认1)
+    :param str format:  导出格式(默认csv)
+    :param bool show:   终端显示导出数据(默认False)
    """
-    def __init__(self, target, brute=False, port='medium', valid=1, path=None, format='xlsx', output=False):
+    def __init__(self, target, brute=None, verify=None, port='medium', valid=1,
+                 format='csv', takeover=True, show=False):
        self.target = target
        self.port = port
        self.domains = set()
-        self.domain = ''
+        self.domain = str()
        self.datas = list()
-        self.brute = brute or config.enable_brute_module
+        self.brute = brute
+        self.verify = verify
+        self.takeover = takeover
        self.valid = valid
-        self.path = path
        self.format = format
-        self.output = output
+        self.show = show
+
+    def main(self):
+        if self.brute is None:
+            self.brute = config.enable_brute_module
+        if self.verify is None:
+            self.verify = config.enable_verify_subdomain
+        rename_table = self.domain + '_last'
+        collect = Collect(self.domain, export=False)
+        collect.run()
+        if self.brute:
+            # 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
+            brute = AIOBrute(self.domain, export=False)
+            brute.run()
+
+        db = Database()
+        db.copy_table(self.domain, self.domain+'_ori')
+        db.remove_invalid(self.domain)
+        db.deduplicate_subdomain(self.domain)
+        # 不验证子域的情况
+        if not self.verify:
+            # 数据库导出
+            self.valid = None
+            dbexport.export(self.domain, valid=self.valid, format=self.format,
+                            show=self.show)
+            db.drop_table(rename_table)
+            db.rename_table(self.domain, rename_table)
+            return
+        # 开始验证子域工作
+        self.datas = db.get_data(self.domain).as_dict()
+        loop = asyncio.get_event_loop()
+        asyncio.set_event_loop(loop)
+
+        # 解析域名地址
+        task = resolve.bulk_query_a(self.datas)
+        self.datas = loop.run_until_complete(task)
+
+        # 保存解析结果
+        resolve_table = self.domain + '_res'
+        db.drop_table(resolve_table)
+        db.create_table(resolve_table)
+        db.save_db(resolve_table, self.datas, 'resolve')
+
+        # 请求域名地址
+        task = request.bulk_get_request(self.datas, self.port)
+        self.datas = loop.run_until_complete(task)
+        # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
+        loop.run_until_complete(asyncio.sleep(0.25))
+
+        db.clear_table(self.domain)
+        db.save_db(self.domain, self.datas)
+
+        # 数据库导出
+        dbexport.export(self.domain, valid=self.valid, format=self.format,
+                        show=self.show)
+        db.drop_table(rename_table)
+        db.rename_table(self.domain, rename_table)
+        db.close()
+        # 子域接管检查
+
+        if self.takeover:
+            subdomains = set(map(lambda x: x.get('subdomain'), self.datas))
+            takeover = Takeover(subdomains)
+            takeover.run()

    def run(self):
+        print(banner)
+        dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        print(f'[*] Starting OneForAll @ {dt}\n')
        logger.log('INFOR', f'开始运行OneForAll')
        self.domains = utils.get_domains(self.target)
        if self.domains:
            for self.domain in self.domains:
-                collect = Collect(self.domain, export=False)
-                collect.run()
-                if self.brute:
-                    # 由于爆破会有大量dns解析请求 并发常常会导致其他任务中的网络请求超时
-                    brute = AIOBrute(self.domain)
-                    brute.run()
-                table_name = self.domain.replace('.', '_')
-                db_conn = database.connect_db()
-                self.datas = database.get_data(db_conn, table_name).as_dict()
-                loop = asyncio.get_event_loop()
-                asyncio.set_event_loop(loop)
-                self.datas = loop.run_until_complete(resolve.bulk_query_a(self.datas))
-                self.datas = loop.run_until_complete(request.bulk_get_request(self.datas, self.port))
-                loop.run_until_complete(asyncio.sleep(0.25))  # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
-                loop.close()
-                database.clear_table(db_conn, table_name)
-                database.save_db(db_conn, table_name, self.datas)
-                # database.deduplicate_subdomain(db_conn, table_name)
-                # database.remove_invalid(db_conn, table_name)
-                # 数据库导出
-                if not self.path:
-                    self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
-                dbexport.export(table_name, db_conn, self.valid, self.path, self.format, self.output)
-                db_conn.close()
+                self.main()
        else:
            logger.log('FATAL', f'获取域名失败')
        logger.log('INFOR', f'结束运行OneForAll')
@@ -95,4 +164,5 @@ class OneForAll(object):

 if __name__ == '__main__':
    fire.Fire(OneForAll)
-    # OneForAll('owasp.org').run()
+    # OneForAll('example.com').run()
+    # OneForAll('./domains.txt').run()
@@ -0,0 +1,2 @@
+www.example.com
+www.hackfun.org
@@ -0,0 +1,169 @@
+#!/usr/bin/python3
+# coding=utf-8
+
+"""
+OneForAll子域接管模块
+
+:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
+:license: GNU General Public License v3.0, see LICENSE for more details.
+"""
+import time
+import json
+from threading import Thread
+from queue import Queue
+
+import fire
+from tablib import Dataset
+from tqdm import tqdm
+
+import config
+from config import logger
+from common import resolve, utils
+from common.module import Module
+from common.domain import Domain
+
+
+def get_fingerprint():
+    path = config.data_storage_path.joinpath('fingerprints.json')
+    with open(path) as file:
+        fingerprints = json.load(file)
+    return fingerprints
+
+
+def get_cname(subdomain):
+    resolver = resolve.dns_resolver()
+    try:
+        answers = resolver.query(subdomain, 'CNAME')
+    except Exception as e:
+        logger.log('DEBUG', e.args)
+        return None
+    for answer in answers:
+        return answer.to_text()  # 一个子域只有一个CNAME记录
+
+
+def get_maindomain(subdomain):
+    return Domain(subdomain).registered()
+
+
+class Takeover(Module):
+    """
+    OneForAll多线程子域接管风险检查模块
+
+    Example:
+        python3 takeover.py --target www.example.com  --format csv run
+        python3 takeover.py --target ./subdomains.txt --thread 10 run
+
+    Note:
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+        参数dpath为None默认使用OneForAll结果目录
+
+    :param str target:  单个子域或者每行一个子域的文件路径(必需参数)
+    :param int thread:  线程数(默认100)
+    :param str format:  导出格式(默认csv)
+    :param str dpath:   导出目录(默认None)
+    """
+    def __init__(self, target, thread=100, dpath=None, format='csv'):
+        Module.__init__(self)
+        self.subdomains = set()
+        self.module = 'Check'
+        self.source = 'Takeover'
+        self.target = target
+        self.thread = thread
+        self.dpath = dpath
+        self.format = format
+        self.fingerprints = None
+        self.subdomainq = Queue()
+        self.cnames = list()
+        self.results = Dataset()
+
+    def save(self):
+        logger.log('INFOR', '正在保存检查结果')
+        if self.format == 'txt':
+            data = str(self.results)
+        else:
+            data = self.results.export(self.format)
+        fpath = self.dpath.joinpath(f'takeover.{self.format}')
+        utils.save_data(fpath, data)
+
+    def compare(self, subdomain, cname, responses):
+        domain_resp = self.get('http://' + subdomain, check=False)
+        cname_resp = self.get('http://'+cname, check=False)
+        if domain_resp is None or cname_resp is None:
+            return
+
+        for resp in responses:
+            if resp in domain_resp.text and resp in cname_resp.text:
+                logger.log('ALERT', f'{subdomain}存在子域接管风险')
+                self.results.append([subdomain, cname])
+                break
+
+    def worker(self, subdomain):
+        cname = get_cname(subdomain)
+        if cname is None:
+            return
+        maindomain = get_maindomain(cname)
+        for fingerprint in self.fingerprints:
+            cnames = fingerprint.get('cname')
+            if maindomain not in cnames:
+                continue
+            responses = fingerprint.get('response')
+            self.compare(subdomain, cname, responses)
+
+    def check(self):
+        while not self.subdomainq.empty():  # 保证域名队列遍历结束后能退出线程
+            subdomain = self.subdomainq.get()  # 从队列中获取域名
+            self.worker(subdomain)
+            self.subdomainq.task_done()
+
+    def progress(self):
+        # 设置进度
+        bar = tqdm()
+        bar.total = len(self.subdomains)
+        bar.desc = 'Progress'
+        bar.ncols = True
+        while True:
+            done = bar.total - self.subdomainq.qsize()
+            bar.n = done
+            bar.update()
+            if done == bar.total:  # 完成队列中所有子域的检查退出
+                break
+        bar.close()
+
+    def run(self):
+        start = time.time()
+        logger.log('INFOR', f'开始执行{self.source}模块')
+        self.format = utils.check_format(self.format)
+        self.dpath = utils.check_dpath(self.dpath)
+        self.subdomains = utils.get_domains(self.target)
+        if self.subdomains:
+            logger.log('INFOR', f'正在检查子域接管风险')
+            self.fingerprints = get_fingerprint()
+            self.results.headers = ['subdomain', 'cname']
+            # 创建待检查的子域队列
+            for domain in self.subdomains:
+                self.subdomainq.put(domain)
+            # 检查线程
+            for _ in range(self.thread):
+                check_thread = Thread(target=self.check, daemon=True)
+                check_thread.start()
+            # 进度线程
+            progress_thread = Thread(target=self.progress, daemon=True)
+            progress_thread.start()
+
+            self.subdomainq.join()
+            self.save()
+        else:
+            logger.log('FATAL', f'获取域名失败')
+        end = time.time()
+        elapsed = round(end - start, 1)
+        logger.log('INFOR', f'{self.source}模块耗时{elapsed}秒'
+                            f'发现{len(self.results)}个子域存在接管风险')
+        logger.log('DEBUG', f'结束执行{self.source}模块')
+
+
+if __name__ == '__main__':
+    fire.Fire(Takeover)
+    # takeover = Takeover('www.example.com')
+    # takeover = Takeover('./subdomains.txt')
+    # takeover.run()
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
shmilylty	786be39398	v0.0.7	2019-10-18 01:06:05 +08:00
shmilylty	c92794263b	更新依赖	2019-10-18 00:34:21 +08:00
shmilylty	953b7a4e9b	添加百度云观测接口查询	2019-10-18 00:33:31 +08:00
shmilylty	88b7089d1b	修复一些语法错误	2019-10-14 22:21:37 +08:00
shmilylty	d4745fe7cf	修正一些语法错误	2019-10-14 22:09:37 +08:00
shmilylty	94423d2d12	添加英文Readme	2019-10-14 21:37:54 +08:00
shmilylty	f5476ab381	修正	2019-10-14 21:37:14 +08:00
shmilylty	1cd52ce2d6	修改为批量存储	2019-10-13 17:11:18 +08:00
Jing Ling	071c115d70	删除无用输出	2019-09-27 01:09:57 +08:00
Jing Ling	20eea3518a	更新依赖	2019-09-26 23:11:05 +08:00
Jing Ling	0a049e991d	修复#23问题	2019-09-26 22:14:08 +08:00
shmilylty	6ef7c96a93	日志等级修正(FETAL->FATAL)	2019-09-23 22:14:44 +08:00
shmilylty	edcdaa2f05	处理获取data为None情况	2019-09-17 14:01:27 +08:00
shmilylty	f7c03656bf	处理resp.json()异常	2019-09-16 16:41:48 +08:00
shmilylty	a9ff58061e	添加8106端口	2019-09-10 16:29:32 +08:00
shmilylty	5a269fc601	certdb_api修改为spyse_api	2019-09-09 18:53:18 +08:00
shmilylty	57b581c85b	修复一些链接404错误	2019-09-08 13:11:05 +08:00
shmilylty	dcb1d99b26	添加结果中出现乱码问题的解决方法	2019-09-03 16:45:51 +08:00
shmilylty	372a2de406	忽略解码错误	2019-08-28 17:59:48 +08:00
shmilylty	c8fb3a2faf	优化标题获取	2019-08-28 17:16:14 +08:00
shmilylty	c9208b61dd	修改为单次存储	2019-08-28 17:15:30 +08:00
shmilylty	e74a738565	更新文档	2019-08-28 12:13:40 +08:00
shmilylty	5d6b7c1b7b	v0.0.6	2019-08-27 17:56:37 +08:00
shmilylty	bc9b5b0afe	重命名	2019-08-27 17:39:54 +08:00
shmilylty	a332d3bdc4	重写标题获取	2019-08-27 17:36:19 +08:00
shmilylty	c3f6f0b08d	优化标题获取	2019-08-26 18:27:10 +08:00
shmilylty	515bd34c7e	修复问题15 https://github.com/shmilylty/OneForAll/issues/15	2019-08-25 18:01:50 +08:00
shmilylty	32330fa8b2	修复问题16 https://github.com/shmilylty/OneForAll/issues/16	2019-08-25 17:51:16 +08:00
shmilylty	16d3530ac3	修复问题16 https://github.com/shmilylty/OneForAll/issues/16	2019-08-25 17:48:34 +08:00
Jing Ling	cc10ce78d3	Update issue templates	2019-08-23 22:34:29 +08:00
shmilylty	f35239de40	v0.0.6	2019-08-23 18:55:16 +08:00
shmilylty	daaf7891d6	更新文档	2019-08-23 18:51:22 +08:00
Jing Ling	46d447392c	Update fofa_api.py	2019-08-23 18:35:47 +08:00
shmilylty	b4432dfd47	utf-8	2019-08-23 17:10:17 +08:00
shmilylty	1b9b9cb11d	添加brotlipy依赖	2019-08-23 17:04:13 +08:00
shmilylty	f4b4f06333	一直请求bug修复	2019-08-23 15:48:56 +08:00
shmilylty	fca97c7d5c	修复需要登录问题	2019-08-23 14:54:10 +08:00
shmilylty	818af18355	去掉无用打印	2019-08-23 10:18:03 +08:00
shmilylty	69f94193f6	默认导出格式修改为csv	2019-08-23 10:17:33 +08:00
shmilylty	403a3fefcb	添加Github子域搜索模块	2019-08-22 19:04:31 +08:00
shmilylty	8ce39a9127	修复编码问题	2019-08-22 14:42:19 +08:00
shmilylty	f3a1b2bada	更新文档	2019-08-22 11:27:09 +08:00
shmilylty	41422f4826	修改为正确的doctest格式	2019-08-21 18:24:36 +08:00
Jing Ling	6189bc73ac	错字纠正	2019-08-21 16:52:01 +08:00
Jing Ling	45d9ae2f2d	修复检查线程无限等待问题	2019-08-21 16:50:17 +08:00
Jing Ling	c329f6c77e	添加PassiveDNS查询	2019-08-21 16:45:40 +08:00
Jing Ling	9fa77a16b6	添加passivedns_api配置	2019-08-21 16:44:23 +08:00
Jing Ling	2c581c31aa	删除无用判断	2019-08-21 16:42:57 +08:00
shmilylty	76fc12c365	添加绕过cloudFlare验证	2019-08-20 21:44:30 +08:00
shmilylty	3413869bf1	禁用SSL安全警告信息	2019-08-20 21:29:13 +08:00
shmilylty	84aea67aa0	修复链接	2019-08-20 12:07:19 +08:00
shmilylty	73d92a92e1	文档更新	2019-08-20 11:59:00 +08:00
shmilylty	9c8d7e937e	更新文档	2019-08-20 11:47:28 +08:00
shmilylty	b165670ed3	合并building_fail_solution.md	2019-08-20 11:41:17 +08:00
shmilylty	4e6dd23129	使用推荐使用socks5h	2019-08-19 21:26:09 +08:00
shmilylty	f0a3b44c53	v0.0.5	2019-08-19 20:05:01 +08:00
shmilylty	472da35a36	tqdm修改	2019-08-19 19:58:00 +08:00
shmilylty	b60d86fdc3	tqdm配置	2019-08-19 19:45:11 +08:00
shmilylty	9b99b9fd35	添加不能正常安装依赖问题的解决方法	2019-08-19 19:00:29 +08:00
shmilylty	dde3da84fd	更新依赖	2019-08-19 18:56:08 +08:00
shmilylty	147f1030c3	添加takeover使用帮助	2019-08-19 18:44:23 +08:00
shmilylty	43d4b90782	pep8	2019-08-19 18:21:47 +08:00
shmilylty	dee7ee2d2c	添加github自动接管	2019-08-19 18:11:36 +08:00
shmilylty	bf8df3bc02	添加进度条修复线程退出卡死bug	2019-08-19 18:10:44 +08:00
shmilylty	82973c295c	标题信息获取处理	2019-08-19 02:24:00 +08:00
shmilylty	ac654f9bab	添加默认子域接管检查目标列表	2019-08-18 19:35:26 +08:00
shmilylty	310c40d7aa	添加子域接管检查功能	2019-08-18 19:34:16 +08:00
shmilylty	b3b528966c	重构	2019-08-18 19:33:23 +08:00
shmilylty	e31ff755f4	重构响应检查	2019-08-18 19:32:30 +08:00
shmilylty	101742de3c	添加部分公用函数	2019-08-18 19:31:50 +08:00
shmilylty	25e8541964	添加进度条	2019-08-18 19:31:22 +08:00
shmilylty	1251efd752	使用回调	2019-08-18 19:30:54 +08:00
shmilylty	61075f40ac	添加超时	2019-08-18 19:29:06 +08:00
shmilylty	39b92b79d9	pep8	2019-08-18 19:27:33 +08:00
shmilylty	0e88591a3b	修改默认导出格式为xls	2019-08-18 19:27:09 +08:00
shmilylty	32b06362bd	修改默认请求超时为30秒	2019-08-18 19:26:30 +08:00
shmilylty	298ef2072c	修改默认目前域名	2019-08-18 19:11:51 +08:00
shmilylty	7390e941a7	重命名	2019-08-18 19:02:54 +08:00
shmilylty	1fad2a413e	pep8	2019-08-18 19:01:43 +08:00
shmilylty	de96822c0d	添加请求超时	2019-08-18 02:45:32 +08:00
shmilylty	4c6af1a925	pep8	2019-08-18 02:12:40 +08:00
shmilylty	7ccddb93f7	修复误把domain替换为subdomain的问题	2019-08-18 02:12:12 +08:00
shmilylty	3883a14365	pep8	2019-08-18 00:03:08 +08:00
shmilylty	837b2f4d51	pep8	2019-08-18 00:02:25 +08:00
Jing Ling	03126872bb	Merge pull request #13 from Echocipher/master subdomain takeover	2019-08-17 14:57:30 +08:00
shmilylty	8eaf1bd60b	修改parsed.names为正确值，解决[问题14](https://github.com/shmilylty/OneForAll/issues/14 )。	2019-08-16 23:03:49 +08:00
shmilylty	7f9fbac68b	添加threatcrowd模块	2019-08-16 19:49:39 +08:00
shmilylty	335f6b6c0c	添加常见问题与问答文档	2019-08-16 19:41:21 +08:00
shmilylty	f52768632a	pep8	2019-08-16 19:13:19 +08:00
shmilylty	8d46ce0862	添加绕过cloudFlare检查	2019-08-16 18:33:11 +08:00
shmilylty	11cf46a006	添加cloudscraper和Js2Py依赖	2019-08-16 16:48:51 +08:00
Echocipher	f1e2b739e1	Update takeover.py	2019-08-16 16:44:21 +08:00
Echocipher	3e4a067c6b	Update takeover.py	2019-08-16 16:43:48 +08:00
shmilylty	d2dc83ebb2	添加ThreatCrowd接口	2019-08-16 16:22:33 +08:00
echocipher	c040501ee4	subdomain takeover	2019-08-16 15:35:40 +08:00
echocipher	70b1ddfda2	subdomain takeover	2019-08-16 15:31:45 +08:00
shmilylty	cf2e91bfcc	添加其他支持格式说明	2019-08-15 01:57:11 +08:00
Jing Ling	daed4d5a7c	Set theme jekyll-theme-architect	2019-08-14 23:58:17 +08:00
shmilylty	e99497fdc7	美化排版	2019-08-14 23:29:04 +08:00
shmilylty	cb7df51209	添加码云厂库链接和说明	2019-08-14 23:12:12 +08:00
shmilylty	a3c8834f05	美化排版	2019-08-14 22:22:30 +08:00
shmilylty	3f4336cca9	修改默认导出格式为xls，添加txt,rst,jira格式输出。	2019-08-14 19:07:52 +08:00
shmilylty	da4e3bbd2a	修复[问题10](https://github.com/shmilylty/OneForAll/issues/10 )	2019-08-14 16:35:29 +08:00
shmilylty	20e582135d	移除freebuf.com	2019-08-14 16:15:49 +08:00
shmilylty	bfb069509e	添加subdomains为None情况判断	2019-08-14 16:13:27 +08:00
shmilylty	d9c1cb3937	解决批量获取子域问题	2019-08-14 16:07:30 +08:00
shmilylty	f3540ee07b	8443端口使用https协议	2019-08-14 13:44:32 +08:00
shmilylty	794fec26db	修改使用帮助	2019-08-14 03:36:53 +08:00
shmilylty	46282e37a4	修复使用帮助链接	2019-08-14 03:26:12 +08:00
shmilylty	87749f1452	更友好的使用帮助	2019-08-14 03:25:15 +08:00
shmilylty	4428790766	添加使用帮助文档	2019-08-14 03:24:44 +08:00
shmilylty	51c42f2958	pep8	2019-08-14 03:22:00 +08:00
shmilylty	aaf5863d9a	对齐	2019-08-14 02:39:34 +08:00
shmilylty	88e084e695	添加详细的安装依赖文档	2019-08-14 02:02:31 +08:00
shmilylty	96487aec2e	模块优化	2019-08-13 19:15:13 +08:00
shmilylty	1179f5f118	添加limit_open_conn检查	2019-08-13 19:14:47 +08:00
shmilylty	ef3a1f5245	pep8	2019-08-13 19:13:56 +08:00
shmilylty	f223d9cf1b	参数优化	2019-08-13 19:13:08 +08:00
shmilylty	78832b4790	响应解码问题处理	2019-08-13 17:35:28 +08:00
shmilylty	3c065f56ae	修复字典路径问题	2019-08-12 22:31:17 +08:00
shmilylty	a65840b013	修复self.verify值的获取	2019-08-12 22:30:54 +08:00
shmilylty	49e4278647	修改output参数为show参数	2019-08-12 18:41:58 +08:00
shmilylty	5a324971ed	添加验证子域有效性参数	2019-08-12 18:41:18 +08:00
shmilylty	f3e76f7ee5	添加跳出判断	2019-08-12 11:26:50 +08:00
shmilylty	e1d5515c44	v0.0.4	2019-08-11 21:34:29 +08:00
shmilylty	bf10c1a652	v0.0.4	2019-08-11 21:32:04 +08:00
shmilylty	33d7b6cc02	参数调优	2019-08-11 20:49:38 +08:00
shmilylty	903b18a058	生成请求地址中443端口的特殊处理	2019-08-11 20:39:58 +08:00
shmilylty	38117f06d1	参数调优	2019-08-11 20:20:42 +08:00
shmilylty	f9117ff795	添加resp.json()异常处理	2019-08-11 20:10:23 +08:00
shmilylty	d2219bdf8c	添加搜索退出判断	2019-08-11 20:08:06 +08:00
shmilylty	3c8627035d	设置域传送超时为30秒	2019-08-11 19:58:10 +08:00
shmilylty	4bb52f4183	参数调优	2019-08-11 15:53:49 +08:00
shmilylty	f426b8f2f6	重构	2019-08-11 03:48:36 +08:00
shmilylty	ebd81b83e9	pep8	2019-08-10 14:54:08 +08:00
shmilylty	d70ebe637e	升级tqdm版本	2019-08-10 12:02:32 +08:00
shmilylty	d10bc9c8d9	默认不使用代理	2019-08-10 03:30:52 +08:00
shmilylty	a7b189f311	调整执行finish函数位置	2019-08-10 00:48:51 +08:00
shmilylty	98642da4f3	拆分出执行域传送函数	2019-08-09 23:46:09 +08:00
shmilylty	f17aea1f44	删除无用日志输出	2019-08-09 23:20:21 +08:00
shmilylty	611580aa97	修正模块名字	2019-08-09 21:02:27 +08:00
shmilylty	b21e31a223	database.py修改为类	2019-08-09 18:47:01 +08:00
shmilylty	4c63bd672f	修正valid参数使用描述	2019-08-09 16:56:03 +08:00
shmilylty	c7f86fdefa	新增2个模块	2019-08-09 16:54:10 +08:00
shmilylty	c026dc550e	新增检查sitemap收集子域名模块	2019-08-09 16:43:09 +08:00
shmilylty	f2b08f6b02	新增检查robots.txt收集子域名模块	2019-08-09 16:42:03 +08:00
Jing Ling	f0f9f9b75e	删除无用模板	2019-08-09 15:40:04 +08:00
Jing Ling	a9347b8ef0	修改提交Bug提示	2019-08-09 15:32:46 +08:00
shmilylty	c2679a6ceb	暂时不探测https	2019-08-09 15:13:22 +08:00
shmilylty	9197aaf52b	pep8	2019-08-09 14:25:21 +08:00
shmilylty	272662fa82	修改pip3安装命令	2019-08-09 14:24:50 +08:00
shmilylty	623a21a11e	移除fake-useragent依赖	2019-08-09 11:39:21 +08:00
shmilylty	05040d3b8d	resolve_callback函数添加说明并做容错处理	2019-08-09 01:12:52 +08:00
shmilylty	e248826327	v0.0.3	2019-08-08 23:37:09 +08:00
shmilylty	07de3ecc2b	解决报WRONG_VERSION_NUMBER问题	2019-08-08 22:34:56 +08:00
shmilylty	518b27e386	Python 3.8 on Xenial Linux	2019-08-08 21:57:17 +08:00
shmilylty	8148c56fbc	不使用回调	2019-08-08 17:30:02 +08:00
shmilylty	a0269b1814	调整对齐	2019-08-08 16:14:42 +08:00
shmilylty	efaa806260	修改pipenv安装命令	2019-08-08 16:10:52 +08:00
shmilylty	d4e77aeac2	设置系统变量PYTHONIOENCODING为utf-8 去掉使用reconfigure函数设置	2019-08-08 13:59:20 +08:00
shmilylty	f1fdb32b56	添加终端输出为utf-8编码	2019-08-08 12:11:40 +08:00
shmilylty	f5ba141097	终端日志输出改为sys.stderr	2019-08-08 11:53:38 +08:00
shmilylty	3e004dd069	pip8	2019-08-08 01:23:34 +08:00
shmilylty	47cfaca9ee	pep8	2019-08-07 23:52:02 +08:00
Jing Ling	c6ddd83700	更新配置信息	2019-08-07 19:06:23 +08:00
Jing Ling	827c3f5f93	更新群链接	2019-08-07 18:38:59 +08:00
Jing Ling	de8c9a5c3b	Merge pull request #3 from Windforce17/master [README]fix wrong url format	2019-08-07 18:32:50 +08:00
shmilylty	28bfb80dc4	移除fake-useragent依赖	2019-08-07 18:28:36 +08:00
shmilylty	3e3b852830	移除Windows python支持（travisci官方不支持）	2019-08-07 16:37:57 +08:00
shmilylty	8f676e2ebe	添加Windows，linux和MacOSX系统测试	2019-08-07 16:04:00 +08:00
shmilylty	0fee65dd36	except BaseException	2019-08-07 15:50:13 +08:00
shmilylty	cfd60b366c	pip8	2019-08-07 15:49:27 +08:00
shmilylty	18851408c2	pip8格式化使用ssl=False	2019-08-07 15:17:40 +08:00
shmilylty	f74c45a67d	pep6	2019-08-07 15:16:55 +08:00
zhichen.Wang	a0163ed582	[README]fix wrong url format	2019-08-07 15:10:09 +08:00
shmilylty	85aa7d2ec4	添加字典	2019-08-07 01:44:08 +08:00
shmilylty	b628859d32	pep8	2019-08-07 01:32:49 +08:00
shmilylty	8ba3b926ce	pep8	2019-08-06 22:52:44 +08:00
shmilylty	a5cbcd08fa	修改错误模块名	2019-08-06 12:58:25 +08:00
shmilylty	7945d8024b	添加爆破模块和segment参数说明	2019-08-06 12:37:38 +08:00
shmilylty	d9c12ef9c3	deal_results函数里添加响应结果判断	2019-08-06 12:07:02 +08:00
shmilylty	23d4765e8e	Merge branch 'master' of https://github.com/shmilylty/OneForAll	2019-08-06 02:16:20 +08:00
shmilylty	f17a2636ae	修改为默认不使用代理，默认请求超时30秒。	2019-08-06 02:14:45 +08:00
shmilylty	84dd464a02	添加任务分组参数	2019-08-06 02:13:39 +08:00
Jing Ling	8429f0f015	添加Bug报告模板	2019-08-06 01:33:41 +08:00
shmilylty	62d2b1e4bd	修改默认使用字典文件	2019-08-06 01:09:29 +08:00
shmilylty	ca09ea5870	移除参数ssl=False	2019-08-06 00:43:30 +08:00
shmilylty	365162a474	修改程序入口为fire	2019-08-06 00:42:49 +08:00
shmilylty	60d8a2a88a	向TCPConnector函数添加ssl=False禁用SSL证书检查	2019-08-06 00:22:28 +08:00
shmilylty	cd22034415	修改为正确执行resp.json()的条件	2019-08-06 00:13:35 +08:00
shmilylty	e706c51b74	修改加群链接	2019-08-06 00:03:35 +08:00
shmilylty	8420258ff7	增加判断响应体不为空才执行resp.json()	2019-08-06 00:00:55 +08:00
shmilylty	31666fdd75	移除aiobrute向aiodns_query_a()函数传semaphore参数	2019-08-05 23:38:53 +08:00
shmilylty	e3f9d5e149	修复aiobrute中没有传semaphore参数问题	2019-08-05 22:28:17 +08:00
Jing Ling	a651c2148c	修改安装命令	2019-08-05 21:42:31 +08:00
shmilylty	e8781c7a29	添加响应判断	2019-08-05 19:11:40 +08:00
shmilylty	edd6b35388	添加更新说明	2019-08-05 18:33:07 +08:00
shmilylty	d8b3fe4bd8	new_table_name使用双引号包裹	2019-08-05 18:12:39 +08:00
shmilylty	5ccf57d595	表名用双引号包裹解决issues/2	2019-08-05 18:01:48 +08:00
shmilylty	11642edf6e	优化代码结构删除无用队列结果集参数	2019-08-05 18:00:01 +08:00
shmilylty	53213304e2	修改为api调用	2019-08-05 17:58:35 +08:00
shmilylty	bf20197631	添加api检查	2019-08-05 16:00:46 +08:00
shmilylty	8bad8b74d6	修改编译失败解决方法链接地址	2019-08-05 14:29:36 +08:00
shmilylty	a313c2f3e4	添加编译失败解决方法	2019-08-05 14:27:49 +08:00
shmilylty	e3312533e9	优化cdx和csp模块，这两个模块默认使用代理。	2019-08-05 12:20:58 +08:00
shmilylty	a899753db9	添加.gitignore文件	2019-08-05 11:39:05 +08:00
shmilylty	1855b4714f	解决请求响应为空resp.json()报错问题	2019-08-05 11:38:20 +08:00
Jing Ling	33e382af58	修复徽章排列问题	2019-08-04 21:29:59 +08:00
shmilylty	89e5943e6d	v0.0.2	2019-08-04 21:15:26 +08:00
shmilylty	205286bcd7	完善文档	2019-08-04 21:08:52 +08:00
shmilylty	95bfa6030a	修改自动集成化设置	2019-08-04 19:39:17 +08:00
shmilylty	84b64f8f99	添加有关集成化管理和代码覆盖率检查设置	2019-08-04 18:58:26 +08:00
shmilylty	45e955ae4e	上传新的演示svg	2019-08-04 05:55:34 +08:00
shmilylty	93f0fcd972	修复演示svg问题	2019-08-04 05:54:07 +08:00
shmilylty	ed0a1ef9cd	进一步完善文档	2019-08-04 05:15:18 +08:00
shmilylty	6a49ee22ab	修改Fire为程序入口	2019-08-04 03:36:33 +08:00
shmilylty	e76ee57c8a	添加运行输出banner	2019-08-04 03:20:59 +08:00
shmilylty	666942640e	修改日志输出格式	2019-08-04 03:20:25 +08:00
shmilylty	ecc6bd955f	修改参数说明	2019-08-04 00:57:16 +08:00
shmilylty	8eda8702ea	修改日志输出	2019-08-03 22:57:04 +08:00
shmilylty	2f1c1fed97	修改日志输出	2019-08-03 22:34:59 +08:00
shmilylty	3635f8f0a6	修改收集模块默认超时为请求超时秒	2019-08-03 22:05:47 +08:00
shmilylty	8bf9f2f0f6	修改Fire为程序入口	2019-08-03 21:28:08 +08:00
shmilylty	439fc9aa8e	修改日志输出格式，降低代码重复率。	2019-08-03 21:21:48 +08:00
shmilylty	6241dd16db	修复Typo问题	2019-08-03 18:38:13 +08:00
shmilylty	87de10478f	更改搜索地址	2019-08-03 18:20:08 +08:00
shmilylty	e5b9e30f00	统一为半角括号	2019-08-03 17:40:10 +08:00
shmilylty	98cea782b8	移除无用图片	2019-08-03 17:00:12 +08:00
shmilylty	142f1d13fb	移除使用通用字典选项	2019-08-03 16:51:11 +08:00
shmilylty	d9b6dd5be9	修改指定使用模块示例	2019-08-03 16:41:18 +08:00
shmilylty	e9553d1332	修改为标准的DocString格式	2019-08-03 16:39:57 +08:00
shmilylty	197712f1d1	完善收集模块说明文档	2019-08-03 15:17:57 +08:00
shmilylty	a76db09621	移除brotlipy依赖，更新fire到0.2.1解决报错。	2019-08-03 14:25:02 +08:00
Jing Ling	df808bcd53	Fixed Typo	2019-08-02 23:25:20 +08:00
Jing Ling	895cdb91da	添加A-Team链接地址	2019-08-02 22:32:52 +08:00
Jing Ling	125fed228a	Update README.md	2019-08-02 22:31:40 +08:00
shmilylty	ff4e413569	修改为中文格式	2019-08-02 19:07:54 +08:00
shmilylty	bfa8aecb0d	修改贡献者Black Star链接，增加了iceMatcha贡献者	2019-08-02 19:07:16 +08:00
shmilylty	0db3791f0f	入口修改为使用Fire	2019-08-02 19:05:35 +08:00
shmilylty	32c1e154de	降低tqdm版本	2019-08-02 19:04:36 +08:00
shmilylty	055e84e48d	添加说明文档	2019-08-02 19:04:04 +08:00
shmilylty	b693859d6b	移除无用的oneforall/requirements.txt	2019-08-02 18:58:31 +08:00
Jing Ling	87d30bd363	修复链接地址	2019-08-02 01:07:18 +08:00