v0.0.5

tqdm修改
tqdm配置
2026-08-26 12:57:50 +08:00 · 2019-08-19 20:05:01 +08:00 · 2019-08-19 19:58:00 +08:00 · 2019-08-19 19:45:11 +08:00 · 2019-08-19 19:00:29 +08:00 · 2019-08-19 18:56:08 +08:00
85 changed files with 26542 additions and 20332 deletions
@@ -0,0 +1,35 @@
+---
+name: 提交Bug
+about: "请务必按照模板提交Bug\U0001F64F"
+title: 请填写BUG标题
+labels: bug
+assignees: shmilylty
+
+---
+
+**是否使用了最新代码**
+是或否（如果不是的话尝试克隆最新的代码再跑一下）
+
+**Bug描述**
+清晰而简洁的Bug描述
+
+**如何复现**
+复现步骤（可不写）
+复现命令
+
+**预期结果**
+清晰而简洁的预期结果描述（可不写）
+
+**实际结果**
+清晰而简洁的实际结果描述（如出现什么错误）
+
+**屏幕截图**
+
+
+**运行环境**
+ - 系统：[例如Windows 10 x64]
+ - Python版本：[例如3.7.1]
+
+
+**报错文本**
+复制完整的报错文本
@@ -0,0 +1,238 @@
+
+# Created by https://www.gitignore.io/api/python,windows,pycharm
+# Edit at https://www.gitignore.io/?templates=python,windows,pycharm
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+.idea/sonarlint
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# End of https://www.gitignore.io/api/python,windows,pycharm
@@ -1,24 +1,39 @@
-sudo: false
+sudo: true

 notifications:
  email: false

-dist: xenial
-
 language: python

-python:
-  - "3.6"
-  - "3.7"
-  - "3.8-dev"
+matrix:
+  include:
+    - name: "Python 3.6 on Linux"
+      python: 3.6           # this works for Linux but is ignored on macOS or Windows
+    - name: "Python 3.7 on Xenial Linux"
+      python: 3.7           # this works for Linux but is ignored on macOS or Windows
+      dist: xenial          # required for Python >= 3.7
+    - name: "Python 3.8 on Xenial Linux"
+      python: 3.8-dev           # this works for Linux but is ignored on macOS or Windows
+      dist: xenial          # required for Python >= 3.7
+    - name: "Python 3.7 on macOS"
+      os: osx
+      osx_image: xcode10.2  # Python 3.7 running on macOS 10.14.3
+      language: shell       # 'language: python' is an error on Travis CI macOS
+    - name: "Python 3.7 on Windows"
+      os: windows           # Windows 10.0.17134 N/A Build 17134
+      language: shell
+      before_install:
+        - choco install python
+        - python -m pip install --upgrade pip
+      env: PATH=/c/Python37:/c/Python37/Scripts:$PATH

 install:
-  - pip install -U pip
-  - pip install codecov
-  - pip install -r requirements.txt
+  - pip3 install -U pip
+  - pip3 install codecov
+  - pip3 install -r requirements.txt

 script:
 - coverage run oneforall/example.py

 after_success:
- - codecov
+ - codecov
@@ -6,6 +6,22 @@ OneForAll的更新日志格式基于[Keep a Changelog](https://keepachangelog.co
 OneForAll遵守[语义化版本格式](https://semver.org/)。

 ## Unreleased
+## [0.0.5](https://github.com/shmilylty/oneforall/releases/tag/v0.0.5) - 2019-08-19
+- 修复一些已知Bugs
+- 优化各子域收集接口并添加新的子域收集接口
+- 添加子域DNS解析和子域HTTP探测进度条
+- 添加子域接管风险检查模块及其使用说明
+- 更新OneForAll依赖
+
+## [0.0.4](https://github.com/shmilylty/oneforall/releases/tag/v0.0.4) - 2019-08-11
+### 修复
+- 修复一些已知Bugs
+
+## [0.0.3](https://github.com/shmilylty/oneforall/releases/tag/v0.0.3) - 2019-08-08
+### 修改
+- 代码PEP8格式化
+### 修改
+- 修改一些已知Bugs

 ## [0.0.2](https://github.com/shmilylty/oneforall/releases/tag/v0.0.2) - 2019-08-04
 ### 新增
@@ -16,12 +16,14 @@ records = "*"
 tldextract = "*"
 exrex = "*"
 aiohttp = "*"
-fire = "==0.2.1"
-fake-useragent = "*"
+fire = "*"
 bs4 = "*"
 cchardet = "*"
 lxml = "*"
 pysocks = "*"
+cloudscraper = "*"
+js2py = "*"
+tablib = "*"

 [requires]
 python_version = "3.7"
@@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "cf68a50128345e403c4d2c25c59cf22490b83db80fc98da10dc41f515972e37e"
+            "sha256": "0e5757dad136b3c0900ba481fad3b90e1b346b8cf8974e94676890f603f2a452"
        },
        "pipfile-spec": 6,
        "requires": {
@@ -59,6 +59,13 @@
            "index": "pypi",
            "version": "==0.6.0"
        },
+        "asn1crypto": {
+            "hashes": [
+                "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87",
+                "sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49"
+            ],
+            "version": "==0.24.0"
+        },
        "async-timeout": {
            "hashes": [
                "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
@@ -88,6 +95,39 @@
            ],
            "version": "==4.8.0"
        },
+        "brotli": {
+            "hashes": [
+                "sha256:0538dc1744fd17c314d2adc409ea7d1b779783b89fd95bcfb0c2acc93a6ea5a7",
+                "sha256:0970a47f471782912d7705160b2b0a9306e68e6fadf9cffcaeb42d8f0951e26c",
+                "sha256:113f51658e6fe548dce4b3749f6ef6c24de4184ba9c10a909cbee4261c2a5da0",
+                "sha256:1e1aa9c4d1558889f42749c8baf846007953bfd32c8209230cf1cd1f5ef33495",
+                "sha256:2f2f4f78f29ac4a45d15b3d9fc3fd9705e0ad313a44b129f6e1d0c6916bad0e2",
+                "sha256:3269f6de1dd150fd0cce1c158b61ff5ac06d627fd3ae9c6ea03aed26fbbff7ea",
+                "sha256:50dd9ad2a2bb12da4e9002a438672d182f98e546e99952de80280a1e1729664f",
+                "sha256:5519a4b01b1a4f965083cbfa2ef2b9774c5a5f352341c47b50776ad109423d72",
+                "sha256:5eb27722d320370315971c427eb8aa7cc0791f2a458840d357ac653bd0ad3a14",
+                "sha256:5f06b4d5b6f58e5b5c220c2f23cad034dc5efa51b01fde2351ced1605bd980e2",
+                "sha256:72848d25a5f9e736db4af4512e0c3feecc094d57d241f8f1ae959115a2c39756",
+                "sha256:743001bca75f4a6b4454be3510feca46f9d61a0c782a9bc2bc684bdb245e279e",
+                "sha256:9d1c2dd27a1083fefd05b1b2f8df4a6bc2aaa6c21dd82cd41c8ae5e7c23a87f8",
+                "sha256:a13ce9b419fe9f277c63f700efb0e444331509d1881b5610d2ba7e9080606967",
+                "sha256:a19ef0952b9d2803df88dff07f45a6c92d5676afb9b8d69cf32232d684036d11",
+                "sha256:ad766ca8b8c1419b71a22756b45264f45725c86133dc80a7cbe30b6b78c75620",
+                "sha256:ad7963f261988ee0883816b6b9f206f11461c9b3cb5cfbca0c9ab5adc406d395",
+                "sha256:c16201060c5a3f8742e3deae759014251ac92f382f82bc2a41dc079ff18c3f24",
+                "sha256:c43b202f65891861a9a336984a103de25de235f756de69e32db893156f767013",
+                "sha256:c675c6cce4295cb1a692f3de7416aacace7314e064b94bc86e93aceefce7fd3e",
+                "sha256:d17cec0b992b1434f5f9df9986563605a4d1b1acd5574c87fc2ac014bcbd3316",
+                "sha256:dc91f6129953861a73d9a65c52a8dd682b561a9ebaf65283541645cab6489917",
+                "sha256:e2f4cbd1760d2bf2f30e396c2301999aab0191aec031a6a8a04950b2f575a536",
+                "sha256:f192e6d3556714105c10486bbd6d045e38a0c04d9da3cef21e0a8dfd8e162df4",
+                "sha256:f775b07026af2b1b0b5a8b05e41571cdcf3a315a67df265d60af301656a5425b",
+                "sha256:f969ec7f56ba9636679e69ca07fba548312ccaca37412ee823c7f413541ad7e0",
+                "sha256:f9dc52cd70907aafb99a773b66b156f2f995c7a0d284397c487c8b71ddbef2f9",
+                "sha256:fc7212e36ebeb81aebf7949c92897b622490d7c0e333a479c0395591e7994600"
+            ],
+            "version": "==1.0.7"
+        },
        "bs4": {
            "hashes": [
                "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"
@@ -171,6 +211,14 @@
            ],
            "version": "==3.0.4"
        },
+        "cloudscraper": {
+            "hashes": [
+                "sha256:7080a4c9ea6fe244422551172eb28a1d6c042da4c9d56d15ff319dbbac2ab6d7",
+                "sha256:79d04cd3c7d782035b404b5093a0cdbacd219cb883134017f4f7ba703f12a93c"
+            ],
+            "index": "pypi",
+            "version": "==1.1.40"
+        },
        "colorama": {
            "hashes": [
                "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d",
@@ -179,6 +227,27 @@
            "markers": "sys_platform == 'win32'",
            "version": "==0.4.1"
        },
+        "cryptography": {
+            "hashes": [
+                "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c",
+                "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643",
+                "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216",
+                "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799",
+                "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a",
+                "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9",
+                "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc",
+                "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8",
+                "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53",
+                "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1",
+                "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609",
+                "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292",
+                "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e",
+                "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6",
+                "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed",
+                "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d"
+            ],
+            "version": "==2.7"
+        },
        "defusedxml": {
            "hashes": [
                "sha256:6687150770438374ab581bb7a1b327a847dd9c5749e396102de3fad4e8a3ef93",
@@ -213,13 +282,6 @@
            "index": "pypi",
            "version": "==0.10.5"
        },
-        "fake-useragent": {
-            "hashes": [
-                "sha256:c104998b750eb097eefc28ae28e92d66397598d2cf41a31aa45d5559ef1adf35"
-            ],
-            "index": "pypi",
-            "version": "==0.1.11"
-        },
        "fire": {
            "hashes": [
                "sha256:6865fefc6981a713d2ce56a2a2c92c56c729269f74a6cddd6f4b94d16ae084c9"
@@ -241,6 +303,14 @@
            ],
            "version": "==1.4.1"
        },
+        "js2py": {
+            "hashes": [
+                "sha256:6e5628abfff2fb4051e8e77a353e44831f474e2ceb865278271897f7f326aeb6",
+                "sha256:bf87cb4432944470f11fed9c1cb8d0312dd505e7b867362f55102f24379ab94f"
+            ],
+            "index": "pypi",
+            "version": "==0.66"
+        },
        "loguru": {
            "hashes": [
                "sha256:b6fad0d7aed357b5c147edcc6982606b933754338950b72d8123f48c150c5a4f",
@@ -251,31 +321,31 @@
        },
        "lxml": {
            "hashes": [
-                "sha256:06e5599b9c54f797a3c0f384c67705a0d621031007aa2400a6c7d17300fdb995",
-                "sha256:092237cfe4ece074401b75001a2e525fa6e1fb9d40fee8b7b132b1947d3bd2f8",
-                "sha256:0b6d49d0a26fe8207df8dd27c40b75be4deb2277173903aa76ec3e82df77cbe7",
-                "sha256:0f77061c20b4f32b1cf39e8f661c74e966344084c996e7b23c3a94e472461df0",
-                "sha256:0fef86edfa2f146b4b0ae2c6c05c3e4a8f3388b3655eafbc4aab3247f4dabb24",
-                "sha256:2f163c8844db4ed06a230ef092e2461ad01830972a896b8f3cf8b5bac70ae85d",
-                "sha256:350333190052bbfbc3222b1805b59b7979d7276e57af2257367e15a2db27082d",
-                "sha256:3b57dc5ed7b6a7d852c961f2389ca99404c2b59fd2088baec6fbaca02f688be4",
-                "sha256:3e86e5df4a8edd6f725f3c76f1d45e046d4f3aa40478092e4f5f373ad1f526e2",
-                "sha256:43dac60d10341d3e56be089cd0798b70e70d45ce32279f4c3190d8cbd71350e4",
-                "sha256:4665ee84ac8ba11d58f1ed517e29ea8536b4ae4e0c6fb6c7d3dce70abcd279f0",
-                "sha256:5033cf606a7cb559db967689b1b2e743994000f783607ba4c484e90917395ad7",
-                "sha256:75d731af05bf40f808d7716e0d26b4b02913402f861c032ce8c36efca350ae72",
-                "sha256:7720174604c7647e357566ac9e4d135c137caed5e7b01223551a4c81c8dc8b9a",
-                "sha256:b33ec641309bcea40c76c1b105f988e4e8f9a2f1ee1486aa5c0eeef33956c9bb",
-                "sha256:d1135dc0ac197242028ede085b693ba1f2bff7f0f9b91080e2540348312bfa53",
-                "sha256:d5a61e9c2322b45f259909a02b76bc98c4641214e22a37191d00c151aa9cdb9a",
-                "sha256:da22c4b17bc17dad9c8faf6d94c8fe568ac71c867a56631ab874da418fc7f8f7",
-                "sha256:da5c48ec9f8d8b5df42d328b6d1fb8d9413cd664a2367ef4f6f7cc48ee5b82c0",
-                "sha256:db2794bad21b7b30b6849b4e1537171cae8a7087711d958d69c233470dc612e7",
-                "sha256:f1c2f67df727034f94ccb590142d1d110f3dd38f638a4f1567fdd9f39892ba05",
-                "sha256:f840dddded8b046edc774c88ed8d2442cdb231a68894c42c74e3a809450fae76"
+                "sha256:02ca7bf899da57084041bb0f6095333e4d239948ad3169443f454add9f4e9cb4",
+                "sha256:096b82c5e0ea27ce9138bcbb205313343ee66a6e132f25c5ed67e2c8d960a1bc",
+                "sha256:0a920ff98cf1aac310470c644bc23b326402d3ef667ddafecb024e1713d485f1",
+                "sha256:17cae1730a782858a6e2758fd20dd0ef7567916c47757b694a06ffafdec20046",
+                "sha256:17e3950add54c882e032527795c625929613adbd2ce5162b94667334458b5a36",
+                "sha256:1f4f214337f6ee5825bf90a65d04d70aab05526c08191ab888cb5149501923c5",
+                "sha256:2e8f77db25b0a96af679e64ff9bf9dddb27d379c9900c3272f3041c4d1327c9d",
+                "sha256:4dffd405390a45ecb95ab5ab1c1b847553c18b0ef8ed01e10c1c8b1a76452916",
+                "sha256:6b899931a5648862c7b88c795eddff7588fb585e81cecce20f8d9da16eff96e0",
+                "sha256:726c17f3e0d7a7200718c9a890ccfeab391c9133e363a577a44717c85c71db27",
+                "sha256:760c12276fee05c36f95f8040180abc7fbebb9e5011447a97cdc289b5d6ab6fc",
+                "sha256:796685d3969815a633827c818863ee199440696b0961e200b011d79b9394bbe7",
+                "sha256:891fe897b49abb7db470c55664b198b1095e4943b9f82b7dcab317a19116cd38",
+                "sha256:a471628e20f03dcdfde00770eeaf9c77811f0c331c8805219ca7b87ac17576c5",
+                "sha256:a63b4fd3e2cabdcc9d918ed280bdde3e8e9641e04f3c59a2a3109644a07b9832",
+                "sha256:b0b84408d4eabc6de9dd1e1e0bc63e7731e890c0b378a62443e5741cfd0ae90a",
+                "sha256:be78485e5d5f3684e875dab60f40cddace2f5b2a8f7fede412358ab3214c3a6f",
+                "sha256:c27eaed872185f047bb7f7da2d21a7d8913457678c9a100a50db6da890bc28b9",
+                "sha256:c81cb40bff373ab7a7446d6bbca0190bccc5be3448b47b51d729e37799bb5692",
+                "sha256:d11874b3c33ee441059464711cd365b89fa1a9cf19ae75b0c189b01fbf735b84",
+                "sha256:e9c028b5897901361d81a4718d1db217b716424a0283afe9d6735fe0caf70f79",
+                "sha256:fe489d486cd00b739be826e8c1be188ddb74c7a1ca784d93d06fda882a6a1681"
            ],
            "index": "pypi",
-            "version": "==4.4.0"
+            "version": "==4.4.1"
        },
        "multidict": {
            "hashes": [
@@ -347,6 +417,20 @@
            ],
            "version": "==2.19"
        },
+        "pyjsparser": {
+            "hashes": [
+                "sha256:2b12842df98d83f65934e0772fa4a5d8b123b3bc79f1af1789172ac70265dd21",
+                "sha256:be60da6b778cc5a5296a69d8e7d614f1f870faf94e1b1b6ac591f2ad5d729579"
+            ],
+            "version": "==2.7.1"
+        },
+        "pyopenssl": {
+            "hashes": [
+                "sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200",
+                "sha256:c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6"
+            ],
+            "version": "==19.0.0"
+        },
        "pysocks": {
            "hashes": [
                "sha256:15d38914b60dbcb231d276f64882a20435c049450160e953ca7d313d1405f16f",
@@ -356,6 +440,13 @@
            "index": "pypi",
            "version": "==1.7.0"
        },
+        "pytz": {
+            "hashes": [
+                "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32",
+                "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7"
+            ],
+            "version": "==2019.2"
+        },
        "pyyaml": {
            "hashes": [
                "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9",
@@ -397,6 +488,13 @@
            ],
            "version": "==1.4.3"
        },
+        "requests-toolbelt": {
+            "hashes": [
+                "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f",
+                "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"
+            ],
+            "version": "==0.9.1"
+        },
        "six": {
            "hashes": [
                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
@@ -406,23 +504,24 @@
        },
        "soupsieve": {
            "hashes": [
-                "sha256:72b5f1aea9101cf720a36bb2327ede866fd6f1a07b1e87c92a1cc18113cbc946",
-                "sha256:e4e9c053d59795e440163733a7fec6c5972210e1790c507e4c7b051d6c5259de"
+                "sha256:8662843366b8d8779dec4e2f921bebec9afd856a5ff2e82cd419acc5054a1a92",
+                "sha256:a5a6166b4767725fd52ae55fee8c8b6137d9a51e9f1edea461a062a759160118"
            ],
-            "version": "==1.9.2"
+            "version": "==1.9.3"
        },
        "sqlalchemy": {
            "hashes": [
-                "sha256:217e7fc52199a05851eee9b6a0883190743c4fb9c8ac4313ccfceaffd852b0ff"
+                "sha256:0459bf0ea6478f3e904de074d65769a11d74cdc34438ab3159250c96d089aef0"
            ],
            "markers": "python_version >= '3.0'",
-            "version": "==1.3.6"
+            "version": "==1.3.7"
        },
        "tablib": {
            "hashes": [
                "sha256:0f88a9cebdaa1a2cc29ae57387082ee81015d1149ecd34e48a8c8d3b4dd21670",
                "sha256:5f33c079b07eb10cf9c4b4696add2ecf32c89db7729240546ecdcd5c92f67e13"
            ],
+            "index": "pypi",
            "version": "==0.13.0"
        },
        "termcolor": {
@@ -441,11 +540,18 @@
        },
        "tqdm": {
            "hashes": [
-                "sha256:14a285392c32b6f8222ecfbcd217838f88e11630affe9006cd0e94c7eff3cb61",
-                "sha256:25d4c0ea02a305a688e7e9c2cdc8f862f989ef2a4701ab28ee963295f5b109ab"
+                "sha256:438d6a735167099d75e5fd9a55175c6727c4dbba345ae406b2886c2728fe3e80",
+                "sha256:ebc205051d79b49989140f5f6c73ec23fce5f590cbc4d9cd6e4c47f168fa0f10"
            ],
            "index": "pypi",
-            "version": "==4.32.2"
+            "version": "==4.34.0"
+        },
+        "tzlocal": {
+            "hashes": [
+                "sha256:11c9f16e0a633b4b60e1eede97d8a46340d042e67b670b290ca526576e039048",
+                "sha256:949b9dd5ba4be17190a80c0268167d7e6c92c62b30026cf9764caf3e308e5590"
+            ],
+            "version": "==2.0.0"
        },
        "urllib3": {
            "hashes": [
@@ -5,7 +5,7 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](./LICENSE)
 [![python](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue)](./)
-[![python](https://img.shields.io/badge/release-v0.0.2-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/release-v0.0.5-brightgreen)](https://github.com/shmilylty/OneForAll/releases)

 👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](./README.en.md)

@@ -24,24 +24,24 @@

 为了解决以上痛点，此项目应用而生，OneForAll一词是来自我喜欢的一部日漫《[我的英雄学院](https://manhua.fzdm.com/131/)》，它是一种通过一代代的传承不断变强的潜力无穷的顶级个性，目前[番剧](https://www.bilibili.com/bangumi/media/md7452/)也更新到了第三季了，欢迎大佬们入坑😄。正如其名，我希望OneForAll是一款集百家之长，功能强大的全面快速子域收集终极神器🔨。

-目前OneForAll还在开发中，肯定有不少问题和需要改进的地方，欢迎大佬们提交[Issues](https://github.com/shmilylty/OneForAll/issues)和[PR](https://github.com/shmilylty/OneForAll/pulls)，用着还行给个小星星✨吧，目前有一个专门用于OneForAll交流和反馈QQ群👨‍👨‍👦‍👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=3fb9de888e3dbac91abb5731fabf4cdac6a7c0de3db665ca8e79c2cd239a102d)，也可以给我发邮件📧[admin@hackfun.org]。
+目前OneForAll还在开发中，肯定有不少问题和需要改进的地方，欢迎大佬们提交[Issues](https://github.com/shmilylty/OneForAll/issues)和[PR](https://github.com/shmilylty/OneForAll/pulls)，用着还行给个小星星✨吧，目前有一个专门用于OneForAll交流和反馈QQ群👨‍👨‍👦‍👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)，也可以给我发邮件📧[admin@hackfun.org]。

 ## 👍功能特性

-* **收集能力强大**，详细模块请阅读[搜索模块说明](./docs/collection_modules.md)。
+* **收集能力强大**，详细模块请阅读[收集模块说明](./docs/collection_modules.md)。
  1. 利用证书透明度收集子域（目前有6个模块：`censys_api`，`certdb_api`，`certspotter`，`crtsh`，`entrust`，`google`）

-  2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，后续会添加检查NSEC记录，NSEC记录等模块）
+  2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，检查robots文件`robots`，检查sitemap文件`sitemap`，后续会添加检查NSEC记录，NSEC3记录等模块）

  3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善）

-  4. 利用DNS数据集收集子域（目前有16个模块：`binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `ptrarchive`, `sitedossier`）
+  4. 利用DNS数据集收集子域（目前有17个模块：`binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `ptrarchive`, `sitedossier`,`threatcrowd`）

  5. 利用DNS查询收集子域（目前有1个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，该模块还有待添加和完善）

-  6. 利用威胁情报平台数据收集子域（目前有5个模块：`riskiq`，`threatbook`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）
+  6. 利用威胁情报平台数据收集子域（目前有5个模块：`riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）

-  7. 利用搜索引擎发现子域（目前有15个模块：`ask`, `bing_api`, `fofa`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
+  7. 利用搜索引擎发现子域（目前有15个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。

 * **处理功能强大**，发现的子域结果支持自动去除，自动DNS解析，HTTP请求探测，自动移除无效子域，拓展子域的Banner信息，最终支持的导出格式有`csv`, `tsv`, `json`, `yaml`, `html`, `xls`, `xlsx`, `dbf`, `latex`, `ods`。

@@ -49,61 +49,86 @@

 ## 🚀上手指南

-由于该项目**处于开发中**，会不断进行更新迭代，下载使用最好**克隆**最新项目。
+📢由于该项目**处于开发中**，会不断进行更新迭代，下载使用最好**克隆**最新项目，请务必花一点时间阅读此文档，有助于你快速熟悉OneForAll！

 **🐍安装要求**

-1. Python 3.6-3.7
+OneForAll是基于CPython开发的，所以你需要Python环境才能运行，如果你的系统还没有Pythin环境你可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)，理论上Python 3.6，3.7和3.8都可以正常运行OneForAll，**但是**许多测试都是在Python 3.7上进行的，所以**推荐**你使用**Python 3.7**版本运行OneForAll。运行以下命令检查Python和pip3版本：
+```bash
+python -V
+pip3 -V
+```
+如果你看到以下类似输出便说明Python环境没有问题：
+```bash
+Python 3.7.4
+pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.7)
+```

 **✔安装步骤**

-1.  下载
+1. **下载**
+   本项目已经在[码云](https://gitee.com/shmilylty/OneForAll.git)(Gitee)镜像了一份，国内推荐使用码云进行克隆比较快：

   ```bash
-git clone https://github.com/shmilylty/OneForAll.git
+   git clone https://gitee.com/shmilylty/OneForAll.git
+   ```
+   或者：
+   ```bash
+   git clone https://github.com/shmilylty/OneForAll.git
   ```

-   或者到[Releases](https://github.com/shmilylty/OneForAll/releases)手动下载。
+2. **安装**
+   首先运行以下命令
+   你可以通过pip3安装OneForAll的依赖（如果你熟悉[pipenv](https://docs.pipenv.org/en/latest/)，那么推荐你使用[pipenv安装依赖]((./docs/Installation_dependency.md))），以下为**Windows系统**下使用**pip3**安装依赖的示例：（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python37`，那么请以管理员身份运行命令提示符cmd执行以下命令！）

-2. 安装依赖
+   ```bash
+   cd OneForAll/
+   python -m pip install --user  -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+   pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+   cd oneforall/
+   python oneforall.py --help
+   ```
+其他系统平台的请参考[依赖安装](./docs/installation_dependency.md)，如果在安装依赖过程中发现编译某个依赖库失败时可以参考[编译失败解决方法](./docs/building_fail_solution.md)，如果还没有解决欢迎加群反馈。

-   * 使用pipenv
+3. **更新**
+   ❗注意：如果你之前已经克隆了项目运行之前请**备份**自己修改过的文件到项目外的地方（如**config.py**），然后执行以下命令**更新**项目：

-     ```bash
-     pip3 install pipenv
-     cd OneForAll/oneforall
-     pipenv install python 3.7.4
-     pipenv run python oneforall.py --help
-     ```
-
-   * 使用pip3
-
-     ```bash
-     cd OneForAll/oneforall
-     pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
-     python3 oneforall.py --help
-     ```
+   ```bash
+   git fetch --all
+   git reset --hard origin/master
+   git pull
+   ```

 **✨使用演示**

-```bash
-python3 oneforall.py --target example.com run
-```
+1. 如果你是通过pip3安装的依赖则使用以下命令运行示例：   
+    ```bash
+    cd oneforall/
+    python3 oneforall.py --target example.com run
+    ```

-![Example](./docs/usage_example.svg)
+    ![Example](./docs/usage_example.svg)
+
+2. 如果你通过pipenv安装的依赖则使用以下命令运行示例：
+   ```bash
+   cd oneforall/
+   pipenv run python oneforall.py --target example.com run
+   ```

 **🤔使用帮助**

-命令行参数只提供了一些常用参数，更多详细的参数配置请见[config.py](./oneforall/config.py)，如果你认为有些参数是命令界面经常使用到的或缺少了什么参数等问题非常欢迎反馈。由于众所周知的原因，如果要使用一些被墙的收集接口请先到[config.py](./oneforall/config.py)配置代理，有些收集模块需要提供API（大多都是可以注册账号免费获取），如果需要使用请到[config.py](./oneforall/config.py)配置API信息，如果不使用请忽略有关报错提示。（详细模块请阅读[搜索模块说明](./docs/collection_modules.md)）
+命令行参数只提供了一些常用参数，更多详细的参数配置请见[config.py](./oneforall/config.py)，如果你认为有些参数是命令界面经常使用到的或缺少了什么参数等问题非常欢迎反馈。由于众所周知的原因，如果要使用一些被墙的收集接口请先到[config.py](./oneforall/config.py)配置代理，有些收集模块需要提供API（大多都是可以注册账号免费获取），如果需要使用请到[config.py](./oneforall/config.py)配置API信息，如果不使用请忽略有关报错提示。（详细模块请阅读[收集模块说明](./docs/collection_modules.md)）

 OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实现，有关Fire更高级使用方法请参阅[使用Fire CLI](https://github.com/google/python-fire/blob/master/docs/using-cli.md)，有任何使用疑惑欢迎加群交流。

 oneforall.py是主程序入口，oneforall.py里有调用aiobrute.py和dbexport.py，为了方便进行子域爆破和数据库导出独立出了aiobrute.py和dbexport.py，这两个文件可以单独运行，并且所接受参数要更丰富一点。

-1. oneforall.py使用帮助
+❗注意：当你在使用过程中遇到一些问题或者疑惑时，请先到[Issues](https://github.com/shmilylty/OneForAll/issues)里使用搜索找找答案，还可以参阅[常见问题与回答](./docs/Q&A.md)。
+
+1. **oneforall.py使用帮助**

   ```bash
-   pipenv run python oneforall.py --help
+   python oneforall.py --help
   ```
   ```bash
   NAME
@@ -113,43 +138,58 @@ oneforall.py是主程序入口，oneforall.py里有调用aiobrute.py和dbexport.
       oneforall.py --target=TARGET <flags>
   
   DESCRIPTION
-       Version: 0.0.2
-       Project: https://github.com/shmilylty/OneForAll/
+       Version: 0.0.5
+       Project: https://git.io/fjHT1
   
       Example:
-           python oneforall.py --target example.com run
-           python oneforall.py --target example.com --brute True --port medium valid 1 run
-           python oneforall.py --target ./domains.txt --format csv --path= ./result.csv  --output True run
+           python3 oneforall.py --target example.com run
+           python3 oneforall.py --target ./domains.txt run
+           python3 oneforall.py --target example.com --brute True run
+           python3 oneforall.py --target example.com --verify False run
+           python3 oneforall.py --target example.com --valid None run
+           python3 oneforall.py --target example.com --port medium run
+           python3 oneforall.py --target example.com --format csv run
+           python3 oneforall.py --target example.com --show True run
   
       Note:
-           参数valid可选值有1，0，None，分别表示导出有效，无效，全部子域
+           参数valid可选值1，0，None分别表示导出有效，无效，全部子域
+           参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
           参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
-           参数format可选格式有'csv','tsv','json','yaml','html','xls','xlsx','dbf','latex','ods'
+           参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                             'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
   
   ARGUMENTS
       TARGET
-           单个域名或者每行一个域名的文件路径
+           单个域名或者每行一个域名的文件路径(必需参数)
   
   FLAGS
       --brute=BRUTE
-           是否使用爆破模块（默认禁用）
+           使用爆破模块(默认False)
+       --verify=VERIFY
+           验证子域有效性(默认True)
       --port=PORT
-           HTTP请求验证的端口范围（默认medium）
+           请求验证的端口范围(默认medium)
       --valid=VALID
-           导出子域的有效性（默认1）
+           导出子域的有效性(默认1)
       --path=PATH
           导出路径(默认None)
       --format=FORMAT
-           导出格式（默认xlsx）
-       --output=OUTPUT
-           是否将导出数据输出到终端（默认False）
+           导出格式(默认xlsx)
+       --show=SHOW
+           终端显示导出数据(默认False)
   ```

-2. aiobrute.py使用帮助
+2. **aiobrute.py使用帮助**
+
+   关于泛解析问题处理程序首先会访问一个随机的子域判断是否泛解析，如果使用了泛解析则是通过以下判断处理：
+   - 一是主要是与泛解析的IP集合和TTL值做对比，可以参考[这篇文章](http://sh3ll.me/archives/201704041222.txt)。
+   - 二是多次解析到同一IP集合次数（默认设置为10，可以在config.py设置大小）
+   - 考虑爆破效率问题目前还没有加上HTTP响应体相似度对比和响应体内容判断
+   经过测试在16核心的CPU，使用16进程64协程，100M带宽的环境下，设置任务分割为50000，跑两百万字典大概10分钟左右跑完，大概3333个子域每秒。

   ```bash
-   pipenv run python aiobrute.py --help
+   python aiobrute.py --help
   ```

   ```bash
@@ -161,23 +201,35 @@ oneforall.py是主程序入口，oneforall.py里有调用aiobrute.py和dbexport.
   
   DESCRIPTION
       Example：
-           python aiobrute.py --target example.com run
-           python aiobrute.py --target ./domains.txt run
-           python aiobrute.py --target example.com --processes 4 --coroutine 64 --wordlist data/subdomains.txt run
-           python aiobrute.py --target example.com --recursive True --depth 2 --namelist data/next_subdomains.txt run
-           python aiobrute.py --target www.{fuzz}.example.com --fuzz True --rule [a-z][0-9] run
+           python3 aiobrute.py --target example.com run
+           python3 aiobrute.py --target ./domains.txt run
+           python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
+           python3 aiobrute.py --target example.com --wordlist subdomains.txt run
+           python3 aiobrute.py --target example.com --recursive True --depth 2 run
+           python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
+   
+       Note:
+           参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为任务组，
+           当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+           十万字典建议设置为5000，百万字典设置为50000
+           参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+           参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                             'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
   
   ARGUMENTS
       TARGET
           单个域名或者每行一个域名的文件路径
   
   FLAGS
-       --processes=PROCESSES
+       --process=PROCESS
           爆破的进程数(默认CPU核心数)
       --coroutine=COROUTINE
-           每个爆破进程下的协程数(默认16)
+           每个爆破进程下的协程数(默认64)
       --wordlist=WORDLIST
           指定爆破所使用的字典路径(默认使用config.py配置)
+       --segment=SEGMENT
+           爆破任务分割(默认500)
       --recursive=RECURSIVE
           是否使用递归爆破(默认False)
       --depth=DEPTH
@@ -185,54 +237,23 @@ oneforall.py是主程序入口，oneforall.py里有调用aiobrute.py和dbexport.
       --namelist=NAMELIST
           指定递归爆破所使用的字典路径(默认使用config.py配置)
       --fuzz=FUZZ
-           是否使用fuzz模式进行爆破(默认False，开启必须指定fuzz正则规则)
+           是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
       --rule=RULE
           fuzz模式使用的正则规则(默认使用config.py配置)
-   ```
-
-3. dbexport.py使用帮助
-
-   ```bash
-   pipenv run python dbexport.py --help
-   ```
-
-   ```bash
-   NAME
-       dbexport.py - OneForAll数据库导出模块
-   
-   SYNOPSIS
-       dbexport.py TABLE <flags>
-   
-   DESCRIPTION
-       Example:
-           python dbexport.py --db result.db --table name --format csv --output False
-           python dbexport.py --db result.db --table name --format csv --path= ./result.csv
-   
-       Note:
-           参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
-           参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
-           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
-   
-   POSITIONAL ARGUMENTS
-       TABLE
-           要导出的表
-   
-   FLAGS
-       --db=DB
-           要导出的数据库路径(默认为results/result.sqlite3)
+       --export=EXPORT
+           是否导出爆破结果(默认True)
       --valid=VALID
           导出子域的有效性(默认None)
-       --path=PATH
-           导出路径(默认None)
       --format=FORMAT
           导出格式(默认xlsx)
-       --output=OUTPUT
-           是否将导出数据输出到终端(默认False)
-   
-   NOTES
-       You can also use flags syntax for POSITIONAL ARGUMENTS
+       --path=PATH
+           导出路径(默认None)
+       --show=SHOW
+           终端显示导出数据(默认False)
   ```

+3. 其他模块使用请参考[使用帮助](./docs/using_help.md)
+
 ## 👏主要框架

 * [aiodns](https://github.com/saghul/aiodns) - 简单DNS异步解析库。
@@ -0,0 +1 @@
+theme: jekyll-theme-architect
@@ -0,0 +1,64 @@
+# 安装依赖
+
+你可以通过pip3和pipenv两种方法安装OneForAll的依赖（如果你熟悉[pipenv](https://docs.pipenv.org/en/latest/)，那么推荐使用你使用pipenv）：
+
+* **Windows系统**（注意：如果你的Python3安装在系统Program Files目录下，如：`C:\Program Files\Python37`，那么请以管理员身份运行命令提示符cmd执行以下命令！）
+
+    1. 使用pipenv
+
+    ```bash
+    cd OneForAll/
+    python -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
+    pipenv install --user --python 3.7
+    cd oneforall
+    pipenv run python oneforall.py --help
+    ```
+
+    2. 使用pip3
+
+    ```bash
+    cd OneForAll/
+    python -m pip install --user  -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+    cd oneforall/
+    python oneforall.py --help
+    ```
+* **Linux系统**
+
+    1. 使用pipenv
+    ```bash
+    cd OneForAll/
+    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    sudo pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
+    sudo pipenv install --user --python 3.7
+    cd oneforall
+    pipenv run python3 oneforall.py --help
+    ```
+    2. 使用pip3
+    ```bash
+    cd OneForAll/
+    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+    cd oneforall/
+    python3 oneforall.py --help
+    ```
+* **Darwin系统**
+
+    1. 使用pipenv
+    ```bash
+    cd OneForAll/
+    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user pipenv -i https://mirrors.aliyun.com/pypi/simple/
+    pipenv install --user --python 3.7
+    cd oneforall
+    pipenv run python3 oneforall.py --help
+    ```
+    2. 使用pip3
+    ```bash
+    cd OneForAll/
+    python3 -m pip install --user -U pip setuptools wheel -i https://mirrors.aliyun.com/pypi/simple/
+    pip3 install --user -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+    cd oneforall/
+    python3 oneforall.py --help
+    ```
@@ -0,0 +1,12 @@
+# 常见问题与回答
+
+## 使用问题
+
+1. 为什么运行OneForAll之后最终结果为空？
+
+   有几种可能性：第一可能目标域名没有子域。第二由于OneForAll默认会自动验证子域，在导出是只会有效子域，所以存在导出时没有有效子域的情况，你可以在运行OneForAll使用--valid=None指定导出所有发现的子域，你也可以使用--verify=False指定不验证子域的有效性。
+
+2. 安装依赖时出现以下类似报错
+   Cannot uninstall 'PyYAML'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall.
+   
+   安装依赖时尝试加上--ignore-installed参数
@@ -0,0 +1,13 @@
+如果在安装依赖过程遇到编译某个依赖库失败时可以尝试以下方法：
+
+1. 到提供编译好的whl文件的第三方平台，找到对应库手动下载安装。第三方平台平台有： 
+ * [https://www.lfd.uci.edu/~gohlke/pythonlibs](https://www.lfd.uci.edu/~gohlke/pythonlibs)
+ * [https://pythonwheels.com/](https://pythonwheels.com/)
+
+选择好对应版本执行以下命令手动安装。举个例子，当编译pycares时失败时，找到[https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares](https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares)，由于我的系统是Windows 10 64位，使用的Python 3.7便下载`pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl`（一般来说下载最新版本的），然后手动安装：
+
+```bash
+pip3 install pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl
+```
+
+2. 到库的项目地址issues和wiki等找找有没有解决方法，如果没有就给他们提issues发邮件😜。
@@ -31,14 +31,16 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
      | google      | 是           | 否          |                                                    |


-  2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，后续会添加检查NSEC记录，NSEC记录等模块）
+  2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，后续会添加检查NSEC记录，NSEC3记录等模块）

-      | 模块名称 | 是否需要代理 | 是否需要API | 其他说明         |
-      | -------- | ------------ | ----------- | ---------------- |
-      | axfr     | 否           | 否          | 域传送漏洞利用   |
-      | cdx      | 由域名决定   | 否          | 检查跨域策略文件 |
-      | cert     | 否           | 否          | 检查HTTPS证书    |
-      | csp      | 由域名决定   | 否          | 检查内容安全策略 |
+      | 模块名称 | 是否需要代理           | 是否需要API | 其他说明           |
+      | -------- | ---------------------- | ----------- | ------------------ |
+      | axfr     | 否                     | 否          | 域传送漏洞利用     |
+      | cdx      | 手动设置（默认不使用） | 否          | 检查跨域策略文件   |
+      | cert     | 否                     | 否          | 检查HTTPS证书      |
+      | csp      | 手动设置（默认不使用） | 否          | 检查内容安全策略   |
+      | robots   | 手动设置（默认不使用） | 否          | 检查robots.txt文件 |
+      | sitemap  | 手动设置（默认不使用） | 否          | 检查sitemap文件    |
  3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善）

      | 模块名称     | 是否需要代理 | 是否需要API | 其他说明 |
@@ -46,57 +48,59 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
      | archivecrawl | 否           | 否          |          |
      | commoncrawl  | 否           | 否          |          |

-  4. 利用DNS数据集收集子域（目前有16个模块：`binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `ptrarchive`, `sitedossier`）
+  4. 利用DNS数据集收集子域（目前有17个模块：`binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `ptrarchive`, `sitedossier`,`threatcrowd`）

-      | 模块名称           | 是否需要代理 | 是否需要API | 其他说明                                            |
-      | ------------------ | ------------ | ----------- | --------------------------------------------------- |
-      | binaryedge_api     | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py)  |
-      | circl_api          | 否           | 是          | API使用和申请见[config.py](.../oneforall/config.py) |
-      | hackertarget       | 否           | 否          |                                                     |
-      | riddler            | 否           | 否          |                                                     |
-      | bufferover         | 否           | 否          |                                                     |
-      | dnsdb              | 否           | 否          |                                                     |
-      | ipv4info           | 否           | 否          |                                                     |
-      | robtex             | 否           | 否          |                                                     |
-      | chinaz             | 否           | 否          |                                                     |
-      | dnsdb_api          | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py)  |
-      | netcraft           | 否           | 否          |                                                     |
-      | securitytrails_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py)  |
-      | chinaz_api         | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py)  |
-      | dnsdumpster        | 否           | 否          |                                                     |
-      | sitedossier        | 否           | 否          |                                                     |
+      | 模块名称           | 是否需要代理 | 是否需要API | 其他说明                                           |
+      | ------------------ | ------------ | ----------- | -------------------------------------------------- |
+      | binaryedge_api     | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | bufferover         | 否           | 否          |                                                    |
+      | chinaz             | 否           | 否          |                                                    |
+      | chinaz_api         | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | circl_api          | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | dnsdb              | 否           | 否          |                                                    |
+      | dnsdb_api          | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | dnsdumpster        | 否           | 否          |                                                    |
+      | hackertarget       | 否           | 否          |                                                    |
+      | ipv4info           | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | netcraft           | 否           | 否          |                                                    |
+      | ptrarchive         | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | riddler            | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | robtex             | 否           | 否          |                                                    |
+      | securitytrails_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | sitedossier        | 否           | 否          |                                                    |
+      | threatcrowd        | 否           | 否          |                                                    |
  5. 利用DNS查询收集子域（目前有1个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，该模块还有待添加和完善）

      | 模块名称 | 是否需要代理 | 是否需要API | 其他说明                      |
      | -------- | ------------ | ----------- | ----------------------------- |
      | srv      | 否           | 否          | 枚举域名常见的SRV记录发现子域 |
-  6. 利用威胁平台数据收集子域（目前有5个模块：`riskiq`，`threatbook`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）
+  6. 利用威胁平台数据收集子域（目前有5个模块：`riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）

      | 模块名称       | 是否需要代理 | 是否需要API | 其他说明                                           |
      | -------------- | ------------ | ----------- | -------------------------------------------------- |
-      | riskiq         | 否           | 否          |                                                    |
-      | threatbook     | 否           | 否          |                                                    |
+      | riskiq_api     | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | threatbook_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
      | threatminer    | 否           | 否          |                                                    |
      | virustotal     | 否           | 否          |                                                    |
      | virustotal_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
-  7. 利用搜索引擎发现子域（目前有15个模块：`ask`, `bing_api`, `fofa`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`）
+  7. 利用搜索引擎发现子域（目前有15个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`）

     除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。

-     | 模块        | 是否需要代理           | 是否需要API | 其他说明                                          |
-     | ----------- | ---------------------- | ----------- | ------------------------------------------------- |
-     | ask         | 是                     | 否          |                                                   |
-     | baidu       | 否                     | 否          |                                                   |
-     | bing        | 否                     | 否          |                                                   |
+     | 模块        | 是否需要代理           | 是否需要API | 其他说明                                           |
+     | ----------- | ---------------------- | ----------- | -------------------------------------------------- |
+     | ask         | 是                     | 否          |                                                    |
+     | baidu       | 否                     | 否          |                                                    |
+     | bing        | 否                     | 否          |                                                    |
     | bing_api    | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py) |
-     | duckduckgo  | 是                     | 否          |                                                   |
-     | exalead     | 否，最好使用国外代理。 | 否          |                                                   |
-     | fofa        | 否                     | 否          |                                                   |
-     | google      | 是                     | 否          |                                                   |
+     | duckduckgo  | 是                     | 否          |                                                    |
+     | exalead     | 否，最好使用国外代理。 | 否          |                                                    |
+     | fofa_api    | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+     | google      | 是                     | 否          |                                                    |
     | google_api  | 是                     | 是          | API使用和申请见[config.py](../oneforall/config.py) |
     | shodan_api  | 否，最好使用国外代理。 | 是          | API使用和申请见[config.py](../oneforall/config.py) |
-     | so          | 否                     | 否          |                                                   |
-     | sogou       | 否                     | 否          |                                                   |
-     | yahoo       | 是                     | 否          |                                                   |
-     | yandex      | 是                     | 否          |                                                   |
+     | so          | 否                     | 否          |                                                    |
+     | sogou       | 否                     | 否          |                                                    |
+     | yahoo       | 是                     | 否          |                                                    |
+     | yandex      | 是                     | 否          |                                                    |
     | zoomeye_api | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py) |
@@ -0,0 +1,212 @@
+# 使用帮助
+
+oneforall.py是主程序入口，oneforall.py里有调用aiobrute.py和dbexport.py，为了方便进行子域爆破和数据库导出独立出了aiobrute.py和dbexport.py，这两个文件可以单独运行，并且所接受参数要更丰富一点。
+
+1. oneforall.py使用帮助
+
+   ```bash
+   python oneforall.py --help
+   ```
+   ```bash
+   NAME
+       oneforall.py - OneForAll是一款功能强大的子域收集工具
+   
+   SYNOPSIS
+       oneforall.py --target=TARGET <flags>
+   
+   DESCRIPTION
+       Version: 0.0.4
+       Project: https://git.io/fjHT1
+   
+       Example:
+           python3 oneforall.py --target example.com run
+           python3 oneforall.py --target ./domains.txt run
+           python3 oneforall.py --target example.com --brute True run
+           python3 oneforall.py --target example.com --verify False run
+           python3 oneforall.py --target example.com --valid None run
+           python3 oneforall.py --target example.com --port medium run
+           python3 oneforall.py --target example.com --format csv run
+           python3 oneforall.py --target example.com --show True run
+   
+       Note:
+           参数valid可选值1，0，None分别表示导出有效，无效，全部子域
+           参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
+           参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+           参数format可选格式有'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
+                             'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   ARGUMENTS
+       TARGET
+           单个域名或者每行一个域名的文件路径(必需参数)
+   
+   FLAGS
+       --brute=BRUTE
+           使用爆破模块(默认False)
+       --verify=VERIFY
+           验证子域有效性(默认True)
+       --port=PORT
+           请求验证的端口范围(默认medium)
+       --valid=VALID
+           导出子域的有效性(默认1)
+       --path=PATH
+           导出路径(默认None)
+       --format=FORMAT
+           导出格式(默认xlsx)
+       --show=SHOW
+           终端显示导出数据(默认False)
+   ```
+
+2. aiobrute.py使用帮助
+
+   关于泛解析问题处理程序首先会访问一个随机的子域判断是否泛解析，如果使用了泛解析则是通过以下判断处理：
+   - 一是主要是与泛解析的IP集合和TTL值做对比，可以参考[这篇文章](http://sh3ll.me/archives/201704041222.txt)。
+   - 二是多次解析到同一IP集合次数（默认设置为10，可以在config.py设置大小）
+   - 考虑爆破效率问题目前还没有加上HTTP响应体相似度对比和响应体内容判断
+   经过测试在16核心的CPU，使用16进程64协程，100M带宽的环境下，设置任务分割为50000，跑两百万字典大概10分钟左右跑完，大概3333个子域每秒。
+
+   ```bash
+   python aiobrute.py --help
+   ```
+
+   ```bash
+   NAME
+       aiobrute.py - OneForAll多进程多协程异步子域爆破模块
+   
+   SYNOPSIS
+       aiobrute.py --target=TARGET <flags>
+   
+   DESCRIPTION
+       Example：
+           python3 aiobrute.py --target example.com run
+           python3 aiobrute.py --target ./domains.txt run
+           python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
+           python3 aiobrute.py --target example.com --wordlist subdomains.txt run
+           python3 aiobrute.py --target example.com --recursive True --depth 2 run
+           python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
+   
+       Note:
+           参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为任务组，
+           当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+           十万字典建议设置为5000，百万字典设置为50000
+           参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+           参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
+                             'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   ARGUMENTS
+       TARGET
+           单个域名或者每行一个域名的文件路径
+   
+   FLAGS
+       --process=PROCESS
+           爆破的进程数(默认CPU核心数)
+       --coroutine=COROUTINE
+           每个爆破进程下的协程数(默认64)
+       --wordlist=WORDLIST
+           指定爆破所使用的字典路径(默认使用config.py配置)
+       --segment=SEGMENT
+           爆破任务分割(默认500)
+       --recursive=RECURSIVE
+           是否使用递归爆破(默认False)
+       --depth=DEPTH
+           递归爆破的深度(默认2)
+       --namelist=NAMELIST
+           指定递归爆破所使用的字典路径(默认使用config.py配置)
+       --fuzz=FUZZ
+           是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
+       --rule=RULE
+           fuzz模式使用的正则规则(默认使用config.py配置)
+       --export=EXPORT
+           是否导出爆破结果(默认True)
+       --valid=VALID
+           导出子域的有效性(默认None)
+       --format=FORMAT
+           导出格式(默认xlsx)
+       --path=PATH
+           导出路径(默认None)
+       --show=SHOW
+           终端显示导出数据(默认False)
+   
+   ```
+
+
+3. takeover.py使用帮助
+
+   ```bash
+   python takeover.py --help
+   ```
+   
+   ```bash
+   NAME
+    takeover.py - OneForAll多线程子域接管风险检查模块
+   
+   
+    SYNOPSIS
+        takeover.py COMMAND | --target=TARGET <flags>
+   
+    DESCRIPTION
+        Example:
+            python3 takeover.py --target www.example.com  --format csv run
+            python3 takeover.py --target ./subdomains.txt --thread 10 run
+   
+        Note:
+            参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                              'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+            参数dpath为None默认使用OneForAll结果目录
+   
+    ARGUMENTS
+        TARGET
+            单个子域或者每行一个子域的文件路径(必需参数)
+   
+    FLAGS
+        --thread=THREAD
+            线程数(默认100)
+        --dpath=DPATH
+            导出目录(默认None)
+        --format=FORMAT
+            导出格式(默认xls)
+   
+   ```
+
+
+4. dbexport.py使用帮助
+
+   ```bash
+   python dbexport.py --help
+   ```
+
+   ```bash
+   NAME
+       dbexport.py - OneForAll数据库导出模块
+   
+   SYNOPSIS
+       dbexport.py TABLE <flags>
+   
+   DESCRIPTION
+       Example:
+           python3 dbexport.py --table name --format csv --path= ./result.csv
+           python3 dbexport.py --db result.db --table name --show False
+   
+       Note:
+           参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+           参数format可选格式有'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
+                             'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   POSITIONAL ARGUMENTS
+       TABLE
+           要导出的表
+   
+   FLAGS
+       --db=DB
+           要导出的数据库路径(默认为results/result.sqlite3)
+       --valid=VALID
+           导出子域的有效性(默认None)
+       --path=PATH
+           导出路径(默认None)
+       --format=FORMAT
+           导出格式(默认xlsx)
+       --show=SHOW
+           终端显示导出数据(默认False)
+   ```
@@ -9,20 +9,21 @@ OneForAll多进程多协程异步子域爆破模块
 """

 import asyncio
-import os
 import queue
 import secrets
 import signal
 import time

-import aiomultiprocess
+import aiomultiprocess as aiomp
 import exrex
 import fire
 import tqdm

 import config
-from common import database, resolve, utils
+import dbexport
+from common import resolve, utils
 from common.module import Module
+from common.database import Database
 from config import logger


@@ -30,23 +31,20 @@ def init_worker():
    signal.signal(signal.SIGINT, signal.SIG_IGN)


-def get_wordlist(name):
-    return config.data_storage_path.joinpath(name)
-
-
 def detect_wildcard(domain):
    """
    探测域名是否使用泛解析

    :param str domain: 域名
-    :return: 如果没有使用泛解析返回False 使用返回泛解析的IP集合和ttl整型值
+    :return: 如果没有使用泛解析返回False 反之返回泛解析的IP集合和ttl整型值
    """
    logger.log('INFOR', f'正在探测{domain}是否使用泛解析')
    token = secrets.token_hex(16)
    random_subdomain = f'{token}.{domain}'
    try:
        answers = resolve.dns_query_a(random_subdomain)
-    except Exception as e:  # 如果查询随机域名A记录出错 说明不存在随机子域的A记录 即没有开启泛解析
+    # 如果查询随机域名A记录出错 说明不存在随机子域的A记录 即没有开启泛解析
+    except Exception as e:
        logger.log('DEBUG', e)
        logger.log('INFOR', f'{domain}没有使用泛解析')
        return False, None, None
@@ -134,39 +132,63 @@ class AIOBrute(Module):
    OneForAll多进程多协程异步子域爆破模块

    Example：
-        python aiobrute.py --target example.com run
-        python aiobrute.py --target ./domains.txt run
-        python aiobrute.py --target example.com --processes 4 --coroutine 64 --wordlist data/subdomains.txt run
-        python aiobrute.py --target example.com --recursive True --depth 2 --namelist data/next_subdomains.txt run
-        python aiobrute.py --target www.{fuzz}.example.com --fuzz True --rule [a-z][0-9] run
+        python3 aiobrute.py --target subdomain.com run
+        python3 aiobrute.py --target ./subdomains.txt run
+        python3 aiobrute.py --target example.com --process 4 --coroutine 64 run
+        python3 aiobrute.py --target example.com --wordlist subnames.txt run
+        python3 aiobrute.py --target example.com --recursive True --depth 2 run
+        python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
+
+    Note:
+        参数segment的设置受CPU性能，网络带宽，运营商限制等限制，默认500个子域为任务组，
+        当你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+        十万字典建议设置为5000，百万字典设置为50000
+        参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件

    :param str target:       单个域名或者每行一个域名的文件路径
-    :param int processes:    爆破的进程数(默认CPU核心数)
-    :param int coroutine:    每个爆破进程下的协程数(默认16)
+    :param int process:      爆破的进程数(默认CPU核心数)
+    :param int coroutine:    每个爆破进程下的协程数(默认64)
    :param str wordlist:     指定爆破所使用的字典路径(默认使用config.py配置)
+    :param int segment:      爆破任务分割(默认500)
    :param bool recursive:   是否使用递归爆破(默认False)
    :param int depth:        递归爆破的深度(默认2)
    :param str namelist:     指定递归爆破所使用的字典路径(默认使用config.py配置)
-    :param bool fuzz:        是否使用fuzz模式进行爆破(默认False，开启必须指定fuzz正则规则)
+    :param bool fuzz:        是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
    :param str rule:         fuzz模式使用的正则规则(默认使用config.py配置)
+    :param bool export:      是否导出爆破结果(默认True)
+    :param int valid:        导出子域的有效性(默认None)
+    :param str format:       导出格式(默认xls)
+    :param str path:         导出路径(默认None)
+    :param bool show:        终端显示导出数据(默认False)
    """

-    def __init__(self, target, processes=None, coroutine=64, wordlist=None,
-                 recursive=False, depth=2, namelist=None, fuzz=False, rule=None):
+    def __init__(self, target, process=None, coroutine=64, wordlist=None,
+                 segment=500, recursive=False, depth=2, namelist=None,
+                 fuzz=False, rule=None, export=True, valid=None, format='xls',
+                 path=None, show=False):
        Module.__init__(self)
        self.domains = set()
        self.domain = str()
        self.module = 'Brute'
        self.source = 'AIOBrute'
        self.target = target
-        self.processes = processes or config.brute_processes_num or os.cpu_count()
+        self.process = process or config.brute_process_num
        self.coroutine = coroutine or config.brute_coroutine_num
-        self.wordlist = wordlist or config.brute_wordlist_path or get_wordlist('subdomains.txt')
+        self.wordlist = wordlist or config.brute_wordlist_path
+        self.segment = segment or config.brute_task_segment
        self.recursive_brute = recursive or config.enable_recursive_brute
        self.recursive_depth = depth or config.brute_recursive_depth
-        self.recursive_namelist = namelist or config.recursive_namelist_path or get_wordlist('next_subdomains.txt')
+        self.recursive_namelist = namelist or config.recursive_namelist_path
        self.fuzz = fuzz or config.enable_fuzz
        self.rule = rule or config.fuzz_rule
+        self.export = export
+        self.valid = valid
+        self.format = format
+        self.path = path
+        self.show = show
        self.nameservers = config.resolver_nameservers
        self.ips_times = dict()  # IP集合出现次数
        self.enable_wildcard = False  # 当前域名是否使用泛解析
@@ -174,15 +196,18 @@ class AIOBrute(Module):
        self.wildcard_ttl = int()  # 泛解析TTL整型值

    def gen_tasks(self, domain):
-        logger.log('INFOR', f'正在生成{domain}的字典')
-        if self.domain != domain:  # 如果domain不是self.domain，而是self.domain的子域 生成递归爆破字典
+        # 如果domain不是self.subdomain，而是self.domain的子域 生成递归爆破字典
+        if self.domain != domain:
+            logger.log('INFOR', f'使用{self.recursive_namelist}字典')
            domains = gen_brute_domains(domain, self.recursive_namelist)
        elif self.fuzz and self.rule:  # 开启fuzz模式并指定了fuzz正则规则
+            logger.log('INFOR', f'正在生成{domain}的fuzz字典')
            domains = gen_fuzz_domains(domain, self.rule)
        else:
+            logger.log('INFOR', f'使用{self.wordlist}字典')
            domains = gen_brute_domains(domain, self.wordlist)
        domains = list(domains)
-        return utils.split_list(domains, 500)  # 分割任务组 500个子域为一组任务
+        return utils.split_list(domains, self.segment)  # 分割任务组

    def deal_results(self, results):
        for result in results:
@@ -191,27 +216,39 @@ class AIOBrute(Module):
                continue
            if isinstance(result, tuple):
                subdomain, answers = result
+                if not answers:
+                    continue
                ips = {record.host for record in answers}
-                value = self.ips_times.setdefault(str(ips), 0)  # 取值 如果是首次出现的IP集合 出现次数先赋值0
+                # 取值 如果是首次出现的IP集合 出现次数先赋值0
+                value = self.ips_times.setdefault(str(ips), 0)
                self.ips_times[str(ips)] = value + 1
                ttl = answers[0].ttl
                if self.enable_wildcard:
-                    if wildcard_by_compare(ips, ttl, self.wildcard_ips, self.wildcard_ttl):
+                    if wildcard_by_compare(ips,
+                                           ttl,
+                                           self.wildcard_ips,
+                                           self.wildcard_ttl):
                        continue
                if wildcard_by_times(ips, self.ips_times):
                    continue
-                logger.log('INFOR', f'发现{self.domain}的子域: {subdomain} 解析IP: {ips} TTL: {ttl}')
+                logger.log('INFOR', f'发现{self.domain}的子域: {subdomain} '
+                                    f'解析IP: {ips} TTL: {ttl}')
                self.subdomains.add(subdomain)
                self.records[subdomain] = str(ips)

    async def main(self, domain, rx_queue):
        if not self.fuzz:  # fuzz模式不探测域名是否使用泛解析
-            self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl = detect_wildcard(domain)
+            self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl \
+                = detect_wildcard(domain)
        tasks = self.gen_tasks(domain)
        logger.log('INFOR', f'正在爆破{domain}的域名')
-        for task in tqdm.tqdm(tasks, desc='Progress', smoothing=1.0, ncols=True):
-            async with aiomultiprocess.Pool(processes=self.processes, initializer=init_worker,
-                                            childconcurrency=self.coroutine) as pool:
+        for task in tqdm.tqdm(tasks,
+                              desc='Progress',
+                              smoothing=1.0,
+                              ncols=True):
+            async with aiomp.Pool(processes=self.process,
+                                  initializer=init_worker,
+                                  childconcurrency=self.coroutine) as pool:
                try:
                    results = await pool.map(resolve.aiodns_query_a, task)
                except KeyboardInterrupt:
@@ -221,50 +258,66 @@ class AIOBrute(Module):
                    self.gen_result()
                    rx_queue.put(self.results)
                    return
-                else:
-                    self.deal_results(results)
-            self.save_json()
-            self.gen_result()
-            rx_queue.put(self.results)
+                self.deal_results(results)
+                self.save_json()
+                self.gen_result()
+                rx_queue.put(self.results)

    def run(self, rx_queue=None):
        self.domains = utils.get_domains(self.target)
        while self.domains:
            self.domain = self.domains.pop()
            start = time.time()
-            db_conn = database.connect_db()
-            table_name = self.domain.replace('.', '_')
-            database.create_table(db_conn, table_name)
+            db = Database()
+            db.create_table(self.domain)
            if not rx_queue:
                rx_queue = queue.Queue()
            logger.log('INFOR', f'开始执行{self.source}模块爆破域名{self.domain}')
-            logger.log('INFOR', f'{self.source}模块使用{self.processes}个进程乘{self.coroutine}个协程')
-            # logger.log('INFOR', f'{self.source}模块使用个进程乘{self.coroutine}个协程')
-            if self.recursive_brute and not self.fuzz:  # fuzz模式不使用递归爆破
+            logger.log('INFOR', f'使用{self.process}进程乘{self.coroutine}协程')
+            # fuzz模式不使用递归爆破
+            if self.recursive_brute and not self.fuzz:
                logger.log('INFOR', f'开始递归爆破{self.domain}的第1层子域')
            loop = asyncio.get_event_loop()
            asyncio.set_event_loop(loop)
            loop.run_until_complete(self.main(self.domain, rx_queue))

            # 递归爆破下一层的子域
-            if self.recursive_brute and not self.fuzz:  # fuzz模式不使用递归爆破
-                for layer_num in range(1, self.recursive_depth):  # 之前已经做过1层子域爆破 当前实际递归层数是layer+1
-                    logger.log('INFOR', f'开始递归爆破{self.domain}的第{layer_num + 1}层子域')
+            # fuzz模式不使用递归爆破
+            if self.recursive_brute and not self.fuzz:
+                for layer_num in range(1, self.recursive_depth):
+                    # 之前已经做过1层子域爆破 当前实际递归层数是layer+1
+                    logger.log('INFOR', f'开始递归爆破{self.domain}的'
+                                        f'第{layer_num + 1}层子域')
                    for subdomain in self.subdomains.copy():
-                        if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域爆破的限制条件
-                            loop.run_until_complete(self.main(subdomain, rx_queue))
-
-            while not rx_queue.empty():  # 队列不空就一直取数据存数据库
-                database.save_db(db_conn, table_name, rx_queue.get())  # 将结果存入数据库中
-            database.copy_table(db_conn, table_name)
-            database.deduplicate_subdomain(db_conn, table_name)
-            database.remove_invalid(db_conn, table_name)
+                        # 进行下一层子域爆破的限制条件
+                        if subdomain.count('.') - self.domain.count('.') \
+                                == layer_num:
+                            loop.run_until_complete(self.main(subdomain,
+                                                              rx_queue))
+            # 队列不空就一直取数据存数据库
+            while not rx_queue.empty():
+                source, results = rx_queue.get()
+                # 将结果存入数据库中
+                db.save_db(self.domain, results, source)

            end = time.time()
            self.elapsed = round(end - start, 1)
            logger.log('INFOR', f'结束执行{self.source}模块爆破域名{self.domain}')
-            logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒发现{self.domain}的域名{len(self.subdomains)}个')
-            logger.log('DEBUG', f'{self.source}模块发现{self.domain}的的域名 {self.subdomains}')
+            length = len(self.subdomains)
+            logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒'
+                                f'发现{self.domain}的域名{length}个')
+            logger.log('DEBUG', f'{self.source}模块发现{self.domain}的域名:\n'
+                                f'{self.subdomains}')
+            # 数据库导出
+            if self.export:
+                if not self.path:
+                    name = f'{self.domain}_brute.{self.format}'
+                    self.path = config.result_save_path.joinpath(name)
+                dbexport.export(self.domain,
+                                valid=self.valid,
+                                dpath=self.path,
+                                format=self.format,
+                                show=self.show)


 def do(domain, result):  # 统一入口名字 方便多线程调用
@@ -1,14 +1,8 @@
-# coding=utf-8
-"""
-被动收集类
-"""
 import time
-import queue
 import threading
 import importlib
 import config
 import dbexport
-from common import database
 from config import logger


@@ -23,21 +17,24 @@ class Collect(object):
        self.collect_func = []
        self.path = None
        self.export = export
-        self.format = 'xlsx'
+        self.format = 'xls'

    def get_mod(self):
        """
        获取要运行的模块
-        :return: None
        """
        if config.enable_all_module:
-            # modules = ['brute', 'certificates', 'crawl', 'datasets', 'intelligence', 'search']
-            modules = ['certificates', 'check', 'datasets', 'dnsquery', 'intelligence', 'search']  # crawl模块还有点问题
+            # modules = ['brute', 'certificates', 'crawl',
+            # 'datasets', 'intelligence', 'search']
+            # crawl模块还有点问题
+            modules = ['certificates', 'check', 'datasets',
+                       'dnsquery', 'intelligence', 'search']
            # modules = ['intelligence']  # crawl模块还有点问题
            for module in modules:
                module_path = config.oneforall_module_path.joinpath(module)
                for path in module_path.rglob('*.py'):
-                    import_module = ('modules.' + module, path.stem)  # 需要导入的类
+                    # 需要导入的类
+                    import_module = ('modules.' + module, path.stem)
                    self.modules.append(import_module)
        else:
            self.modules = config.enable_partial_module
@@ -50,7 +47,7 @@ class Collect(object):
            import_object = importlib.import_module('.'+name, package)
            self.collect_func.append(getattr(import_object, 'do'))

-    def run(self, rx_queue=None):
+    def run(self):
        """
        类运行入口
        """
@@ -59,12 +56,12 @@ class Collect(object):
        self.get_mod()
        self.import_func()

-        if not rx_queue:
-            rx_queue = queue.Queue(maxsize=len(self.collect_func))  # 结果集队列
        threads = []
        # 创建多个子域收集线程
        for collect_func in self.collect_func:
-            thread = threading.Thread(target=collect_func, args=(self.domain, rx_queue), daemon=True)
+            thread = threading.Thread(target=collect_func,
+                                      args=(self.domain,),
+                                      daemon=True)
            threads.append(thread)
        # 启动所有线程
        for thread in threads:
@@ -73,18 +70,12 @@ class Collect(object):
        for thread in threads:
            thread.join()

-        db_conn = database.connect_db()
-        table_name = self.domain.replace('.', '_')
-        database.create_table(db_conn, table_name)
-        database.copy_table(db_conn, table_name)
-        database.deduplicate_subdomain(db_conn, table_name)
-        database.remove_invalid(db_conn, table_name)
-        db_conn.close()
        # 数据库导出
        if self.export:
            if not self.path:
-                self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
-            dbexport.export(table_name, path=self.path, format=self.format)
+                name = f'{self.domain}.{self.format}'
+                self.path = config.result_save_path.joinpath(name)
+            dbexport.export(self.domain, dpath=self.path, format=self.format)
        end = time.time()
        self.elapsed = round(end - start, 1)

@@ -11,160 +11,200 @@ from records import Connection
 from config import logger


-def connect_db(db_path=None):
-    """
-    获取数据库对象
+class Database(object):
+    def __init__(self, db_path=None):
+        self.conn = self.get_conn(db_path)

-    :param db_path: 数据库连接或路径
-    :return: SQLite数据库
-    """
-    logger.log('DEBUG', f'正在获取数据库连接')
-    if isinstance(db_path, Connection):
-        return db_path
-    protocol = 'sqlite:///'
-    if not db_path:  # 数据库路径为空连接默认数据库
-        db_path = f'{protocol}{config.result_save_path}/result.sqlite3'
-    else:
-        db_path = protocol + db_path
-    db = records.Database(db_path)  # 不存在数据库时会新建一个数据库
-    logger.log('DEBUG', f'使用数据库: {db_path}')
-    return db.get_connection()
+    @staticmethod
+    def get_conn(db_path):
+        """
+        获取数据库对象

+        :param db_path: 数据库连接或路径
+        :return: SQLite数据库
+        """
+        logger.log('DEBUG', f'正在获取数据库连接')
+        if isinstance(db_path, Connection):
+            return db_path
+        protocol = 'sqlite:///'
+        if not db_path:  # 数据库路径为空连接默认数据库
+            db_path = f'{protocol}{config.result_save_path}/result.sqlite3'
+        else:
+            db_path = protocol + db_path
+        db = records.Database(db_path)  # 不存在数据库时会新建一个数据库
+        logger.log('DEBUG', f'使用数据库: {db_path}')
+        return db.get_connection()

-def create_table(db_conn, table_name):
-    """
-    初始化数据库
+    def create_table(self, table_name):
+        """
+        初始化数据库

-    :param db_conn: 数据库连接
-    :param str table_name: 要创建的表名
-    """
-    logger.log('DEBUG', f'正在创建{table_name}表')
-    try:
-        db_conn.query(f'create table if not exists {table_name} ('
-                      f'id integer primary key,'
-                      f'url text,'
-                      f'subdomain text,'
-                      f'port int,'
-                      f'ips text,'
-                      f'status int,'
-                      f'reason text,'
-                      f'valid int,'
-                      f'title text,'
-                      f'banner text,'
-                      f'module text,'
-                      f'source text,'
-                      f'elapsed float,'
-                      f'count int)')
-    except Exception as e:
-        logger.log('ERROR', e)
-
-
-def save_db(db_conn, table_name, results, module_name=None):
-    """
-    将各模块结果存入数据库
-
-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    :param list results: 结果列表
-    :param str module_name: 模块名
-    """
-    logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域结果存入数据库')
-    if results:
+        :param str table_name: 要创建的表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在创建{table_name}表')
        try:
-            db_conn.bulk_query(f'insert into {table_name} (id, url, subdomain, port, ips, status,'
-                               f'reason, valid, title, banner, module, source, elapsed, count)'
-                               f'values (:id, :url, :subdomain, :port, :ips, :status, :reason, :valid,'
-                               f':title, :banner, :module, :source, :elapsed, :count)', results)
+            self.conn.query(f'create table if not exists "{table_name}" ('
+                            f'id integer primary key,'
+                            f'url text,'
+                            f'subdomain text,'
+                            f'port int,'
+                            f'ips text,'
+                            f'status int,'
+                            f'reason text,'
+                            f'valid int,'
+                            f'title text,'
+                            f'banner text,'
+                            f'module text,'
+                            f'source text,'
+                            f'elapsed float,'
+                            f'count int)')
        except Exception as e:
            logger.log('ERROR', e)

+    def save_db(self, table_name, results, module_name=None):
+        """
+        将各模块结果存入数据库

-def copy_table(db_conn, table_name):
-    """
-    复制表创建备份
+        :param str table_name: 表名
+        :param list results: 结果列表
+        :param str module_name: 模块名
+        """
+        logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域结果存入数据库')
+        table_name = table_name.replace('.', '_')
+        if results:
+            try:
+                self.conn.bulk_query(
+                    f'insert into "{table_name}" ('
+                    f'id, url, subdomain, port, ips, status, reason, valid,'
+                    f'title, banner, module, source, elapsed, count)'
+                    f'values (:id, :url, :subdomain, :port, :ips, :status,'
+                    f':reason, :valid, :title, :banner, :module, :source,'
+                    f':elapsed, :count)',
+                    results)
+            except Exception as e:
+                logger.log('ERROR', e)

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    new_table_name = table_name + '_bak'
-    logger.log('DEBUG', f'正在将{table_name}表复制到{new_table_name}新表')
-    try:
-        db_conn.query(f'drop table if exists {new_table_name}')
-        db_conn.query(f'create table {new_table_name} as select * from {table_name}')
-    except Exception as e:
-        logger.log('ERROR', e)
+    def copy_table(self, table_name, bak_table_name):
+        """
+        复制表创建备份

+        :param str table_name: 表名
+        :param str bak_table_name: 新表名
+        """
+        table_name = table_name.replace('.', '_')
+        bak_table_name = bak_table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在将{table_name}表复制到{bak_table_name}新表')
+        try:
+            self.conn.query(f'drop table if exists "{bak_table_name}"')
+            self.conn.query(f'create table "{bak_table_name}" '
+                            f'as select * from "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

-def clear_table(db_conn, table_name):
-    """
-    清空表中数据
+    def clear_table(self, table_name):
+        """
+        清空表中数据

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    logger.log('DEBUG', f'正在清空{table_name}表中的数据')
-    try:
-        db_conn.query(f'delete from {table_name}')
-    except Exception as e:
-        logger.log('ERROR', e)
-
-
-def deduplicate_subdomain(db_conn, table_name):
-    """
-    去重表中的子域并删除空值和无效值
-
-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    logger.log('DEBUG', f'正在去重{table_name}表中的子域')
-    try:
-        db_conn.query(f'delete from {table_name} where id not in (select min(id) from {table_name} group by subdomain)')
-    except Exception as e:
-        logger.log('ERROR', e)
-
-
-def remove_invalid(db_conn, table_name):
-    """
-        去除表中的空值或无效子域
-
-        :param db_conn: 数据库连接
        :param str table_name: 表名
        """
-    logger.log('DEBUG', f'正在去除{table_name}表中的无效子域')
-    try:
-        db_conn.query(f'delete from {table_name} where subdomain is null or valid == 0')
-    except Exception as e:
-        logger.log('ERROR', e)
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在清空{table_name}表中的数据')
+        try:
+            self.conn.query(f'delete from "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

+    def drop_table(self, table_name):
+        """
+        删除表

-def get_data(db_conn, table_name):
-    """
-    获取表中的所有数据
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在删除{table_name}表')
+        try:
+            self.conn.query(f'drop table if exists "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    logger.log('DEBUG', f'获取{table_name}表中的所有数据')
-    try:
-        rows = db_conn.query(f'select * from {table_name}')
-    except Exception as e:
-        logger.log('ERROR', e)
-    else:
-        return rows
+    def rename_table(self, table_name, new_table_name):
+        """
+        复制表创建备份

+        :param str table_name: 表名
+        :param str new_table_name: 新表名
+        """
+        table_name = table_name.replace('.', '_')
+        new_table_name = new_table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在将{table_name}表重命名为{table_name}表')
+        try:
+            self.conn.query(f'alter table "{table_name}" '
+                            f'rename to "{new_table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

-def get_subdomain(db_conn, table_name, valid):
-    """
-    获取表中的子域数据
+    def deduplicate_subdomain(self, table_name):
+        """
+        去重表中的子域

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    :param int valid: 是否有效
-    """
-    logger.log('DEBUG', f'获取{table_name}表中的所有数据')
-    try:
-        rows = db_conn.query(f'select * from {table_name} where valid = {valid}')
-    except Exception as e:
-        logger.log('ERROR', e)
-    else:
-        return rows
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在去重{table_name}表中的子域')
+        try:
+            self.conn.query(
+                f'delete from "{table_name}" where id not in (select min(id) '
+                f'from "{table_name}" group by subdomain)')
+        except Exception as e:
+            logger.log('ERROR', e)
+
+    def remove_invalid(self, table_name):
+        """
+        去除表中的空值或无效子域
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在去除{table_name}表中的无效子域')
+        try:
+            self.conn.query(
+                f'delete from "{table_name}" where '
+                f'subdomain is null or valid == 0')
+        except Exception as e:
+            logger.log('ERROR', e)
+
+    def get_data(self, table_name):
+        """
+        获取表中的所有数据
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'获取{table_name}表中的所有数据')
+        try:
+            rows = self.conn.query(f'select * from "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)
+        else:
+            return rows
+
+    def get_subdomain(self, table_name, valid):
+        """
+        获取表中的子域数据
+
+        :param str table_name: 表名
+        :param int valid: 是否有效
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'获取{table_name}表中的所有数据')
+        try:
+            rows = self.conn.query(
+                f'select * from "{table_name}" where valid = {valid}')
+        except Exception as e:
+            logger.log('ERROR', e)
+        else:
+            return rows
+
+    def close(self):
+        self.conn.close()
@@ -13,7 +13,7 @@ import config
 from config import logger
 from . import utils
 from .domain import Domain
-from common import database
+from common.database import Database


 lock = threading.Lock()
@@ -35,7 +35,18 @@ class Module(object):
        self.results = list()  # 存放模块结果
        self.start = time.time()  # 模块开始执行时间
        self.end = None
-        self.elapsed = time.time() - self.start  # 模块执行耗时
+        self.elapsed = None  # 模块执行耗时
+
+    def check(self, *apis):
+        """
+        简单检查是否配置了api信息
+        :param apis: api信息元组
+        :return: 检查结果
+        """
+        if not all(apis):
+            logger.log('ALERT', f'{self.source}模块API配置有误跳过执行')
+            return False
+        return True

    def begin(self):
        """
@@ -50,57 +61,66 @@ class Module(object):
        self.end = time.time()
        self.elapsed = round(self.end - self.start, 1)
        logger.log('DEBUG', f'结束执行{self.source}模块收集{self.domain}的子域')
-        logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒发现子域{len(self.subdomains)}个')
-        logger.log('DEBUG', f'{self.source}模块发现{self.domain}的子域 {self.subdomains}')
+        logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒发现子域'
+                   f'{len(self.subdomains)}个')
+        logger.log('DEBUG', f'{self.source}模块发现{self.domain}的子域\n'
+                   f'{self.subdomains}')

-    def get(self, url, params=None, **kwargs):
+    def get(self, url, params=None, check=True, **kwargs):
        """
        自定义get请求

        :param str url: 请求地址
        :param dict params: 请求参数
+        :param bool check: 检查响应
        :param kwargs: 其他参数
        :return: requests响应对象
        """
        try:
-            resp = requests.get(url, params=params, cookies=self.cookie, headers=self.header,
-                                proxies=self.proxy, timeout=self.timeout, verify=self.verify, **kwargs)
+            resp = requests.get(url,
+                                params=params,
+                                cookies=self.cookie,
+                                headers=self.header,
+                                proxies=self.proxy,
+                                timeout=self.timeout,
+                                verify=self.verify,
+                                **kwargs)
        except Exception as e:
            logger.log('ERROR', e)
            return None
-        if resp.status_code != 200:
-            logger.log('ALERT', f'GET {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
-            content_type = resp.headers.get('Content-Type')
-            if content_type:
-                if 'json' in content_type:
-                    logger.log('ALERT', resp.json())
-            return None
-        logger.log('DEBUG', f'GET {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
-        return resp
+        if not check:
+            return resp
+        if utils.check_response('GET', resp):
+            return resp
+        return None

-    def post(self, url, data=None, **kwargs):
+    def post(self, url, data=None, check=True, **kwargs):
        """
        自定义post请求

        :param str url: 请求地址
        :param dict data: 请求数据
+        :param bool check: 检查响应
        :param kwargs: 其他参数
        :return: requests响应对象
        """
        try:
-            resp = requests.post(url, data=data, cookies=self.cookie, headers=self.header,
-                                 proxies=self.proxy, timeout=self.timeout, verify=self.verify, **kwargs)
+            resp = requests.post(url,
+                                 data=data,
+                                 cookies=self.cookie,
+                                 headers=self.header,
+                                 proxies=self.proxy,
+                                 timeout=self.timeout,
+                                 verify=self.verify,
+                                 **kwargs)
        except Exception as e:
            logger.log('ERROR', e)
            return None
-        if resp.status_code != 200:
-            content_type = resp.headers.get('Content-Type')
-            if content_type:
-                if 'json' in content_type:
-                    logger.log('ALERT', resp.json())
-            return None
-        logger.log('DEBUG', f'POST {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
-        return resp
+        if not check:
+            return resp
+        if utils.check_response('GET', resp):
+            return resp
+        return None

    def get_header(self):
        """
@@ -146,12 +166,14 @@ class Module(object):
        :rtype: set or list
        """
        logger.log('DEBUG', f'正则匹配响应体中的子域')
-        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
+                 r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
                 + domain.replace('.', r'\.')
        result = re.findall(regexp, html, re.I)
        if not result:
            return set()
-        deal = map(lambda s: re.sub(r'(?:http://|https://)', '', s[1:].lower(), flags=re.I), result)
+        regexp = r'(?:http://|https://)'
+        deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
        if distinct:
            return set(deal)
        else:
@@ -173,38 +195,66 @@ class Module(object):
        """
        logger.log('DEBUG', f'将{self.source}模块发现的子域结果保存为json文件')
        if config.save_module_result:
-            dirpath = config.result_save_path.joinpath(self.domain, self.module)
-            dirpath.mkdir(parents=True, exist_ok=True)
+            dpath = config.result_save_path.joinpath(self.domain, self.module)
+            dpath.mkdir(parents=True, exist_ok=True)
            name = self.source + '.json'
-            path = dirpath.joinpath(name)
+            path = dpath.joinpath(name)
            with open(path, mode='w', encoding='utf-8') as file:
-                result = {'domain': self.domain, 'name': self.module, 'source': self.source, 'elapsed': self.elapsed,
-                          'count': len(self.subdomains), 'subdomains': list(self.subdomains), 'records': self.records}
+                result = {'domain': self.domain,
+                          'name': self.module,
+                          'source': self.source,
+                          'elapsed': self.elapsed,
+                          'count': len(self.subdomains),
+                          'subdomains': list(self.subdomains),
+                          'records': self.records}
                json.dump(result, file, ensure_ascii=False, indent=4)

    def gen_result(self):
        results = list()
        if not len(self.subdomains):  # 一个子域都没有发现的情况
-            result = {'id': None, 'url': None, 'subdomain': None, 'port': None, 'ips': None, 'status': None,
-                      'reason': None, 'valid': 1, 'title': None, 'banner': None, 'module': self.module,
-                      'source': self.source, 'elapsed': self.elapsed, 'count': 0}
+            result = {'id': None,
+                      'url': None,
+                      'subdomain': None,
+                      'port': None,
+                      'ips': None,
+                      'status': None,
+                      'reason': None,
+                      'valid': None,
+                      'title': None,
+                      'banner': None,
+                      'module': self.module,
+                      'source': self.source,
+                      'elapsed': self.elapsed,
+                      'count': 0}
            results.append(result)
            self.results = (self.source, results)
        else:
            for subdomain in self.subdomains:
                url = 'http://' + subdomain
                ips = self.records.get(subdomain)
-                result = {'id': None, 'url': url, 'subdomain': subdomain, 'port': None, 'ips': ips, 'status': None,
-                          'reason': None, 'valid': 1, 'title': None, 'banner': None, 'module': self.module,
-                          'source': self.source, 'elapsed': self.elapsed, 'count': len(self.subdomains)}
+                result = {'id': None,
+                          'url': url,
+                          'subdomain': subdomain,
+                          'port': None,
+                          'ips': ips,
+                          'status': None,
+                          'reason': None,
+                          'valid': None,
+                          'title': None,
+                          'banner': None,
+                          'module': self.module,
+                          'source': self.source,
+                          'elapsed': self.elapsed,
+                          'count': len(self.subdomains)}
                results.append(result)
            self.results = (self.source, results)

    def save_db(self):
        lock.acquire()
-        db_conn = database.connect_db()
-        table_name = self.domain.replace('.', '_')
-        database.create_table(db_conn, table_name)
+        db = Database()
+        db.create_table(self.domain)
        source, results = self.results
-        database.save_db(db_conn, table_name, results, source)  # 将结果存入数据库中
+        # 将结果存入数据库中
+        db.save_db(self.domain, results, source)
+        db.close()
        lock.release()
@@ -2,7 +2,9 @@

 import asyncio
 import functools
+
 import aiohttp
+import tqdm
 from aiohttp import ClientSession
 from aiohttp.resolver import AsyncResolver
 from bs4 import BeautifulSoup
@@ -31,13 +33,19 @@ def get_ports(port):
 def gen_new_datas(datas, ports):
    logger.log('INFOR', f'正在生成请求地址')
    new_datas = []
-    protocols = ['http://', 'https://']
+    protocols = ['http://']
    for data in datas:
-        if data.get('valid'):  # 有效的子域才进行http请求探测
+        valid = data.get('valid')
+        if valid is None:  # 子域有效性未知的才进行http请求探测
            subdomain = data.get('subdomain')
            for port in ports:
                for protocol in protocols:
-                    url = f'{protocol}{subdomain}:{port}'
+                    if port == 443:
+                        url = f'https://{subdomain}:{port}'
+                    elif port == 8443:
+                        url = f'https://{subdomain}:{port}'
+                    else:
+                        url = f'{protocol}{subdomain}:{port}'
                    data['id'] = None
                    data['url'] = url
                    data['port'] = port
@@ -52,64 +60,100 @@ async def fetch(session, url, semaphore):

    :param session: session对象
    :param url: url地址
-    :param semaphore: 同步对象(控制并发量)
+    :param semaphore: 并发信号量
    :return: 响应对象和响应文本
    """
    timeout = aiohttp.ClientTimeout(total=config.get_timeout)
    async with semaphore:
-        async with session.get(url, allow_redirects=config.get_redirects,
-                               timeout=timeout, proxy=config.get_proxy) as resp:
-            text = await resp.text()
-            return resp, text
+        async with session.get(url,
+                               ssl=config.verify_ssl,
+                               allow_redirects=config.get_redirects,
+                               timeout=timeout,
+                               proxy=config.get_proxy) as resp:
+
+            try:
+                text = await resp.text(encoding='gb2312')  # 先尝试用fb2312解码
+            except UnicodeDecodeError:
+                text = await resp.text()
+        return resp, text


 def request_callback(future, index, datas):
    try:
-        resp, text = future.result()
+        result = future.result()
    except Exception as e:
        logger.log('DEBUG', e.args)
        datas[index]['reason'] = str(e.args)
        datas[index]['valid'] = 0
    else:
+        resp, text = result
        datas[index]['reason'] = resp.reason
        datas[index]['status'] = resp.status
-        if resp.status == 400 or resp.status >= 500:
+        if resp.status >= 500:
            datas[index]['valid'] = 0
        else:
+            datas[index]['valid'] = 1
            headers = resp.headers
-            banner = str({'Server': headers.get('Server'), 'Via': headers.get('Via'),
+            banner = str({'Server': headers.get('Server'),
+                          'Via': headers.get('Via'),
                          'X-Powered-By': headers.get('X-Powered-By')})
            datas[index]['banner'] = banner
            soup = BeautifulSoup(text, 'lxml')
            title = soup.title
+            desc = soup.find('meta', attrs={'name': 'description'})
            head = soup.head
            if title:
-                datas[index]['title'] = title.text
+                datas[index]['title'] = title.text.strip()
+            elif desc:
+                datas[index]['title'] = desc['content'].strip()
            elif head:
-                datas[index]['title'] = head.text
-            else:
-                datas[index]['title'] = text
+                datas[index]['title'] = head.text.strip()
+            elif len(text) <= 200:
+                datas[index]['title'] = text.strip()


 async def bulk_get_request(datas, port):
-    logger.log('INFOR', f'正在异步进行子域的GET请求')
    ports = get_ports(port)
    new_datas = gen_new_datas(datas, ports)
+    logger.log('INFOR', f'正在异步进行子域的GET请求')
+
+    limit_open_conn = config.limit_open_conn
+    if limit_open_conn is None:  # 默认情况
+        limit_open_conn = utils.get_semaphore()
+    elif not isinstance(limit_open_conn, int):  # 如果传入不是数字的情况
+        limit_open_conn = utils.get_semaphore()
+    # 使用异步域名解析器 自定义域名服务器
+    resolver = AsyncResolver(nameservers=config.resolver_nameservers)
+    conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
+                                limit=limit_open_conn,
+                                limit_per_host=config.limit_per_host,
+                                resolver=resolver)
+
+    semaphore = asyncio.Semaphore(limit_open_conn)
    header = None
    if config.fake_header:
        header = utils.gen_fake_header()
-    resolver = AsyncResolver(nameservers=config.resolver_nameservers)  # 使用异步域名解析器 自定义域名服务器
-    conn = aiohttp.TCPConnector(verify_ssl=config.verify_ssl, limit=config.limit_open_conn,
-                                limit_per_host=config.limit_per_host, resolver=resolver)
-    semaphore = asyncio.Semaphore(utils.get_semaphore())
    async with ClientSession(connector=conn, headers=header) as session:
        tasks = []
        for i, data in enumerate(new_datas):
            url = data.get('url')
            task = asyncio.ensure_future(fetch(session, url, semaphore))
-            task.add_done_callback(functools.partial(request_callback, index=i, datas=new_datas))
+            task.add_done_callback(functools.partial(request_callback,
+                                                     index=i,
+                                                     datas=new_datas))
            tasks.append(task)
        if tasks:  # 任务列表里有任务不空时才进行解析
-            await asyncio.wait(tasks)  # 等待所有task完成
+            # 等待所有task完成 错误聚合到结果列表里
+            futures = asyncio.as_completed(tasks)
+            for future in tqdm.tqdm(futures,
+                                    total=len(tasks),
+                                    desc='Progress',
+                                    smoothing=1.0,
+                                    ncols=True):
+                try:
+                    await future
+                except:
+                    pass
+
    logger.log('INFOR', f'完成异步进行子域的GET请求')
    return new_datas
@@ -4,6 +4,8 @@ import functools

 import dns.resolver
 import aiodns
+import tqdm
+
 import config
 from common import utils
 from config import logger
@@ -39,7 +41,7 @@ def aiodns_resolver():
                              timeout=config.resolver_timeout)


-async def aiodns_query_a(hostname, semaphore):
+async def aiodns_query_a(hostname, semaphore=None):
    """
    异步查询A记录

@@ -47,23 +49,37 @@ async def aiodns_query_a(hostname, semaphore):
    :param semaphore: 并发查询数量
    :return: 主机名或查询结果或查询异常
    """
-    async with semaphore:
+    if semaphore is None:
        resolver = aiodns_resolver()
        answers = await resolver.query(hostname, 'A')
        return hostname, answers
+    else:
+        async with semaphore:
+            resolver = aiodns_resolver()
+            answers = await resolver.query(hostname, 'A')
+            return hostname, answers


 def resolve_callback(future, index, datas):
+    """
+    解析结果回调处理
+    :param future: future对象
+    :param index: 下标
+    :param datas: 结果集
+    """
    try:
        result = future.result()
-    except aiodns.error.DNSError as e:
+    except Exception as e:
        datas[index]['ips'] = str(e.args)
        datas[index]['valid'] = 0
    else:
        if isinstance(result, tuple):
            _, answers = result
-            ips = {record.host for record in answers}
-            datas[index]['ips'] = str(ips)
+            if answers:
+                ips = {record.host for record in answers}
+                datas[index]['ips'] = str(ips)
+            else:
+                datas[index]['ips'] = 'No answers'


 async def bulk_query_a(datas):
@@ -75,14 +91,26 @@ async def bulk_query_a(datas):
    """
    logger.log('INFOR', '正在异步查询子域的A记录')
    tasks = []
-    semaphore = asyncio.Semaphore(utils.get_semaphore())
+    semaphore = asyncio.Semaphore(config.limit_resolve_conn)
    for i, data in enumerate(datas):
        if not data.get('ips'):
            subdomain = data.get('subdomain')
            task = asyncio.ensure_future(aiodns_query_a(subdomain, semaphore))
-            task.add_done_callback(functools.partial(resolve_callback, index=i, datas=datas))  # 回调
+            task.add_done_callback(functools.partial(resolve_callback,
+                                                     index=i,
+                                                     datas=datas))  # 回调
            tasks.append(task)
    if tasks:  # 任务列表里有任务不空时才进行解析
-        await asyncio.wait(tasks)  # 等待所有task完成
+        futures = asyncio.as_completed(tasks)
+        for future in tqdm.tqdm(futures,
+                                total=len(tasks),
+                                desc='Progress',
+                                smoothing=1.0,
+                                ncols=True):
+            try:
+                await future
+            except:
+                pass
+        # await asyncio.wait(tasks)  # 等待所有task完成
    logger.log('INFOR', '完成异步查询子域的A记录')
    return datas
@@ -28,10 +28,12 @@ class Search(Module):
        :rtype: str
        """
        statements_list = []
-        subdomains_temp = set(map(lambda x: x + '.' + domain, config.subdomains_common))
+        subdomains_temp = set(map(lambda x: x + '.' + domain,
+                                  config.subdomains_common))
        subdomains_temp = list(subdomain.intersection(subdomains_temp))
        for i in range(0, len(subdomains_temp), 2):  # 同时排除2个子域
-            statements_list.append(''.join(set(map(lambda s: ' -site:' + s, subdomains_temp[i:i + 2]))))
+            statements_list.append(''.join(set(map(lambda s: ' -site:' + s,
+                                                   subdomains_temp[i:i + 2]))))
        return statements_list

    def match_location(self, domain, url):
@@ -45,6 +47,7 @@ class Search(Module):
        :return: 匹配的子域
        :rtype set
        """
-        resp = requests.head(url, headers=self.header, proxies=self.proxy, timeout=self.timeout, allow_redirects=False)
+        resp = requests.head(url, headers=self.header, proxies=self.proxy,
+                             timeout=self.timeout, allow_redirects=False)
        location = resp.headers.get('location')
        return set(utils.match_subdomain(domain, location))
@@ -1,14 +1,25 @@
 # coding=utf-8
 import re
-import pathlib
 import random
 import ipaddress
 import platform
 import config
-from fake_useragent import UserAgent
+from pathlib import Path
 from common.domain import Domain
 from config import logger

+user_agents = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/68.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:61.0) '
+    'Gecko/20100101 Firefox/68.0',
+    'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0']
+

 def match_subdomain(domain, text, distinct=True):
    """
@@ -20,7 +31,8 @@ def match_subdomain(domain, text, distinct=True):
    :return: 匹配结果
    :rtype: set or list
    """
-    regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' + domain.replace('.', r'\.')
+    regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+             + domain.replace('.', r'\.')
    result = re.findall(regexp, text, re.I)
    if not result:
        return set()
@@ -45,10 +57,11 @@ def gen_fake_header():
    """
    生成伪造请求头
    """
-    ua = UserAgent()
+    ua = random.choice(user_agents)
    ip = gen_random_ip()
    headers = {
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept': 'text/html,application/xhtml+xml,'
+                  'application/xml;q=0.9,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
        'Cache-Control': 'max-age=0',
@@ -56,7 +69,7 @@ def gen_fake_header():
        'DNT': '1',
        'Referer': 'https://www.google.com/',
        'Upgrade-Insecure-Requests': '1',
-        'User-Agent': ua.random,
+        'User-Agent': ua,
        'X-Forwarded-For': ip,
        'X-Real-IP': ip
    }
@@ -86,7 +99,7 @@ def split_list(ls, size):
    """
    if size == 0:
        return ls
-    return [ls[i:i+size] for i in range(0, len(ls), size)]
+    return [ls[i:i + size] for i in range(0, len(ls), size)]


 def get_domains(target):
@@ -96,20 +109,22 @@ def get_domains(target):
    :param set or str target:
    :return: 域名集合
    """
-    domains = set()
-    logger.log('INFOR', f'正在获取域名')
-    if isinstance(target, set):
+    domains = list()
+    logger.log('DEBUG', f'正在获取域名')
+    if isinstance(target, (set, tuple)):
+        domains = list(target)
+    elif isinstance(target, list):
        domains = target
    elif isinstance(target, str):
-        path = pathlib.Path(target)
+        path = Path(target)
        if path.is_file():
            with open(target) as file:
                for line in file:
                    domain = Domain(line.strip()).match()
                    if domain:
-                        domains.add(domain)
-        if Domain(target).match():
-            domains = {target}
+                        domains.append(domain)
+        elif Domain(target).match():
+            domains = [target]
    logger.log('INFOR', f'获取到{len(domains)}个域名')
    return domains

@@ -122,9 +137,73 @@ def get_semaphore():
    """
    system = platform.system()
    if system == 'Windows':
-        return 500
+        return 300
    elif system == 'Linux':
-        return 1000
+        return 800
    elif system == 'Darwin':
-        return 1000
+        return 800

+
+def check_dpath(dpath):
+    """
+    检查目录路径
+
+    :param dpath: 传入的目录路径
+    :return: 目录路径
+    """
+    if isinstance(dpath, str):
+        dpath = Path(dpath)
+    else:
+        dpath = config.result_save_path
+    if not dpath.is_dir():
+        logger.log('FATAL', f'{dpath}不是目录')
+    if not dpath.exists():
+        logger.log('ALERT', f'不存在{dpath}将会新建此目录')
+        dpath.mkdir(parents=True, exist_ok=True)
+    return dpath
+
+
+def check_format(format):
+    """
+    检查导出格式
+
+    :param format: 传入的导出格式
+    :return: 导出格式
+    """
+    formats = ['txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+               'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods']
+    if format in formats:
+        return format
+    else:
+        logger.log('ALERT', f'不支持{format}格式导出')
+        logger.log('ALERT', '默认使用csv格式导出')
+        return 'xls'
+
+
+def save_data(fpath, data):
+    try:
+        with open(fpath, 'w', encoding="utf-8", newline='') as file:
+            file.write(data)
+            logger.log('ALERT', fpath)
+    except TypeError:
+        with open(fpath, 'wb') as file:
+            file.write(data)
+            logger.log('ALERT', fpath)
+    except Exception as e:
+        logger.log('ERROR', e)
+
+
+def check_response(method, resp):
+    if resp.status_code == 200 and resp.content:
+        return True
+    logger.log('ALERT', f'{method} {resp.url} {resp.status_code} - '
+                        f'{resp.reason} {len(resp.content)}')
+    content_type = resp.headers.get('Content-Type')
+    if content_type and 'json' in content_type and resp.content:
+        try:
+            msg = resp.json()
+        except Exception as e:
+            logger.log('DEBUG', e.args)
+        else:
+            logger.log('ALERT', msg)
+    return False
@@ -2,7 +2,7 @@
 """
 OneForAll配置
 """
-
+import os
 import sys
 import pathlib
 from loguru import logger
@@ -15,38 +15,52 @@ data_storage_path = oneforall_relpath.joinpath('data')  # 数据存放目录
 result_save_path = oneforall_relpath.joinpath('results')  # 结果保存目录

 # 模块设置
-save_module_result = True  # 保存模块中各脚本结果 默认保存
-enable_all_module = True  # 启用所有模块 默认启用
+save_module_result = True  # 保存模块中各脚本结果(默认True)
+enable_all_module = True  # 启用所有模块(默认True)
 enable_partial_module = []  # 启用部分模块 必须禁用enable_all_module才能生效
-# enable_partial_module = [('modules.search', 'ask')，('modules.search', 'baidu')]  # 只使用ask和baidu搜索引擎收集子域
+# 只使用ask和baidu搜索引擎收集子域
+# enable_partial_module = [('modules.search', 'ask')
+#                          ('modules.search', 'baidu')]


 # 爆破模块设置
 enable_brute_module = False  # 使用爆破模块(默认禁用)
+enable_verify_subdomain = True  # 验证子域有效性(默认True)
 enable_wildcard_check = True  # 开启泛解析检测 会去掉泛解析的子域
-brute_processes_num = None  # 爆破时使用的进程数(根据系统中CPU数量情况设置 不宜大于CPU数量 默认None为系统中的CPU数量)
+# 爆破时使用的进程数(根据系统中CPU数量情况设置 不宜大于CPU数量 默认为系统中的CPU数量)
+brute_process_num = os.cpu_count()
 brute_coroutine_num = 128  # 爆破时每个进程下的协程数(不宜大于1000)
-brute_wordlist_path = None  # 爆破所使用的字典路径 默认data/subdomains.dict
+# 爆破所使用的字典路径 默认data/subdomains.txt
+brute_wordlist_path = data_storage_path.joinpath('subnames.txt')
+brute_task_segment = 500
+# 参数segment的设置受CPU性能，网络带宽，运营商限制等限制，默认500个子域为一任务组，
+# 当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+# 十万字典建议设置为5000，百万字典设置为50000
 enable_recursive_brute = False  # 是否使用递归爆破(默认禁用)
 brute_recursive_depth = 2  # 递归爆破深度(默认2层)
-recursive_namelist_path = None  # 爆破下一层子域所使用的字典路径 默认data/next_subdomains.dict
+# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt
+recursive_namelist_path = data_storage_path.joinpath('next_subnames.txt')
 enable_fuzz = False  # 是否使用fuzz模式枚举域名
 fuzz_rule = ''  # fuzz域名的正则 示例：[a-z][0-9] 第一位是字母 第二位是数字
 ips_appear_maximum = 10  # 同一IP集合出现次数超过10认为是泛解析

 # 代理设置
-enable_proxy = True  # 是否使用代理 全局开关
+enable_proxy = False  # 是否使用代理(全局开关)
 proxy_all_module = False  # 代理所有模块
-proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch', 'GoogleAPISearch',
-                        'GoogleSearch', 'YahooSearch', 'YandexSearch']  # 代理自定义的模块
-proxy_pool = [{'http': 'http://127.0.0.1:1080', 'https': 'https://127.0.0.1:1080'}]  # 代理池
-# proxy_pool = [{'http': 'socks5://127.0.0.1:10808', 'https': 'socks5://127.0.0.1:10808'}]  # 代理池
+proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
+                        'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
+                        'YandexSearch', 'CrossDomainXml',
+                        'ContentSecurityPolicy']  # 代理自定义的模块
+proxy_pool = [{'http': 'http://127.0.0.1:1080',
+               'https': 'https://127.0.0.1:1080'}]  # 代理池
+# proxy_pool = [{'http': 'socks5://127.0.0.1:10808',
+#                'https': 'socks5://127.0.0.1:10808'}]  # 代理池


 # 网络请求设置
 enable_fake_header = True  # 启用伪造请求头
 request_delay = 1  # 请求时延
-request_timeout = 60  # 请求超时
+request_timeout = 30  # 请求超时
 request_verify = True  # 请求SSL验证

 # 搜索模块设置
@@ -65,6 +79,7 @@ resolver_nameservers = [
 ]  # 指定查询的DNS域名服务器
 resolver_timeout = 5.0  # 解析超时时间
 resolver_lifetime = 30.0  # 解析存活时间
+limit_resolve_conn = 50  # 限制同一时间解析的数量(默认50)

 # http探测设置
 small_ports = {80, 443}
@@ -79,14 +94,18 @@ xlarge_ports = {80, 81, 300, 443, 591, 593, 832, 981, 1010, 1311, 2082,
                8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080, 9090,
                9091, 9200, 9443, 9800, 9981, 12443, 16080, 18091, 18092,
                20720, 28017}
-ports = {'small': small_ports, 'medium': medium_ports, 'large': large_ports, 'xlarge': xlarge_ports}
+ports = {'small': small_ports, 'medium': medium_ports,
+         'large': large_ports, 'xlarge': xlarge_ports}
 verify_ssl = False
-get_proxy = None  # aiohttp 支持 HTTP/HTTPS形式的代理 proxy="http://user:pass@some.proxy.com"
-get_timeout = 10  # http请求探测总超时时间 None或者0则表示不检测超时
+# aiohttp 支持 HTTP/HTTPS形式的代理
+get_proxy = None  # proxy="http://user:pass@some.proxy.com"
+get_timeout = 120  # http请求探测总超时时间 None或者0则表示不检测超时
 get_redirects = True  # 允许请求跳转
 fake_header = True  # 使用伪造请求头
-limit_open_conn = 100  # 限制同一时间打开的连接数(默认100)，0表示不限制
-limit_per_host = 0  # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数(默认0表示不限制)
+# 限制同一时间打开的连接数(默认None，根据系统不同设置，Windows系统400 其他系统800)
+limit_open_conn = None
+# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
+limit_per_host = 0  # 默认0表示不限制


 # 模块API配置
@@ -98,10 +117,11 @@ censys_api_secret = ''
 # 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
 binaryedge_api = ''

-# Binaryedge可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
+# Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
 chinaz_api = ''

-# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/cognitive-services/bing-web-search-api/#web-json
+# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/
+# cognitive-services/bing-web-search-api/#web-json
 bing_api_id = ''
 bing_api_key = ''

@@ -112,7 +132,8 @@ securitytrails_api = ''
 fofa_api_email = ''  # fofa用户邮箱
 fofa_api_key = ''  # fofa用户key

-# Google可以免费注册获取API: https://developers.google.com/custom-search/v1/overview
+# Google可以免费注册获取API:
+# https://developers.google.com/custom-search/v1/overview
 # 免费的API只能查询前100条结果
 google_api_key = ''  # Google API搜索key
 google_api_cx = ''  # Google API搜索cx
@@ -148,19 +169,28 @@ dnsdb_api_key = ''
 # 免费的API有效期只有2天，到期之后可以再次生成，每天可以查询50次。
 ipv4info_api_key = ''

-subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs', 'web', 'www', 'job', 'docs', 'news', 'blog',
-                     'data', 'help', 'live', 'mall', 'blogs', 'files', 'forum', 'store', 'mobile'}
+# Github Token可以访问https://github.com/settings/tokens生成,user为Github用户名
+github_api_user = ''
+github_api_token = ''
+
+subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
+                     'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
+                     'help', 'live', 'mall', 'blogs', 'files', 'forum',
+                     'store', 'mobile'}

 # 日志配置
+# 终端日志输出格式
 stdout_fmt = '<cyan>{time:HH:mm:ss,SSS}</cyan> ' \
          '[<level>{level: <5}</level>] ' \
          '<blue>{module}</blue>:<cyan>{line}</cyan> - ' \
          '<level>{message}</level>'
+# 日志文件记录格式
 logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
          '[<level>{level: <5}</level>] ' \
-          '<cyan>{process.name}</cyan>:<cyan>{thread.name: <10}</cyan> | ' \
-          '<blue>{module}</blue>.<blue>{function}</blue>:<blue>{line}</blue> - ' \
-          '<level>{message}</level>'
+          '<cyan>{process.name}({process.id})</cyan>:' \
+          '<cyan>{thread.name: <10}({thread.id: <5})</cyan> | ' \
+          '<blue>{module}</blue>.<blue>{function}</blue>:' \
+          '<blue>{line}</blue> - <level>{message}</level>'

 log_path = result_save_path.joinpath('oneforall.log')

@@ -172,13 +202,9 @@ logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️')
 logger.level(name='ERROR', no=40, color='<red><bold>', icon='❌️')
 logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')

-logger.add(sys.stdout, level='INFOR', format=stdout_fmt, enqueue=True)
-logger.add(log_path, level='TRACE', format=logfile_fmt, enqueue=True, encoding='utf-8')
+if not os.environ.get('PYTHONIOENCODING'):  # 设置编码
+    os.environ['PYTHONIOENCODING'] = 'utf-8'

-# 调试模式
-# import urllib3
-# urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-# request_proxy = [{'http': 'http://127.0.0.1:8080', 'https': 'https://127.0.0.1:8080'}]
-# request_verify = False
-# enable_all_module = False  # 启用所有模块 默认启用
-# enable_partial_module = [('modules.certificates', 'censys_api')]  # 启用部分模块 必须禁用enable_all_module才能生效
+logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True)
+logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True,
+           encoding='utf-8')
@@ -0,0 +1,249 @@
+[
+    {
+        "name":"github", 
+        "cname":["github.io", "github.map.fastly.net"], 
+        "response":["There isn't a GitHub Pages site here.", "For root URLs (like http://example.com/) you must provide an index.html file"]
+    },
+    {
+        "name":"heroku", 
+        "cname":["herokudns.com", "herokussl.com", "herokuapp.com"], 
+        "response":["There's nothing here, yet.", "herokucdn.com/error-pages/no-such-app.html", "<title>No such app</title>"]
+    },
+    {
+        "name":"unbounce",
+        "cname":["unbouncepages.com"],
+        "response":["Sorry, the page you were looking for doesn’t exist.", "The requested URL was not found on this server"]
+    },
+    {
+        "name":"tumblr",
+        "cname":["tumblr.com"],
+        "response":["There's nothing here.", "Whatever you were looking for doesn't currently exist at this address."]
+    },
+    {
+        "name":"shopify",
+        "cname":["myshopify.com"],
+        "response":["Sorry, this shop is currently unavailable.", "Only one step left!"]
+    },
+    {
+        "name":"instapage",
+        "cname":["pageserve.co", "secure.pageserve.co", "https://instapage.com/"],
+        "response":["Looks Like You're Lost","The page you're looking for is no longer available."]
+    },
+    {
+        "name":"desk",
+        "cname":["desk.com"],
+        "response":["Please try again or try Desk.com free for 14 days.", "Sorry, We Couldn't Find That Page"]
+    },
+    {
+        "name":"campaignmonitor",
+        "cname":["createsend.com", "name.createsend.com"],
+        "response":["Double check the URL", "<strong>Trying to access your account?</strong>"]
+    },
+    {
+        "name":"cargocollective",
+        "cname":["cargocollective.com"],
+        "response":["404 Not Found"]
+    },
+    {
+        "name":"statuspage",
+        "cname":["statuspage.io"],
+        "response":["Better Status Communication", "You are being <a href=\"https://www.statuspage.io\">redirected"]
+    },
+    {
+        "name":"amazonaws",
+        "cname":["amazonaws.com"],
+        "response":["NoSuchBucket", "The specified bucket does not exist"]
+    },
+    {
+        "name":"bitbucket",
+        "cname":["bitbucket.org"],  
+        "response":["The page you have requested does not exist","Repository not found"]
+    },
+    {
+        "name":"smartling",
+        "cname":["smartling.com"],
+        "response":["Domain is not configured"]
+    },
+    {
+        "name":"acquia",
+        "cname":["acquia.com"],
+        "response":["If you are an Acquia Cloud customer and expect to see your site at this address","The site you are looking for could not be found."]
+    },
+    {
+        "name":"fastly",
+        "cname":["fastly.net"],
+        "response":["Please check that this domain has been added to a service", "Fastly error: unknown domain"]
+    },
+    {
+        "name":"pantheon",
+        "cname":["pantheonsite.io"],
+        "response":["The gods are wise", "The gods are wise, but do not know of the site which you seek."]
+    },
+    {
+        "name":"zendesk",
+        "cname":["zendesk.com"],
+        "response":["Help Center Closed"]
+    },
+    {
+        "name":"uservoice",
+        "cname":["uservoice.com"],
+        "response":["This UserVoice subdomain is currently available!"]
+    },
+    {
+        "name":"ghost",
+        "cname":["ghost.io"],
+        "response":["The thing you were looking for is no longer here", "The thing you were looking for is no longer here, or never was"]
+    },
+    {
+        "name":"pingdom",
+        "cname":["stats.pingdom.com"],
+        "response":["pingdom"]
+    },
+    {
+        "name":"tilda",
+        "cname":["tilda.ws"],
+        "response":["Domain has been assigned"]
+    },
+    {
+        "name":"wordpress",
+        "cname":["wordpress.com"],  
+        "response":["Do you want to register"]
+    },
+    {
+        "name":"teamwork",
+        "cname":["teamwork.com"],
+        "response":["Oops - We didn't find your site."]
+    },
+    {
+        "name":"helpjuice",
+        "cname":["helpjuice.com"],
+        "response":["We could not find what you're looking for."]
+    },
+    {
+        "name":"helpscout",
+        "cname":["helpscoutdocs.com"],
+        "response":["No settings were found for this company:"]
+    },
+    {
+        "name":"cargo",
+        "cname":["cargocollective.com"],
+        "response":["If you're moving your domain away from Cargo you must make this configuration through your registrar's DNS control panel."]
+    },
+    {
+        "name":"feedpress",
+        "cname":["redirect.feedpress.me"],
+        "response":["The feed has not been found."]
+    },
+    {
+        "name":"surge",
+        "cname":["surge.sh"],
+        "response":["project not found"]
+    },
+    {
+        "name":"surveygizmo",
+        "cname":["privatedomain.sgizmo.com", "privatedomain.surveygizmo.eu", "privatedomain.sgizmoca.com"],
+        "response":["data-html-name"]
+    },
+    {
+        "name":"mashery",
+        "cname":["mashery.com"],
+        "response":["Unrecognized domain <strong>"]
+    },
+    {
+        "name":"intercom",
+        "cname":["custom.intercom.help"],
+        "response":["This page is reserved for artistic dogs.","<h1 class=\"headline\">Uh oh. That page doesn’t exist.</h1>"]
+    },
+    {
+        "name":"webflow",
+        "cname":["proxy.webflow.io"],
+        "response":["<p class=\"description\">The page you are looking for doesn't exist or has been moved.</p>"]
+    },
+    {
+        "name":"kajabi",
+        "cname":["endpoint.mykajabi.com"],
+        "response":["<h1>The page you were looking for doesn't exist.</h1>"]
+    },
+    {
+        "name":"thinkific",
+        "cname":["thinkific.com"],
+        "response":["You may have mistyped the address or the page may have moved."]
+    },
+    {
+        "name":"tave",
+        "cname":["clientaccess.tave.com"],
+        "response":["<h1>Error 404: Page Not Found</h1>"]
+    },
+    {
+        "name":"wishpond",
+        "cname":["wishpond.com"],
+        "response":["https://www.wishpond.com/404?campaign=true"]
+    },
+    {
+        "name":"aftership",
+        "cname":["aftership.com"],
+        "response":["Oops.</h2><p class=\"text-muted text-tight\">The page you're looking for doesn't exist."]
+    },
+    {
+        "name":"aha",
+        "cname":["ideas.aha.io"],
+        "response":["There is no portal here ... sending you back to Aha!"]
+    },
+    {
+        "name":"brightcove",
+        "cname":["brightcovegallery.com", "gallery.video", "bcvp0rtal.com"],
+        "response":["<p class=\"bc-gallery-error-code\">Error Code: 404</p>"]
+    },
+    {
+        "name":"bigcartel",
+        "cname":["bigcartel.com"],
+        "response":["<h1>Oops! We couldn&#8217;t find that page.</h1>"]
+    },
+    {
+        "name":"activecompaign",
+        "cname":["activehosted.com"],
+        "response":["alt=\"LIGHTTPD - fly light.\""]
+    },
+    {
+        "name":"compaignmonitor",
+        "cname":["createsend.com"],
+        "response":["Double check the URL or <a href=\"mailto:help@createsend.com"]
+    },
+    {
+        "name":"simplebooklet",
+        "cname":["simplebooklet.com"],
+        "response":["We can't find this <a href=\"https://simplebooklet.com"]
+    },
+    {
+        "name":"getresponse",
+        "cname":[".gr8.com"],
+        "response":["With GetResponse Landing Pages, lead generation has never been easier"]
+    },
+    {
+        "name":"vend",
+        "cname":["vendecommerce.com"],
+        "response":["Looks like you've traveled too far into cyberspace."]
+    },
+    {
+        "name":"jetbrains",
+        "cname":["myjetbrains.com"],
+        "response":["is not a registered InCloud YouTrack.","is not a registered InCloud YouTrack."]
+    },
+    {
+        "name":"azure",
+        "cname":["azurewebsites.net",
+            ".cloudapp.net",
+            ".cloudapp.azure.com",
+            ".trafficmanager.net",
+            ".blob.core.windows.net",
+            ".azure-api.net",
+            ".azurehdinsight.net",
+            ".azureedge.net"],
+        "response":["404 Web Site not found"]
+    },
+    {
+        "name":"readme",
+        "cname":["readme.io"],
+        "response":["Project doesnt exist... yet!"]
+    }
+]
@@ -1,163 +1,163 @@
-test
-test2
-t
-dev
-1
-2
-3
-s1
-s2
-s3
-admin
-adm
-a
-b
-c
-m
-ht
-adminht
-webht
-web
-gm
-sys
-system
-manage
-manager
-mgr
-passport
-bata
-wei
-weixin
-wechat
-wx
-wiki
-upload
-ftp
-pic
-jira
-zabbix
-nagios
-bug
-bugzilla
-sql
-mysql
-db
-stmp
-pop
-imap
-mail
-zimbra
-exchange
-forum
-bbs
-list
-count
-counter
-img
-img01
-img02
-img03
-img04
-api
-cache
-js
-css
-app
-apps
-wap
-sms
-zip
-monitor
-proxy
-update
-upgrade
-stat
-stats
-data
-portal
-blog
-autodiscover
-en
-search
-so
-oa
-database
-home
-sso
-help
-vip
-s
-w
-down
-download
-downloads
-dl
-svn
-git
-log
-staff
-vpn
-sslvpn
-ssh
-scanner
-sandbox
-ldap
-lab
-go
-demo
-console
-cms
-auth
-crm
-erp
-res
-static
-old
-new
-beta
-image
-service
-login
-3g
-docs
-it
-e
-live
-library
-files
-i
-d
-cp
-connect
-gateway
-lib
-preview
-backup
-share
-status
-assets
-user
-vote
-bugs
-cas
-feedback
-id
-edm
-survey
-union
-ceshi
-dev1
-updates
-phpmyadmin
-pma
-edit
-master
-xml
-control
-profile
-zhidao
-tool
-toolbox
-boss
-activity
-www
+test
+test2
+t
+dev
+1
+2
+3
+s1
+s2
+s3
+admin
+adm
+a
+b
+c
+m
+ht
+adminht
+webht
+web
+gm
+sys
+system
+manage
+manager
+mgr
+passport
+bata
+wei
+weixin
+wechat
+wx
+wiki
+upload
+ftp
+pic
+jira
+zabbix
+nagios
+bug
+bugzilla
+sql
+mysql
+db
+stmp
+pop
+imap
+mail
+zimbra
+exchange
+forum
+bbs
+list
+count
+counter
+img
+img01
+img02
+img03
+img04
+api
+cache
+js
+css
+app
+apps
+wap
+sms
+zip
+monitor
+proxy
+update
+upgrade
+stat
+stats
+data
+portal
+blog
+autodiscover
+en
+search
+so
+oa
+database
+home
+sso
+help
+vip
+s
+w
+down
+download
+downloads
+dl
+svn
+git
+log
+staff
+vpn
+sslvpn
+ssh
+scanner
+sandbox
+ldap
+lab
+go
+demo
+console
+cms
+auth
+crm
+erp
+res
+static
+old
+new
+beta
+image
+service
+login
+3g
+docs
+it
+e
+live
+library
+files
+i
+d
+cp
+connect
+gateway
+lib
+preview
+backup
+share
+status
+assets
+user
+vote
+bugs
+cas
+feedback
+id
+edm
+survey
+union
+ceshi
+dev1
+updates
+phpmyadmin
+pma
+edit
+master
+xml
+control
+profile
+zhidao
+tool
+toolbox
+boss
+activity
+www
@@ -9,55 +9,51 @@ OneForAll数据库导出模块
 """

 import fire
-from common import database
-from config import logger
+from common import utils
+from common.database import Database


-def export(table, db=None, valid=None, path=None, format='xlsx', output=False):
+def export(table, db=None, valid=None, dpath=None, format='xls', show=False):
    """
    OneForAll数据库导出模块

    Example:
-        python dbexport.py --db result.db --table name --format csv --output False
-        python dbexport.py --db result.db --table name --format csv --path= ./result.csv
+        python3 dbexport.py --table name --format csv --dir= ./result.csv
+        python3 dbexport.py --db result.db --table name --show False

    Note:
-        参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
-        参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
-        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+        参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+        参数dpath为None默认使用OneForAll结果目录

    :param str table:   要导出的表
    :param str db:      要导出的数据库路径(默认为results/result.sqlite3)
    :param int valid:   导出子域的有效性(默认None)
-    :param str format:  导出格式(默认xlsx)
-    :param str path:    导出路径(默认None)
-    :param bool output: 是否将导出数据输出到终端(默认False)
+    :param str format:  导出格式(默认xls)
+    :param str dpath:    导出目录(默认None)
+    :param bool show:   终端显示导出数据(默认False)
    """
-    db_conn = database.connect_db(db)
+    format = utils.check_format(format)
+    dpath = utils.check_dpath(dpath)
+    database = Database(db)
    if valid is None:
-        rows = database.get_data(db_conn, table)
+        rows = database.get_data(table)
    elif isinstance(valid, int):
-        rows = database.get_subdomain(db_conn, table, valid)
+        rows = database.get_subdomain(table, valid)
    else:
-        rows = database.get_data(db_conn, table)  # 意外情况导出全部子域
-    if output:
+        rows = database.get_data(table)  # 意外情况导出全部子域
+    if show:
        print(rows.dataset)
-    if not path:
-        path = 'export.' + format
-    logger.log('INFOR', f'正在将数据库中{table}表导出')
-    try:
-        with open(path, 'w') as file:
-            file.write(rows.export(format))
-            logger.log('INFOR', '成功完成导出')
-            logger.log('INFOR', path)
-    except TypeError:
-        with open(path, 'wb') as file:
-            file.write(rows.export(format))
-            logger.log('INFOR', '成功完成导出')
-            logger.log('INFOR', path)
-    except Exception as e:
-        logger.log('ERROR', e)
+    if format == 'txt':
+        data = str(rows.dataset)
+    else:
+        data = rows.export(format)
+    database.close()
+    fpath = dpath.joinpath(f'{table}.{format}')
+    utils.save_data(fpath, data)


 if __name__ == '__main__':
    fire.Fire(export)
+    # save('example_com_last', format='txt')
@@ -1 +1,2 @@
-example.com
+example.com
+hackfun.org
@@ -1 +0,0 @@
-# coding=utf-8
@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+"""
+github自动接管
+"""
+
+import json
+import base64
+import requests
+import config
+
+HEADERS = {
+    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "Accept-Language": "zh-CN,zh;q=0.9",
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
+}
+
+
+def github_takeover(url):
+    # 读取config配置文件
+    repo_name = url
+    print('[*]正在读取配置文件...')
+    user = config.github_api_user
+    token = config.github_api_token
+    CHECK_HEADERS = {
+        "Authorization": 'token ' + token,
+        "Accept": "application/vnd.github.switcheroo-preview+json"
+    }
+    repos_url = 'https://api.github.com/repos/' + user + '/' + repo_name
+    repos_r = requests.get(url=repos_url, headers=CHECK_HEADERS)
+    # 验证token是否正确
+    if 'message' in repos_r.json():
+        if repos_r.json()['message'] == 'Bad credentials':
+            print('[*]请检查Token是否正确')
+        elif repos_r.json()['message'] == 'Not Found':
+            print('[*]正在生成接管库...')  # 生成接管库
+            creat_repo_dict = {
+                "name": repo_name,
+                "description": "This is a subdomain takeover Repository",
+            }
+            creat_repo_url = 'https://api.github.com/user/repos'
+            creat_repo_r = requests.post(url=creat_repo_url,
+                                         headers=CHECK_HEADERS,
+                                         data=json.dumps(creat_repo_dict))
+            creat_repo_status = creat_repo_r.status_code
+            if creat_repo_status == 201:
+                print('[*]创建接管库' + repo_name + '成功，正在进行自动接管...')
+                # 接管文件生成
+                # index.html文件
+                html = b'''
+                <html>
+                    <p>Subdomain Takerover Test!</>
+                </html>
+                '''
+                html64 = base64.b64encode(html).decode('utf-8')
+                html_dict = {
+                    "message": "my commit message",
+                    "committer": {
+                        "name": "user",  # 提交id，非必改项
+                        "email": "user@163.com"  # 同上
+                    },
+                    "content": html64
+                }
+                # CNAME文件
+                cname_url = bytes(url, encoding='utf-8')
+                cname_url64 = base64.b64encode(cname_url).decode('utf-8')
+                url_dict = {
+                    "message": "my commit message",
+                    "committer": {
+                        "name": "user",
+                        "email": "user@163.com"
+                    },
+                    "content": cname_url64
+                }
+                html_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/index.html'
+                url_url = 'https://api.github.com/repos/' + user + '/' + repo_name + '/contents/CNAME'
+                html_r = requests.put(url=html_url, data=json.dumps(html_dict),
+                                      headers=CHECK_HEADERS)  # 上传index.html
+                cname_r = requests.put(url=url_url, data=json.dumps(url_dict),
+                                       headers=CHECK_HEADERS)  # 上传CNAME
+                rs = cname_r.status_code
+                if rs == 201:
+                    print('[*]生成接管库成功，正在开启Github pages...')
+                    page_url = "https://api.github.com/repos/" + user + "/" + url + "/pages"
+                    page_dict = {
+                        "source": {
+                            "branch": "master"
+                        }
+                    }
+                    page_r = requests.post(url=page_url,
+                                           data=json.dumps(page_dict),
+                                           headers=CHECK_HEADERS)  # 开启page
+                    if page_r.status_code == 201:
+                        print('[+]自动接管成功，请稍后访问http://' + str(url) + '查看结果')
+                    else:
+                        print('[+]开启Github pages失败，请检查网络或稍后重试...')
+                else:
+                    print('[+]生成接管库失败，请检查网络或稍后重试...')
+    elif url in repos_r.json()['name']:
+        print('[*]生成接管库失败，请检查https://github.com/' + user +
+              '?tab=repositories是否存在同名接管库...')
@@ -1,6 +1,4 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
 from config import logger
@@ -24,58 +22,53 @@ class CensysAPI(Query):
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        data = {
-            'query': 'parsed.names: example.com',
+            'query': f'parsed.names: {self.domain}',
            'page': 1,
            'fields': ['parsed.subject_dn'],
            'flatten': True}
        resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
        if not resp:
            return
-        resp_json = resp.json()
-        status = resp_json.get('status')
+        data = resp.json()
+        status = data.get('status')
        if status != 'ok':
            logger.log('ALERT', status)
            return
-        subdomains_find = self.match(self.domain, str(resp_json))
-        self.subdomains = self.subdomains.union(subdomains_find)
-        pages = resp_json.get('metadata').get('pages')
+        subdomains = self.match(self.domain, str(data))
+        self.subdomains = self.subdomains.union(subdomains)
+        pages = data.get('metadata').get('pages')
        for page in range(2, pages + 1):
            time.sleep(self.delay)
            data['page'] = page
            resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
            if not resp:
                return
-            subdomains_find = self.match(self.domain, str(resp.json()))
-            self.subdomains = self.subdomains.union(subdomains_find)
+            subdomains = self.match(self.domain, str(resp.json()))
+            self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not (self.id and self.secret):
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.id, self.secret):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = CensysAPI(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,10 +1,7 @@
-# coding=utf-8
 import time
-import queue
 import config
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class CertDBAPI(Query):
@@ -12,7 +9,7 @@ class CertDBAPI(Query):
        Query.__init__(self)
        self.domain = domain
        self.module = 'Certificate'
-        self.source = 'CertDBQuery'
+        self.source = 'CertDBAPIQuery'
        self.addr = 'https://api.spyse.com/v1/subdomains'
        self.token = config.certdb_api_token

@@ -25,45 +22,46 @@ class CertDBAPI(Query):
            time.sleep(self.delay)
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
-            params = {'domain': self.domain, 'api_token': self.token, 'page': page_num}
+            params = {'domain': self.domain,
+                      'api_token': self.token,
+                      'page': page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
-            resp_json = resp.json()
-            subdomains_find = utils.match_subdomain(self.domain, str(resp_json))
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            json = resp.json()
+            subdomains = utils.match_subdomain(self.domain, str(json))
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
            page_num += 1
-            if resp_json.get('count') < 30:  # 默认每次查询最多返回30条 当前条数小于30条说明已经查完
+            # 默认每次查询最多返回30条 当前条数小于30条说明已经查完
+            if json.get('count') < 30:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.token:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.token):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = CertDBAPI(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,7 +1,4 @@
-# coding=utf-8
-import queue
 import time
-
 from common import utils
 from common.query import Query

@@ -21,37 +18,38 @@ class CertSpotter(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'domain': self.domain, 'include_subdomains': 'true', 'expand': 'dns_names'}
+        params = {'domain': self.domain,
+                  'include_subdomains': 'true',
+                  'expand': 'dns_names'}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = utils.match_subdomain(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
+
    """
    query = CertSpotter(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,7 +1,4 @@
-# coding=utf-8
-import queue
 import time
-
 from common import utils
 from common.query import Query

@@ -25,33 +22,30 @@ class Crtsh(Query):
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = utils.match_subdomain(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Crtsh(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,7 +1,4 @@
-# coding=utf-8
-import queue
 import time
-
 from common import utils
 from common.query import Query

@@ -21,37 +18,36 @@ class Entrust(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'fields': 'subjectDN', 'domain': self.domain, 'includeExpired': 'true'}
+        params = {'fields': 'subjectDN',
+                  'domain': self.domain,
+                  'includeExpired': 'true'}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = utils.match_subdomain(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Entrust(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,7 +1,4 @@
-# coding=utf-8
-import queue
 import time
-
 from common import utils
 from common.query import Query

@@ -12,7 +9,8 @@ class Google(Query):
        self.domain = self.register(domain)
        self.module = 'Certificate'
        self.source = 'GoogleQuery'
-        self.addr = 'https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch'
+        self.addr = 'https://transparencyreport.google.com/' \
+                    'transparencyreport/api/v3/httpsreport/ct/certsearch'

    def query(self):
        """
@@ -21,37 +19,37 @@ class Google(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'include_expired': 'true', 'include_subdomains': 'true', 'domain': self.domain}
+        params = {'include_expired': 'true',
+                  'include_subdomains': 'true',
+                  'domain': self.domain}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = utils.match_subdomain(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Google(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,13 +1,12 @@
-# coding=utf-8
 """
-查询域名的NS记录(域名服务器记录，记录该域名由哪台域名服务器解析)，
-检查查出的域名服务器是否开启DNS域传送，如果开启且没做访问控制和身份验证便加以利用获取域名的所有记录
+查询域名的NS记录(域名服务器记录，记录该域名由哪台域名服务器解析)，检查查出的域名服务器是
+否开启DNS域传送，如果开启且没做访问控制和身份验证便加以利用获取域名的所有记录。

-DNS域传送(DNS zone transfer)指的是一台备用域名服务器使用来自主域名服务器的数据刷新自己的域数据库，
-目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
-当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获取某个域的所有记录。
+DNS域传送(DNS zone transfer)指的是一台备用域名服务器使用来自主域名服务器的数据刷新自己
+的域数据库，目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
+当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获
+取某个域的所有记录。
 """
-import queue
 import dns.resolver
 import dns.zone

@@ -25,9 +24,34 @@ class CheckAXFR(Module):
        self.domain = self.register(domain)
        self.module = 'Check'
        self.source = 'AXFRCheck'
-        self.nsservers = []
        self.results = []

+    def axfr(self, server):
+        """
+        执行域传送
+
+        :param server: 域名服务器
+        """
+        logger.log('DEBUG', f'尝试对{self.domain}的域名服务器{server}进行域传送')
+        try:
+            xfr = dns.query.xfr(server, self.domain, timeout=30.0)
+            zone = dns.zone.from_xfr(xfr)
+        except Exception as e:
+            logger.log('DEBUG', str(e))
+            logger.log('DEBUG', f'对{self.domain}的域名服务器{server}进行域传送失败')
+            return
+        names = zone.nodes.keys()
+        for name in names:
+            full_domain = str(name) + '.' + self.domain
+            subdomain = utils.match_subdomain(self.domain, full_domain)
+            self.subdomains = self.subdomains.union(subdomain)
+            record = zone[name].to_text(name)
+            self.results.append(record)
+        if self.results:
+            logger.log('DEBUG', f'发现{self.domain}在{server}上的域传送记录')
+            logger.log('DEBUG', '\n'.join(self.results))
+            self.results = []
+
    def check(self):
        """
        正则匹配响应头中的内容安全策略字段以发现子域名
@@ -38,58 +62,35 @@ class CheckAXFR(Module):
        except Exception as e:
            logger.log('ERROR', e)
            return
-        self.nsservers = [str(answer) for answer in answers]
-        if not len(self.nsservers):
+        nsservers = [str(answer) for answer in answers]
+        if not len(nsservers):
            logger.log('ALERT', f'没有找到{self.domain}的NS域名服务器记录')
            return
-        for nsserver in self.nsservers:
-            logger.log('DEBUG', f'正在尝试对{self.domain}的域名服务器{nsserver}进行域传送')
-            try:
-                xfr = dns.query.xfr(nsserver, self.domain)
-                zone = dns.zone.from_xfr(xfr)
-            except Exception as e:
-                logger.log('DEBUG', str(e))
-                logger.log('DEBUG', f'对{self.domain}的域名服务器{nsserver}进行域传送失败')
-                continue
-            else:
-                names = zone.nodes.keys()
-                for name in names:
-                    subdomain = utils.match_subdomain(self.domain, str(name) + '.' + self.domain)
-                    self.subdomains = self.subdomains.union(subdomain)
-                    record = zone[name].to_text(name)
-                    self.results.append(record)
-            if self.results:
-                logger.log('DEBUG', f'发现{self.domain}在{nsserver}上的域传送记录')
-                logger.log('DEBUG', '\n'.join(self.results))
-                self.results = []
+        for nsserver in nsservers:
+            self.axfr(nsserver)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}的域传送漏洞')
        self.check()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}的域传送漏洞')
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    check = CheckAXFR(domain)
-    check.run(rx_queue)
+    check.run()


 if __name__ == '__main__':
-    # do('ZoneTransfer.me')
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('ZoneTransfer.me')
+    do('example.com')
@@ -1,19 +1,15 @@
-# coding=utf-8
 """
 检查crossdomain.xml文件收集子域名
 """
-import queue

 from common.module import Module
-from common.utils import match_subdomain
-from config import logger
+from common import utils


 class CheckCDX(Module):
    """
    检查crossdomain.xml文件收集子域名
    """
-
    def __init__(self, domain: str):
        Module.__init__(self)
        self.domain = self.register(domain)
@@ -23,41 +19,43 @@ class CheckCDX(Module):
    def check(self):
        """
        检查crossdomain.xml收集子域名
-        :return:
        """
-        url = f'http://{self.domain}/crossdomain.xml'
-        self.header = self.get_header()
-        self.proxy = self.get_proxy(self.source)
-        resp = self.get(url)
-        if not resp:
+        urls = [f'http://{self.domain}/crossdomain.xml',
+                f'https://{self.domain}/crossdomain.xml',
+                f'http://www.{self.domain}/crossdomain.xml',
+                f'https://www.{self.domain}/crossdomain.xml']
+        response = None
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            response = self.get(url)
+            if response:
+                break
+        if not response:
            return
-        self.subdomains = match_subdomain(self.domain, resp.text)
+        self.subdomains = utils.match_subdomain(self.domain, response.text)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的crossdomain.xml')
        self.check()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的crossdomain.xml')
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口
+
    :param domain: 域名
-    :param rx_queue: 结果集队列
    """
    check = CheckCDX(domain)
-    check.run(rx_queue)
+    check.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('163.com', result_queue)
+    do('example.com')
@@ -1,10 +1,8 @@
 #!/usr/bin/env python3
-# coding=utf-8

 """
 检查域名证书收集子域名
 """
-import queue
 import socket
 import ssl

@@ -33,34 +31,30 @@ class CheckCert(Module):
        except Exception as e:
            logger.log('ERROR', e)
            return
-        subdomains_find = utils.match_subdomain(self.domain, str(cert_dict))
-        self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = utils.match_subdomain(self.domain, str(cert_dict))
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的证书中的子域')
+        self.begin()
        self.check()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的证书中的子域')
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    check = CheckCert(domain)
-    check.run(rx_queue)
+    check.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,9 +1,8 @@
-# coding=utf-8
 """
 检查内容安全策略收集子域名收集子域名
 """
-import queue
 import requests
+
 from common import utils
 from common.module import Module
 from config import logger
@@ -18,7 +17,7 @@ class CheckCSP(Module):
        Module.__init__(self)
        self.domain = self.register(domain)
        self.module = 'Check'
-        self.source = 'Content-Security-Policy'
+        self.source = 'ContentSecurityPolicy'
        self.header = header

    def check(self):
@@ -26,11 +25,20 @@ class CheckCSP(Module):
        正则匹配响应头中的内容安全策略字段以发现子域名
        """
        if not self.header:
-            url = f'http://www.{self.domain}'
-            resp = self.get(url)
-            if not resp:
+            urls = [f'http://{self.domain}',
+                    f'https://{self.domain}',
+                    f'http://www.{self.domain}',
+                    f'https://www.{self.domain}']
+            response = None
+            for url in urls:
+                self.header = self.get_header()
+                self.proxy = self.get_proxy(self.source)
+                response = self.get(url)
+                if response:
+                    break
+            if not response:
                return
-            self.header = resp.headers
+            self.header = response.headers
        csp = self.header.get('Content-Security-Policy')
        if not csp:
            logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段')
@@ -38,34 +46,29 @@ class CheckCSP(Module):
        logger.log('DEBUG', f'{self.domain}域的响应头存在内容安全策略字段')
        self.subdomains = utils.match_subdomain(self.domain, csp)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
+        self.begin()
        self.check()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
-        self.finish()


-def do(domain, rx_queue, header=None):  # 统一入口名字 方便多线程调用
+def do(domain, header=None):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    :param dict or None header: 响应头
    """
    check = CheckCSP(domain, header)
-    check.run(rx_queue)
+    check.run()


 if __name__ == '__main__':
-    # resp = requests.get('https://content-security-policy.com/')
-    result_queue = queue.Queue()
-    resp = requests.get('https://www.baidu.com/')
-    do('google-analytics.com', result_queue, resp.headers)
+    resp = requests.get('https://content-security-policy.com/')
+    do('google-analytics.com', resp.headers)
@@ -0,0 +1,64 @@
+"""
+检查内容安全策略收集子域名收集子域名
+"""
+import requests
+
+from common.module import Module
+from common import utils
+from config import logger
+
+
+class CheckRobots(Module):
+    """
+    检查robots.txt收集子域名
+    """
+
+    def __init__(self, domain):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Check'
+        self.source = 'Robots'
+
+    def check(self):
+        """
+        正则匹配域名的robots.txt文件中的子域
+        """
+        urls = [f'http://{self.domain}/robots.txt',
+                f'https://{self.domain}/robots.txt',
+                f'http://www.{self.domain}/robots.txt',
+                f'https://www.{self.domain}/robots.txt']
+        response = None
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            response = self.get(url, allow_redirects=False)
+            if response:
+                break
+        if not response:
+            return
+        self.subdomains = utils.match_subdomain(self.domain, response.text)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    check = CheckRobots(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    do('qq.com')
@@ -0,0 +1,77 @@
+"""
+检查内容安全策略收集子域名收集子域名
+"""
+import requests
+
+from common.module import Module
+from common import utils
+from config import logger
+
+
+class CheckRobots(Module):
+    """
+    检查sitemap收集子域名
+    """
+
+    def __init__(self, domain):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Check'
+        self.source = 'Sitemap'
+
+    def check(self):
+        """
+        正则匹配域名的sitemap文件中的子域
+        """
+        urls = [f'http://{self.domain}/sitemap.xml',
+                f'https://{self.domain}/sitemap.xml',
+                f'http://www.{self.domain}/sitemap.xml',
+                f'https://www.{self.domain}/sitemap.xml',
+                f'http://{self.domain}/sitemap.txt',
+                f'https://{self.domain}/sitemap.txt',
+                f'http://www.{self.domain}/sitemap.txt',
+                f'https://www.{self.domain}/sitemap.txt',
+                f'http://{self.domain}/sitemap.html',
+                f'https://{self.domain}/sitemap.html',
+                f'http://www.{self.domain}/sitemap.html',
+                f'https://www.{self.domain}/sitemap.html',
+                f'http://{self.domain}/sitemap_index.xml',
+                f'https://{self.domain}/sitemap_index.xml',
+                f'http://www.{self.domain}/sitemap_index.xml',
+                f'https://www.{self.domain}/sitemap_index.xml']
+        response = None
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            self.timeout = 10
+            response = self.get(url, allow_redirects=False)
+            if response:
+                break
+        if not response:
+            return
+        self.subdomains = utils.match_subdomain(self.domain, response.text)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    check = CheckRobots(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    do('qq.com')
@@ -1,5 +1,3 @@
-# coding=utf-8
-import queue
 import cdx_toolkit
 from common.crawl import Crawl
 from config import logger
@@ -28,10 +26,12 @@ class ArchiveCrawl(Crawl):
        for resp in cdx.iter(url, limit=limit):
            if resp.data.get('status') not in ['301', '302']:
                url = resp.data.get('url')
-                subdomains_find = self.match(self.register(domain), url + resp.text)
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                subdomains = self.match(self.register(domain),
+                                             url + resp.text)
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
@@ -41,26 +41,21 @@ class ArchiveCrawl(Crawl):
        for subdomain in self.subdomains:
            if subdomain != self.domain:
                self.crawl(subdomain, 10)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    crawl = ArchiveCrawl(domain)
-    crawl.run(rx_queue)
-    logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
-    logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
+    crawl.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,6 +1,3 @@
-# coding=utf-8
-import queue
-
 import cdx_toolkit
 from tqdm import tqdm

@@ -30,10 +27,11 @@ class CommonCrawl(Crawl):

        for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
            if resp.data.get('status') not in ['301', '302']:
-                subdomains_find = self.match(self.register(domain), resp.text)
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                subdomains = self.match(self.register(domain), resp.text)
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
@@ -43,26 +41,21 @@ class CommonCrawl(Crawl):
        for subdomain in self.subdomains:
            if subdomain != self.domain:
                self.crawl(subdomain, 10)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    crawl = CommonCrawl(domain)
-    crawl.run(rx_queue)
-    logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
-    logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
+    crawl.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class BinaryEdgeAPI(Query):
@@ -27,37 +24,32 @@ class BinaryEdgeAPI(Query):
        resp = self.get(url)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)
+        subdomains = self.match(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.api:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.api):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = BinaryEdgeAPI(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,6 +1,4 @@
-# coding=utf-8
 import time
-import queue
 from common.query import Query


@@ -23,33 +21,31 @@ class BufferOver(Query):
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = self.match(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = BufferOver(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,7 +1,4 @@
-# coding=utf-8
-import queue
 import time
-
 from common.query import Query


@@ -24,33 +21,31 @@ class Chinaz(Query):
        resp = self.get(self.addr)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = self.match(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Chinaz(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class ChinazAPI(Query):
@@ -26,37 +23,33 @@ class ChinazAPI(Query):
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.api:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.api):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = ChinazAPI(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,6 +1,4 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query

@@ -25,34 +23,33 @@ class CirclAPI(Query):
        resp = self.get(self.addr + self.domain, auth=(self.user, self.pwd))
        if not resp:
            return
-        subdomains_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
+        if not self.check(self.user, self.pwd):
+            return
        self.begin()
-        if self.user and self.pwd:
-            self.query()
-            self.save_json()
-            self.gen_result()
-            self.save_db()
-            rx_queue.put(self.results)
+        self.query()
        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = CirclAPI(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,11 +1,9 @@
-# coding=utf-8
-import queue
-import random
 import time
-
+import random
+import cloudscraper
 from bs4 import BeautifulSoup
-
 from common.query import Query
+from config import logger


 class DNSdb(Query):
@@ -14,59 +12,81 @@ class DNSdb(Query):
        self.domain = self.register(domain)
        self.module = 'Dataset'
        self.source = 'DNSdbQuery'
-        self.addr = 'https://www.dnsdb.org/'
+        self.addr = 'http://www.dnsdb.org/'
+        self.url = f'{self.addr}{self.domain}/'
+
+    def get_tokens(self):
+        """
+        绕过cloudFlare验证并获取taken
+
+        :return: 绕过失败返回None 成功返回tokens
+        """
+        scraper = cloudscraper.create_scraper()
+        scraper.interpreter = 'js2py'
+        scraper.proxies = self.get_proxy(self.source)
+        scraper.timeout = 10
+        try:
+            tokens = scraper.get_tokens(self.url)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return None
+        if len(tokens) != 2:
+            return None
+        return tokens

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        self.header = self.get_header()
-        self.header.update({'Referer': self.addr})
-        self.proxy = self.get_proxy(self.source)
-        url = self.addr + self.domain + '/'
-        resp = self.get(url)
+        tokens = self.get_tokens()
+        if not tokens:
+            logger.log('ALERT', f'{self.source}模块绕过cloudFlare检查失败')
+            return False
+        self.cookie = tokens[0]
+        self.header = {'User-Agent': tokens[1]}
+        self.timeout = 10
+        resp = self.get(self.url)
        if not resp:
            return
-        if resp.status_code == 200:
-            if 'index' in resp.text:
-                soup = BeautifulSoup(resp.text, features='lxml')
-                index_urls = set(map(lambda x: self.addr + self.domain + x.text, soup.find_all('a')))
-                for url in index_urls:
-                    self.delay = random.randint(2, 5)  # 休眠绕过CloudFlare的DDoS保护
-                    time.sleep(self.delay)
-                    resp = self.get(url)
-                    if not resp:
-                        return
-                    subdomains_find = self.match(self.domain, resp.text)
-                    self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
-            else:
-                subdomains_find = self.match(self.domain, resp.text)
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        if 'index' in resp.text:
+            soup = BeautifulSoup(resp.text, features='lxml')
+            base = self.addr+self.domain
+            urls = list(map(lambda a: base + '/' + a.get('href'),
+                            soup.find_all('a')))
+            urls = urls[:-1]  # idn域名暂时不考虑
+            for url in urls:
+                resp = self.get(url)
+                if not resp:
+                    return
+                subdomains = self.match(self.domain, resp.text)
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains)
+        else:
+            subdomains = self.match(self.domain, resp.text)
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = DNSdb(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,10 +1,7 @@
-# coding=utf-8
 import time
-import queue
 import config
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class DNSdbAPI(Query):
@@ -29,37 +26,34 @@ class DNSdbAPI(Query):
        if not resp:
            return
        if resp.status_code == 200:
-            subdomains_find = utils.match_subdomain(self.domain, resp.text)
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            subdomains = utils.match_subdomain(self.domain, resp.text)
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.api:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.api):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
+
    """
    query = DNSdbAPI(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,16 +1,9 @@
-# coding=utf-8
-import queue
 import time
-
 from common import utils
 from common.query import Query


 class DNSdumpster(Query):
-    """
-
-    """
-
    def __init__(self, domain):
        Query.__init__(self)
        self.domain = self.register(domain)
@@ -30,38 +23,38 @@ class DNSdumpster(Query):
        if not resp:
            return
        self.cookie = resp.cookies
-        data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'), 'targetip': self.domain}
+        data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'),
+                'targetip': self.domain}
        resp = self.post(self.addr, data)
        if not resp:
            return
-        subdomains_find = utils.match_subdomain(self.domain, resp.text)
-        if subdomains_find:
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = utils.match_subdomain(self.domain, resp.text)
+        if subdomains:
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = DNSdumpster(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+
+    do('example.com')
@@ -1,6 +1,3 @@
-# coding=utf-8
-import queue
-
 from common import utils
 from common.query import Query

@@ -24,34 +21,32 @@ class HackerTarget(Query):
        if not resp:
            return
        if resp.status_code == 200:
-            subdomains_find = utils.match_subdomain(self.domain, resp.text)
-            if subdomains_find:
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            subdomains = utils.match_subdomain(self.domain, resp.text)
+            if subdomains:
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = HackerTarget(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,6 +1,3 @@
-# coding=utf-8
-import queue
-
 import config
 from common.query import Query

@@ -22,48 +19,50 @@ class IPv4InfoAPI(Query):
        while True:
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
-            params = {'type': 'SUBDOMAINS', 'key': self.api, 'value': self.domain, 'page': page}
+            params = {'type': 'SUBDOMAINS', 'key': self.api,
+                      'value': self.domain, 'page': page}
            resp = self.get(self.addr, params)
            if not resp:
                return
            if resp.status_code != 200:
                break  # 请求不正常通常网络是有问题，不再继续请求下去
-            resp_json = resp.json()
-            subdomains_find = self.match(self.domain, str(resp_json))
-            if not subdomains_find:
-                break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
-            subdomains = resp_json.get('Subdomains')  # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
-            if len(subdomains) < 300:  # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
+            data = resp.json()
+            subdomains = self.match(self.domain, str(data))
+            if not subdomains:
                break
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
+            subdomains = data.get('Subdomains')
+            if subdomains:
+                # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
+                if len(subdomains) < 300:
+                    break
            page += 1
            if page >= 50:  # ipv4info子域查询接口最多允许查询50页
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = IPv4InfoAPI(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,6 +1,4 @@
-# coding=utf-8
 import hashlib
-import queue
 import re
 import time
 from urllib import parse
@@ -24,16 +22,21 @@ class NetCraft(Query):
        绕过NetCraft的JS验证
        """
        self.header = self.get_header()  # Netcraft会检查User-Agent
-        self.cookie = self.get(self.init).cookies
+        resp = self.get(self.init)
+        if not resp:
+            return None
+        self.cookie = resp.cookies
        cookie_value = self.cookie['netcraft_js_verification_challenge']
-        verify_taken = hashlib.sha1(parse.unquote(cookie_value).encode('utf-8')).hexdigest()
+        cookie_encode = parse.unquote(cookie_value).encode('utf-8')
+        verify_taken = hashlib.sha1(cookie_encode).hexdigest()
        self.cookie['netcraft_js_verification_response'] = verify_taken

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        self.bypass_verification()
+        if not self.bypass_verification():
+            return
        last = ''
        while True:
            time.sleep(self.delay)
@@ -43,39 +46,37 @@ class NetCraft(Query):
            resp = self.get(self.addr + last, params)
            if not resp:
                return
-            subdomains_find = self.match(self.domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
            if 'Next page' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                break
            last = re.search(r'&last=.*' + self.domain, resp.text).group(0)
            self.page_num += self.per_page_num

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = NetCraft(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,5 +1,3 @@
-# coding=utf-8
-import queue
 import random

 from common import utils
@@ -12,7 +10,7 @@ class PTRArchive(Query):
        self.domain = self.register(domain)
        self.module = 'Dataset'
        self.source = "PTRArchiveQuery"
-        self.addr = 'http://ptrarchive.com/tools/search3.htm'
+        self.addr = 'http://ptrarchive.com/tools/search4.htm'

    def query(self):
        """
@@ -20,40 +18,39 @@ class PTRArchive(Query):
        """
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        self.cookie = {'pa_id': str(random.randint(0, 1000000000))}  # 绕过主页前端JS验证
+        # 绕过主页前端JS验证
+        self.cookie = {'pa_id': str(random.randint(0, 1000000000))}
        params = {'label': self.domain, 'date': 'ALL'}
        resp = self.get(self.addr, params)
        if not resp:
            return
        if resp.status_code == 200:
-            subdomains_find = utils.match_subdomain(self.domain, resp.text)
-            if subdomains_find:
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            subdomains = utils.match_subdomain(self.domain, resp.text)
+            if subdomains:
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = PTRArchive(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,7 +1,4 @@
-# coding=utf-8
-import queue
 import time
-
 from common.query import Query


@@ -24,33 +21,31 @@ class Riddler(Query):
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = self.match(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Riddler(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,6 +1,4 @@
-# coding=utf-8
 import json
-import queue
 import time

 from common.query import Query
@@ -34,34 +32,32 @@ class Robtex(Query):
                resp = self.get(url)
                if not resp:
                    return
-                subdomains_find = self.match(self.domain, resp.text)
-                if subdomains_find:
-                    self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                subdomains = self.match(self.domain, resp.text)
+                if subdomains:
+                    # 合并搜索子域名搜索结果
+                    self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Robtex(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class SecurityTrailsAPI(Query):
@@ -28,39 +25,35 @@ class SecurityTrailsAPI(Query):
        resp = self.get(url, params)
        if not resp:
            return
-        subdomains_prefix = resp.json()['subdomains']
-        subdomains_find = [f'{prefix}.{self.domain}' for prefix in subdomains_prefix]
-        if subdomains_find:
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        prefixs = resp.json()['subdomains']
+        subdomains = [f'{prefix}.{self.domain}' for prefix in prefixs]
+        if subdomains:
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.api:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.api):
            return
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = SecurityTrailsAPI(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,5 +1,3 @@
-# coding=utf-8
-import queue
 import time

 from common.query import Query
@@ -28,38 +26,37 @@ class SiteDossier(Query):
            resp = self.get(url)
            if not resp:
                return
-            subdomains_find = self.match(self.domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
-            if 'Show next 100 items' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            # 搜索页面没有出现下一页时停止搜索
+            if 'Show next 100 items' not in resp.text:
                break
            self.page_num += self.per_page_num

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = SiteDossier(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -0,0 +1,55 @@
+import cloudscraper
+
+from common.query import Query
+from config import logger
+
+
+class ThreatCrowd(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.source = 'ThreatCrowdQuery'
+        self.addr = 'https://www.threatcrowd.org/searchApi' \
+                    '/v2/domain/report?domain='
+
+    def query(self):
+        # 绕过cloudFlare验证
+        scraper = cloudscraper.create_scraper()
+        scraper.interpreter = 'js2py'
+        scraper.proxies = self.get_proxy(self.source)
+        url = self.addr + self.domain
+        try:
+            resp = scraper.get(url, timeout=self.timeout)
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = ThreatCrowd(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# coding=utf-8

 """
 通过枚举域名常见的SRV记录并做查询来发现子域
@@ -7,7 +6,6 @@

 import asyncio
 import json
-import queue

 import aiodns

@@ -62,39 +60,36 @@ class BruteSRV(Module):
        for result in results:
            if result:
                for answer in result:
-                    subdomain = utils.match_subdomain(self.domain, answer.host)
-                    if subdomain:
-                        self.subdomains = self.subdomains.union(subdomain)
+                    subdomains = utils.match_subdomain(self.domain, answer.host)
+                    if subdomains:
+                        self.subdomains = self.subdomains.union(subdomains)
                    else:
                        logger.log('DEBUG', f'{answer.host}不是{self.domain}的子域')
        if not len(self.subdomains):
            logger.log('DEBUG', f'没有找到{self.domain}的SRV记录')

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
-        logger.log('DEBUG', f'开始枚举{self.domain}域的SRV记录')
        self.brute()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束枚举{self.domain}域的SRV记录')
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口
+
    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    brute = BruteSRV(domain)
-    brute.run(rx_queue)
+    brute.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+
+    do('example.com')
@@ -1,59 +1,56 @@
-# coding=utf-8
-import queue
-
-import config
-from common.query import Query
-
-
-class RiskIQ(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = self.register(domain)
-        self.module = 'Intelligence'
-        self.source = 'RiskIQQuery'
-        self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
-        self.username = config.riskiq_api_username
-        self.key = config.riskiq_api_key
-
-    def query(self):
-        """
-        向接口查询子域并做子域匹配
-        """
-        self.header = self.get_header()
-        self.proxy = self.get_proxy(self.source)
-        params = {'query': self.domain}
-        resp = self.get(url=self.addr, params=params, auth=(self.username, self.key))
-        if not resp:
-            return
-        resp_json = resp.json()
-        subdomains_find = resp_json.get('subdomains')
-        if subdomains_find:
-            self.subdomains = set(map(lambda x: x + '.' + self.domain, subdomains_find))
-
-    def run(self, rx_queue):
-        """
-        类执行入口
-        """
-        self.begin()
-        self.query()
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()
-
-
-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    :param rx_queue: 结果集队列
-    """
-    query = RiskIQ(domain)
-    query.run(rx_queue)
-
-
-if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+import config
+from common.query import Query
+
+
+class RiskIQ(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Intelligence'
+        self.source = 'RiskIQAPIQuery'
+        self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
+        self.user = config.riskiq_api_username
+        self.key = config.riskiq_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'query': self.domain}
+        resp = self.get(url=self.addr,
+                        params=params,
+                        auth=(self.user, self.key))
+        if not resp:
+            return
+        data = resp.json()
+        names = data.get('subdomains')
+        self.subdomains = set(map(lambda sub: f'{sub}.{self.domain}', names))
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.user, self.key):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = RiskIQ(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,61 +1,54 @@
-# coding=utf-8
-import queue
-
-import config
-from common.query import Query
-from config import logger
-
-
-class ThreatBookAPI(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = self.register(domain)
-        self.module = 'Intelligence'
-        self.source = 'ThreatBookAPIQuery'
-        self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
-        self.key = config.threatbook_api_key
-
-    def query(self, domain):
-        """
-        向接口查询子域并做子域匹配
-        """
-        self.header = self.get_header()
-        self.proxy = self.get_proxy(self.source)
-        params = {'apikey': self.key, 'domain': domain, 'field': 'sub_domains'}
-        resp = self.post(self.addr, params)
-        if not resp:
-            return
-        subdomain_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomain_find)
-
-    def run(self, rx_queue):
-        """
-        类执行入口
-        """
-        if not self.key:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
-            return
-        self.begin()
-        self.query(self.domain)
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()
-
-
-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    :param rx_queue: 结果集队列
-    """
-    query = ThreatBookAPI(domain)
-    query.run(rx_queue)
-
-
-if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+import config
+from common.query import Query
+
+
+class ThreatBookAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Intelligence'
+        self.source = 'ThreatBookAPIQuery'
+        self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
+        self.key = config.threatbook_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'apikey': self.key,
+                  'domain': self.domain,
+                  'field': 'sub_domains'}
+        resp = self.post(self.addr, params)
+        if not resp:
+            return
+        subdomains = self.match(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.key):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = ThreatBookAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,5 +1,3 @@
-# coding=utf-8
-import queue
 import time

 from common.query import Query
@@ -20,37 +18,36 @@ class ThreatMiner(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'e': 'subdomains_container', 'q': self.domain, 't': 0, 'rt': 10}
+        params = {'e': 'subdomains_container',
+                  'q': self.domain, 't': 0, 'rt': 10}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        subdomains_find = self.match(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        subdomains = self.match(self.domain, resp.text)
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = ThreatMiner(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,5 +1,3 @@
-# coding=utf-8
-import queue
 import time

 from common.query import Query
@@ -17,7 +15,7 @@ class VirusTotal(Query):
        self.addr = 'https://www.virustotal.com/ui/domains/{}/subdomains'
        self.domain = self.register(domain)

-    def query(self, domain):
+    def query(self):
        """
        向接口查询子域并做子域匹配
        """
@@ -29,51 +27,48 @@ class VirusTotal(Query):
                                'TE': 'Trailers'})
            self.proxy = self.get_proxy(self.source)
            params = {'limit': '40', 'cursor': next_cursor}
-            resp = self.get(url=self.addr.format(domain), params=params)
+            resp = self.get(url=self.addr.format(self.domain), params=params)
            if not resp:
                return
-            resp_json = resp.json()
-            subdomain_find = set()
-            datas = resp_json.get('data')
+            data = resp.json()
+            subdomains = set()
+            datas = data.get('data')

            if datas:
                for data in datas:
                    subdomain = data.get('id')
                    if subdomain:
-                        subdomain_find.add(subdomain)
+                        subdomains.add(subdomain)
            else:
                break
-            self.subdomains = self.subdomains.union(subdomain_find)
-            meta = resp_json.get('meta')
+            self.subdomains = self.subdomains.union(subdomains)
+            meta = data.get('meta')
            if meta:
                next_cursor = meta.get('cursor')
            else:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
        self.begin()
-        self.query(self.domain)
+        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = VirusTotal(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,9 +1,5 @@
-# coding=utf-8
-import queue
-
 import config
 from common.query import Query
-from config import logger


 class VirusTotalAPI(Query):
@@ -15,48 +11,43 @@ class VirusTotalAPI(Query):
        self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'
        self.key = config.virustotal_api_key

-    def query(self, domain):
+    def query(self):
        """
        向接口查询子域并做子域匹配
        """
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'apikey': self.key, 'domain': domain}
+        params = {'apikey': self.key, 'domain': self.domain}
        resp = self.get(self.addr, params)
        if not resp:
            return
-        resp_json = resp.json()
-        subdomain_find = set(resp_json.get('subdomains'))
-        self.subdomains = self.subdomains.union(subdomain_find)
+        data = resp.json()
+        subdomains = set(data.get('subdomains'))
+        self.subdomains = self.subdomains.union(subdomains)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.key:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.key):
            return
        self.begin()
-        self.query(self.domain)
+        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        self.finish()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = VirusTotalAPI(domain)
-    query.run(rx_queue)
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.search import Search
-from config import logger


 class Ask(Search):
@@ -33,54 +30,52 @@ class Ask(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += 1
            if '>Next<' not in resp.text:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-
+        self.begin()
        self.search(self.domain, full_search=True)
-
        # 排除同一子域搜索结果过多的子域以发现新的子域
        for statement in self.filter(self.domain, self.subdomains):
            self.search(self.domain, filtered_subdomain=statement)

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Ask(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.search import Search
 from bs4 import BeautifulSoup
-from config import logger
+from common.search import Search


 class Baidu(Search):
@@ -25,7 +22,8 @@ class Baidu(Search):
        """
        bs = BeautifulSoup(html, features='lxml')
        subdomains_all = set()
-        for find_res in bs.find_all('a', {'class': 'c-showurl'}):  # 获取搜索结果中所有的跳转URL地址
+        # 获取搜索结果中所有的跳转URL地址
+        for find_res in bs.find_all('a', {'class': 'c-showurl'}):
            url = find_res.get('href')
            subdomain = self.match_location(domain, url)
            subdomains_all = subdomains_all.union(subdomain)
@@ -45,31 +43,36 @@ class Baidu(Search):
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'wd': query, 'pn': self.page_num, 'rn': self.per_page_num}
+            params = {'wd': query, 'pn': self.page_num,
+                      'rn': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
            if len(domain) > 12:  # 解决百度搜索结果中域名过长会显示不全的问题
-                subdomains_find = self.redirect_match(domain, resp.text)  # 获取百度跳转URL响应头的Location字段获取直链
+                # 获取百度跳转URL响应头的Location字段获取直链
+                subdomains = self.redirect_match(domain, resp.text)
            else:
-                subdomains_find = self.match(domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+                subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += self.per_page_num
-            if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 搜索页面没有出现下一页时停止搜索
+            if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:
                break
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        self.begin()

        self.search(self.domain, full_search=True)

@@ -79,29 +82,28 @@ class Baidu(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Baidu(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.search import Search
-from config import logger


 class Bing(Search):
@@ -34,29 +31,32 @@ class Bing(Search):
            time.sleep(self.delay)
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'q': query, 'first': self.page_num, 'count': self.per_page_num}
+            params = {'q': query, 'first': self.page_num,
+                      'count': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomains_find = self.match(domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
-            if '<div class="sw_next>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
+            # 搜索页面没有出现下一页时停止搜索
+            if '<div class="sw_next>' not in resp.text:
                break
            self.page_num += self.per_page_num
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-
+        self.begin()
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -65,29 +65,28 @@ class Bing(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Bing(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.search import Search
-from config import logger


 class BingAPI(Search):
@@ -11,8 +8,9 @@ class BingAPI(Search):
        Search.__init__(self)
        self.domain = domain
        self.module = 'Search'
-        self.source = 'BingCustomSearch'
-        self.addr = 'https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search'
+        self.source = 'BingAPISearch'
+        self.addr = 'https://api.cognitive.microsoft.com/' \
+                    'bingcustomsearch/v7.0/search'
        self.id = config.bing_api_id
        self.key = config.bing_api_key
        self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数
@@ -38,27 +36,26 @@ class BingAPI(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomains_find = self.match(domain, str(resp.json()))
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(domain, str(resp.json()))
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += self.per_page_num
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not (self.id and self.key):
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.id, self.key):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-
+        self.begin()
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -67,29 +64,28 @@ class BingAPI(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = BingAPI(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,7 +1,5 @@
-# coding=utf-8
 import re
 import time
-import queue
 from common.search import Search
 from config import logger

@@ -32,26 +30,28 @@ class DuckDuckGO(Search):
            resp = self.post(self.addr, data)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            try:
                s = re.findall(r'name="s" value="(\d.*)"', resp.text)[-1]
                dc = re.findall(r'name="dc" value="(\d.*)"', resp.text)
            except Exception as e:
                logger.error(e)
                break
-            data.update({'s': s, 'nextParams': '', 'o': 'json', 'dc': dc, 'api': '/d.js'})
+            data.update({'s': s, 'nextParams': '', 'o': 'json',
+                         'dc': dc, 'api': '/d.js'})

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        self.begin()

        self.search(self.domain, full_search=True)

@@ -61,30 +61,30 @@ class DuckDuckGO(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    # return  # 暂时还有点问题
    search = DuckDuckGO(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
-import time
-import queue
 import random
+import time
 from common.search import Search
-from config import logger


 class Exalead(Search):
@@ -30,28 +27,29 @@ class Exalead(Search):
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'q': query, 'elements_per_page': '30', "start_index": self.page_num}
+            params = {'q': query, 'elements_per_page': '30',
+                      "start_index": self.page_num}
            resp = self.get(url=self.addr, params=params)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += self.per_page_num
            if self.page_num > 1999:
                break
            if 'title="Go to the next page"' not in resp.text:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        self.begin()

        self.search(self.domain, full_search=True)

@@ -62,29 +60,28 @@ class Exalead(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Exalead(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,68 +1,69 @@
-# coding=utf-8
-import base64
-import queue
-import time
-
-import config
-from common.search import Search
-from config import logger
-
-
-class FoFa(Search):
-    def __init__(self, domain):
-        Search.__init__(self)
-        self.domain = domain
-        self.module = 'Search'
-        self.source = 'FoFaSearch'
-        self.addr = 'https://fofa.so/api/v1/search/all'
-        self.delay = 1
-        self.email = config.fofa_api_email
-        self.key = config.fofa_api_key
-
-    def search(self):
-        """
-        发送搜索请求并做子域匹配
-        """
-        self.page_num = 1
-        query_base64 = base64.b64encode(f'domain={self.domain}'.encode('utf-8'))
-        while True:
-            time.sleep(self.delay)
-            self.header = self.get_header()
-            self.proxy = self.get_proxy(self.source)
-            query = {'email': self.email, 'key': self.key, 'qbase64': query_base64, 'page': self.page_num}
-            resp = self.get(self.addr, query)
-            if not resp:
-                return
-            subdomain_find = self.match(self.domain, resp.text)
-            self.subdomains = self.subdomains.union(subdomain_find)
-            self.page_num += 1
-
-    def run(self, rx_queue):
-        """
-        类执行入口
-        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-
-        self.search()
-
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
-
-
-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    :param rx_queue: 结果集队列
-    """
-    search = FoFa(domain)
-    search.run(rx_queue)
-
-
-if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+import base64
+import time
+
+import config
+from common.search import Search
+from config import logger
+
+
+class FoFa(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'FoFaAPISearch'
+        self.addr = 'https://fofa.so/api/v1/search/all'
+        self.delay = 1
+        self.email = config.fofa_api_email
+        self.key = config.fofa_api_key
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        self.page_num = 1
+        subdomain_encode = f'subdomain={self.domain}'.encode('utf-8')
+        query_data = base64.b64encode(subdomain_encode)
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = {'email': self.email,
+                     'key': self.key,
+                     'qbase64': query_data,
+                     'page': self.page_num}
+            resp = self.get(self.addr, query)
+            if not resp:
+                return
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains = self.subdomains.union(subdomains)
+            self.page_num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.email, self.key):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = FoFa(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
-import time
-import queue
 import random
+import time
 from common.search import Search
-from config import logger


 class Google(Search):
@@ -43,24 +40,24 @@ class Google(Search):
            resp = self.get(url=self.addr, params=payload)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            page_num += per_page_num
            if 'start=' + str(page_num) not in resp.text:
                break
            if '302 Moved' in resp.text:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        self.begin()

        self.search(self.domain, full_search=True)

@@ -70,29 +67,28 @@ class Google(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Google(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.search import Search
-from config import logger


 class GoogleAPI(Search):
@@ -32,32 +29,30 @@ class GoogleAPI(Search):
            time.sleep(self.delay)
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
-            params = {'key': self.key, 'cx': self.cx, 'q': word, 'fields': 'items/link',
+            params = {'key': self.key, 'cx': self.cx,
+                      'q': word, 'fields': 'items/link',
                      'start': self.page_num, 'num': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomain_find = self.match(domain, str(resp.json()))
-            if not subdomain_find:
+            subdomains = self.match(domain, str(resp.json()))
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += self.per_page_num
            if self.page_num > 100:  # 免费的API只能查询前100条结果
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not (self.cx and self.key):
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.cx, self.key):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-
+        self.begin()
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -66,29 +61,28 @@ class GoogleAPI(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = GoogleAPI(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,10 +1,5 @@
-# coding=utf-8
-import queue
-
 import config
-# from shodan import Shodan
 from common.search import Search
-from config import logger


 class ShodanAPI(Search):
@@ -12,7 +7,7 @@ class ShodanAPI(Search):
        Search.__init__(self)
        self.domain = self.register(domain)
        self.module = 'Search'
-        self.source = 'ShodanSearch'
+        self.source = 'ShodanAPISearch'
        self.addr = 'https://api.shodan.io/shodan/host/search'
        self.key = config.shodan_api_key

@@ -25,45 +20,41 @@ class ShodanAPI(Search):
        query = 'hostname:.' + self.domain
        page = 1
        while True:
-            params = {'key': self.key, 'page': page, 'query': query, 'minify': True, 'facets': {'hostnames'}}
+            params = {'key': self.key, 'page': page, 'query': query,
+                      'minify': True, 'facets': {'hostnames'}}
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomain_find = self.match(self.domain, resp.text)
-            if subdomain_find:
-                self.subdomains = self.subdomains.union(subdomain_find)
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            if subdomains:
+                self.subdomains = self.subdomains.union(subdomains)
            page += 1

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.key:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.key):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-
+        self.begin()
        self.search()
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = ShodanAPI(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    results = queue.Queue()
-    do('qq.com', results)
+    do('example.com')
@@ -1,8 +1,6 @@
-# coding=utf-8
 import time
-import queue
+
 from common.search import Search
-from config import logger


 class So(Search):
@@ -33,25 +31,27 @@ class So(Search):
            resp = self.get(url=self.addr, params=payload)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            page_num += 1
-            if '<a id="snext"' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 搜索页面没有出现下一页时停止搜索
+            if '<a id="snext"' not in resp.text:
                break
-            if self.page_num * self.per_page_num >= self.limit_num:  # 搜索条数限制
+            # 搜索条数限制
+            if self.page_num * self.per_page_num >= self.limit_num:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-
+        self.begin()
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -60,29 +60,28 @@ class So(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = So(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,8 +1,4 @@
-# coding=utf-8
-import queue
-
 from common.search import Search
-from config import logger


 class Sogou(Search):
@@ -27,28 +23,32 @@ class Sogou(Search):
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            word = 'site:' + domain + filtered_subdomain
-            payload = {'query': word, 'page': self.page_num, "num": self.per_page_num}
+            payload = {'query': word, 'page': self.page_num,
+                       "num": self.per_page_num}
            resp = self.get(self.addr, payload)
            if not resp:
                return
-            subdomain_find = self.match(domain, resp.text)
-            if not subdomain_find:
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomain_find)
+            self.subdomains = self.subdomains.union(subdomains)
            self.page_num += 1
-            if '<a id="sogou_next"' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 搜索页面没有出现下一页时停止搜索
+            if '<a id="sogou_next"' not in resp.text:
                break
-            if self.page_num * self.per_page_num >= self.limit_num:  # 搜索条数限制
+            # 搜索条数限制
+            if self.page_num * self.per_page_num >= self.limit_num:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        self.begin()

        self.search(self.domain, full_search=True)

@@ -58,29 +58,28 @@ class Sogou(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Sogou(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.search import Search
-from config import logger


 class Yahoo(Search):
@@ -39,24 +36,26 @@ class Yahoo(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomains_find = self.match(domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
            if '>Next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                break
            self.page_num += self.per_page_num
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        self.begin()

        self.search(self.domain, full_search=True)

@@ -66,29 +65,28 @@ class Yahoo(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Yahoo(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.search import Search
-from config import logger


 class Yandex(Search):
@@ -25,34 +22,40 @@ class Yandex(Search):
        :param bool full_search: 全量搜索
        """
        self.page_num = 0  # 二次搜索重新置0
-        self.cookie = self.get(self.init).cookies  # 获取cookie bing在搜索时需要带上cookie
+        resp = self.get(self.init)
+        if not resp:
+            return
+        self.cookie = resp.cookies  # 获取cookie
        while True:
            time.sleep(self.delay)
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'text': query, 'p': self.page_num, 'numdoc': self.per_page_num}
+            params = {'text': query, 'p': self.page_num,
+                      'numdoc': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomains_find = self.match(domain, resp.text)
-            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
+            subdomains = self.match(domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains)
            if '>next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                break
            self.page_num += 1
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
+        self.begin()

        self.search(self.domain, full_search=True)

@@ -62,29 +65,28 @@ class Yandex(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Yandex(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -1,6 +1,4 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.search import Search
 from config import logger
@@ -20,7 +18,6 @@ class ZoomEyeAPI(Search):
    def login(self):
        """
        登陆获取查询taken
-        :return:
        """
        url = 'https://api.zoomeye.org/user/login'
        data = {'username': self.user, 'password': self.pwd}
@@ -28,12 +25,12 @@ class ZoomEyeAPI(Search):
        if not resp:
            logger.log('FETAL', f'登录失败无法获取{self.source}的访问token')
            return
-        resp_json = resp.json()
+        data = resp.json()
        if resp.status_code == 200:
-            # print('登陆成功')
-            return resp_json.get('access_token')
+            logger.log('DEBUG', f'{self.source}模块登录成功')
+            return data.get('access_token')
        else:
-            logger.log('ALERT', resp_json.get('message'))
+            logger.log('ALERT', data.get('message'))
            exit(1)

    def search(self):
@@ -51,44 +48,39 @@ class ZoomEyeAPI(Search):
            resp = self.get(self.addr, params)
            if not resp:
                return
-            subdomain_find = self.match(self.domain, resp.text)
-            self.subdomains = self.subdomains.union(subdomain_find)
+            subdomains = self.match(self.domain, resp.text)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains = self.subdomains.union(subdomains)
            page_num += 1
            if page_num > 500:
                break
            if resp.status_code == 403:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not (self.user and self.pwd):
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.user, self.pwd):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-
+        self.begin()
        self.search()
-
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = ZoomEyeAPI(domain)
-    search.run(rx_queue)
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+    do('example.com')
@@ -9,6 +9,7 @@ OneForAll是一款功能强大的子域收集工具
 """

 import asyncio
+
 import fire
 import config
 import dbexport
@@ -16,54 +17,136 @@ from datetime import datetime
 from config import logger
 from collect import Collect
 from aiobrute import AIOBrute
-from common import utils, database, resolve, request
+from common import utils, resolve, request
+from common.database import Database
+from takeover import Takeover

+yellow = '\033[01;33m'
+white = '\033[01;37m'
+green = '\033[01;32m'
+blue = '\033[01;34m'
+red = '\033[1;31m'
+end = '\033[0m'

-banner = """\033[01;33m
+version = white + '{' + red + 'v0.0.5#dev' + white + '}'
+
+banner = f"""{yellow}
             ___             _ _ 
- ___ ___ ___|  _|___ ___ ___| | | \033[01;37m{\033[1;31mv0.0.2#dev\033[01;37m}\033[01;32m
-| . |   | -_|  _| . |  _| .'| | | \033[01;34m
-|___|_|_|___|_| |___|_| |__,|_|_| \033[0m\033[4;37mgit.io/fjHT1\033[0m\n
-        """
+ ___ ___ ___|  _|___ ___ ___| | | {version}{green}
+| . |   | -_|  _| . |  _| .'| | | {blue}
+|___|_|_|___|_| |___|_| |__,|_|_| {white}git.io/fjHT1{end}
+"""


 class OneForAll(object):
    """
    OneForAll是一款功能强大的子域收集工具

-    Version: 0.0.2
-    Project: https://github.com/shmilylty/OneForAll/
+    Version: 0.0.5
+    Project: https://git.io/fjHT1

    Example:
-        python oneforall.py --target example.com run
-        python oneforall.py --target example.com --brute True --port medium valid 1 run
-        python oneforall.py --target ./domains.txt --format csv --path= ./result.csv  --output True run
+        python3 oneforall.py --target example.com run
+        python3 oneforall.py --target ./subdomains.txt run
+        python3 oneforall.py --target example.com --valid None run
+        python3 oneforall.py --target example.com --brute True run
+        python3 oneforall.py --target example.com --port medium run
+        python3 oneforall.py --target example.com --format csv run
+        python3 oneforall.py --target example.com --verify False run
+        python3 oneforall.py --target example.com --takeover False run
+        python3 oneforall.py --target example.com --show True run

    Note:
-        参数valid可选值有1，0，None，分别表示导出有效，无效，全部子域
+        参数valid可选值1，0，None分别表示导出有效，无效，全部子域
+        参数verify为True会尝试解析和请求子域并根据结果给子域有效性打上标签
        参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
-        参数format可选格式有'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
-        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'

-    :param str target:  单个域名或者每行一个域名的文件路径
-    :param bool brute:  是否使用爆破模块(默认禁用)
-    :param str port:    HTTP请求验证的端口范围(默认medium)
+    :param str target:  单个域名或者每行一个域名的文件路径(必需参数)
+    :param bool brute:  使用爆破模块(默认False)
+    :param bool verify: 验证子域有效性(默认True)
+    :param str port:    请求验证的端口范围(默认medium)
    :param int valid:   导出子域的有效性(默认1)
-    :param str format:  导出格式(默认xlsx)
-    :param str path:    导出路径(默认None)
-    :param bool output: 是否将导出数据输出到终端(默认False)
+    :param str format:  导出格式(默认xls)
+    :param bool show:   终端显示导出数据(默认False)
    """
-    def __init__(self, target, brute=False, port='medium', valid=1, path=None, format='xlsx', output=False):
+    def __init__(self, target, brute=None, verify=None, port='medium', valid=1,
+                 format='xls', takeover=True, show=False):
        self.target = target
        self.port = port
        self.domains = set()
-        self.domain = ''
+        self.domain = str()
        self.datas = list()
-        self.brute = brute or config.enable_brute_module
+        self.brute = brute
+        self.verify = verify
+        self.takeover = takeover
        self.valid = valid
-        self.path = path
        self.format = format
-        self.output = output
+        self.show = show
+
+    def main(self):
+        if self.brute is None:
+            self.brute = config.enable_brute_module
+        if self.verify is None:
+            self.verify = config.enable_verify_subdomain
+        rename_table = self.domain + '_last'
+        collect = Collect(self.domain, export=False)
+        collect.run()
+        if self.brute:
+            # 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
+            brute = AIOBrute(self.domain, export=False)
+            brute.run()
+
+        db = Database()
+        db.copy_table(self.domain, self.domain+'_ori')
+        db.remove_invalid(self.domain)
+        db.deduplicate_subdomain(self.domain)
+        # 不验证子域的情况
+        if not self.verify:
+            # 数据库导出
+            self.valid = None
+            dbexport.export(self.domain, valid=self.valid, format=self.format,
+                            show=self.show)
+            db.drop_table(rename_table)
+            db.rename_table(self.domain, rename_table)
+            return
+        # 开始验证子域工作
+        self.datas = db.get_data(self.domain).as_dict()
+        loop = asyncio.get_event_loop()
+        asyncio.set_event_loop(loop)
+
+        # 解析域名地址
+        task = resolve.bulk_query_a(self.datas)
+        self.datas = loop.run_until_complete(task)
+
+        # 保存解析结果
+        resolve_table = self.domain + '_res'
+        db.drop_table(resolve_table)
+        db.create_table(resolve_table)
+        db.save_db(resolve_table, self.datas, 'resolve')
+
+        # 请求域名地址
+        task = request.bulk_get_request(self.datas, self.port)
+        self.datas = loop.run_until_complete(task)
+        # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
+        loop.run_until_complete(asyncio.sleep(0.25))
+
+        db.clear_table(self.domain)
+        db.save_db(self.domain, self.datas)
+
+        # 数据库导出
+        dbexport.export(self.domain, valid=self.valid, format=self.format,
+                        show=self.show)
+        db.drop_table(rename_table)
+        db.rename_table(self.domain, rename_table)
+        db.close()
+        # 子域接管检查
+
+        if self.takeover:
+            subdomains = set(map(lambda x: x.get('subdomain'), self.datas))
+            takeover = Takeover(subdomains)
+            takeover.run()

    def run(self):
        print(banner)
@@ -73,28 +156,7 @@ class OneForAll(object):
        self.domains = utils.get_domains(self.target)
        if self.domains:
            for self.domain in self.domains:
-                collect = Collect(self.domain, export=False)
-                collect.run()
-                if self.brute:
-                    # 由于爆破会有大量dns解析请求 并发常常会导致其他任务中的网络请求超时
-                    brute = AIOBrute(self.domain)
-                    brute.run()
-                table_name = self.domain.replace('.', '_')
-                db_conn = database.connect_db()
-                self.datas = database.get_data(db_conn, table_name).as_dict()
-                loop = asyncio.get_event_loop()
-                asyncio.set_event_loop(loop)
-                self.datas = loop.run_until_complete(resolve.bulk_query_a(self.datas))
-                self.datas = loop.run_until_complete(request.bulk_get_request(self.datas, self.port))
-                loop.run_until_complete(asyncio.sleep(0.25))  # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
-                loop.close()
-                database.clear_table(db_conn, table_name)
-                database.save_db(db_conn, table_name, self.datas)
-                # 数据库导出
-                if not self.path:
-                    self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
-                dbexport.export(table_name, db_conn, self.valid, self.path, self.format, self.output)
-                db_conn.close()
+                self.main()
        else:
            logger.log('FATAL', f'获取域名失败')
        logger.log('INFOR', f'结束运行OneForAll')
@@ -103,3 +165,4 @@ class OneForAll(object):
 if __name__ == '__main__':
    fire.Fire(OneForAll)
    # OneForAll('example.com').run()
+    # OneForAll('./domains.txt').run()
@@ -0,0 +1,2 @@
+www.example.com
+www.hackfun.org
@@ -0,0 +1,166 @@
+#!/usr/bin/python3
+# coding=utf-8
+
+"""
+OneForAll子域接管模块
+
+:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
+:license: GNU General Public License v3.0, see LICENSE for more details.
+"""
+import time
+import json
+from threading import Thread
+from queue import Queue
+
+import fire
+from tablib import Dataset
+from tqdm import tqdm
+
+import config
+from config import logger
+from common import resolve, utils
+from common.module import Module
+from common.domain import Domain
+
+
+def get_fingerprint():
+    path = config.data_storage_path.joinpath('fingerprints.json')
+    with open(path) as file:
+        fingerprints = json.load(file)
+    return fingerprints
+
+
+def get_cname(subdomain):
+    resolver = resolve.dns_resolver()
+    try:
+        answers = resolver.query(subdomain, 'CNAME')
+    except Exception as e:
+        logger.log('DEBUG', e.args)
+        return None
+    for answer in answers:
+        return answer.to_text()  # 一个子域只有一个CNAME记录
+
+
+def get_maindomain(subdomain):
+    return Domain(subdomain).registered()
+
+
+class Takeover(Module):
+    """
+    OneForAll多线程子域接管风险检查模块
+
+    Example:
+        python3 takeover.py --target www.example.com  --format csv run
+        python3 takeover.py --target ./subdomains.txt --thread 10 run
+
+    Note:
+        参数format可选格式有'txt', 'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+                          'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+        参数dpath为None默认使用OneForAll结果目录
+
+    :param str target:  单个子域或者每行一个子域的文件路径(必需参数)
+    :param int thread:  线程数(默认100)
+    :param str format:  导出格式(默认xls)
+    :param str dpath:   导出目录(默认None)
+    """
+    def __init__(self, target, thread=100, dpath=None, format='xls'):
+        Module.__init__(self)
+        self.subdomains = set()
+        self.module = 'Check'
+        self.source = 'Takeover'
+        self.target = target
+        self.thread = thread
+        self.dpath = dpath
+        self.format = format
+        self.fingerprints = None
+        self.subdomainq = Queue()
+        self.bar = tqdm()
+        self.cnames = list()
+        self.results = Dataset()
+
+    def save(self):
+        logger.log('INFOR', '正在保存检查结果')
+        if self.format == 'txt':
+            data = str(self.results)
+        else:
+            data = self.results.export(self.format)
+        fpath = self.dpath.joinpath(f'takeover.{self.format}')
+        utils.save_data(fpath, data)
+
+    def compare(self, subdomain, cname, responses):
+        domain_resp = self.get('http://' + subdomain, check=False)
+        cname_resp = self.get('http://'+cname, check=False)
+        if domain_resp is None or cname_resp is None:
+            return
+
+        for resp in responses:
+            if resp in domain_resp.text and resp in cname_resp.text:
+                logger.log('ALERT', f'{subdomain}存在子域接管风险')
+                self.results.append([subdomain, cname])
+                break
+
+    def check(self):
+        while not self.subdomainq.empty():  # 保证域名队列遍历结束后能退出线程
+            subdomain = self.subdomainq.get()  # 从队列中获取域名
+            cname = get_cname(subdomain)
+            maindomain = get_maindomain(cname)
+            if cname is None:
+                continue
+            for fingerprint in self.fingerprints:
+                cnames = fingerprint.get('cname')
+                if maindomain not in cnames:
+                    continue
+                responses = fingerprint.get('response')
+                self.compare(subdomain, cname, responses)
+
+    def progress(self):
+        while not self.subdomainq.empty():
+            done = self.bar.total - self.subdomainq.qsize()
+            self.bar.n = done
+            self.bar.update()
+        self.bar.close()
+
+    def run(self):
+        start = time.time()
+        logger.log('INFOR', f'开始执行{self.source}模块')
+        self.format = utils.check_format(self.format)
+        self.dpath = utils.check_dpath(self.dpath)
+        self.subdomains = utils.get_domains(self.target)
+        if self.subdomains:
+            logger.log('INFOR', f'正在检查子域接管风险')
+            self.fingerprints = get_fingerprint()
+            self.results.headers = ['subdomain', 'cname']
+            # 创建待检查的子域队列
+            for domain in self.subdomains:
+                self.subdomainq.put(domain)
+            # 设置进度
+            self.bar.total = self.subdomainq.qsize()
+            self.bar.desc = 'Progress'
+            self.bar.ncols = True
+            # 进度线程
+            threads = []
+            thread = Thread(target=self.progress, daemon=True)
+            thread.start()
+            threads.append(thread)
+            # 检查线程
+            for _ in range(self.thread):
+                thread = Thread(target=self.check, daemon=True)
+                thread.start()
+                threads.append(thread)
+            for thread in threads:
+                thread.join()
+            self.save()
+        else:
+            logger.log('FATAL', f'获取域名失败')
+        end = time.time()
+        elapsed = round(end - start, 1)
+        logger.log('INFOR', f'{self.source}模块耗时{elapsed}秒'
+                            f'发现{len(self.results)}个子域存在接管风险')
+        logger.log('DEBUG', f'结束执行{self.source}模块')
+
+
+if __name__ == '__main__':
+    fire.Fire(Takeover)
+    # takeover = Takeover('www.example.com')
+    # takeover = Takeover('./subdomains.txt')
+    # takeover.run()
Author	SHA1	Message	Date
shmilylty	f0a3b44c53	v0.0.5	2019-08-19 20:05:01 +08:00
shmilylty	472da35a36	tqdm修改	2019-08-19 19:58:00 +08:00
shmilylty	b60d86fdc3	tqdm配置	2019-08-19 19:45:11 +08:00
shmilylty	9b99b9fd35	添加不能正常安装依赖问题的解决方法	2019-08-19 19:00:29 +08:00
shmilylty	dde3da84fd	更新依赖	2019-08-19 18:56:08 +08:00
shmilylty	147f1030c3	添加takeover使用帮助	2019-08-19 18:44:23 +08:00
shmilylty	43d4b90782	pep8	2019-08-19 18:21:47 +08:00
shmilylty	dee7ee2d2c	添加github自动接管	2019-08-19 18:11:36 +08:00
shmilylty	bf8df3bc02	添加进度条修复线程退出卡死bug	2019-08-19 18:10:44 +08:00
shmilylty	82973c295c	标题信息获取处理	2019-08-19 02:24:00 +08:00
shmilylty	ac654f9bab	添加默认子域接管检查目标列表	2019-08-18 19:35:26 +08:00
shmilylty	310c40d7aa	添加子域接管检查功能	2019-08-18 19:34:16 +08:00
shmilylty	b3b528966c	重构	2019-08-18 19:33:23 +08:00
shmilylty	e31ff755f4	重构响应检查	2019-08-18 19:32:30 +08:00
shmilylty	101742de3c	添加部分公用函数	2019-08-18 19:31:50 +08:00
shmilylty	25e8541964	添加进度条	2019-08-18 19:31:22 +08:00
shmilylty	1251efd752	使用回调	2019-08-18 19:30:54 +08:00
shmilylty	61075f40ac	添加超时	2019-08-18 19:29:06 +08:00
shmilylty	39b92b79d9	pep8	2019-08-18 19:27:33 +08:00
shmilylty	0e88591a3b	修改默认导出格式为xls	2019-08-18 19:27:09 +08:00
shmilylty	32b06362bd	修改默认请求超时为30秒	2019-08-18 19:26:30 +08:00
shmilylty	298ef2072c	修改默认目前域名	2019-08-18 19:11:51 +08:00
shmilylty	7390e941a7	重命名	2019-08-18 19:02:54 +08:00
shmilylty	1fad2a413e	pep8	2019-08-18 19:01:43 +08:00
shmilylty	de96822c0d	添加请求超时	2019-08-18 02:45:32 +08:00
shmilylty	4c6af1a925	pep8	2019-08-18 02:12:40 +08:00
shmilylty	7ccddb93f7	修复误把domain替换为subdomain的问题	2019-08-18 02:12:12 +08:00
shmilylty	3883a14365	pep8	2019-08-18 00:03:08 +08:00
shmilylty	837b2f4d51	pep8	2019-08-18 00:02:25 +08:00
Jing Ling	03126872bb	Merge pull request #13 from Echocipher/master subdomain takeover	2019-08-17 14:57:30 +08:00
shmilylty	8eaf1bd60b	修改parsed.names为正确值，解决[问题14](https://github.com/shmilylty/OneForAll/issues/14 )。	2019-08-16 23:03:49 +08:00
shmilylty	7f9fbac68b	添加threatcrowd模块	2019-08-16 19:49:39 +08:00
shmilylty	335f6b6c0c	添加常见问题与问答文档	2019-08-16 19:41:21 +08:00
shmilylty	f52768632a	pep8	2019-08-16 19:13:19 +08:00
shmilylty	8d46ce0862	添加绕过cloudFlare检查	2019-08-16 18:33:11 +08:00
shmilylty	11cf46a006	添加cloudscraper和Js2Py依赖	2019-08-16 16:48:51 +08:00
Echocipher	f1e2b739e1	Update takeover.py	2019-08-16 16:44:21 +08:00
Echocipher	3e4a067c6b	Update takeover.py	2019-08-16 16:43:48 +08:00
shmilylty	d2dc83ebb2	添加ThreatCrowd接口	2019-08-16 16:22:33 +08:00
echocipher	c040501ee4	subdomain takeover	2019-08-16 15:35:40 +08:00
echocipher	70b1ddfda2	subdomain takeover	2019-08-16 15:31:45 +08:00
shmilylty	cf2e91bfcc	添加其他支持格式说明	2019-08-15 01:57:11 +08:00
Jing Ling	daed4d5a7c	Set theme jekyll-theme-architect	2019-08-14 23:58:17 +08:00
shmilylty	e99497fdc7	美化排版	2019-08-14 23:29:04 +08:00
shmilylty	cb7df51209	添加码云厂库链接和说明	2019-08-14 23:12:12 +08:00
shmilylty	a3c8834f05	美化排版	2019-08-14 22:22:30 +08:00
shmilylty	3f4336cca9	修改默认导出格式为xls，添加txt,rst,jira格式输出。	2019-08-14 19:07:52 +08:00
shmilylty	da4e3bbd2a	修复[问题10](https://github.com/shmilylty/OneForAll/issues/10 )	2019-08-14 16:35:29 +08:00
shmilylty	20e582135d	移除freebuf.com	2019-08-14 16:15:49 +08:00
shmilylty	bfb069509e	添加subdomains为None情况判断	2019-08-14 16:13:27 +08:00
shmilylty	d9c1cb3937	解决批量获取子域问题	2019-08-14 16:07:30 +08:00
shmilylty	f3540ee07b	8443端口使用https协议	2019-08-14 13:44:32 +08:00
shmilylty	794fec26db	修改使用帮助	2019-08-14 03:36:53 +08:00
shmilylty	46282e37a4	修复使用帮助链接	2019-08-14 03:26:12 +08:00
shmilylty	87749f1452	更友好的使用帮助	2019-08-14 03:25:15 +08:00
shmilylty	4428790766	添加使用帮助文档	2019-08-14 03:24:44 +08:00
shmilylty	51c42f2958	pep8	2019-08-14 03:22:00 +08:00
shmilylty	aaf5863d9a	对齐	2019-08-14 02:39:34 +08:00
shmilylty	88e084e695	添加详细的安装依赖文档	2019-08-14 02:02:31 +08:00
shmilylty	96487aec2e	模块优化	2019-08-13 19:15:13 +08:00
shmilylty	1179f5f118	添加limit_open_conn检查	2019-08-13 19:14:47 +08:00
shmilylty	ef3a1f5245	pep8	2019-08-13 19:13:56 +08:00
shmilylty	f223d9cf1b	参数优化	2019-08-13 19:13:08 +08:00
shmilylty	78832b4790	响应解码问题处理	2019-08-13 17:35:28 +08:00
shmilylty	3c065f56ae	修复字典路径问题	2019-08-12 22:31:17 +08:00
shmilylty	a65840b013	修复self.verify值的获取	2019-08-12 22:30:54 +08:00
shmilylty	49e4278647	修改output参数为show参数	2019-08-12 18:41:58 +08:00
shmilylty	5a324971ed	添加验证子域有效性参数	2019-08-12 18:41:18 +08:00
shmilylty	f3e76f7ee5	添加跳出判断	2019-08-12 11:26:50 +08:00
shmilylty	e1d5515c44	v0.0.4	2019-08-11 21:34:29 +08:00
shmilylty	bf10c1a652	v0.0.4	2019-08-11 21:32:04 +08:00
shmilylty	33d7b6cc02	参数调优	2019-08-11 20:49:38 +08:00
shmilylty	903b18a058	生成请求地址中443端口的特殊处理	2019-08-11 20:39:58 +08:00
shmilylty	38117f06d1	参数调优	2019-08-11 20:20:42 +08:00
shmilylty	f9117ff795	添加resp.json()异常处理	2019-08-11 20:10:23 +08:00
shmilylty	d2219bdf8c	添加搜索退出判断	2019-08-11 20:08:06 +08:00
shmilylty	3c8627035d	设置域传送超时为30秒	2019-08-11 19:58:10 +08:00
shmilylty	4bb52f4183	参数调优	2019-08-11 15:53:49 +08:00
shmilylty	f426b8f2f6	重构	2019-08-11 03:48:36 +08:00
shmilylty	ebd81b83e9	pep8	2019-08-10 14:54:08 +08:00
shmilylty	d70ebe637e	升级tqdm版本	2019-08-10 12:02:32 +08:00
shmilylty	d10bc9c8d9	默认不使用代理	2019-08-10 03:30:52 +08:00
shmilylty	a7b189f311	调整执行finish函数位置	2019-08-10 00:48:51 +08:00
shmilylty	98642da4f3	拆分出执行域传送函数	2019-08-09 23:46:09 +08:00
shmilylty	f17aea1f44	删除无用日志输出	2019-08-09 23:20:21 +08:00
shmilylty	611580aa97	修正模块名字	2019-08-09 21:02:27 +08:00
shmilylty	b21e31a223	database.py修改为类	2019-08-09 18:47:01 +08:00
shmilylty	4c63bd672f	修正valid参数使用描述	2019-08-09 16:56:03 +08:00
shmilylty	c7f86fdefa	新增2个模块	2019-08-09 16:54:10 +08:00
shmilylty	c026dc550e	新增检查sitemap收集子域名模块	2019-08-09 16:43:09 +08:00
shmilylty	f2b08f6b02	新增检查robots.txt收集子域名模块	2019-08-09 16:42:03 +08:00
Jing Ling	f0f9f9b75e	删除无用模板	2019-08-09 15:40:04 +08:00
Jing Ling	a9347b8ef0	修改提交Bug提示	2019-08-09 15:32:46 +08:00
shmilylty	c2679a6ceb	暂时不探测https	2019-08-09 15:13:22 +08:00
shmilylty	9197aaf52b	pep8	2019-08-09 14:25:21 +08:00
shmilylty	272662fa82	修改pip3安装命令	2019-08-09 14:24:50 +08:00
shmilylty	623a21a11e	移除fake-useragent依赖	2019-08-09 11:39:21 +08:00
shmilylty	05040d3b8d	resolve_callback函数添加说明并做容错处理	2019-08-09 01:12:52 +08:00
shmilylty	e248826327	v0.0.3	2019-08-08 23:37:09 +08:00
shmilylty	07de3ecc2b	解决报WRONG_VERSION_NUMBER问题	2019-08-08 22:34:56 +08:00
shmilylty	518b27e386	Python 3.8 on Xenial Linux	2019-08-08 21:57:17 +08:00
shmilylty	8148c56fbc	不使用回调	2019-08-08 17:30:02 +08:00
shmilylty	a0269b1814	调整对齐	2019-08-08 16:14:42 +08:00
shmilylty	efaa806260	修改pipenv安装命令	2019-08-08 16:10:52 +08:00
shmilylty	d4e77aeac2	设置系统变量PYTHONIOENCODING为utf-8 去掉使用reconfigure函数设置	2019-08-08 13:59:20 +08:00
shmilylty	f1fdb32b56	添加终端输出为utf-8编码	2019-08-08 12:11:40 +08:00
shmilylty	f5ba141097	终端日志输出改为sys.stderr	2019-08-08 11:53:38 +08:00
shmilylty	3e004dd069	pip8	2019-08-08 01:23:34 +08:00
shmilylty	47cfaca9ee	pep8	2019-08-07 23:52:02 +08:00
Jing Ling	c6ddd83700	更新配置信息	2019-08-07 19:06:23 +08:00
Jing Ling	827c3f5f93	更新群链接	2019-08-07 18:38:59 +08:00
Jing Ling	de8c9a5c3b	Merge pull request #3 from Windforce17/master [README]fix wrong url format	2019-08-07 18:32:50 +08:00
shmilylty	28bfb80dc4	移除fake-useragent依赖	2019-08-07 18:28:36 +08:00
shmilylty	3e3b852830	移除Windows python支持（travisci官方不支持）	2019-08-07 16:37:57 +08:00
shmilylty	8f676e2ebe	添加Windows，linux和MacOSX系统测试	2019-08-07 16:04:00 +08:00
shmilylty	0fee65dd36	except BaseException	2019-08-07 15:50:13 +08:00
shmilylty	cfd60b366c	pip8	2019-08-07 15:49:27 +08:00
shmilylty	18851408c2	pip8格式化使用ssl=False	2019-08-07 15:17:40 +08:00
shmilylty	f74c45a67d	pep6	2019-08-07 15:16:55 +08:00
zhichen.Wang	a0163ed582	[README]fix wrong url format	2019-08-07 15:10:09 +08:00
shmilylty	85aa7d2ec4	添加字典	2019-08-07 01:44:08 +08:00
shmilylty	b628859d32	pep8	2019-08-07 01:32:49 +08:00
shmilylty	8ba3b926ce	pep8	2019-08-06 22:52:44 +08:00
shmilylty	a5cbcd08fa	修改错误模块名	2019-08-06 12:58:25 +08:00
shmilylty	7945d8024b	添加爆破模块和segment参数说明	2019-08-06 12:37:38 +08:00
shmilylty	d9c12ef9c3	deal_results函数里添加响应结果判断	2019-08-06 12:07:02 +08:00
shmilylty	23d4765e8e	Merge branch 'master' of https://github.com/shmilylty/OneForAll	2019-08-06 02:16:20 +08:00
shmilylty	f17a2636ae	修改为默认不使用代理，默认请求超时30秒。	2019-08-06 02:14:45 +08:00
shmilylty	84dd464a02	添加任务分组参数	2019-08-06 02:13:39 +08:00
Jing Ling	8429f0f015	添加Bug报告模板	2019-08-06 01:33:41 +08:00
shmilylty	62d2b1e4bd	修改默认使用字典文件	2019-08-06 01:09:29 +08:00
shmilylty	ca09ea5870	移除参数ssl=False	2019-08-06 00:43:30 +08:00
shmilylty	365162a474	修改程序入口为fire	2019-08-06 00:42:49 +08:00
shmilylty	60d8a2a88a	向TCPConnector函数添加ssl=False禁用SSL证书检查	2019-08-06 00:22:28 +08:00
shmilylty	cd22034415	修改为正确执行resp.json()的条件	2019-08-06 00:13:35 +08:00
shmilylty	e706c51b74	修改加群链接	2019-08-06 00:03:35 +08:00
shmilylty	8420258ff7	增加判断响应体不为空才执行resp.json()	2019-08-06 00:00:55 +08:00
shmilylty	31666fdd75	移除aiobrute向aiodns_query_a()函数传semaphore参数	2019-08-05 23:38:53 +08:00
shmilylty	e3f9d5e149	修复aiobrute中没有传semaphore参数问题	2019-08-05 22:28:17 +08:00
Jing Ling	a651c2148c	修改安装命令	2019-08-05 21:42:31 +08:00
shmilylty	e8781c7a29	添加响应判断	2019-08-05 19:11:40 +08:00
shmilylty	edd6b35388	添加更新说明	2019-08-05 18:33:07 +08:00
shmilylty	d8b3fe4bd8	new_table_name使用双引号包裹	2019-08-05 18:12:39 +08:00
shmilylty	5ccf57d595	表名用双引号包裹解决issues/2	2019-08-05 18:01:48 +08:00
shmilylty	11642edf6e	优化代码结构删除无用队列结果集参数	2019-08-05 18:00:01 +08:00
shmilylty	53213304e2	修改为api调用	2019-08-05 17:58:35 +08:00
shmilylty	bf20197631	添加api检查	2019-08-05 16:00:46 +08:00
shmilylty	8bad8b74d6	修改编译失败解决方法链接地址	2019-08-05 14:29:36 +08:00
shmilylty	a313c2f3e4	添加编译失败解决方法	2019-08-05 14:27:49 +08:00
shmilylty	e3312533e9	优化cdx和csp模块，这两个模块默认使用代理。	2019-08-05 12:20:58 +08:00
shmilylty	a899753db9	添加.gitignore文件	2019-08-05 11:39:05 +08:00
shmilylty	1855b4714f	解决请求响应为空resp.json()报错问题	2019-08-05 11:38:20 +08:00