v0.0.4

参数调优
2026-08-26 04:47:48 +08:00 · 2019-08-11 21:34:29 +08:00 · 2019-08-11 21:32:04 +08:00 · 2019-08-11 20:49:38 +08:00 · 2019-08-11 20:39:58 +08:00 · 2019-08-11 20:20:42 +08:00
83 changed files with 11219 additions and 109718 deletions
@@ -0,0 +1,35 @@
+---
+name: 提交Bug
+about: "请务必按照模板提交Bug\U0001F64F"
+title: 请填写BUG标题
+labels: bug
+assignees: shmilylty
+
+---
+
+**是否使用了最新代码**
+是或否（如果不是的话尝试克隆最新的代码再跑一下）
+
+**Bug描述**
+清晰而简洁的Bug描述
+
+**如何复现**
+复现步骤（可不写）
+复现命令
+
+**预期结果**
+清晰而简洁的预期结果描述（可不写）
+
+**实际结果**
+清晰而简洁的实际结果描述（如出现什么错误）
+
+**屏幕截图**
+
+
+**运行环境**
+ - 系统：[例如Windows 10 x64]
+ - Python版本：[例如3.7.1]
+
+
+**报错文本**
+复制完整的报错文本
@@ -0,0 +1,238 @@
+
+# Created by https://www.gitignore.io/api/python,windows,pycharm
+# Edit at https://www.gitignore.io/?templates=python,windows,pycharm
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+.idea/sonarlint
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# End of https://www.gitignore.io/api/python,windows,pycharm
@@ -0,0 +1,39 @@
+sudo: true
+
+notifications:
+  email: false
+
+language: python
+
+matrix:
+  include:
+    - name: "Python 3.6 on Linux"
+      python: 3.6           # this works for Linux but is ignored on macOS or Windows
+    - name: "Python 3.7 on Xenial Linux"
+      python: 3.7           # this works for Linux but is ignored on macOS or Windows
+      dist: xenial          # required for Python >= 3.7
+    - name: "Python 3.8 on Xenial Linux"
+      python: 3.8-dev           # this works for Linux but is ignored on macOS or Windows
+      dist: xenial          # required for Python >= 3.7
+    - name: "Python 3.7 on macOS"
+      os: osx
+      osx_image: xcode10.2  # Python 3.7 running on macOS 10.14.3
+      language: shell       # 'language: python' is an error on Travis CI macOS
+    - name: "Python 3.7 on Windows"
+      os: windows           # Windows 10.0.17134 N/A Build 17134
+      language: shell
+      before_install:
+        - choco install python
+        - python -m pip install --upgrade pip
+      env: PATH=/c/Python37:/c/Python37/Scripts:$PATH
+
+install:
+  - pip3 install -U pip
+  - pip3 install codecov
+  - pip3 install -r requirements.txt
+
+script:
+ - coverage run oneforall/example.py
+
+after_success:
+ - codecov
@@ -1,18 +1,38 @@
-# Changelog
+# 更新日志
 OneForAll的所有值得注意的更改都将记录在此文件中。

 OneForAll的更新日志格式基于[Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/)。

 OneForAll遵守[语义化版本格式](https://semver.org/)。

-## [Unreleased](https://github.com/shmilylty/onefroall/compare/v1.0.0...HEAD)
+## Unreleased
+## [0.0.4](https://github.com/shmilylty/oneforall/releases/tag/v0.0.4) - 2019-08-11
+### 修复
+- 修复一些已知Bugs

-## [0.0.1](https://github.com/shmilylty/onefroall/releases/tag/v0.0.1) - 2019-08-01
-### Added
- 添加检查crossdomain.xml收集子域功能
- 添加检查域名证书收集子域功能
- 添加检查内容安全策略头收集子域功能
- 添加域传送利用功能
- 添加子域收集功能（搜索引擎，DNS数据集，证书透明度，网上爬虫档案）
- 添加子域爆破功能
- 添加数据库导出功能
+## [0.0.3](https://github.com/shmilylty/oneforall/releases/tag/v0.0.3) - 2019-08-08
+### 修改
+- 代码PEP8格式化
+### 修改
+- 修改一些已知Bugs
+
+## [0.0.2](https://github.com/shmilylty/oneforall/releases/tag/v0.0.2) - 2019-08-04
+### 新增
+- 新增有关文档
+### 修改
+- 修改有关日志输出格式和信息
+### 修复
+- 升级fire库版本解决运行报错问题
+### 移除
+- 移除brotlipy依赖
+
+
+## [0.0.1](https://github.com/shmilylty/oneforall/releases/tag/v0.0.1) - 2019-08-02
+### 新增
+- 新增检查crossdomain.xml收集子域功能
+- 新增检查域名证书收集子域功能
+- 新增检查内容安全策略头收集子域功能
+- 新增域传送利用功能
+- 新增子域收集功能（搜索引擎，DNS数据集，证书透明度，网上爬虫档案）
+- 新增子域爆破功能
+- 新增数据库导出功能
@@ -3,5 +3,9 @@
 * **[Jing Ling](https://github.com/shmilylty)**
  * 核心开发

-* **[Black Star](https://github.com/shmilylty)**
-  * 模块贡献
+* **[Black Star](https://github.com/blackstar24)**
+  * 模块贡献
+
+* [**iceMatcha**](https://github.com/iceMatcha)
+  * bug调试
+
@@ -16,12 +16,10 @@ records = "*"
 tldextract = "*"
 exrex = "*"
 aiohttp = "*"
-fire = "*"
-fake-useragent = "*"
+fire = "==0.2.1"
 bs4 = "*"
 cchardet = "*"
 lxml = "*"
-brotlipy = "*"
 pysocks = "*"

 [requires]
@@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "c2d9fe3a7303466070733472b7b06b491d35bbbcf3db8863152e721d797e9195"
+            "sha256": "d95e37615ca48fe39a66e8c24eff76910c66b01b0739c0ab00ffa0f4df022107"
        },
        "pipfile-spec": 6,
        "requires": {
@@ -88,41 +88,6 @@
            ],
            "version": "==4.8.0"
        },
-        "brotlipy": {
-            "hashes": [
-                "sha256:07194f4768eb62a4f4ea76b6d0df6ade185e24ebd85877c351daa0a069f1111a",
-                "sha256:091b299bf36dd6ef7a06570dbc98c0f80a504a56c5b797f31934d2ad01ae7d17",
-                "sha256:09ec3e125d16749b31c74f021aba809541b3564e5359f8c265cbae442810b41a",
-                "sha256:0be698678a114addcf87a4b9496c552c68a2c99bf93cf8e08f5738b392e82057",
-                "sha256:0fa6088a9a87645d43d7e21e32b4a6bf8f7c3939015a50158c10972aa7f425b7",
-                "sha256:1ea4e578241504b58f2456a6c69952c88866c794648bdc74baee74839da61d44",
-                "sha256:2699945a0a992c04fc7dc7fa2f1d0575a2c8b4b769f2874a08e8eae46bef36ae",
-                "sha256:2a80319ae13ea8dd60ecdc4f5ccf6da3ae64787765923256b62c598c5bba4121",
-                "sha256:2e5c64522364a9ebcdf47c5744a5ddeb3f934742d31e61ebfbbc095460b47162",
-                "sha256:36def0b859beaf21910157b4c33eb3b06d8ce459c942102f16988cca6ea164df",
-                "sha256:3a3e56ced8b15fbbd363380344f70f3b438e0fd1fcf27b7526b6172ea950e867",
-                "sha256:3c1d5e2cf945a46975bdb11a19257fa057b67591eb232f393d260e7246d9e571",
-                "sha256:50ca336374131cfad20612f26cc43c637ac0bfd2be3361495e99270883b52962",
-                "sha256:5de6f7d010b7558f72f4b061a07395c5c3fd57f0285c5af7f126a677b976a868",
-                "sha256:637847560d671657f993313ecc6c6c6666a936b7a925779fd044065c7bc035b9",
-                "sha256:653faef61241bf8bf99d73ca7ec4baa63401ba7b2a2aa88958394869379d67c7",
-                "sha256:786afc8c9bd67de8d31f46e408a3386331e126829114e4db034f91eacb05396d",
-                "sha256:79aaf217072840f3e9a3b641cccc51f7fc23037496bd71e26211856b93f4b4cb",
-                "sha256:7e31f7adcc5851ca06134705fcf3478210da45d35ad75ec181e1ce9ce345bb38",
-                "sha256:8b39abc3256c978f575df5cd7893153277216474f303e26f0e43ba3d3969ef96",
-                "sha256:9448227b0df082e574c45c983fa5cd4bda7bfb11ea6b59def0940c1647be0c3c",
-                "sha256:96bc59ff9b5b5552843dc67999486a220e07a0522dddd3935da05dc194fa485c",
-                "sha256:a07647886e24e2fb2d68ca8bf3ada398eb56fd8eac46c733d4d95c64d17f743b",
-                "sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2",
-                "sha256:b4c98b0d2c9c7020a524ca5bbff42027db1004c6571f8bc7b747f2b843128e7a",
-                "sha256:c6cc0036b1304dd0073eec416cb2f6b9e37ac8296afd9e481cac3b1f07f9db25",
-                "sha256:d2c1c724c4ac375feb2110f1af98ecdc0e5a8ea79d068efb5891f621a5b235cb",
-                "sha256:dc6c5ee0df9732a44d08edab32f8a616b769cc5a4155a12d2d010d248eb3fb07",
-                "sha256:fd1d1c64214af5d90014d82cee5d8141b13d44c92ada7a0c0ec0679c6f15a471"
-            ],
-            "index": "pypi",
-            "version": "==0.7.0"
-        },
        "bs4": {
            "hashes": [
                "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"
@@ -248,19 +213,12 @@
            "index": "pypi",
            "version": "==0.10.5"
        },
-        "fake-useragent": {
-            "hashes": [
-                "sha256:c104998b750eb097eefc28ae28e92d66397598d2cf41a31aa45d5559ef1adf35"
-            ],
-            "index": "pypi",
-            "version": "==0.1.11"
-        },
        "fire": {
            "hashes": [
-                "sha256:cdae28ea728d6cad487c900d534426d2cd09ddd2993287a885ff7c8accb097c8"
+                "sha256:6865fefc6981a713d2ce56a2a2c92c56c729269f74a6cddd6f4b94d16ae084c9"
            ],
            "index": "pypi",
-            "version": "==0.2.0"
+            "version": "==0.2.1"
        },
        "idna": {
            "hashes": [
@@ -1,5 +1,353 @@
 # OneForAll
-Subdomain Collection One For All：Full Cowl-100% Smash

-# Developing
-OneForAll is under development, please look forward to it!
+[![Build Status](https://travis-ci.org/shmilylty/OneForAll.svg?branch=master)](https://travis-ci.org/shmilylty/OneForAll)
+[![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
+[![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
+[![License](https://img.shields.io/github/license/shmilylty/OneForAll)](./LICENSE)
+[![python](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue)](./)
+[![python](https://img.shields.io/badge/release-v0.0.4-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+
+👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](./README.en.md)
+
+![](./images/All_Might.jpg)
+
+## 🎉项目简介
+
+在渗透测试中信息收集的重要性不言而喻，子域收集是信息收集中必不可少且非常重要的一环，目前网上也开源了许多子域收集的工具，但是总是存在以下部分问题：
+
+* **不够强大**，子域收集的接口不够多，不能做到对批量子域自动收集，没有自动子域解析，验证，FUZZ以及信息拓展等功能。
+* **不够友好**，固然命令行模块比较方便，但是当可选的参数很多，要实现的操作复杂，用命令行模式就有点不够友好，如果有交互良好，高可操作的前端那么使用体验就会好很多。
+
+* **缺少维护**，很多工具几年没有更新过一次，issues和PR是啥，不存在的。
+
+* **效率问题**，没有利用多进程，多线程以及异步协程技术，速度较慢。
+
+为了解决以上痛点，此项目应用而生，OneForAll一词是来自我喜欢的一部日漫《[我的英雄学院](https://manhua.fzdm.com/131/)》，它是一种通过一代代的传承不断变强的潜力无穷的顶级个性，目前[番剧](https://www.bilibili.com/bangumi/media/md7452/)也更新到了第三季了，欢迎大佬们入坑😄。正如其名，我希望OneForAll是一款集百家之长，功能强大的全面快速子域收集终极神器🔨。
+
+目前OneForAll还在开发中，肯定有不少问题和需要改进的地方，欢迎大佬们提交[Issues](https://github.com/shmilylty/OneForAll/issues)和[PR](https://github.com/shmilylty/OneForAll/pulls)，用着还行给个小星星✨吧，目前有一个专门用于OneForAll交流和反馈QQ群👨‍👨‍👦‍👦：:[**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771)，也可以给我发邮件📧[admin@hackfun.org]。
+
+## 👍功能特性
+
+* **收集能力强大**，详细模块请阅读[收集模块说明](./docs/collection_modules.md)。
+  1. 利用证书透明度收集子域（目前有6个模块：`censys_api`，`certdb_api`，`certspotter`，`crtsh`，`entrust`，`google`）
+
+  2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，检查robots文件`robots`，检查sitemap文件`sitemap`，后续会添加检查NSEC记录，NSEC3记录等模块）
+
+  3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善）
+
+  4. 利用DNS数据集收集子域（目前有16个模块：`binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `ptrarchive`, `sitedossier`）
+
+  5. 利用DNS查询收集子域（目前有1个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，该模块还有待添加和完善）
+
+  6. 利用威胁情报平台数据收集子域（目前有5个模块：`riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）
+
+  7. 利用搜索引擎发现子域（目前有15个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
+
+* **处理功能强大**，发现的子域结果支持自动去除，自动DNS解析，HTTP请求探测，自动移除无效子域，拓展子域的Banner信息，最终支持的导出格式有`csv`, `tsv`, `json`, `yaml`, `html`, `xls`, `xlsx`, `dbf`, `latex`, `ods`。
+
+* **速度极速**，[收集模块](./oneforall/collect.py)使用多线程调用，[爆破模块](./oneforall/aiobrute.py)使用异步多进程多协程，DNS解析和HTTP请求使用异步多协程。
+
+## 🚀上手指南
+
+由于该项目**处于开发中**，会不断进行更新迭代，下载使用最好**克隆**最新项目。
+
+**🐍安装要求**
+
+1. Python 3.6-3.7
+
+**✔安装步骤**
+
+1. 下载更新
+
+   ```bash
+   git clone https://github.com/shmilylty/OneForAll.git
+   ```
+
+   ❗如果你之前已经克隆了项目运行之前请**备份**自己修改过的文件到项目外的地方（如**config.py**），然后执行以下命令**更新**项目：
+
+   ```bash
+   git fetch --all
+   git reset --hard origin/master
+   git pull
+   ```
+
+2. 安装依赖
+
+   * 使用pipenv
+
+     ```bash
+     pip3 install pipenv
+     cd OneForAll/
+     pipenv install --python 3.7
+     cd oneforall
+     pipenv run python3 oneforall.py --help
+     ```
+
+   * 使用pip3
+   
+     ```bash
+     cd OneForAll/
+     pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+     cd oneforall/
+     python3 oneforall.py --help
+     ```
+
+
+如果在安装依赖过程中发现编译某个依赖库失败时可以参考[编译失败解决方法](./docs/building_fail_solution.md)，如果还没有解决欢迎加群反馈。
+
+**✨使用演示**
+
+```bash
+python3 oneforall.py --target example.com run
+```
+
+![Example](./docs/usage_example.svg)
+
+**🤔使用帮助**
+
+命令行参数只提供了一些常用参数，更多详细的参数配置请见[config.py](./oneforall/config.py)，如果你认为有些参数是命令界面经常使用到的或缺少了什么参数等问题非常欢迎反馈。由于众所周知的原因，如果要使用一些被墙的收集接口请先到[config.py](./oneforall/config.py)配置代理，有些收集模块需要提供API（大多都是可以注册账号免费获取），如果需要使用请到[config.py](./oneforall/config.py)配置API信息，如果不使用请忽略有关报错提示。（详细模块请阅读[收集模块说明](./docs/collection_modules.md)）
+
+OneForAll命令行界面基于[Fire](https://github.com/google/python-fire/)实现，有关Fire更高级使用方法请参阅[使用Fire CLI](https://github.com/google/python-fire/blob/master/docs/using-cli.md)，有任何使用疑惑欢迎加群交流。
+
+oneforall.py是主程序入口，oneforall.py里有调用aiobrute.py和dbexport.py，为了方便进行子域爆破和数据库导出独立出了aiobrute.py和dbexport.py，这两个文件可以单独运行，并且所接受参数要更丰富一点。
+
+1. oneforall.py使用帮助
+
+   ```bash
+   pipenv run python oneforall.py --help
+   ```
+   ```bash
+   NAME
+       oneforall.py - OneForAll是一款功能强大的子域收集工具
+   
+   SYNOPSIS
+       oneforall.py --target=TARGET <flags>
+   
+   DESCRIPTION
+       Version: 0.0.2
+       Project: https://github.com/shmilylty/OneForAll/
+   
+       Example:
+           python oneforall.py --target example.com run
+           python oneforall.py --target example.com --brute True --port medium --valid 1 run
+           python oneforall.py --target ./domains.txt --format csv --path= ./result.csv  --output True run
+   
+       Note:
+           参数valid可选值有1，0，None，分别表示导出有效，无效，全部子域
+           参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
+           参数format可选格式有'csv','tsv','json','yaml','html','xls','xlsx','dbf','latex','ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   ARGUMENTS
+       TARGET
+           单个域名或者每行一个域名的文件路径
+   
+   FLAGS
+       --brute=BRUTE
+           是否使用爆破模块（默认禁用）
+       --port=PORT
+           HTTP请求验证的端口范围（默认medium）
+       --valid=VALID
+           导出子域的有效性（默认1）
+       --path=PATH
+           导出路径(默认None)
+       --format=FORMAT
+           导出格式（默认xlsx）
+       --output=OUTPUT
+           是否将导出数据输出到终端（默认False）
+   ```
+
+2. aiobrute.py使用帮助
+
+   关于泛解析问题处理程序首先会访问一个随机的子域判断是否泛解析，如果使用了泛解析则是通过以下判断处理：
+   - 一是主要是与泛解析的IP集合和TTL值做对比，可以参考[这篇文章](http://sh3ll.me/archives/201704041222.txt)。
+   - 二是多次解析到同一IP集合次数（默认设置为10，可以在config.py设置大小）
+   - 考虑爆破效率问题目前还没有加上HTTP响应体相似度对比和响应体内容判断
+   经过测试在16核心的CPU，使用16进程64协程，100M带宽的环境下，设置任务分割为50000，跑两百万字典大概10分钟左右跑完，大概3333个子域每秒。
+
+   ```bash
+   pipenv run python aiobrute.py --help
+   ```
+
+   ```bash
+    NAME
+        aiobrute.py - OneForAll多进程多协程异步子域爆破模块
+
+    SYNOPSIS
+        aiobrute.py COMMAND | --target=TARGET <flags>
+
+    DESCRIPTION
+        Example：
+            python aiobrute.py --target example.com run
+            python aiobrute.py --target ./domains.txt run
+            python aiobrute.py --target example.com --processes 4 --coroutine 64 --wordlist data/subdomains.txt run
+            python aiobrute.py --target example.com --recursive True --depth 2 --namelist data/next_subdomains.txt run
+            python aiobrute.py --target www.{fuzz}.example.com --fuzz True --rule [a-z][0-9] run
+
+        Note:
+            参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为一任务组，
+            当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+            十万字典建议设置为5000，百万字典设置为50000
+
+    ARGUMENTS
+        TARGET
+            单个域名或者每行一个域名的文件路径
+
+    FLAGS
+        --processes=PROCESSES
+            爆破的进程数(默认CPU核心数)
+        --coroutine=COROUTINE
+            每个爆破进程下的协程数(默认16)
+        --wordlist=WORDLIST
+            指定爆破所使用的字典路径(默认使用config.py配置)
+        --segment=SEGMENT
+            爆破任务分割(默认500)
+        --recursive=RECURSIVE
+            是否使用递归爆破(默认False)
+        --depth=DEPTH
+            递归爆破的深度(默认2)
+        --namelist=NAMELIST
+            指定递归爆破所使用的字典路径(默认使用config.py配置)
+        --fuzz=FUZZ
+            是否使用fuzz模式进行爆破(默认False，开启必须指定fuzz正则规则)
+        --rule=RULE
+            fuzz模式使用的正则规则(默认使用config.py配置)
+
+   ```
+
+3. dbexport.py使用帮助
+
+   ```bash
+   pipenv run python dbexport.py --help
+   ```
+
+   ```bash
+   NAME
+       dbexport.py - OneForAll数据库导出模块
+   
+   SYNOPSIS
+       dbexport.py TABLE <flags>
+   
+   DESCRIPTION
+       Example:
+           python dbexport.py --db result.db --table name --format csv --output False
+           python dbexport.py --db result.db --table name --format csv --path= ./result.csv
+   
+       Note:
+           参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+           参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
+           参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+   
+   POSITIONAL ARGUMENTS
+       TABLE
+           要导出的表
+   
+   FLAGS
+       --db=DB
+           要导出的数据库路径(默认为results/result.sqlite3)
+       --valid=VALID
+           导出子域的有效性(默认None)
+       --path=PATH
+           导出路径(默认None)
+       --format=FORMAT
+           导出格式(默认xlsx)
+       --output=OUTPUT
+           是否将导出数据输出到终端(默认False)
+   
+   NOTES
+       You can also use flags syntax for POSITIONAL ARGUMENTS
+   ```
+
+## 👏主要框架
+
+* [aiodns](https://github.com/saghul/aiodns) - 简单DNS异步解析库。
+* [aiohttp](https://github.com/aio-libs/aiohttp) - 异步http客户端/服务器框架
+* [aiomultiprocess](https://github.com/jreese/aiomultiprocess) - 将Python代码提升到更高的性能水平(multiprocessing和asyncio结合，实现异步多进程多协程)
+* [beautifulsoup4](https://pypi.org/project/beautifulsoup4/) - 可以轻松从HTML或XML文件中提取数据的Python库
+* [fire](https://github.com/google/python-fire) - Python Fire是一个纯粹根据任何Python对象自动生成命令行界面（CLI）的库
+* [loguru](https://github.com/Delgan/loguru) - 旨在带来愉快的日志记录Python库
+* [records](https://github.com/kennethreitz/records) - Records是一个非常简单但功能强大的库，用于对大多数关系数据库进行最原始SQL查询。
+* [requests](https://github.com/psf/requests) - Requests 唯一的一个非转基因的 Python HTTP 库，人类可以安全享用。
+* [tqdm](https://github.com/tqdm/tqdm) - 适用于Python和CLI的快速，可扩展的进度条库
+
+感谢这些伟大优秀的Python库！
+
+## 🌲目录结构
+
+```bash
+D:.
+|
+---.github
+---docs
+|       collection_modules.md 收集模块说明
+---images
+\---oneforall
+    |   aiobrute.py   异步多进程多协程子域爆破模块，可以单独运行
+    |   collect.py    各个收集模块上层调用
+    |   config.py     配置文件
+    |   dbexport.py   数据库导出模块，可以单独运行
+    |   domains.txt   要批量爆破的域名列表
+    |   oneforall.py  OneForAll主入口，可以单独运行
+    |   __init__.py
+    |
+    +---common 公共调用模块
+    +---data   存放一些所需数据
+    |       next_subdomains.txt     下一层子域字典
+    |       public_suffix_list.dat  顶级域名后缀 
+    |       srv_names.json          常见SRV记录前缀名
+    |       subdomains.txt          子域爆破常见字典
+    |
+    \---modules 
+        +---certificates     利用证书透明度收集子域模块
+        +---check            常规检查收集子域模块
+        +---crawl            利用网上爬虫档案收集子域模块
+        +---datasets         利用DNS数据集收集子域模块
+        +---dnsquery         利用DNS查询收集子域模块
+        +---intelligence     利用威胁情报平台数据收集子域模块
+        \---search           利用搜索引擎发现子域模块
+
+```
+
+## 🙏贡献
+
+非常热烈欢迎各位大佬一起完善本项目！
+
+## ⌛后续计划
+
+- [ ] 子域收集模块优化
+- [ ] 子域接管功能实现
+- [ ] 子域收集爬虫实现
+- [ ] 操作强大交互人性的前端界面实现
+
+更多详细信息请阅读[TODO.md](./TODO.md)。
+
+## 🔖版本控制
+
+该项目使用[SemVer](https://semver.org/)语言化版本格式进行版本管理，你可以在[Releases](https://github.com/shmilylty/OneForAll/releases)查看可用版本。
+
+## 👨‍💻作者
+
+* **[Jing Ling](https://github.com/shmilylty)**
+  * 核心开发
+
+* **[Black Star](https://github.com/blackstar24)**
+  * 模块贡献
+
+* [**iceMatcha**](https://github.com/iceMatcha)
+  * bug调试
+
+*你也可以在[CONTRIBUTORS.md](./CONTRIBUTORS.md)中参看所有参与该项目的开发者。*
+
+## 📄版权
+
+该项目签署了GPL-3.0授权许可，详情请参阅[LICENSE.md](./LICENSE.md)。
+
+## 😘鸣谢
+
+感谢网上开源的各个子域收集项目！
+
+感谢[A-Team](https://github.com/QAX-A-Team)大哥们热情无私的问题解答！
+
+## 📜免责声明 ##
+
+本工具仅限于安全研究与教学使用，用户使用本工具所造成的所有后果，由用户承担全部法律及连带责任，本项目所有作者和贡献者不承担任何法律及连带责任。
@@ -0,0 +1,13 @@
+如果在安装依赖过程遇到编译某个依赖库失败时可以尝试以下方法：
+
+1. 到提供编译好的whl文件的第三方平台，找到对应库手动下载安装。第三方平台平台有： 
+ * [https://www.lfd.uci.edu/~gohlke/pythonlibs](https://www.lfd.uci.edu/~gohlke/pythonlibs)
+ * [https://pythonwheels.com/](https://pythonwheels.com/)
+
+选择好对应版本执行以下命令手动安装。举个例子，当编译pycares时失败时，找到[https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares](https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycares)，由于我的系统是Windows 10 64位，使用的Python 3.7便下载`pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl`（一般来说下载最新版本的），然后手动安装：
+
+```bash
+pip3 install pycares‑3.0.0‑cp37‑cp37m‑win_amd64.whl
+```
+
+2. 到库的项目地址issues和wiki等找找有没有解决方法，如果没有就给他们提issues发邮件😜。
@@ -0,0 +1,105 @@
+#  收集模块说明 #
+
+如果要使用通过API收集子域的模块请先到[config.py](../oneforall/config.py)配置相关信息，大多平台的API都是可以注册账号免费获取的。
+
+如果你指定使用某些模块可以在[config.py](../oneforall/config.py)中设置：
+
+```python
+enable_all_module = False  # 不开启所有模块
+enable_partial_module = [('modules.search', 'ask')，('modules.search', 'baidu')]  # 只使用ask和baidu搜索引擎收集子域
+```
+
+如果你指定使用某些模块使用代理可以在[config.py](../oneforall/config.py)中设置：
+
+```python
+enable_proxy = True  # 使用代理
+proxy_all_module = False  # 不代理所有模块
+proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和AskSearch模块（各个模块的source属性值）
+```
+
+以下是各个模块说明：
+
+ 1. 利用证书透明度收集子域（目前有6个模块：`censys_api`，`certdb_api`，`certspotter`，`crtsh`，`entrust`，`google`）
+    
+      | 模块名称    | 是否需要代理 | 是否需要API | 其他说明                                           |
+      | ----------- | ------------ | ----------- | -------------------------------------------------- |
+      | censys_api  | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | certdb_api  | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | certspotter | 否           | 否          |                                                    |
+      | crtsh       | 否           | 否          |                                                    |
+      | entrust     | 否           | 否          |                                                    |
+      | google      | 是           | 否          |                                                    |
+
+
+  2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，后续会添加检查NSEC记录，NSEC3记录等模块）
+
+      | 模块名称 | 是否需要代理           | 是否需要API | 其他说明           |
+      | -------- | ---------------------- | ----------- | ------------------ |
+      | axfr     | 否                     | 否          | 域传送漏洞利用     |
+      | cdx      | 手动设置（默认不使用） | 否          | 检查跨域策略文件   |
+      | cert     | 否                     | 否          | 检查HTTPS证书      |
+      | csp      | 手动设置（默认不使用） | 否          | 检查内容安全策略   |
+      | robots   | 手动设置（默认不使用） | 否          | 检查robots.txt文件 |
+      | sitemap  | 手动设置（默认不使用） | 否          | 检查sitemap文件    |
+  3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善）
+
+      | 模块名称     | 是否需要代理 | 是否需要API | 其他说明 |
+      | ------------ | ------------ | ----------- | -------- |
+      | archivecrawl | 否           | 否          |          |
+      | commoncrawl  | 否           | 否          |          |
+
+  4. 利用DNS数据集收集子域（目前有16个模块：`binaryedge_api`, `circl_api`, `hackertarget`, `riddler`, `bufferover`, `dnsdb`, `ipv4info`, `robtex`, `chinaz`, `dnsdb_api`, `netcraft`, `securitytrails_api`, `chinaz_api`, `dnsdumpster`, `ptrarchive`, `sitedossier`）
+
+      | 模块名称           | 是否需要代理 | 是否需要API | 其他说明                                           |
+      | ------------------ | ------------ | ----------- | -------------------------------------------------- |
+      | binaryedge_api     | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | bufferover         | 否           | 否          |                                                    |
+      | chinaz             | 否           | 否          |                                                    |
+      | chinaz_api         | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | circl_api          | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | dnsdb              | 否           | 否          |                                                    |
+      | dnsdb_api          | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | dnsdumpster        | 否           | 否          |                                                    |
+      | hackertarget       | 否           | 否          |                                                    |
+      | ipv4info           | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | netcraft           | 否           | 否          |                                                    |
+      | ptrarchive         | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | riddler            | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | robtex             | 否           | 否          |                                                    |
+      | securitytrails_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | sitedossier        | 否           | 否          |                                                    |
+  5. 利用DNS查询收集子域（目前有1个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，该模块还有待添加和完善）
+
+      | 模块名称 | 是否需要代理 | 是否需要API | 其他说明                      |
+      | -------- | ------------ | ----------- | ----------------------------- |
+      | srv      | 否           | 否          | 枚举域名常见的SRV记录发现子域 |
+  6. 利用威胁平台数据收集子域（目前有5个模块：`riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）
+
+      | 模块名称       | 是否需要代理 | 是否需要API | 其他说明                                           |
+      | -------------- | ------------ | ----------- | -------------------------------------------------- |
+      | riskiq_api     | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | threatbook_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+      | threatminer    | 否           | 否          |                                                    |
+      | virustotal     | 否           | 否          |                                                    |
+      | virustotal_api | 否           | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+  7. 利用搜索引擎发现子域（目前有15个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`）
+
+     除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
+
+     | 模块        | 是否需要代理           | 是否需要API | 其他说明                                           |
+     | ----------- | ---------------------- | ----------- | -------------------------------------------------- |
+     | ask         | 是                     | 否          |                                                    |
+     | baidu       | 否                     | 否          |                                                    |
+     | bing        | 否                     | 否          |                                                    |
+     | bing_api    | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+     | duckduckgo  | 是                     | 否          |                                                    |
+     | exalead     | 否，最好使用国外代理。 | 否          |                                                    |
+     | fofa_api    | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+     | google      | 是                     | 否          |                                                    |
+     | google_api  | 是                     | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+     | shodan_api  | 否，最好使用国外代理。 | 是          | API使用和申请见[config.py](../oneforall/config.py) |
+     | so          | 否                     | 否          |                                                    |
+     | sogou       | 否                     | 否          |                                                    |
+     | yahoo       | 是                     | 否          |                                                    |
+     | yandex      | 是                     | 否          |                                                    |
+     | zoomeye_api | 否                     | 是          | API使用和申请见[config.py](../oneforall/config.py) |
@@ -1 +0,0 @@
-# coding=utf-8
@@ -1,42 +1,50 @@
+#!/usr/bin/python3
 # coding=utf-8
-import os
-import time
-import queue
-import signal
-import pathlib
+
+"""
+OneForAll多进程多协程异步子域爆破模块
+
+:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
+:license: GNU General Public License v3.0, see LICENSE for more details.
+"""
+
 import asyncio
+import queue
+import secrets
+import signal
+import time
+
+import aiomultiprocess
+import exrex
 import fire
 import tqdm
-import exrex
-import secrets
-import aiomultiprocess
+
 import config
-from config import logger
-from common import utils, database, resolve
+import dbexport
+from common import resolve, utils
 from common.module import Module
+from common.database import Database
+from config import logger


 def init_worker():
    signal.signal(signal.SIGINT, signal.SIG_IGN)


-def get_wordlist(name):
-    return config.data_storage_path.joinpath(name)
-
-
 def detect_wildcard(domain):
    """
    探测域名是否使用泛解析

    :param str domain: 域名
-    :return: 如果没有使用泛解析返回False 使用返回泛解析的IP集合和ttl整型值
+    :return: 如果没有使用泛解析返回False 反之返回泛解析的IP集合和ttl整型值
    """
    logger.log('INFOR', f'正在探测{domain}是否使用泛解析')
    token = secrets.token_hex(16)
    random_subdomain = f'{token}.{domain}'
    try:
        answers = resolve.dns_query_a(random_subdomain)
-    except Exception as e:  # 如果查询随机域名A记录出错 说明不存在随机子域的A记录 即没有开启泛解析
+    # 如果查询随机域名A记录出错 说明不存在随机子域的A记录 即没有开启泛解析
+    except Exception as e:
        logger.log('DEBUG', e)
        logger.log('INFOR', f'{domain}没有使用泛解析')
        return False, None, None
@@ -121,42 +129,65 @@ def gen_brute_domains(domain, path):

 class AIOBrute(Module):
    """
-    多进程多协程异步子域爆破
-
-    :param str target:       单个域名或者每行一个域名的文件路径
-    :param int processes:    爆破的进程数（默认CPU核心数）
-    :param int coroutine:    每个爆破进程下的协程数（默认16）
-    :param str wordlist:     指定爆破所使用的字典路径（默认使用config.py配置）
-    :param bool recursive:   是否使用递归爆破（默认禁用）
-    :param int depth:        递归爆破的深度（默认2）
-    :param str namelist:     指定递归爆破所使用的字典路径（默认使用config.py配置）
-    :param bool fuzz:        是否使用fuzz模式进行爆破（默认禁用，开启必须指定fuzz正则规则）
-    :param str rule:         fuzz模式使用的正则规则（默认使用config.py配置）
+    OneForAll多进程多协程异步子域爆破模块

    Example：
-    python aiobrute.py --target example.com run
-    python aiobrute.py --target ./domains.txt run
-    python aiobrute.py --target example.com --processes 4 --coroutine 64 --wordlist data/subdomains.txt run
-    python aiobrute.py --target example.com --recursive True --depth 2 --namelist data/next_subdomains.txt run
-    python aiobrute.py --target www.{fuzz}.example.com --fuzz True --rule [a-z][0-9] run
+        python3 aiobrute.py --target example.com run
+        python3 aiobrute.py --target ./domains.txt run
+        python3 aiobrute.py --target example.com --processes 4 --coroutine 64
+        python3 aiobrute.py --target example.com --wordlist subdomains.txt run
+        python3 aiobrute.py --target example.com --recursive True --depth 2 run
+        python3 aiobrute.py --target m.{fuzz}.a.bz --fuzz True --rule [a-z] run
+
+    Note:
+        参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为任务组，
+        当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+        十万字典建议设置为5000，百万字典设置为50000
+        参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
+        参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
+                         'dbf', 'latex', 'ods'
+        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件
+
+    :param str target:       单个域名或者每行一个域名的文件路径
+    :param int processes:    爆破的进程数(默认CPU核心数)
+    :param int coroutine:    每个爆破进程下的协程数(默认64)
+    :param str wordlist:     指定爆破所使用的字典路径(默认使用config.py配置)
+    :param int segment:      爆破任务分割(默认500)
+    :param bool recursive:   是否使用递归爆破(默认False)
+    :param int depth:        递归爆破的深度(默认2)
+    :param str namelist:     指定递归爆破所使用的字典路径(默认使用config.py配置)
+    :param bool fuzz:        是否使用fuzz模式进行爆破(默认False，开启须指定fuzz正则规则)
+    :param str rule:         fuzz模式使用的正则规则(默认使用config.py配置)
+    :param bool export:      是否导出爆破结果(默认True)
+    :param int valid:        导出子域的有效性(默认None)
+    :param str format:       导出格式(默认xlsx)
+    :param str path:         导出路径(默认None)
+    :param
    """

    def __init__(self, target, processes=None, coroutine=64, wordlist=None,
-                 recursive=False, depth=2, namelist=None, fuzz=False, rule=None):
+                 segment=500, recursive=False, depth=2, namelist=None,
+                 fuzz=False, rule=None, export=True, valid=None, format='xlsx',
+                 path=None):
        Module.__init__(self)
        self.domains = set()
        self.domain = str()
        self.module = 'Brute'
        self.source = 'AIOBrute'
        self.target = target
-        self.processes = processes or config.brute_processes_num or os.cpu_count()
+        self.processes = processes or config.brute_processes_num
        self.coroutine = coroutine or config.brute_coroutine_num
-        self.wordlist = wordlist or config.brute_wordlist_path or get_wordlist('subdomains.txt')
+        self.wordlist = wordlist or config.brute_wordlist_path
+        self.segment = segment or config.brute_task_segment
        self.recursive_brute = recursive or config.enable_recursive_brute
        self.recursive_depth = depth or config.brute_recursive_depth
-        self.recursive_namelist = namelist or config.recursive_namelist_path or get_wordlist('next_subdomains.txt')
+        self.recursive_namelist = namelist or config.recursive_namelist_path
        self.fuzz = fuzz or config.enable_fuzz
        self.rule = rule or config.fuzz_rule
+        self.export = export
+        self.valid = valid
+        self.format = format
+        self.path = path
        self.nameservers = config.resolver_nameservers
        self.ips_times = dict()  # IP集合出现次数
        self.enable_wildcard = False  # 当前域名是否使用泛解析
@@ -164,15 +195,18 @@ class AIOBrute(Module):
        self.wildcard_ttl = int()  # 泛解析TTL整型值

    def gen_tasks(self, domain):
-        logger.log('INFOR', f'正在生成{domain}的字典')
-        if self.domain != domain:  # 如果domain不是self.domain，而是self.domain的子域 生成递归爆破字典
+        # 如果domain不是self.domain，而是self.domain的子域 生成递归爆破字典
+        if self.domain != domain:
+            logger.log('INFOR', f'使用{self.recursive_namelist}字典')
            domains = gen_brute_domains(domain, self.recursive_namelist)
        elif self.fuzz and self.rule:  # 开启fuzz模式并指定了fuzz正则规则
+            logger.log('INFOR', f'正在生成{domain}的fuzz字典')
            domains = gen_fuzz_domains(domain, self.rule)
        else:
+            logger.log('INFOR', f'使用{self.wordlist}字典')
            domains = gen_brute_domains(domain, self.wordlist)
        domains = list(domains)
-        return utils.split_list(domains, 500)  # 分割任务组 500个子域为一组任务
+        return utils.split_list(domains, self.segment)  # 分割任务组

    def deal_results(self, results):
        for result in results:
@@ -181,16 +215,22 @@ class AIOBrute(Module):
                continue
            if isinstance(result, tuple):
                subdomain, answers = result
+                if not answers:
+                    continue
                ips = {record.host for record in answers}
-                value = self.ips_times.setdefault(str(ips), 0)  # 取值 如果是首次出现的IP集合 出现次数先赋值0
+                # 取值 如果是首次出现的IP集合 出现次数先赋值0
+                value = self.ips_times.setdefault(str(ips), 0)
                self.ips_times[str(ips)] = value + 1
                ttl = answers[0].ttl
                if self.enable_wildcard:
-                    if wildcard_by_compare(ips, ttl, self.wildcard_ips, self.wildcard_ttl):
+                    if wildcard_by_compare(ips, ttl,
+                                           self.wildcard_ips,
+                                           self.wildcard_ttl):
                        continue
                if wildcard_by_times(ips, self.ips_times):
                    continue
-                logger.log('INFOR', f'发现{self.domain}的子域: {subdomain} 解析IP: {ips} TTL: {ttl}')
+                logger.log('INFOR', f'发现{self.domain}的子域: {subdomain} '
+                                    f'解析IP: {ips} TTL: {ttl}')
                self.subdomains.add(subdomain)
                self.records[subdomain] = str(ips)

@@ -199,9 +239,12 @@ class AIOBrute(Module):
            self.enable_wildcard, self.wildcard_ips, self.wildcard_ttl = detect_wildcard(domain)
        tasks = self.gen_tasks(domain)
        logger.log('INFOR', f'正在爆破{domain}的域名')
-        for task in tqdm.tqdm(tasks, desc='Progress', smoothing=1.0, ncols=True):
-            async with aiomultiprocess.Pool(processes=self.processes, initializer=init_worker,
-                                            childconcurrency=self.coroutine) as pool:
+        for task in tqdm.tqdm(tasks, desc='Progress',
+                              smoothing=1.0, ncols=True):
+            async with aiomultiprocess.Pool(processes=self.processes,
+                                            initializer=init_worker,
+                                            childconcurrency=self.coroutine)\
+                    as pool:
                try:
                    results = await pool.map(resolve.aiodns_query_a, task)
                except KeyboardInterrupt:
@@ -211,51 +254,64 @@ class AIOBrute(Module):
                    self.gen_result()
                    rx_queue.put(self.results)
                    return
-                else:
-                    self.deal_results(results)
-            self.save_json()
-            self.gen_result()
-            rx_queue.put(self.results)
+                self.deal_results(results)
+                self.save_json()
+                self.gen_result()
+                rx_queue.put(self.results)

    def run(self, rx_queue=None):
        self.domains = utils.get_domains(self.target)
        while self.domains:
            self.domain = self.domains.pop()
            start = time.time()
-            db_conn = database.connect_db()
-            table_name = self.domain.replace('.', '_')
-            database.create_table(db_conn, table_name)
+            db = Database()
+            db.create_table(self.domain)
            if not rx_queue:
                rx_queue = queue.Queue()
            logger.log('INFOR', f'开始执行{self.source}模块爆破域名{self.domain}')
-            logger.log('INFOR', f'{self.source}模块使用{self.processes}个进程乘{self.coroutine}个协程')
-            # logger.log('INFOR', f'{self.source}模块使用个进程乘{self.coroutine}个协程')
-            if self.recursive_brute and not self.fuzz:  # fuzz模式不使用递归爆破
+            logger.log('INFOR', f'使用{self.processes}进程乘{self.coroutine}协程')
+            # fuzz模式不使用递归爆破
+            if self.recursive_brute and not self.fuzz:
                logger.log('INFOR', f'开始递归爆破{self.domain}的第1层子域')
            loop = asyncio.get_event_loop()
            asyncio.set_event_loop(loop)
            loop.run_until_complete(self.main(self.domain, rx_queue))

            # 递归爆破下一层的子域
-            if self.recursive_brute and not self.fuzz:  # fuzz模式不使用递归爆破
-                for layer_num in range(1, self.recursive_depth):  # 之前已经做过1层子域爆破 当前实际递归层数是layer+1
-                    logger.log('INFOR', f'开始递归爆破{self.domain}的第{layer_num+1}层子域')
+            # fuzz模式不使用递归爆破
+            if self.recursive_brute and not self.fuzz:
+                for layer_num in range(1, self.recursive_depth):
+                    # 之前已经做过1层子域爆破 当前实际递归层数是layer+1
+                    logger.log('INFOR', f'开始递归爆破{self.domain}的'
+                                        f'第{layer_num + 1}层子域')
                    for subdomain in self.subdomains.copy():
-                        if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域爆破的限制条件
-                            loop.run_until_complete(self.main(subdomain, rx_queue))
-
-
-            while not rx_queue.empty():  # 队列不空就一直取数据存数据库
-                database.save_db(db_conn, table_name, rx_queue.get())  # 将结果存入数据库中
-            database.copy_table(db_conn, table_name)
-            database.deduplicate_subdomain(db_conn, table_name)
-            database.remove_invalid(db_conn, table_name)
+                        # 进行下一层子域爆破的限制条件
+                        if subdomain.count('.') - self.domain.count('.') == layer_num:
+                            loop.run_until_complete(self.main(subdomain,
+                                                              rx_queue))
+            # 队列不空就一直取数据存数据库
+            while not rx_queue.empty():
+                source, results = rx_queue.get()
+                # 将结果存入数据库中
+                db.save_db(self.domain, results, source)

            end = time.time()
            self.elapsed = round(end - start, 1)
            logger.log('INFOR', f'结束执行{self.source}模块爆破域名{self.domain}')
-            logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒发现{self.domain}的域名{len(self.subdomains)}个')
-            logger.log('DEBUG', f'{self.source}模块发现{self.domain}的的域名 {self.subdomains}')
+            length = len(self.subdomains)
+            logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒'
+                                f'发现{self.domain}的域名{length}个')
+            logger.log('DEBUG', f'{self.source}模块发现{self.domain}的域名:\n'
+                                f'{self.subdomains}')
+            # 数据库导出
+            if self.export:
+                if not self.path:
+                    name = f'{self.domain}_brute.{self.format}'
+                    self.path = config.result_save_path.joinpath(name)
+                dbexport.export(self.domain,
+                                valid=self.valid,
+                                path=self.path,
+                                format=self.format)


 def do(domain, result):  # 统一入口名字 方便多线程调用
@@ -270,6 +326,6 @@ def do(domain, result):  # 统一入口名字 方便多线程调用


 if __name__ == '__main__':
-    # fire.Fire(AIOBrute)
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    fire.Fire(AIOBrute)
+    # result_queue = queue.Queue()
+    # do('example.com', result_queue)
@@ -1,14 +1,10 @@
-# coding=utf-8
-"""
-被动收集类
-"""
 import time
-import queue
 import threading
 import importlib
 import config
 import dbexport
-from common import database
+from common.database import Database
+from config import logger


 class Collect(object):
@@ -27,16 +23,19 @@ class Collect(object):
    def get_mod(self):
        """
        获取要运行的模块
-        :return: None
        """
        if config.enable_all_module:
-            # modules = ['brute', 'certificates', 'crawl', 'datasets', 'intelligence', 'search']
-            modules = ['certificates', 'check', 'datasets', 'dnsquery', 'intelligence', 'search']  # crawl模块还有点问题
+            # modules = ['brute', 'certificates', 'crawl',
+            # 'datasets', 'intelligence', 'search']
+            # crawl模块还有点问题
+            modules = ['certificates', 'check', 'datasets',
+                       'dnsquery', 'intelligence', 'search']
            # modules = ['intelligence']  # crawl模块还有点问题
            for module in modules:
                module_path = config.oneforall_module_path.joinpath(module)
                for path in module_path.rglob('*.py'):
-                    import_module = ('modules.' + module, path.stem)  # 需要导入的类
+                    # 需要导入的类
+                    import_module = ('modules.' + module, path.stem)
                    self.modules.append(import_module)
        else:
            self.modules = config.enable_partial_module
@@ -49,20 +48,21 @@ class Collect(object):
            import_object = importlib.import_module('.'+name, package)
            self.collect_func.append(getattr(import_object, 'do'))

-    def run(self, rx_queue=None):
+    def run(self):
        """
        类运行入口
        """
        start = time.time()
+        logger.log('INFOR', f'开始收集{self.domain}的子域')
        self.get_mod()
        self.import_func()

-        if not rx_queue:
-            rx_queue = queue.Queue(maxsize=len(self.collect_func))  # 结果集队列
        threads = []
        # 创建多个子域收集线程
        for collect_func in self.collect_func:
-            thread = threading.Thread(target=collect_func, args=(self.domain, rx_queue), daemon=True)
+            thread = threading.Thread(target=collect_func,
+                                      args=(self.domain,),
+                                      daemon=True)
            threads.append(thread)
        # 启动所有线程
        for thread in threads:
@@ -71,18 +71,17 @@ class Collect(object):
        for thread in threads:
            thread.join()

-        db_conn = database.connect_db()
-        table_name = self.domain.replace('.', '_')
-        database.create_table(db_conn, table_name)
-        database.copy_table(db_conn, table_name)
-        database.deduplicate_subdomain(db_conn, table_name)
-        database.remove_invalid(db_conn, table_name)
-        db_conn.close()
+        # db = Database()
+        # db.create_table(self.domain)
+        # db.copy_table(self.domain, self.domain+'_collect')
+        # db.remove_invalid(self.domain)
+        # db.deduplicate_subdomain(self.domain)
        # 数据库导出
        if self.export:
            if not self.path:
-                self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
-            dbexport.export(table_name, path=self.path, format=self.format)
+                name = f'{self.domain}.{self.format}'
+                self.path = config.result_save_path.joinpath(name)
+            dbexport.export(self.domain, path=self.path, format=self.format)
        end = time.time()
        self.elapsed = round(end - start, 1)

@@ -11,160 +11,197 @@ from records import Connection
 from config import logger


-def connect_db(db_path=None):
-    """
-    获取数据库对象
+class Database(object):
+    def __init__(self, db_path=None):
+        self.conn = self.get_connection(db_path)

-    :param db_path: 数据库连接或路径
-    :return: SQLite数据库
-    """
-    logger.log('DEBUG', f'正在获取数据库连接')
-    if isinstance(db_path, Connection):
-        return db_path
-    protocol = 'sqlite:///'
-    if not db_path:  # 数据库路径为空连接默认数据库
-        db_path = f'{protocol}{config.result_save_path}/result.sqlite3'
-    else:
-        db_path = protocol + db_path
-    db = records.Database(db_path)  # 不存在数据库时会新建一个数据库
-    logger.log('DEBUG', f'使用数据库: {db_path}')
-    return db.get_connection()
+    @staticmethod
+    def get_connection(db_path):
+        """
+        获取数据库对象

+        :param db_path: 数据库连接或路径
+        :return: SQLite数据库
+        """
+        logger.log('DEBUG', f'正在获取数据库连接')
+        if isinstance(db_path, Connection):
+            return db_path
+        protocol = 'sqlite:///'
+        if not db_path:  # 数据库路径为空连接默认数据库
+            db_path = f'{protocol}{config.result_save_path}/result.sqlite3'
+        else:
+            db_path = protocol + db_path
+        db = records.Database(db_path)  # 不存在数据库时会新建一个数据库
+        logger.log('DEBUG', f'使用数据库: {db_path}')
+        return db.get_connection()

-def create_table(db_conn, table_name):
-    """
-    初始化数据库
+    def create_table(self, table_name):
+        """
+        初始化数据库

-    :param db_conn: 数据库连接
-    :param str table_name: 要创建的表名
-    """
-    logger.log('DEBUG', f'正在创建{table_name}表')
-    try:
-        db_conn.query(f'create table if not exists {table_name} ('
-                      f'id integer primary key,'
-                      f'url text,'
-                      f'subdomain text,'
-                      f'port int,'
-                      f'ips text,'
-                      f'status int,'
-                      f'reason text,'
-                      f'valid int,'
-                      f'title text,'
-                      f'banner text,'
-                      f'module text,'
-                      f'source text,'
-                      f'elapsed float,'
-                      f'count int)')
-    except Exception as e:
-        logger.log('ERROR', e)
-
-
-def save_db(db_conn, table_name, results, module_name=None):
-    """
-    将各模块结果存入数据库
-
-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    :param list results: 结果列表
-    :param str module_name: 模块名
-    """
-    logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域结果存入数据库')
-    if results:
+        :param str table_name: 要创建的表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在创建{table_name}表')
        try:
-            db_conn.bulk_query(f'insert into {table_name} (id, url, subdomain, port, ips, status,'
-                               f'reason, valid, title, banner, module, source, elapsed, count)'
-                               f'values (:id, :url, :subdomain, :port, :ips, :status, :reason, :valid,'
-                               f':title, :banner, :module, :source, :elapsed, :count)', results)
+            self.conn.query(f'create table if not exists "{table_name}" ('
+                            f'id integer primary key,'
+                            f'url text,'
+                            f'subdomain text,'
+                            f'port int,'
+                            f'ips text,'
+                            f'status int,'
+                            f'reason text,'
+                            f'valid int,'
+                            f'title text,'
+                            f'banner text,'
+                            f'module text,'
+                            f'source text,'
+                            f'elapsed float,'
+                            f'count int)')
        except Exception as e:
            logger.log('ERROR', e)

+    def save_db(self, table_name, results, module_name=None):
+        """
+        将各模块结果存入数据库

-def copy_table(db_conn, table_name):
-    """
-    复制表创建备份
+        :param str table_name: 表名
+        :param list results: 结果列表
+        :param str module_name: 模块名
+        """
+        logger.log('DEBUG', f'正在将{module_name}模块发现{table_name}的子域结果存入数据库')
+        table_name = table_name.replace('.', '_')
+        if results:
+            try:
+                self.conn.bulk_query(
+                    f'insert into "{table_name}" ('
+                    f'id, url, subdomain, port, ips, status, reason, valid,'
+                    f'title, banner, module, source, elapsed, count)'
+                    f'values (:id, :url, :subdomain, :port, :ips, :status,'
+                    f':reason, :valid, :title, :banner, :module, :source,'
+                    f':elapsed, :count)',
+                    results)
+            except Exception as e:
+                logger.log('ERROR', e)

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    new_table_name = table_name + '_bak'
-    logger.log('DEBUG', f'正在将{table_name}表复制到{new_table_name}新表')
-    try:
-        db_conn.query(f'drop table if exists {new_table_name}')
-        db_conn.query(f'create table {new_table_name} as select * from {table_name}')
-    except Exception as e:
-        logger.log('ERROR', e)
+    def copy_table(self, table_name, bak_table_name):
+        """
+        复制表创建备份

+        :param str table_name: 表名
+        :param str bak_table_name: 新表名
+        """
+        table_name = table_name.replace('.', '_')
+        bak_table_name = bak_table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在将{table_name}表复制到{bak_table_name}新表')
+        try:
+            self.conn.query(f'drop table if exists "{bak_table_name}"')
+            self.conn.query(f'create table "{bak_table_name}" '
+                            f'as select * from "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

-def clear_table(db_conn, table_name):
-    """
-    清空表中数据
+    def clear_table(self, table_name):
+        """
+        清空表中数据

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    logger.log('DEBUG', f'正在清空{table_name}表中的数据')
-    try:
-        db_conn.query(f'delete from {table_name}')
-    except Exception as e:
-        logger.log('ERROR', e)
-
-
-def deduplicate_subdomain(db_conn, table_name):
-    """
-    去重表中的子域并删除空值和无效值
-
-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    logger.log('DEBUG', f'正在去重{table_name}表中的子域')
-    try:
-        db_conn.query(f'delete from {table_name} where id not in (select min(id) from {table_name} group by subdomain)')
-    except Exception as e:
-        logger.log('ERROR', e)
-
-
-def remove_invalid(db_conn, table_name):
-    """
-        去除表中的空值或无效子域
-
-        :param db_conn: 数据库连接
        :param str table_name: 表名
        """
-    logger.log('DEBUG', f'正在去除{table_name}表中的无效子域')
-    try:
-        db_conn.query(f'delete from {table_name} where subdomain is null or valid == 0')
-    except Exception as e:
-        logger.log('ERROR', e)
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在清空{table_name}表中的数据')
+        try:
+            self.conn.query(f'delete from "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

+    def drop_table(self, table_name):
+        """
+        删除表

-def get_data(db_conn, table_name):
-    """
-    获取表中的所有数据
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在删除{table_name}表')
+        try:
+            self.conn.query(f'drop table if exists "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    """
-    logger.log('DEBUG', f'获取{table_name}表中的所有数据')
-    try:
-        rows = db_conn.query(f'select * from {table_name}')
-    except Exception as e:
-        logger.log('ERROR', e)
-    else:
-        return rows
+    def rename_table(self, table_name, new_table_name):
+        """
+        复制表创建备份

+        :param str table_name: 表名
+        :param str new_table_name: 新表名
+        """
+        table_name = table_name.replace('.', '_')
+        new_table_name = new_table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在将{table_name}表重命名为{table_name}表')
+        try:
+            self.conn.query(f'alter table "{table_name}" '
+                            f'rename to "{new_table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)

-def get_subdomain(db_conn, table_name, valid):
-    """
-    获取表中的子域数据
+    def deduplicate_subdomain(self, table_name):
+        """
+        去重表中的子域

-    :param db_conn: 数据库连接
-    :param str table_name: 表名
-    :param int valid: 是否有效
-    """
-    logger.log('DEBUG', f'获取{table_name}表中的所有数据')
-    try:
-        rows = db_conn.query(f'select * from {table_name} where valid = {valid}')
-    except Exception as e:
-        logger.log('ERROR', e)
-    else:
-        return rows
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在去重{table_name}表中的子域')
+        try:
+            self.conn.query(
+                f'delete from "{table_name}" where id not in (select min(id) '
+                f'from "{table_name}" group by subdomain)')
+        except Exception as e:
+            logger.log('ERROR', e)
+
+    def remove_invalid(self, table_name):
+        """
+        去除表中的空值或无效子域
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'正在去除{table_name}表中的无效子域')
+        try:
+            self.conn.query(
+                f'delete from "{table_name}" where '
+                f'subdomain is null or valid == 0')
+        except Exception as e:
+            logger.log('ERROR', e)
+
+    def get_data(self, table_name):
+        """
+        获取表中的所有数据
+
+        :param str table_name: 表名
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'获取{table_name}表中的所有数据')
+        try:
+            rows = self.conn.query(f'select * from "{table_name}"')
+        except Exception as e:
+            logger.log('ERROR', e)
+        else:
+            return rows
+
+    def get_subdomain(self, table_name, valid):
+        """
+        获取表中的子域数据
+
+        :param str table_name: 表名
+        :param int valid: 是否有效
+        """
+        table_name = table_name.replace('.', '_')
+        logger.log('DEBUG', f'获取{table_name}表中的所有数据')
+        try:
+            rows = self.conn.query(
+                f'select * from "{table_name}" where valid = {valid}')
+        except Exception as e:
+            logger.log('ERROR', e)
+        else:
+            return rows
@@ -6,13 +6,14 @@
 import json
 import re
 import threading
+import time

 import requests
 import config
 from config import logger
 from . import utils
 from .domain import Domain
-from common import database
+from common.database import Database


 lock = threading.Lock()
@@ -32,7 +33,38 @@ class Module(object):
        self.subdomains = set()  # 存放发现的子域
        self.records = dict()  # 存放子域解析记录
        self.results = list()  # 存放模块结果
-        self.elapsed = 0.0  # 模块执行耗时
+        self.start = time.time()  # 模块开始执行时间
+        self.end = None
+        self.elapsed = None  # 模块执行耗时
+
+    def check(self, *apis):
+        """
+        简单检查是否配置了api信息
+        :param apis: api信息元组
+        :return: 检查结果
+        """
+        if not all(apis):
+            logger.log('ALERT', f'{self.source}模块API配置有误跳过执行')
+            return False
+        return True
+
+    def begin(self):
+        """
+        输出模块开始信息
+        """
+        logger.log('DEBUG', f'开始执行{self.source}模块收集{self.domain}的子域')
+
+    def finish(self):
+        """
+        输出模块结束信息
+        """
+        self.end = time.time()
+        self.elapsed = round(self.end - self.start, 1)
+        logger.log('DEBUG', f'结束执行{self.source}模块收集{self.domain}的子域')
+        logger.log('INFOR', f'{self.source}模块耗时{self.elapsed}秒发现子域'
+                   f'{len(self.subdomains)}个')
+        logger.log('DEBUG', f'{self.source}模块发现{self.domain}的子域\n'
+                   f'{self.subdomains}')

    def get(self, url, params=None, **kwargs):
        """
@@ -44,19 +76,32 @@ class Module(object):
        :return: requests响应对象
        """
        try:
-            resp = requests.get(url, params=params, cookies=self.cookie, headers=self.header,
-                                proxies=self.proxy, timeout=self.timeout, verify=self.verify, **kwargs)
+            resp = requests.get(url,
+                                params=params,
+                                cookies=self.cookie,
+                                headers=self.header,
+                                proxies=self.proxy,
+                                timeout=self.timeout,
+                                verify=self.verify,
+                                **kwargs)
        except Exception as e:
            logger.log('ERROR', e)
            return None
-        if resp.status_code != 200:
-            logger.log('ALERT', f'GET {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
+        # 状态码非200或者响应体为空
+        if resp.status_code != 200 or not resp.content:
+            logger.log('ALERT', f'GET {resp.url} {resp.status_code} - '
+                       f'{resp.reason} {len(resp.content)}')
            content_type = resp.headers.get('Content-Type')
-            if content_type:
-                if 'json' in content_type:
-                    logger.log('ALERT', resp.json())
+            if content_type and 'json' in content_type and resp.content:
+                try:
+                    msg = resp.json()
+                except Exception as e:
+                    logger.log('DEBUG', e.args)
+                else:
+                    logger.log('ALERT', msg)
            return None
-        logger.log('DEBUG', f'GET {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
+        logger.log('DEBUG', f'GET {resp.url} {resp.status_code} - '
+                   f'{resp.reason} {len(resp.content)}')
        return resp

    def post(self, url, data=None, **kwargs):
@@ -69,18 +114,30 @@ class Module(object):
        :return: requests响应对象
        """
        try:
-            resp = requests.post(url, data=data, cookies=self.cookie, headers=self.header,
-                                 proxies=self.proxy, timeout=self.timeout, verify=self.verify, **kwargs)
+            resp = requests.post(url,
+                                 data=data,
+                                 cookies=self.cookie,
+                                 headers=self.header,
+                                 proxies=self.proxy,
+                                 timeout=self.timeout,
+                                 verify=self.verify,
+                                 **kwargs)
        except Exception as e:
            logger.log('ERROR', e)
            return None
-        if resp.status_code != 200:
+        # 状态码非200或者响应体为空
+        if resp.status_code != 200 or not resp.content:
            content_type = resp.headers.get('Content-Type')
-            if content_type:
-                if 'json' in content_type:
-                    logger.log('ALERT', resp.json())
+            if content_type and 'json' in content_type and resp.content:
+                try:
+                    msg = resp.json()
+                except Exception as e:
+                    logger.log('DEBUG', e.args)
+                else:
+                    logger.log('ALERT', msg)
            return None
-        logger.log('DEBUG', f'POST {resp.url} {resp.status_code} - {resp.reason} {len(resp.content)}')
+        logger.log('DEBUG', f'POST {resp.url} {resp.status_code} - '
+                   f'{resp.reason} {len(resp.content)}')
        return resp

    def get_header(self):
@@ -127,12 +184,14 @@ class Module(object):
        :rtype: set or list
        """
        logger.log('DEBUG', f'正则匹配响应体中的子域')
-        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+        regexp = r'(?:\>|\"|\'|\=|\,)(?:http\:\/\/|https\:\/\/)?' \
+                 r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
                 + domain.replace('.', r'\.')
        result = re.findall(regexp, html, re.I)
        if not result:
            return set()
-        deal = map(lambda s: re.sub(r'(?:http://|https://)', '', s[1:].lower(), flags=re.I), result)
+        regexp = r'(?:http://|https://)'
+        deal = map(lambda s: re.sub(regexp, '', s[1:].lower()), result)
        if distinct:
            return set(deal)
        else:
@@ -154,38 +213,65 @@ class Module(object):
        """
        logger.log('DEBUG', f'将{self.source}模块发现的子域结果保存为json文件')
        if config.save_module_result:
-            dirpath = config.result_save_path.joinpath(self.domain, self.module)
-            dirpath.mkdir(parents=True, exist_ok=True)
+            dpath = config.result_save_path.joinpath(self.domain, self.module)
+            dpath.mkdir(parents=True, exist_ok=True)
            name = self.source + '.json'
-            path = dirpath.joinpath(name)
+            path = dpath.joinpath(name)
            with open(path, mode='w', encoding='utf-8') as file:
-                result = {'domain': self.domain, 'name': self.module, 'source': self.source, 'elapsed': self.elapsed,
-                          'count': len(self.subdomains), 'subdomains': list(self.subdomains), 'records': self.records}
+                result = {'domain': self.domain,
+                          'name': self.module,
+                          'source': self.source,
+                          'elapsed': self.elapsed,
+                          'count': len(self.subdomains),
+                          'subdomains': list(self.subdomains),
+                          'records': self.records}
                json.dump(result, file, ensure_ascii=False, indent=4)

    def gen_result(self):
        results = list()
        if not len(self.subdomains):  # 一个子域都没有发现的情况
-            result = {'id': None, 'url': None, 'subdomain': None, 'port': None, 'ips': None, 'status': None,
-                      'reason': None, 'valid': 1, 'title': None, 'banner': None, 'module': self.module,
-                      'source': self.source, 'elapsed': self.elapsed, 'count': 0}
+            result = {'id': None,
+                      'url': None,
+                      'subdomain': None,
+                      'port': None,
+                      'ips': None,
+                      'status': None,
+                      'reason': None,
+                      'valid': None,
+                      'title': None,
+                      'banner': None,
+                      'module': self.module,
+                      'source': self.source,
+                      'elapsed': self.elapsed,
+                      'count': 0}
            results.append(result)
            self.results = (self.source, results)
        else:
            for subdomain in self.subdomains:
                url = 'http://' + subdomain
                ips = self.records.get(subdomain)
-                result = {'id': None, 'url': url, 'subdomain': subdomain, 'port': None, 'ips': ips, 'status': None,
-                          'reason': None, 'valid': 1, 'title': None, 'banner': None, 'module': self.module,
-                          'source': self.source, 'elapsed': self.elapsed, 'count': len(self.subdomains)}
+                result = {'id': None,
+                          'url': url,
+                          'subdomain': subdomain,
+                          'port': None,
+                          'ips': ips,
+                          'status': None,
+                          'reason': None,
+                          'valid': None,
+                          'title': None,
+                          'banner': None,
+                          'module': self.module,
+                          'source': self.source,
+                          'elapsed': self.elapsed,
+                          'count': len(self.subdomains)}
                results.append(result)
            self.results = (self.source, results)

    def save_db(self):
        lock.acquire()
-        db_conn = database.connect_db()
-        table_name = self.domain.replace('.', '_')
-        database.create_table(db_conn, table_name)
+        db = Database()
+        db.create_table(self.domain)
        source, results = self.results
-        database.save_db(db_conn, table_name, results, source)  # 将结果存入数据库中
+        # 将结果存入数据库中
+        db.save_db(self.domain, results, source)
        lock.release()
@@ -1,7 +1,6 @@
 # coding=utf-8

 import asyncio
-import functools
 import aiohttp
 from aiohttp import ClientSession
 from aiohttp.resolver import AsyncResolver
@@ -31,13 +30,17 @@ def get_ports(port):
 def gen_new_datas(datas, ports):
    logger.log('INFOR', f'正在生成请求地址')
    new_datas = []
-    protocols = ['http://', 'https://']
+    protocols = ['http://']
    for data in datas:
-        if data.get('valid'):  # 有效的子域才进行http请求探测
+        valid = data.get('valid')
+        if valid is None:  # 子域有效性未知的才进行http请求探测
            subdomain = data.get('subdomain')
            for port in ports:
                for protocol in protocols:
-                    url = f'{protocol}{subdomain}:{port}'
+                    if port == 443:
+                        url = f'https://{subdomain}:{port}'
+                    else:
+                        url = f'{protocol}{subdomain}:{port}'
                    data['id'] = None
                    data['url'] = url
                    data['port'] = port
@@ -52,64 +55,81 @@ async def fetch(session, url, semaphore):

    :param session: session对象
    :param url: url地址
-    :param semaphore: 同步对象（控制并发量）
+    :param semaphore: 并发信号量
    :return: 响应对象和响应文本
    """
    timeout = aiohttp.ClientTimeout(total=config.get_timeout)
    async with semaphore:
-        async with session.get(url, allow_redirects=config.get_redirects,
-                               timeout=timeout, proxy=config.get_proxy) as resp:
+        async with session.get(url,
+                               ssl=config.verify_ssl,
+                               allow_redirects=config.get_redirects,
+                               timeout=timeout,
+                               proxy=config.get_proxy) as resp:
            text = await resp.text()
-            return resp, text
+        return resp, text


-def request_callback(future, index, datas):
-    try:
-        resp, text = future.result()
-    except Exception as e:
-        logger.log('DEBUG', e.args)
-        datas[index]['reason'] = str(e.args)
-        datas[index]['valid'] = 0
-    else:
-        datas[index]['reason'] = resp.reason
-        datas[index]['status'] = resp.status
-        if resp.status == 400 or resp.status >= 500:
+def deal_results(datas, results):
+    for index, result in enumerate(results):
+        if isinstance(result, Exception):
+            logger.log('DEBUG', result.args)
+            datas[index]['reason'] = str(result.args)
            datas[index]['valid'] = 0
-        else:
-            headers = resp.headers
-            banner = str({'Server': headers.get('Server'), 'Via': headers.get('Via'),
-                          'X-Powered-By': headers.get('X-Powered-By')})
-            datas[index]['banner'] = banner
-            soup = BeautifulSoup(text, 'lxml')
-            title = soup.title
-            head = soup.head
-            if title:
-                datas[index]['title'] = title.text
-            elif head:
-                datas[index]['title'] = head.text
+            continue
+        if isinstance(result, tuple):
+            resp, text = result
+            datas[index]['reason'] = resp.reason
+            datas[index]['status'] = resp.status
+            if resp.status >= 500:
+                datas[index]['valid'] = 0
            else:
-                datas[index]['title'] = text
+                datas[index]['valid'] = 1
+                headers = resp.headers
+                banner = str({'Server': headers.get('Server'),
+                              'Via': headers.get('Via'),
+                              'X-Powered-By': headers.get('X-Powered-By')})
+                datas[index]['banner'] = banner
+                soup = BeautifulSoup(text, 'lxml')
+                title = soup.title
+                head = soup.head
+                if title:
+                    datas[index]['title'] = title.text
+                elif head:
+                    datas[index]['title'] = head.text
+                elif len(text) <= 200:
+                    datas[index]['title'] = text
+    return datas


 async def bulk_get_request(datas, port):
-    logger.log('INFOR', f'正在异步进行子域的GET请求')
    ports = get_ports(port)
    new_datas = gen_new_datas(datas, ports)
+    logger.log('INFOR', f'正在异步进行子域的GET请求')
+
+    limit_open_conn = config.limit_open_conn
+    if not limit_open_conn:
+        limit_open_conn = utils.get_semaphore()
+    # 使用异步域名解析器 自定义域名服务器
+    resolver = AsyncResolver(nameservers=config.resolver_nameservers)
+    conn = aiohttp.TCPConnector(ssl=config.verify_ssl,
+                                limit=limit_open_conn,
+                                limit_per_host=config.limit_per_host,
+                                resolver=resolver)
+
+    semaphore = asyncio.Semaphore(limit_open_conn)
    header = None
    if config.fake_header:
        header = utils.gen_fake_header()
-    resolver = AsyncResolver(nameservers=config.resolver_nameservers)  # 使用异步域名解析器 自定义域名服务器
-    conn = aiohttp.TCPConnector(verify_ssl=config.verify_ssl, limit=config.limit_open_conn,
-                                limit_per_host=config.limit_per_host, resolver=resolver)
-    semaphore = asyncio.Semaphore(utils.get_semaphore())
    async with ClientSession(connector=conn, headers=header) as session:
        tasks = []
        for i, data in enumerate(new_datas):
            url = data.get('url')
            task = asyncio.ensure_future(fetch(session, url, semaphore))
-            task.add_done_callback(functools.partial(request_callback, index=i, datas=new_datas))
            tasks.append(task)
        if tasks:  # 任务列表里有任务不空时才进行解析
-            await asyncio.wait(tasks)  # 等待所有task完成
+            # 等待所有task完成 错误聚合到结果列表里
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+            new_datas = deal_results(new_datas, results)
+
    logger.log('INFOR', f'完成异步进行子域的GET请求')
    return new_datas
@@ -39,7 +39,7 @@ def aiodns_resolver():
                              timeout=config.resolver_timeout)


-async def aiodns_query_a(hostname, semaphore):
+async def aiodns_query_a(hostname, semaphore=None):
    """
    异步查询A记录

@@ -47,23 +47,37 @@ async def aiodns_query_a(hostname, semaphore):
    :param semaphore: 并发查询数量
    :return: 主机名或查询结果或查询异常
    """
-    async with semaphore:
+    if semaphore is None:
        resolver = aiodns_resolver()
        answers = await resolver.query(hostname, 'A')
        return hostname, answers
+    else:
+        async with semaphore:
+            resolver = aiodns_resolver()
+            answers = await resolver.query(hostname, 'A')
+            return hostname, answers


 def resolve_callback(future, index, datas):
+    """
+    解析结果回调处理
+    :param future: future对象
+    :param index: 下标
+    :param datas: 结果集
+    """
    try:
        result = future.result()
-    except aiodns.error.DNSError as e:
+    except Exception as e:
        datas[index]['ips'] = str(e.args)
        datas[index]['valid'] = 0
    else:
        if isinstance(result, tuple):
            _, answers = result
-            ips = {record.host for record in answers}
-            datas[index]['ips'] = str(ips)
+            if answers:
+                ips = {record.host for record in answers}
+                datas[index]['ips'] = str(ips)
+            else:
+                datas[index]['ips'] = 'No answers'


 async def bulk_query_a(datas):
@@ -75,12 +89,14 @@ async def bulk_query_a(datas):
    """
    logger.log('INFOR', '正在异步查询子域的A记录')
    tasks = []
-    semaphore = asyncio.Semaphore(utils.get_semaphore())
+    semaphore = asyncio.Semaphore(config.limit_resolve_conn)
    for i, data in enumerate(datas):
        if not data.get('ips'):
            subdomain = data.get('subdomain')
            task = asyncio.ensure_future(aiodns_query_a(subdomain, semaphore))
-            task.add_done_callback(functools.partial(resolve_callback, index=i, datas=datas))  # 回调
+            task.add_done_callback(functools.partial(resolve_callback,
+                                                     index=i,
+                                                     datas=datas))  # 回调
            tasks.append(task)
    if tasks:  # 任务列表里有任务不空时才进行解析
        await asyncio.wait(tasks)  # 等待所有task完成
@@ -28,16 +28,18 @@ class Search(Module):
        :rtype: str
        """
        statements_list = []
-        subdomains_temp = set(map(lambda x: x + '.' + domain, config.subdomains_common))
+        subdomains_temp = set(map(lambda x: x + '.' + domain,
+                                  config.subdomains_common))
        subdomains_temp = list(subdomain.intersection(subdomains_temp))
        for i in range(0, len(subdomains_temp), 2):  # 同时排除2个子域
-            statements_list.append(''.join(set(map(lambda s: ' -site:' + s, subdomains_temp[i:i + 2]))))
+            statements_list.append(''.join(set(map(lambda s: ' -site:' + s,
+                                                   subdomains_temp[i:i + 2]))))
        return statements_list

    def match_location(self, domain, url):
        """
        匹配跳转之后的url
-        针对部分搜索引擎（如百度搜索）搜索展示url时有显示不全的情况
+        针对部分搜索引擎(如百度搜索)搜索展示url时有显示不全的情况
        此函数会向每条结果的链接发送head请求获取响应头的location值并做子域匹配

        :param str domain: 域名
@@ -45,6 +47,7 @@ class Search(Module):
        :return: 匹配的子域
        :rtype set
        """
-        resp = requests.head(url, headers=self.header, proxies=self.proxy, timeout=self.timeout, allow_redirects=False)
+        resp = requests.head(url, headers=self.header, proxies=self.proxy,
+                             timeout=self.timeout, allow_redirects=False)
        location = resp.headers.get('location')
        return set(utils.match_subdomain(domain, location))
@@ -5,10 +5,21 @@ import random
 import ipaddress
 import platform
 import config
-from fake_useragent import UserAgent
 from common.domain import Domain
 from config import logger

+user_agents = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/68.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:61.0) '
+    'Gecko/20100101 Firefox/68.0',
+    'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0']
+

 def match_subdomain(domain, text, distinct=True):
    """
@@ -20,7 +31,8 @@ def match_subdomain(domain, text, distinct=True):
    :return: 匹配结果
    :rtype: set or list
    """
-    regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' + domain.replace('.', r'\.')
+    regexp = r'(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.){0,}' \
+             + domain.replace('.', r'\.')
    result = re.findall(regexp, text, re.I)
    if not result:
        return set()
@@ -45,10 +57,11 @@ def gen_fake_header():
    """
    生成伪造请求头
    """
-    ua = UserAgent()
+    ua = random.choice(user_agents)
    ip = gen_random_ip()
    headers = {
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept': 'text/html,application/xhtml+xml,'
+                  'application/xml;q=0.9,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
        'Cache-Control': 'max-age=0',
@@ -56,7 +69,7 @@ def gen_fake_header():
        'DNT': '1',
        'Referer': 'https://www.google.com/',
        'Upgrade-Insecure-Requests': '1',
-        'User-Agent': ua.random,
+        'User-Agent': ua,
        'X-Forwarded-For': ip,
        'X-Real-IP': ip
    }
@@ -86,7 +99,7 @@ def split_list(ls, size):
    """
    if size == 0:
        return ls
-    return [ls[i:i+size] for i in range(0, len(ls), size)]
+    return [ls[i:i + size] for i in range(0, len(ls), size)]


 def get_domains(target):
@@ -122,9 +135,8 @@ def get_semaphore():
    """
    system = platform.system()
    if system == 'Windows':
-        return 500
+        return 400
    elif system == 'Linux':
-        return 1000
+        return 800
    elif system == 'Darwin':
-        return 1000
-
+        return 800
@@ -2,7 +2,7 @@
 """
 OneForAll配置
 """
-
+import os
 import sys
 import pathlib
 from loguru import logger
@@ -18,36 +18,48 @@ result_save_path = oneforall_relpath.joinpath('results')  # 结果保存目录
 save_module_result = True  # 保存模块中各脚本结果 默认保存
 enable_all_module = True  # 启用所有模块 默认启用
 enable_partial_module = []  # 启用部分模块 必须禁用enable_all_module才能生效
-# enable_partial_module = [('modules.search', 'google')]  # 启用部分模块 必须禁用enable_all_module才能生效
+# 只使用ask和baidu搜索引擎收集子域
+# enable_partial_module = [('modules.search', 'ask')
+#                          ('modules.search', 'baidu')]


 # 爆破模块设置
-enable_brute_module = False  # 使用爆破模块（默认禁用）
+enable_brute_module = False  # 使用爆破模块(默认禁用)
 enable_wildcard_check = True  # 开启泛解析检测 会去掉泛解析的子域
-brute_processes_num = None  # 爆破时使用的进程数（根据系统中CPU数量情况设置 不宜大于CPU数量 默认None为系统中的CPU数量）
-brute_coroutine_num = 128  # 爆破时每个进程下的协程数（不宜大于1000）
-brute_wordlist_path = None  # 爆破所使用的字典路径 默认data/subdomains.dict
-use_general_wordlist = False  # 是否使用通用字典（默认禁用）
-enable_recursive_brute = False  # 是否使用递归爆破（默认禁用）
-brute_recursive_depth = 2  # 递归爆破深度（默认2层）
-recursive_namelist_path = None  # 爆破下一层子域所使用的字典路径 默认data/next_subdomains.dict
+# 爆破时使用的进程数(根据系统中CPU数量情况设置 不宜大于CPU数量 默认为系统中的CPU数量)
+brute_processes_num = os.cpu_count()
+brute_coroutine_num = 128  # 爆破时每个进程下的协程数(不宜大于1000)
+# 爆破所使用的字典路径 默认data/subdomains.txt
+brute_wordlist_path = 'data/subdomains.txt'
+brute_task_segment = 500
+# 参数segment的设置受CPU性能，网络带宽，运营商限制等问题影响，默认设置500个子域为一任务组，
+# 当你觉得你的环境不受以上因素影响，当前爆破速度较慢，那么强烈建议根据字典大小调整大小：
+# 十万字典建议设置为5000，百万字典设置为50000
+enable_recursive_brute = False  # 是否使用递归爆破(默认禁用)
+brute_recursive_depth = 2  # 递归爆破深度(默认2层)
+# 爆破下一层子域所使用的字典路径 默认data/next_subdomains.txt
+recursive_namelist_path = 'data/next_subdomains.txt'
 enable_fuzz = False  # 是否使用fuzz模式枚举域名
 fuzz_rule = ''  # fuzz域名的正则 示例：[a-z][0-9] 第一位是字母 第二位是数字
 ips_appear_maximum = 10  # 同一IP集合出现次数超过10认为是泛解析

 # 代理设置
-enable_proxy = True  # 是否使用代理 全局开关
+enable_proxy = False  # 是否使用代理(全局开关)
 proxy_all_module = False  # 代理所有模块
-proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch', 'GoogleAPISearch',
-                        'GoogleSearch', 'YahooSearch', 'YandexSearch']  # 代理自定义的模块
-proxy_pool = [{'http': 'http://127.0.0.1:1080', 'https': 'https://127.0.0.1:1080'}]  # 代理池
-# proxy_pool = [{'http': 'socks5://127.0.0.1:10808', 'https': 'socks5://127.0.0.1:10808'}]  # 代理池
+proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
+                        'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
+                        'YandexSearch', 'CrossDomainXml',
+                        'ContentSecurityPolicy']  # 代理自定义的模块
+proxy_pool = [{'http': 'http://127.0.0.1:1080',
+               'https': 'https://127.0.0.1:1080'}]  # 代理池
+# proxy_pool = [{'http': 'socks5://127.0.0.1:10808',
+#                'https': 'socks5://127.0.0.1:10808'}]  # 代理池


 # 网络请求设置
 enable_fake_header = True  # 启用伪造请求头
 request_delay = 1  # 请求时延
-request_timeout = 30  # 请求超时(AskSearch和YahooSearch比较慢)
+request_timeout = 60  # 请求超时
 request_verify = True  # 请求SSL验证

 # 搜索模块设置
@@ -66,6 +78,7 @@ resolver_nameservers = [
 ]  # 指定查询的DNS域名服务器
 resolver_timeout = 5.0  # 解析超时时间
 resolver_lifetime = 30.0  # 解析存活时间
+limit_resolve_conn = 50  # 限制同一时间解析的数量(默认50)

 # http探测设置
 small_ports = {80, 443}
@@ -80,14 +93,18 @@ xlarge_ports = {80, 81, 300, 443, 591, 593, 832, 981, 1010, 1311, 2082,
                8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080, 9090,
                9091, 9200, 9443, 9800, 9981, 12443, 16080, 18091, 18092,
                20720, 28017}
-ports = {'small': small_ports, 'medium': medium_ports, 'large': large_ports, 'xlarge': xlarge_ports}
+ports = {'small': small_ports, 'medium': medium_ports,
+         'large': large_ports, 'xlarge': xlarge_ports}
 verify_ssl = False
-get_proxy = None  # aiohttp 支持 HTTP/HTTPS形式的代理 proxy="http://user:pass@some.proxy.com"
-get_timeout = 10  # http请求探测总超时时间 None或者0则表示不检测超时
+# aiohttp 支持 HTTP/HTTPS形式的代理
+get_proxy = None  # proxy="http://user:pass@some.proxy.com"
+get_timeout = 120  # http请求探测总超时时间 None或者0则表示不检测超时
 get_redirects = True  # 允许请求跳转
 fake_header = True  # 使用伪造请求头
-limit_open_conn = 100  # 限制同一时间打开的连接数（默认100），0表示不限制
-limit_per_host = 0  # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数（默认0表示不限制）
+# 限制同一时间打开的连接数(默认None，根据系统不同设置，Windows系统400 其他系统800)
+limit_open_conn = None
+# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
+limit_per_host = 0  # 默认0表示不限制


 # 模块API配置
@@ -102,7 +119,8 @@ binaryedge_api = ''
 # Binaryedge可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
 chinaz_api = ''

-# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/cognitive-services/bing-web-search-api/#web-json
+# Bing可以免费注册获取API：https://azure.microsoft.com/zh-cn/services/
+# cognitive-services/bing-web-search-api/#web-json
 bing_api_id = ''
 bing_api_key = ''

@@ -113,7 +131,8 @@ securitytrails_api = ''
 fofa_api_email = ''  # fofa用户邮箱
 fofa_api_key = ''  # fofa用户key

-# Google可以免费注册获取API: https://developers.google.com/custom-search/v1/overview
+# Google可以免费注册获取API:
+# https://developers.google.com/custom-search/v1/overview
 # 免费的API只能查询前100条结果
 google_api_key = ''  # Google API搜索key
 google_api_cx = ''  # Google API搜索cx
@@ -149,15 +168,21 @@ dnsdb_api_key = ''
 # 免费的API有效期只有2天，到期之后可以再次生成，每天可以查询50次。
 ipv4info_api_key = ''

-subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs', 'web', 'www', 'job', 'docs', 'news', 'blog',
-                     'data', 'help', 'live', 'mall', 'blogs', 'files', 'forum', 'store', 'mobile'}
+subdomains_common = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
+                     'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
+                     'help', 'live', 'mall', 'blogs', 'files', 'forum',
+                     'store', 'mobile'}

 # 日志配置
-log_fmt = '<light-green>{time:HH:mm:ss,SSS}</light-green> ' \
+stdout_fmt = '<cyan>{time:HH:mm:ss,SSS}</cyan> ' \
+          '[<level>{level: <5}</level>] ' \
+          '<blue>{module}</blue>:<cyan>{line}</cyan> - ' \
+          '<level>{message}</level>'
+logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
          '[<level>{level: <5}</level>] ' \
          '<cyan>{process.name}</cyan>:<cyan>{thread.name: <10}</cyan> | ' \
-          '<blue>{module}</blue>.<blue>{function}</blue>:<blue>{line}</blue> - ' \
-          '<level>{message}</level>'
+          '<blue>{module}</blue>.<blue>{function}</blue>:' \
+          '<blue>{line}</blue> - <level>{message}</level>'

 log_path = result_save_path.joinpath('oneforall.log')

@@ -169,13 +194,9 @@ logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️')
 logger.level(name='ERROR', no=40, color='<red><bold>', icon='❌️')
 logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')

-logger.add(sys.stdout, level='INFOR', format=log_fmt, enqueue=True)
-logger.add(log_path, level='TRACE', format=log_fmt, enqueue=True, encoding='utf-8')
+if not os.environ.get('PYTHONIOENCODING'):  # 设置编码
+    os.environ['PYTHONIOENCODING'] = 'utf-8'

-# 调试模式
-# import urllib3
-# urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-# request_proxy = [{'http': 'http://127.0.0.1:8080', 'https': 'https://127.0.0.1:8080'}]
-# request_verify = False
-# enable_all_module = False  # 启用所有模块 默认启用
-# enable_partial_module = [('modules.certificates', 'censys_api')]  # 启用部分模块 必须禁用enable_all_module才能生效
+logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True)
+logger.add(log_path, level='TRACE', format=logfile_fmt, enqueue=True,
+           encoding='utf-8')
@@ -1,50 +1,61 @@
+#!/usr/bin/python3
 # coding=utf-8

+"""
+OneForAll数据库导出模块
+
+:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
+:license: GNU General Public License v3.0, see LICENSE for more details.
+"""
+
 import fire
-from common import database
+from common.database import Database
 from config import logger


 def export(table, db=None, valid=None, path=None, format='xlsx', output=False):
    """
-    将数据库导出为指定格式文件
+    OneForAll数据库导出模块

-    :param str table:   要导出的表
-    :param str db:      要导出的数据库路径（默认为results/result.sqlite3）
-    :param int valid:   导出子域的有效性（默认None）
-    :param str format:  导出格式（默认xlsx）
-    :param str path:    导出路径(默认None)
-    :param bool output: 是否将导出数据输出到终端（默认False）
+    Example:
+        python dbexport.py --table name --format csv --path= ./result.csv
+        python dbexport.py --db result.db --table name --output False

    Note:
        参数valid可选值1，0，None，分别表示导出有效，无效，全部子域
-        参数format可选格式：'csv','tsv','json','yaml','html','xls','xlsx','dbf','latex','ods'
+        参数format可选格式：'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
+                         'dbf', 'latex', 'ods'
        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件

-    Example:
-        python dbexport.py --db result.db --table name --format csv --output False
-        python dbexport.py --db result.db --table name --format csv --path= ./result.csv
+    :param str table:   要导出的表
+    :param str db:      要导出的数据库路径(默认为results/result.sqlite3)
+    :param int valid:   导出子域的有效性(默认None)
+    :param str format:  导出格式(默认xlsx)
+    :param str path:    导出路径(默认None)
+    :param bool output: 是否将导出数据输出到终端(默认False)
    """
-    db_conn = database.connect_db(db)
+    database = Database(db)
    if valid is None:
-        rows = database.get_data(db_conn, table)
+        rows = database.get_data(table)
    elif isinstance(valid, int):
-        rows = database.get_subdomain(db_conn, table, valid)
+        rows = database.get_subdomain(table, valid)
    else:
-        rows = database.get_data(db_conn, table)  # 意外情况导出全部子域
+        rows = database.get_data(table)  # 意外情况导出全部子域
    if output:
        print(rows.dataset)
    if not path:
        path = 'export.' + format
-    logger.log('INFOR', f'正在将数据库中{table}表导出到{path}')
+    logger.log('INFOR', f'正在将数据库中{table}表导出')
    try:
        with open(path, 'w') as file:
            file.write(rows.export(format))
            logger.log('INFOR', '成功完成导出')
+            logger.log('INFOR', path)
    except TypeError:
        with open(path, 'wb') as file:
            file.write(rows.export(format))
            logger.log('INFOR', '成功完成导出')
+            logger.log('INFOR', path)
    except Exception as e:
        logger.log('ERROR', e)

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+"""
+示例
+"""
+
+import oneforall
+
+test = oneforall.OneForAll(target='example.com')
+test.run()
@@ -1 +0,0 @@
-# coding=utf-8
@@ -1,6 +1,4 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
 from config import logger
@@ -24,11 +22,10 @@ class CensysAPI(Query):
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
        data = {
-            'query': 'parsed.names: owasp.org',
+            'query': 'parsed.names: example.com',
            'page': 1,
            'fields': ['parsed.subject_dn'],
            'flatten': True}
-
        resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
        if not resp:
            return
@@ -40,7 +37,7 @@ class CensysAPI(Query):
        subdomains_find = self.match(self.domain, str(resp_json))
        self.subdomains = self.subdomains.union(subdomains_find)
        pages = resp_json.get('metadata').get('pages')
-        for page in range(2, pages+1):
+        for page in range(2, pages + 1):
            time.sleep(self.delay)
            data['page'] = page
            resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
@@ -49,39 +46,29 @@ class CensysAPI(Query):
            subdomains_find = self.match(self.domain, str(resp.json()))
            self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not (self.id and self.secret):
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.id, self.secret):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = CensysAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,10 +1,7 @@
-# coding=utf-8
 import time
-import queue
 import config
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class CertDBAPI(Query):
@@ -12,7 +9,7 @@ class CertDBAPI(Query):
        Query.__init__(self)
        self.domain = domain
        self.module = 'Certificate'
-        self.source = 'CertDBQuery'
+        self.source = 'CertDBAPIQuery'
        self.addr = 'https://api.spyse.com/v1/subdomains'
        self.token = config.certdb_api_token

@@ -25,50 +22,44 @@ class CertDBAPI(Query):
            time.sleep(self.delay)
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
-            params = {'domain': self.domain, 'api_token': self.token, 'page': page_num}
+            params = {'domain': self.domain,
+                      'api_token': self.token,
+                      'page': page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
-            resp_json = resp.json()
-            subdomains_find = utils.match_subdomain(self.domain, str(resp_json))
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            json = resp.json()
+            subdomains_find = utils.match_subdomain(self.domain, str(json))
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains_find)
            page_num += 1
-            if resp_json.get('count') < 30:  # 默认每次查询最多返回30条 当前条数小于30条说明已经查完
+            # 默认每次查询最多返回30条 当前条数小于30条说明已经查完
+            if json.get('count') < 30:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.token:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.token):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):   # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = CertDBAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class CertSpotter(Query):
@@ -21,42 +18,37 @@ class CertSpotter(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'domain': self.domain, 'include_subdomains': 'true', 'expand': 'dns_names'}
+        params = {'domain': self.domain, 'include_subdomains': 'true',
+                  'expand': 'dns_names'}
        resp = self.get(self.addr, params)
        if not resp:
            return
        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
+
    """
    query = CertSpotter(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class Crtsh(Query):
@@ -28,35 +25,27 @@ class Crtsh(Query):
        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
        self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Crtsh(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class Entrust(Query):
@@ -21,42 +18,35 @@ class Entrust(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'fields': 'subjectDN', 'domain': self.domain, 'includeExpired': 'true'}
+        params = {'fields': 'subjectDN', 'domain': self.domain,
+                  'includeExpired': 'true'}
        resp = self.get(self.addr, params)
        if not resp:
            return
        subdomains_find = utils.match_subdomain(self.domain, str(resp.json()))
        self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Entrust(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class Google(Query):
@@ -12,7 +9,8 @@ class Google(Query):
        self.domain = self.register(domain)
        self.module = 'Certificate'
        self.source = 'GoogleQuery'
-        self.addr = 'https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch'
+        self.addr = 'https://transparencyreport.google.com/' \
+                    'transparencyreport/api/v3/httpsreport/ct/certsearch'

    def query(self):
        """
@@ -21,42 +19,36 @@ class Google(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'include_expired': 'true', 'include_subdomains': 'true', 'domain': self.domain}
+        params = {'include_expired': 'true', 'include_subdomains': 'true',
+                  'domain': self.domain}
        resp = self.get(self.addr, params)
        if not resp:
            return
        subdomains_find = utils.match_subdomain(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Google(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,19 +1,18 @@
-# coding=utf-8
 """
-查询域名的NS记录（域名服务器记录，记录该域名由哪台域名服务器解析），
-检查查出的域名服务器是否开启DNS域传送，如果开启且没做访问控制和身份验证便加以利用获取域名的所有记录
+查询域名的NS记录(域名服务器记录，记录该域名由哪台域名服务器解析)，检查查出的域名服务器是否开启
+DNS域传送，如果开启且没做访问控制和身份验证便加以利用获取域名的所有记录

-DNS域传送（DNS zone transfer）指的是一台备用域名服务器使用来自主域名服务器的数据刷新自己的域数据库，
-目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
-当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获取某个域的所有记录。
+DNS域传送(DNS zone transfer)指的是一台备用域名服务器使用来自主域名服务器的数据刷新自己的域
+数据库，目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
+当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获取某个
+域的所有记录。
 """
-import time
-import queue
 import dns.resolver
 import dns.zone
-from config import logger
-from common import utils, resolve
+
+from common import resolve, utils
 from common.module import Module
+from config import logger


 class CheckAXFR(Module):
@@ -25,13 +24,37 @@ class CheckAXFR(Module):
        self.domain = self.register(domain)
        self.module = 'Check'
        self.source = 'AXFRCheck'
-        self.nsservers = []
        self.results = []

+    def axfr(self, server):
+        """
+        执行域传送
+
+        :param server: 域名服务器
+        """
+        logger.log('DEBUG', f'尝试对{self.domain}的域名服务器{server}进行域传送')
+        try:
+            xfr = dns.query.xfr(server, self.domain, timeout=30.0)
+            zone = dns.zone.from_xfr(xfr)
+        except Exception as e:
+            logger.log('DEBUG', str(e))
+            logger.log('DEBUG', f'对{self.domain}的域名服务器{server}进行域传送失败')
+            return
+        names = zone.nodes.keys()
+        for name in names:
+            full_domain = str(name) + '.' + self.domain
+            subdomain = utils.match_subdomain(self.domain, full_domain)
+            self.subdomains = self.subdomains.union(subdomain)
+            record = zone[name].to_text(name)
+            self.results.append(record)
+        if self.results:
+            logger.log('DEBUG', f'发现{self.domain}在{server}上的域传送记录')
+            logger.log('DEBUG', '\n'.join(self.results))
+            self.results = []
+
    def check(self):
        """
        正则匹配响应头中的内容安全策略字段以发现子域名
-        :return: None
        """
        resolver = resolve.dns_resolver()
        try:
@@ -39,60 +62,35 @@ class CheckAXFR(Module):
        except Exception as e:
            logger.log('ERROR', e)
            return
-        self.nsservers = [str(answer) for answer in answers]
-        if not len(self.nsservers):
+        nsservers = [str(answer) for answer in answers]
+        if not len(nsservers):
            logger.log('ALERT', f'没有找到{self.domain}的NS域名服务器记录')
            return
-        for nsserver in self.nsservers:
-            logger.log('DEBUG', f'正在尝试对{self.domain}的域名服务器{nsserver}进行域传送')
-            try:
-                xfr = dns.query.xfr(nsserver, self.domain)
-                zone = dns.zone.from_xfr(xfr)
-            except Exception as e:
-                logger.log('DEBUG', str(e))
-                logger.log('INFOR', f'对{self.domain}的域名服务器{nsserver}进行域传送失败')
-                continue
-            else:
-                names = zone.nodes.keys()
-                for name in names:
-                    subdomain = utils.match_subdomain(self.domain, str(name)+'.'+self.domain)
-                    self.subdomains = self.subdomains.union(subdomain)
-                    record = zone[name].to_text(name)
-                    self.results.append(record)
-            if self.results:
-                logger.log('INFOR', f'发现{self.domain}在{nsserver}上的域传送记录')
-                logger.log('DEBUG', '\n'.join(self.results))
-                self.results = []
+        for nsserver in nsservers:
+            self.axfr(nsserver)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}的域传送漏洞')
-        start = time.time()
+        self.begin()
        self.check()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}的域传送漏洞')
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口
+
    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    check = CheckAXFR(domain)
-    check.run(rx_queue)
-    logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
-    logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
+    check.run()


 if __name__ == '__main__':
-    # do('ZoneTransfer.me')
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('ZoneTransfer.me')
+    do('example.com')
@@ -1,18 +1,16 @@
-# coding=utf-8
 """
 检查crossdomain.xml文件收集子域名
 """
-import time
-import queue
-from config import logger
-from common.utils import match_subdomain
+
 from common.module import Module
+from common import utils


 class CheckCDX(Module):
    """
    检查crossdomain.xml文件收集子域名
    """
+
    def __init__(self, domain: str):
        Module.__init__(self)
        self.domain = self.register(domain)
@@ -22,43 +20,43 @@ class CheckCDX(Module):
    def check(self):
        """
        检查crossdomain.xml收集子域名
-        :return:
        """
-        url = f'http://{self.domain}/crossdomain.xml'
-        self.header = self.get_header()
-        self.proxy = self.get_proxy(self.source)
-        resp = self.get(url)
-        if not resp:
+        urls = [f'http://{self.domain}/crossdomain.xml',
+                f'https://{self.domain}/crossdomain.xml',
+                f'http://www.{self.domain}/crossdomain.xml',
+                f'https://www.{self.domain}/crossdomain.xml']
+        response = None
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            response = self.get(url)
+            if response:
+                break
+        if not response:
            return
-        self.subdomains = match_subdomain(self.domain, resp.text)
+        self.subdomains = utils.match_subdomain(self.domain, response.text)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的crossdomain.xml')
-        start = time.time()
+        self.begin()
        self.check()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的crossdomain.xml')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口
+
    :param domain: 域名
-    :param rx_queue: 结果集队列
    """
    check = CheckCDX(domain)
-    check.run(rx_queue)
-    logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
-    logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
+    check.run()


 if __name__ == '__main__':
-    do('163.com')
+    do('example.com')
@@ -1,16 +1,14 @@
 #!/usr/bin/env python3
-# coding=utf-8

 """
 检查域名证书收集子域名
 """
-import ssl
-import time
-import queue
 import socket
-from config import logger
+import ssl
+
 from common import utils
 from common.module import Module
+from config import logger


 class CheckCert(Module):
@@ -36,35 +34,27 @@ class CheckCert(Module):
        subdomains_find = utils.match_subdomain(self.domain, str(cert_dict))
        self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域的证书中的子域')
-        start = time.time()
+        self.begin()
        self.check()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域的证书中的子域')
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    check = CheckCert(domain)
-    check.run(rx_queue)
-    logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
-    logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
+    check.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,23 +1,23 @@
-# coding=utf-8
 """
 检查内容安全策略收集子域名收集子域名
 """
-import time
-import queue
-from config import logger
+import requests
+
 from common import utils
 from common.module import Module
+from config import logger


 class CheckCSP(Module):
    """
    检查内容安全策略收集子域名
    """
+
    def __init__(self, domain, header):
        Module.__init__(self)
        self.domain = self.register(domain)
        self.module = 'Check'
-        self.source = 'Content-Security-Policy'
+        self.source = 'ContentSecurityPolicy'
        self.header = header

    def check(self):
@@ -25,11 +25,18 @@ class CheckCSP(Module):
        正则匹配响应头中的内容安全策略字段以发现子域名
        """
        if not self.header:
-            url = f'http://www.{self.domain}'
-            resp = self.get(url)
-            if not resp:
+            urls = [f'http://{self.domain}', f'https://{self.domain}',
+                    f'http://www.{self.domain}', f'https://www.{self.domain}']
+            response = None
+            for url in urls:
+                self.header = self.get_header()
+                self.proxy = self.get_proxy(self.source)
+                response = self.get(url)
+                if response:
+                    break
+            if not response:
                return
-            self.header = resp.headers
+            self.header = response.headers
        csp = self.header.get('Content-Security-Policy')
        if not csp:
            logger.log('DEBUG', f'{self.domain}域的响应头不存在内容安全策略字段')
@@ -37,39 +44,29 @@ class CheckCSP(Module):
        logger.log('DEBUG', f'{self.domain}域的响应头存在内容安全策略字段')
        self.subdomains = utils.match_subdomain(self.domain, csp)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
-        start = time.time()
+        self.begin()
        self.check()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        logger.log('DEBUG', f'结束执行{self.source}检查{self.domain}域响应头中的内容安全策略字段')
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)


-def do(domain, rx_queue, header=None):  # 统一入口名字 方便多线程调用
+def do(domain, header=None):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    :param dict or None header: 响应头
    """
    check = CheckCSP(domain, header)
-    check.run(rx_queue)
-    logger.log('INFOR', f'{check.source}模块耗时{check.elapsed}秒发现子域{len(check.subdomains)}个')
-    logger.log('DEBUG', f'{check.source}模块发现的子域 {check.subdomains}')
+    check.run()


 if __name__ == '__main__':
-    import requests
-    # resp = requests.get('https://content-security-policy.com/')
-    result_queue = queue.Queue()
-    resp = requests.get('https://www.baidu.com/')
-    do('google-analytics.com', result_queue, resp.headers)
+    resp = requests.get('https://content-security-policy.com/')
+    do('google-analytics.com', resp.headers)
@@ -0,0 +1,64 @@
+"""
+检查内容安全策略收集子域名收集子域名
+"""
+import requests
+
+from common.module import Module
+from common import utils
+from config import logger
+
+
+class CheckRobots(Module):
+    """
+    检查robots.txt收集子域名
+    """
+
+    def __init__(self, domain):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Check'
+        self.source = 'Robots'
+
+    def check(self):
+        """
+        正则匹配域名的robots.txt文件中的子域
+        """
+        urls = [f'http://{self.domain}/robots.txt',
+                f'https://{self.domain}/robots.txt',
+                f'http://www.{self.domain}/robots.txt',
+                f'https://www.{self.domain}/robots.txt']
+        response = None
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            response = self.get(url, allow_redirects=False)
+            if response:
+                break
+        if not response:
+            return
+        self.subdomains = utils.match_subdomain(self.domain, response.text)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    check = CheckRobots(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    do('qq.com')
@@ -0,0 +1,77 @@
+"""
+检查内容安全策略收集子域名收集子域名
+"""
+import requests
+
+from common.module import Module
+from common import utils
+from config import logger
+
+
+class CheckRobots(Module):
+    """
+    检查sitemap收集子域名
+    """
+
+    def __init__(self, domain):
+        Module.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Check'
+        self.source = 'Sitemap'
+
+    def check(self):
+        """
+        正则匹配域名的sitemap文件中的子域
+        """
+        urls = [f'http://{self.domain}/sitemap.xml',
+                f'https://{self.domain}/sitemap.xml',
+                f'http://www.{self.domain}/sitemap.xml',
+                f'https://www.{self.domain}/sitemap.xml',
+                f'http://{self.domain}/sitemap.txt',
+                f'https://{self.domain}/sitemap.txt',
+                f'http://www.{self.domain}/sitemap.txt',
+                f'https://www.{self.domain}/sitemap.txt',
+                f'http://{self.domain}/sitemap.html',
+                f'https://{self.domain}/sitemap.html',
+                f'http://www.{self.domain}/sitemap.html',
+                f'https://www.{self.domain}/sitemap.html',
+                f'http://{self.domain}/sitemap_index.xml',
+                f'https://{self.domain}/sitemap_index.xml',
+                f'http://www.{self.domain}/sitemap_index.xml',
+                f'https://www.{self.domain}/sitemap_index.xml']
+        response = None
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            self.timeout = 10
+            response = self.get(url, allow_redirects=False)
+            if response:
+                break
+        if not response:
+            return
+        self.subdomains = utils.match_subdomain(self.domain, response.text)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    check = CheckRobots(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    do('qq.com')
@@ -1,6 +1,3 @@
-# coding=utf-8
-import time
-import queue
 import cdx_toolkit
 from common.crawl import Crawl
 from config import logger
@@ -29,43 +26,36 @@ class ArchiveCrawl(Crawl):
        for resp in cdx.iter(url, limit=limit):
            if resp.data.get('status') not in ['301', '302']:
                url = resp.data.get('url')
-                subdomains_find = self.match(self.register(domain), url + resp.text)
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                subdomains_find = self.match(self.register(domain),
+                                             url + resp.text)
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.crawl(self.domain, 50)
-
        # 爬取已发现的子域以发现新的子域
        for subdomain in self.subdomains:
            if subdomain != self.domain:
                self.crawl(subdomain, 10)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    crawl = ArchiveCrawl(domain)
-    crawl.run(result)
-    logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
-    logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
+    crawl.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,6 @@
-# coding=utf-8
-import time
-import queue
-from tqdm import tqdm
 import cdx_toolkit
+from tqdm import tqdm
+
 from common.crawl import Crawl
 from config import logger

@@ -30,42 +28,34 @@ class CommonCrawl(Crawl):
        for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
            if resp.data.get('status') not in ['301', '302']:
                subdomains_find = self.match(self.register(domain), resp.text)
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.crawl(self.domain, 50)
-
        # 爬取已发现的子域以发现新的子域
        for subdomain in self.subdomains:
            if subdomain != self.domain:
                self.crawl(subdomain, 10)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    crawl = CommonCrawl(domain)
-    crawl.run(result)
-    logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
-    logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
+    crawl.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class BinaryEdgeAPI(Query):
@@ -30,39 +27,29 @@ class BinaryEdgeAPI(Query):
        subdomains_find = self.match(self.domain, str(resp.json()))
        self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.api:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.api):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = BinaryEdgeAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.query import Query
-from config import logger


 class BufferOver(Query):
@@ -25,37 +22,30 @@ class BufferOver(Query):
        if not resp:
            return
        subdomains_find = self.match(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = BufferOver(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.query import Query
-from config import logger


 class Chinaz(Query):
@@ -25,37 +22,30 @@ class Chinaz(Query):
        if not resp:
            return
        subdomains_find = self.match(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Chinaz(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class ChinazAPI(Query):
@@ -27,41 +24,32 @@ class ChinazAPI(Query):
        if not resp:
            return
        subdomains_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.api:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.api):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = ChinazAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class CirclAPI(Query):
@@ -27,38 +24,32 @@ class CirclAPI(Query):
        if not resp:
            return
        subdomains_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        if self.user and self.pwd:
-            start = time.time()
-            self.query()
-            end = time.time()
-            self.elapsed = round(end - start, 1)
-            self.save_json()
-            self.gen_result()
-            self.save_db()
-            rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
+        if not self.check(self.user, self.pwd):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = CirclAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,10 +1,7 @@
-# coding=utf-8
 import random
 import time
-import queue
 from bs4 import BeautifulSoup
 from common.query import Query
-from config import logger


 class DNSdb(Query):
@@ -29,48 +26,44 @@ class DNSdb(Query):
        if resp.status_code == 200:
            if 'index' in resp.text:
                soup = BeautifulSoup(resp.text, features='lxml')
-                index_urls = set(map(lambda x: self.addr + self.domain + x.text, soup.find_all('a')))
-                for url in index_urls:
-                    self.delay = random.randint(2, 5)  # 休眠绕过CloudFlare的DDoS保护
+                urls = set(map(lambda x: self.addr + self.domain + x.text,
+                               soup.find_all('a')))
+                for url in urls:
+                    # 休眠绕过CloudFlare的DDoS保护
+                    self.delay = random.randint(2, 5)
                    time.sleep(self.delay)
                    resp = self.get(url)
                    if not resp:
                        return
                    subdomains_find = self.match(self.domain, resp.text)
-                    self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                    # 合并搜索子域名搜索结果
+                    self.subdomains = self.subdomains.union(subdomains_find)
            else:
                subdomains_find = self.match(self.domain, resp.text)
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = DNSdb(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,10 +1,7 @@
-# coding=utf-8
 import time
-import queue
 import config
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class DNSdbAPI(Query):
@@ -30,41 +27,33 @@ class DNSdbAPI(Query):
            return
        if resp.status_code == 200:
            subdomains_find = utils.match_subdomain(self.domain, resp.text)
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.api:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.api):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
+
    """
    query = DNSdbAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,15 +1,9 @@
-# coding=utf-8
 import time
-import queue
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class DNSdumpster(Query):
-    """
-
-    """
    def __init__(self, domain):
        Query.__init__(self)
        self.domain = self.register(domain)
@@ -29,43 +23,38 @@ class DNSdumpster(Query):
        if not resp:
            return
        self.cookie = resp.cookies
-        data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'), 'targetip': self.domain}
+        data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'),
+                'targetip': self.domain}
        resp = self.post(self.addr, data)
        if not resp:
            return
        subdomains_find = utils.match_subdomain(self.domain, resp.text)
        if subdomains_find:
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = DNSdumpster(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+
+    do('example.com')
@@ -1,9 +1,5 @@
-# coding=utf-8
-import time
-import queue
-from common.query import Query
 from common import utils
-from config import logger
+from common.query import Query


 class HackerTarget(Query):
@@ -27,37 +23,30 @@ class HackerTarget(Query):
        if resp.status_code == 200:
            subdomains_find = utils.match_subdomain(self.domain, resp.text)
            if subdomains_find:
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = HackerTarget(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,5 @@
-# coding=utf-8
-import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class IPv4InfoAPI(Query):
@@ -23,7 +19,8 @@ class IPv4InfoAPI(Query):
        while True:
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
-            params = {'type': 'SUBDOMAINS', 'key': self.api, 'value': self.domain, 'page': page}
+            params = {'type': 'SUBDOMAINS', 'key': self.api,
+                      'value': self.domain, 'page': page}
            resp = self.get(self.addr, params)
            if not resp:
                return
@@ -33,43 +30,38 @@ class IPv4InfoAPI(Query):
            subdomains_find = self.match(self.domain, str(resp_json))
            if not subdomains_find:
                break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
-            subdomains = resp_json.get('Subdomains')  # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
-            if len(subdomains) < 300:  # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains_find)
+            # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
+            subdomains = resp_json.get('Subdomains')
+            # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
+            if len(subdomains) < 300:
                break
            page += 1
            if page >= 50:  # ipv4info子域查询接口最多允许查询50页
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = IPv4InfoAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,11 +1,9 @@
-# coding=utf-8
+import hashlib
 import re
 import time
-import queue
-import hashlib
 from urllib import parse
+
 from common.query import Query
-from config import logger


 class NetCraft(Query):
@@ -24,16 +22,21 @@ class NetCraft(Query):
        绕过NetCraft的JS验证
        """
        self.header = self.get_header()  # Netcraft会检查User-Agent
-        self.cookie = self.get(self.init).cookies
+        resp = self.get(self.init)
+        if not resp:
+            return None
+        self.cookie = resp.cookies
        cookie_value = self.cookie['netcraft_js_verification_challenge']
-        verify_taken = hashlib.sha1(parse.unquote(cookie_value).encode('utf-8')).hexdigest()
+        cookie_encode = parse.unquote(cookie_value).encode('utf-8')
+        verify_taken = hashlib.sha1(cookie_encode).hexdigest()
        self.cookie['netcraft_js_verification_response'] = verify_taken

    def query(self):
        """
        向接口查询子域并做子域匹配
        """
-        self.bypass_verification()
+        if not self.bypass_verification():
+            return
        last = ''
        while True:
            time.sleep(self.delay)
@@ -46,41 +49,34 @@ class NetCraft(Query):
            subdomains_find = self.match(self.domain, resp.text)
            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
                break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains_find)
            if 'Next page' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                break
            last = re.search(r'&last=.*' + self.domain, resp.text).group(0)
            self.page_num += self.per_page_num

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = NetCraft(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,10 +1,7 @@
-# coding=utf-8
-import time
-import queue
 import random
-from common.query import Query
+
 from common import utils
-from config import logger
+from common.query import Query


 class PTRArchive(Query):
@@ -21,7 +18,8 @@ class PTRArchive(Query):
        """
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        self.cookie = {'pa_id': str(random.randint(0, 1000000000))}  # 绕过主页前端JS验证
+        # 绕过主页前端JS验证
+        self.cookie = {'pa_id': str(random.randint(0, 1000000000))}
        params = {'label': self.domain, 'date': 'ALL'}
        resp = self.get(self.addr, params)
        if not resp:
@@ -29,37 +27,30 @@ class PTRArchive(Query):
        if resp.status_code == 200:
            subdomains_find = utils.match_subdomain(self.domain, resp.text)
            if subdomains_find:
-                self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                # 合并搜索子域名搜索结果
+                self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = PTRArchive(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.query import Query
-from config import logger


 class Riddler(Query):
@@ -25,37 +22,30 @@ class Riddler(Query):
        if not resp:
            return
        subdomains_find = self.match(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Riddler(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,7 @@
-# coding=utf-8
-import time
-import queue
 import json
+import time
+
 from common.query import Query
-from config import logger


 class Robtex(Query):
@@ -36,37 +34,30 @@ class Robtex(Query):
                    return
                subdomains_find = self.match(self.domain, resp.text)
                if subdomains_find:
-                    self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+                    # 合并搜索子域名搜索结果
+                    self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = Robtex(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class SecurityTrailsAPI(Query):
@@ -28,44 +25,35 @@ class SecurityTrailsAPI(Query):
        resp = self.get(url, params)
        if not resp:
            return
-        subdomains_prefix = resp.json()['subdomains']
-        subdomains_find = [f'{prefix}.{self.domain}' for prefix in subdomains_prefix]
+        prefixs = resp.json()['subdomains']
+        subdomains_find = [f'{prefix}.{self.domain}' for prefix in prefixs]
        if subdomains_find:
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.api:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.api):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = SecurityTrailsAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,6 @@
-# coding=utf-8
 import time
-import queue
+
 from common.query import Query
-from config import logger


 class SiteDossier(Query):
@@ -31,40 +29,34 @@ class SiteDossier(Query):
            subdomains_find = self.match(self.domain, resp.text)
            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
                break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
-            if 'Show next 100 items' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains_find)
+            # 搜索页面没有出现下一页时停止搜索
+            if 'Show next 100 items' not in resp.text:
                break
            self.page_num += self.per_page_num

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = SiteDossier(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,18 +1,17 @@
 #!/usr/bin/env python3
-# coding=utf-8

 """
 通过枚举域名常见的SRV记录并做查询来发现子域
 """

-import time
-import queue
-import json
 import asyncio
+import json
+
 import aiodns
-from common.module import Module
+
 from common import utils
-from config import logger, data_storage_path, resolver_nameservers
+from common.module import Module
+from config import data_storage_path, logger, resolver_nameservers


 class BruteSRV(Module):
@@ -69,34 +68,28 @@ class BruteSRV(Module):
        if not len(self.subdomains):
            logger.log('DEBUG', f'没有找到{self.domain}的SRV记录')

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始枚举{self.domain}域的SRV记录')
-        start = time.time()
+        self.begin()
        self.brute()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束枚举{self.domain}域的SRV记录')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口
+
    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    brute = BruteSRV(domain)
-    brute.run(rx_queue)
-    logger.log('INFOR', f'{brute.source}模块耗时{brute.elapsed}秒发现{brute.domain}的子域{len(brute.subdomains)}个')
-    logger.log('DEBUG', f'{brute.source}模块发现{brute.domain}的子域 {brute.subdomains}')
+    brute.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+
+    do('example.com')
@@ -1,66 +0,0 @@
-# coding=utf-8
-import time
-import queue
-import config
-from common.query import Query
-from config import logger
-
-
-class RiskIQ(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = self.register(domain)
-        self.module = 'Intelligence'
-        self.source = 'RiskIQQuery'
-        self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
-        self.username = config.riskiq_api_username
-        self.key = config.riskiq_api_key
-
-    def query(self):
-        """
-        向接口查询子域并做子域匹配
-        """
-        self.header = self.get_header()
-        self.proxy = self.get_proxy(self.source)
-        params = {'query': self.domain}
-        resp = self.get(url=self.addr, params=params, auth=(self.username, self.key))
-        if not resp:
-            return
-        resp_json = resp.json()
-        subdomains_find = resp_json.get('subdomains')
-        if subdomains_find:
-            self.subdomains = set(map(lambda x: x + '.' + self.domain, subdomains_find))
-
-    def run(self, rx_queue):
-        """
-        类执行入口
-        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
-        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
-
-
-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    :param rx_queue: 结果集队列
-    """
-    return
-    query = RiskIQ(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
-
-
-if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
@@ -0,0 +1,58 @@
+import config
+from common.query import Query
+
+
+class RiskIQ(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Intelligence'
+        self.source = 'RiskIQAPIQuery'
+        self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
+        self.user = config.riskiq_api_username
+        self.key = config.riskiq_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'query': self.domain}
+        resp = self.get(url=self.addr,
+                        params=params,
+                        auth=(self.user, self.key))
+        if not resp:
+            return
+        resp_json = resp.json()
+        subnames = resp_json.get('subdomains')
+        if subnames:
+            self.subdomains = set(map(lambda sub: f'{sub}.{self.domain}',
+                                      subnames))
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.user, self.key):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = RiskIQ(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,66 +0,0 @@
-# coding=utf-8
-import time
-import queue
-import config
-from common.query import Query
-from config import logger
-
-
-class ThreatBookAPI(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = self.register(domain)
-        self.module = 'Intelligence'
-        self.source = 'ThreatBookAPIQuery'
-        self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
-        self.key = config.threatbook_api_key
-
-    def query(self, domain):
-        """
-        向接口查询子域并做子域匹配
-        """
-        self.header = self.get_header()
-        self.proxy = self.get_proxy(self.source)
-        params = {'apikey': self.key, 'domain': domain, 'field': 'sub_domains'}
-        resp = self.post(self.addr, params)
-        if not resp:
-            return
-        subdomain_find = self.match(self.domain, str(resp.json()))
-        self.subdomains = self.subdomains.union(subdomain_find)
-
-    def run(self, rx_queue):
-        """
-        类执行入口
-        """
-        if not self.key:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
-            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
-        self.query(self.domain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')
-
-
-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    :param rx_queue: 结果集队列
-    """
-    query = ThreatBookAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
-
-
-if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
@@ -0,0 +1,54 @@
+import config
+from common.query import Query
+
+
+class ThreatBookAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = self.register(domain)
+        self.module = 'Intelligence'
+        self.source = 'ThreatBookAPIQuery'
+        self.addr = 'https://x.threatbook.cn/api/v1/domain/query'
+        self.key = config.threatbook_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'apikey': self.key,
+                  'domain': self.domain,
+                  'field': 'sub_domains'}
+        resp = self.post(self.addr, params)
+        if not resp:
+            return
+        subdomain_find = self.match(self.domain, str(resp.json()))
+        self.subdomains = self.subdomains.union(subdomain_find)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.key):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = ThreatBookAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,8 +1,6 @@
-# coding=utf-8
 import time
-import queue
+
 from common.query import Query
-from config import logger


 class ThreatMiner(Query):
@@ -20,42 +18,36 @@ class ThreatMiner(Query):
        time.sleep(self.delay)
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'e': 'subdomains_container', 'q': self.domain, 't': 0, 'rt': 10}
+        params = {'e': 'subdomains_container',
+                  'q': self.domain, 't': 0, 'rt': 10}
        resp = self.get(self.addr, params)
        if not resp:
            return
        subdomains_find = self.match(self.domain, resp.text)
-        self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+        # 合并搜索子域名搜索结果
+        self.subdomains = self.subdomains.union(subdomains_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.query()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = ThreatMiner(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.query import Query
-from config import logger

+from common.query import Query

 '''
 最多查询100条
@@ -18,7 +15,7 @@ class VirusTotal(Query):
        self.addr = 'https://www.virustotal.com/ui/domains/{}/subdomains'
        self.domain = self.register(domain)

-    def query(self, domain):
+    def query(self):
        """
        向接口查询子域并做子域匹配
        """
@@ -30,7 +27,7 @@ class VirusTotal(Query):
                                'TE': 'Trailers'})
            self.proxy = self.get_proxy(self.source)
            params = {'limit': '40', 'cursor': next_cursor}
-            resp = self.get(url=self.addr.format(domain), params=params)
+            resp = self.get(url=self.addr.format(self.domain), params=params)
            if not resp:
                return
            resp_json = resp.json()
@@ -51,35 +48,27 @@ class VirusTotal(Query):
            else:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
-        self.query(self.domain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.begin()
+        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    query = VirusTotal(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,5 @@
-# coding=utf-8
-import time
-import queue
 import config
 from common.query import Query
-from config import logger


 class VirusTotalAPI(Query):
@@ -15,13 +11,13 @@ class VirusTotalAPI(Query):
        self.addr = 'https://www.virustotal.com/vtapi/v2/domain/report'
        self.key = config.virustotal_api_key

-    def query(self, domain):
+    def query(self):
        """
        向接口查询子域并做子域匹配
        """
        self.header = self.get_header()
        self.proxy = self.get_proxy(self.source)
-        params = {'apikey': self.key, 'domain': domain}
+        params = {'apikey': self.key, 'domain': self.domain}
        resp = self.get(self.addr, params)
        if not resp:
            return
@@ -29,40 +25,29 @@ class VirusTotalAPI(Query):
        subdomain_find = set(resp_json.get('subdomains'))
        self.subdomains = self.subdomains.union(subdomain_find)

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.key:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.key):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块查询{self.domain}的子域')
-        start = time.time()
-        self.query(self.domain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.begin()
+        self.query()
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块查询{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
-    return
    query = VirusTotalAPI(domain)
-    query.run(rx_queue)
-    logger.log('INFOR', f'{query.source}模块耗时{query.elapsed}秒发现{query.domain}的子域{len(query.subdomains)}个')
-    logger.log('DEBUG', f'{query.source}模块发现{query.domain}的子域 {query.subdomains}')
+    query.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.search import Search
-from config import logger


 class Ask(Search):
@@ -37,53 +34,48 @@ class Ask(Search):
            if not subdomain_find:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomain_find.issubset(self.subdomains):
                    break
            self.subdomains = self.subdomains.union(subdomain_find)
            self.page_num += 1
            if '>Next<' not in resp.text:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search(self.domain, full_search=True)
-
        # 排除同一子域搜索结果过多的子域以发现新的子域
        for statement in self.filter(self.domain, self.subdomains):
            self.search(self.domain, filtered_subdomain=statement)

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Ask(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
-from common.search import Search
 from bs4 import BeautifulSoup
-from config import logger
+from common.search import Search


 class Baidu(Search):
@@ -25,7 +22,8 @@ class Baidu(Search):
        """
        bs = BeautifulSoup(html, features='lxml')
        subdomains_all = set()
-        for find_res in bs.find_all('a', {'class': 'c-showurl'}):  # 获取搜索结果中所有的跳转URL地址
+        # 获取搜索结果中所有的跳转URL地址
+        for find_res in bs.find_all('a', {'class': 'c-showurl'}):
            url = find_res.get('href')
            subdomain = self.match_location(domain, url)
            subdomains_all = subdomains_all.union(subdomain)
@@ -45,32 +43,37 @@ class Baidu(Search):
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'wd': query, 'pn': self.page_num, 'rn': self.per_page_num}
+            params = {'wd': query, 'pn': self.page_num,
+                      'rn': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
            if len(domain) > 12:  # 解决百度搜索结果中域名过长会显示不全的问题
-                subdomains_find = self.redirect_match(domain, resp.text)  # 获取百度跳转URL响应头的Location字段获取直链
+                # 获取百度跳转URL响应头的Location字段获取直链
+                subdomains_find = self.redirect_match(domain, resp.text)
            else:
                subdomains_find = self.match(domain, resp.text)
            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains_find.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains_find)
            self.page_num += self.per_page_num
-            if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 搜索页面没有出现下一页时停止搜索
+            if '&pn={next_pn}&'.format(next_pn=self.page_num) not in resp.text:
                break
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -79,32 +82,28 @@ class Baidu(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Baidu(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.search import Search
-from config import logger


 class Bing(Search):
@@ -34,7 +31,8 @@ class Bing(Search):
            time.sleep(self.delay)
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'q': query, 'first': self.page_num, 'count': self.per_page_num}
+            params = {'q': query, 'first': self.page_num,
+                      'count': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
@@ -42,21 +40,23 @@ class Bing(Search):
            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains_find.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
-            if '<div class="sw_next>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains_find)
+            # 搜索页面没有出现下一页时停止搜索
+            if '<div class="sw_next>' not in resp.text:
                break
            self.page_num += self.per_page_num
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -65,32 +65,28 @@ class Bing(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Bing(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.search import Search
-from config import logger


 class BingAPI(Search):
@@ -11,8 +8,9 @@ class BingAPI(Search):
        Search.__init__(self)
        self.domain = domain
        self.module = 'Search'
-        self.source = 'BingCustomSearch'
-        self.addr = 'https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search'
+        self.source = 'BingAPISearch'
+        self.addr = 'https://api.cognitive.microsoft.com/' \
+                    'bingcustomsearch/v7.0/search'
        self.id = config.bing_api_id
        self.key = config.bing_api_key
        self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数
@@ -42,23 +40,22 @@ class BingAPI(Search):
            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains_find.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains_find)
            self.page_num += self.per_page_num
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not (self.id and self.key):
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.id, self.key):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -67,32 +64,28 @@ class BingAPI(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = BingAPI(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,7 +1,5 @@
-# coding=utf-8
 import re
 import time
-import queue
 from common.search import Search
 from config import logger

@@ -36,7 +34,8 @@ class DuckDuckGO(Search):
            if not subdomain_find:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomain_find.issubset(self.subdomains):
                    break
            self.subdomains = self.subdomains.union(subdomain_find)
            try:
@@ -45,14 +44,15 @@ class DuckDuckGO(Search):
            except Exception as e:
                logger.error(e)
                break
-            data.update({'s': s, 'nextParams': '', 'o': 'json', 'dc': dc, 'api': '/d.js'})
+            data.update({'s': s, 'nextParams': '', 'o': 'json',
+                         'dc': dc, 'api': '/d.js'})

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -61,33 +61,30 @@ class DuckDuckGO(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    # return  # 暂时还有点问题
    search = DuckDuckGO(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
-import time
-import queue
 import random
+import time
 from common.search import Search
-from config import logger


 class Exalead(Search):
@@ -30,7 +27,8 @@ class Exalead(Search):
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'q': query, 'elements_per_page': '30', "start_index": self.page_num}
+            params = {'q': query, 'elements_per_page': '30',
+                      "start_index": self.page_num}
            resp = self.get(url=self.addr, params=params)
            if not resp:
                return
@@ -47,12 +45,12 @@ class Exalead(Search):
            if 'title="Go to the next page"' not in resp.text:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -62,32 +60,28 @@ class Exalead(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Exalead(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,71 +1,64 @@
-# coding=utf-8
-import time
-import queue
-import json
-import base64
-import config
-from common.search import Search
-from config import logger
-
-
-class FoFa(Search):
-    def __init__(self, domain):
-        Search.__init__(self)
-        self.domain = domain
-        self.module = 'Search'
-        self.source = 'FoFaSearch'
-        self.addr = 'https://fofa.so/api/v1/search/all'
-        self.delay = 1
-        self.email = config.fofa_api_email
-        self.key = config.fofa_api_key
-
-    def search(self):
-        """
-        发送搜索请求并做子域匹配
-        """
-        self.page_num = 1
-        query_base64 = base64.b64encode(f'domain={self.domain}'.encode('utf-8'))
-        while True:
-            time.sleep(self.delay)
-            self.header = self.get_header()
-            self.proxy = self.get_proxy(self.source)
-            query = {'email': self.email, 'key': self.key, 'qbase64': query_base64, 'page': self.page_num}
-            resp = self.get(self.addr, query)
-            if not resp:
-                return
-            subdomain_find = self.match(self.domain, resp.text)
-            self.subdomains = self.subdomains.union(subdomain_find)
-            self.page_num += 1
-
-    def run(self, rx_queue):
-        """
-        类执行入口
-        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
-        self.search()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')
-
-
-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    :param rx_queue: 结果集队列
-    """
-    search = FoFa(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
-
-
-if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+import base64
+import time
+
+import config
+from common.search import Search
+from config import logger
+
+
+class FoFa(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'FoFaAPISearch'
+        self.addr = 'https://fofa.so/api/v1/search/all'
+        self.delay = 1
+        self.email = config.fofa_api_email
+        self.key = config.fofa_api_key
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        self.page_num = 1
+        query_data = base64.b64encode(f'domain={self.domain}'.encode('utf-8'))
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = {'email': self.email, 'key': self.key,
+                     'qbase64': query_data, 'page': self.page_num}
+            resp = self.get(self.addr, query)
+            if not resp:
+                return
+            subdomain_find = self.match(self.domain, resp.text)
+            self.subdomains = self.subdomains.union(subdomain_find)
+            self.page_num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.check(self.email, self.key):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def do(domain):  # 统一入口名字 方便多线程调用
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = FoFa(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
-import time
-import queue
 import random
+import time
 from common.search import Search
-from config import logger


 class Google(Search):
@@ -51,17 +48,17 @@ class Google(Search):
                    break
            self.subdomains = self.subdomains.union(subdomain_find)
            page_num += per_page_num
-            if 'start='+str(page_num) not in resp.text:
+            if 'start=' + str(page_num) not in resp.text:
                break
            if '302 Moved' in resp.text:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -70,32 +67,28 @@ class Google(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Google(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,9 +1,6 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.search import Search
-from config import logger


 class GoogleAPI(Search):
@@ -32,7 +29,8 @@ class GoogleAPI(Search):
            time.sleep(self.delay)
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
-            params = {'key': self.key, 'cx': self.cx, 'q': word, 'fields': 'items/link',
+            params = {'key': self.key, 'cx': self.cx,
+                      'q': word, 'fields': 'items/link',
                      'start': self.page_num, 'num': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
@@ -48,16 +46,13 @@ class GoogleAPI(Search):
            if self.page_num > 100:  # 免费的API只能查询前100条结果
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not (self.cx and self.key):
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.cx, self.key):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -66,32 +61,28 @@ class GoogleAPI(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = GoogleAPI(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,11 +1,5 @@
-# coding=utf-8
-import time
-import queue
-import queue
 import config
-# from shodan import Shodan
 from common.search import Search
-from config import logger


 class ShodanAPI(Search):
@@ -13,7 +7,7 @@ class ShodanAPI(Search):
        Search.__init__(self)
        self.domain = self.register(domain)
        self.module = 'Search'
-        self.source = 'ShodanSearch'
+        self.source = 'ShodanAPISearch'
        self.addr = 'https://api.shodan.io/shodan/host/search'
        self.key = config.shodan_api_key

@@ -26,48 +20,41 @@ class ShodanAPI(Search):
        query = 'hostname:.' + self.domain
        page = 1
        while True:
-            params = {'key': self.key, 'page': page, 'query': query, 'minify': True, 'facets': {'hostnames'}}
+            params = {'key': self.key, 'page': page, 'query': query,
+                      'minify': True, 'facets': {'hostnames'}}
            resp = self.get(self.addr, params)
            if not resp:
                return
            subdomain_find = self.match(self.domain, resp.text)
+            if not subdomain_find:  # 搜索没有发现子域名则停止搜索
+                break
            if subdomain_find:
                self.subdomains = self.subdomains.union(subdomain_find)
            page += 1

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not self.key:
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.key):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = ShodanAPI(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    results = queue.Queue()
-    do('qq.com', results)
+    do('example.com')
@@ -1,8 +1,6 @@
-# coding=utf-8
 import time
-import queue
+
 from common.search import Search
-from config import logger


 class So(Search):
@@ -37,21 +35,23 @@ class So(Search):
            if not subdomain_find:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomain_find.issubset(self.subdomains):
                    break
            self.subdomains = self.subdomains.union(subdomain_find)
            page_num += 1
-            if '<a id="snext"' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 搜索页面没有出现下一页时停止搜索
+            if '<a id="snext"' not in resp.text:
                break
-            if self.page_num * self.per_page_num >= self.limit_num:  # 搜索条数限制
+            # 搜索条数限制
+            if self.page_num * self.per_page_num >= self.limit_num:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -60,32 +60,28 @@ class So(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = So(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,4 @@
-# coding=utf-8
-import time
-import queue
 from common.search import Search
-from config import logger


 class Sogou(Search):
@@ -27,7 +23,8 @@ class Sogou(Search):
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            word = 'site:' + domain + filtered_subdomain
-            payload = {'query': word, 'page': self.page_num, "num": self.per_page_num}
+            payload = {'query': word, 'page': self.page_num,
+                       "num": self.per_page_num}
            resp = self.get(self.addr, payload)
            if not resp:
                return
@@ -35,21 +32,24 @@ class Sogou(Search):
            if not subdomain_find:
                break
            if not full_search:
-                if subdomain_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomain_find.issubset(self.subdomains):
                    break
            self.subdomains = self.subdomains.union(subdomain_find)
            self.page_num += 1
-            if '<a id="sogou_next"' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+            # 搜索页面没有出现下一页时停止搜索
+            if '<a id="sogou_next"' not in resp.text:
                break
-            if self.page_num * self.per_page_num >= self.limit_num:  # 搜索条数限制
+            # 搜索条数限制
+            if self.page_num * self.per_page_num >= self.limit_num:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -58,32 +58,28 @@ class Sogou(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Sogou(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.search import Search
-from config import logger


 class Yahoo(Search):
@@ -11,8 +8,8 @@ class Yahoo(Search):
        self.domain = domain
        self.module = 'Search'
        self.source = 'YahooSearch'
-        self.init = 'https://hk.search.yahoo.com/'
-        self.addr = 'https://hk.search.yahoo.com/search'
+        self.init = 'https://search.yahoo.com/'
+        self.addr = 'https://search.yahoo.com/search'
        self.limit_num = 1000  # 限制搜索条数
        self.delay = 5
        self.per_page_num = 40
@@ -43,21 +40,23 @@ class Yahoo(Search):
            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains_find.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains_find)
            if '>Next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                break
            self.page_num += self.per_page_num
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -66,32 +65,28 @@ class Yahoo(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Yahoo(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,8 +1,5 @@
-# coding=utf-8
 import time
-import queue
 from common.search import Search
-from config import logger


 class Yandex(Search):
@@ -25,13 +22,17 @@ class Yandex(Search):
        :param bool full_search: 全量搜索
        """
        self.page_num = 0  # 二次搜索重新置0
-        self.cookie = self.get(self.init).cookies  # 获取cookie bing在搜索时需要带上cookie
+        resp = self.get(self.init)
+        if not resp:
+            return
+        self.cookie = resp.cookies  # 获取cookie
        while True:
            time.sleep(self.delay)
            self.header = self.get_header()
            self.proxy = self.get_proxy(self.source)
            query = 'site:' + domain + filtered_subdomain
-            params = {'text': query, 'p': self.page_num, 'numdoc': self.per_page_num}
+            params = {'text': query, 'p': self.page_num,
+                      'numdoc': self.per_page_num}
            resp = self.get(self.addr, params)
            if not resp:
                return
@@ -39,21 +40,23 @@ class Yandex(Search):
            if not subdomains_find:  # 搜索没有发现子域名则停止搜索
                break
            if not full_search:
-                if subdomains_find.issubset(self.subdomains):  # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                # 搜索中发现搜索出的结果有完全重复的结果就停止搜索
+                if subdomains_find.issubset(self.subdomains):
                    break
-            self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
+            # 合并搜索子域名搜索结果
+            self.subdomains = self.subdomains.union(subdomains_find)
            if '>next</a>' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
                break
            self.page_num += 1
            if self.page_num >= self.limit_num:  # 搜索条数限制
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
+
        self.search(self.domain, full_search=True)

        # 排除同一子域搜索结果过多的子域以发现新的子域
@@ -62,32 +65,28 @@ class Yandex(Search):

        # 递归搜索下一层的子域
        if self.recursive_search:
-            for layer_num in range(1, self.recursive_times):  # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+            for layer_num in range(1, self.recursive_times):
                for subdomain in self.subdomains:
-                    if subdomain.count('.') - self.domain.count('.') == layer_num:  # 进行下一层子域搜索的限制条件
+                    # 进行下一层子域搜索的限制条件
+                    count = subdomain.count('.') - self.domain.count('.')
+                    if count == layer_num:
                        self.search(subdomain)
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = Yandex(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -1,6 +1,4 @@
-# coding=utf-8
 import time
-import queue
 import config
 from common.search import Search
 from config import logger
@@ -20,7 +18,6 @@ class ZoomEyeAPI(Search):
    def login(self):
        """
        登陆获取查询taken
-        :return:
        """
        url = 'https://api.zoomeye.org/user/login'
        data = {'username': self.user, 'password': self.pwd}
@@ -30,7 +27,7 @@ class ZoomEyeAPI(Search):
            return
        resp_json = resp.json()
        if resp.status_code == 200:
-            # print('登陆成功')
+            logger.log('DEBUG', f'{self.source}模块登录成功')
            return resp_json.get('access_token')
        else:
            logger.log('ALERT', resp_json.get('message'))
@@ -59,39 +56,29 @@ class ZoomEyeAPI(Search):
            if resp.status_code == 403:
                break

-    def run(self, rx_queue):
+    def run(self):
        """
        类执行入口
        """
-        if not (self.user and self.pwd):
-            logger.log('ERROR', f'{self.source}模块API配置错误')
-            logger.log('ALERT', f'不执行{self.source}模块')
+        if not self.check(self.user, self.pwd):
            return
-        logger.log('DEBUG', f'开始执行{self.source}模块搜索{self.domain}的子域')
-        start = time.time()
+        self.begin()
        self.search()
-        end = time.time()
-        self.elapsed = round(end - start, 1)
+        self.finish()
        self.save_json()
        self.gen_result()
        self.save_db()
-        rx_queue.put(self.results)
-        logger.log('DEBUG', f'结束执行{self.source}模块搜索{self.domain}的子域')


-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
    """
    类统一调用入口

    :param str domain: 域名
-    :param rx_queue: 结果集队列
    """
    search = ZoomEyeAPI(domain)
-    search.run(rx_queue)
-    logger.log('INFOR', f'{search.source}模块耗时{search.elapsed}秒发现{search.domain}的子域{len(search.subdomains)}个')
-    logger.log('DEBUG', f'{search.source}模块发现{search.domain}的子域 {search.subdomains}')
+    search.run()


 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('owasp.org', result_queue)
+    do('example.com')
@@ -2,56 +2,78 @@
 # coding=utf-8

 """
-OneForAll是一款强大的子域收集神器
+OneForAll是一款功能强大的子域收集工具
+
+:copyright: Copyright (c) 2019, Jing Ling. All rights reserved.
+:license: GNU General Public License v3.0, see LICENSE for more details.
 """

 import asyncio
+import sys
+
 import fire
 import config
 import dbexport
+from datetime import datetime
 from config import logger
 from collect import Collect
 from aiobrute import AIOBrute
-from common import utils, database, resolve, request
+from common import utils, resolve, request
+from common.database import Database

-__author__ = 'Jing Ling, Black Star'
-__contact__ = 'admin@hackfun.org'
-__copyright__ = 'Copyright (c) 2019, Jing Ling. All rights reserved.'
-__license__ = 'GNU General Public License v3.0'
-__version__ = '0.0.1'
+yellow = '\033[01;33m'
+white = '\033[01;37m'
+green = '\033[01;32m'
+blue = '\033[01;34m'
+red = '\033[1;31m'
+end = '\033[0m'
+
+version = white + '{' + red + 'v0.0.4#dev' + white + '}'
+
+banner = f"""{yellow}
+             ___             _ _ 
+ ___ ___ ___|  _|___ ___ ___| | | {version}{green}
+| . |   | -_|  _| . |  _| .'| | | {blue}
+|___|_|_|___|_| |___|_| |__,|_|_| {white}git.io/fjHT1{end}
+"""


 class OneForAll(object):
    """
-    OneForAll是一款强大的子域收集神器
+    OneForAll是一款功能强大的子域收集工具

-    Version: 0.0.1
-    Project: https://github.com/shmilylty/OneForAll/
+    Version: 0.0.4
+    Project: https://git.io/fjHT1

-    :param str target:  单个域名或者每行一个域名的文件路径
-    :param bool brute:  是否使用爆破模块（默认禁用）
-    :param str port:    HTTP请求验证的端口范围（默认medium）
-    :param int valid:   导出子域的有效性（默认1）
-    :param str format:  导出格式（默认xlsx）
-    :param str path:    导出路径(默认None)
-    :param bool output: 是否将导出数据输出到终端（默认False）
+    Example:
+        python oneforall.py --target example.com run
+        python oneforall.py --target ./domains.txt run
+        python oneforall.py --target example.com --brute True --port medium
+        python oneforall.py --target example.com --valid None run
+        python oneforall.py --target example.com --format csv --path result.csv
+        python oneforall.py --target example.com --output True run

    Note:
        参数valid可选值有1，0，None，分别表示导出有效，无效，全部子域
        参数port可选值有'small', 'medium', 'large', 'xlarge'，详见config.py配置
-        参数format可选格式有'csv','tsv','json','yaml','html','xls','xlsx','dbf','latex','ods'
+        参数format可选格式有'csv', 'tsv', 'json', 'yaml', 'html', 'xls', 'xlsx',
+                         'dbf', 'latex', 'ods'
        参数path为None会根据format参数和域名名称在项目结果目录生成相应文件

-    Example:
-    python oneforall.py --target example.com run
-    python oneforall.py --target example.com --brute True --port medium valid 1 run
-    python oneforall.py --target ./domains.txt --format csv --path= ./result.csv  --output True run
+    :param str target:  单个域名或者每行一个域名的文件路径
+    :param bool brute:  是否使用爆破模块(默认禁用)
+    :param str port:    HTTP请求验证的端口范围(默认medium)
+    :param int valid:   导出子域的有效性(默认1)
+    :param str format:  导出格式(默认xlsx)
+    :param str path:    导出路径(默认None)
+    :param bool output: 是否将导出数据输出到终端(默认False)
    """
-    def __init__(self, target, brute=False, port='medium', valid=1, path=None, format='xlsx', output=False):
+    def __init__(self, target, brute=None, port='medium', valid=1, path=None,
+                 format='xlsx', output=False):
        self.target = target
        self.port = port
        self.domains = set()
-        self.domain = ''
+        self.domain = str()
        self.datas = list()
        self.brute = brute or config.enable_brute_module
        self.valid = valid
@@ -59,35 +81,62 @@ class OneForAll(object):
        self.format = format
        self.output = output

+    def main(self):
+        collect = Collect(self.domain, export=False)
+        collect.run()
+        if self.brute:
+            # 由于爆破会有大量dns解析请求 并发爆破可能会导致其他任务中的网络请求异常
+            brute = AIOBrute(self.domain, export=False)
+            brute.run()
+
+        db = Database()
+        db.copy_table(self.domain, self.domain+'_ori')
+        db.remove_invalid(self.domain)
+        db.deduplicate_subdomain(self.domain)
+        self.datas = db.get_data(self.domain).as_dict()
+        loop = asyncio.get_event_loop()
+        asyncio.set_event_loop(loop)
+
+        # 解析域名地址
+        task = resolve.bulk_query_a(self.datas)
+        self.datas = loop.run_until_complete(task)
+
+        # 保存解析结果
+        resolve_table = self.domain + '_res'
+        db.drop_table(resolve_table)
+        db.create_table(resolve_table)
+        db.save_db(resolve_table, self.datas, 'resolve')
+
+        # 请求域名地址
+        task = request.bulk_get_request(self.datas, self.port)
+        self.datas = loop.run_until_complete(task)
+        # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
+        loop.run_until_complete(asyncio.sleep(0.25))
+        loop.close()
+
+        db.clear_table(self.domain)
+        db.save_db(self.domain, self.datas)
+
+        # 数据库导出
+        if not self.path:
+            name = f'{self.domain}.{self.format}'
+            self.path = config.result_save_path.joinpath(name)
+        dbexport.export(self.domain, db.conn, self.valid, self.path,
+                        self.format, self.output)
+
+        rename_table = self.domain + '_last'
+        db.drop_table(rename_table)
+        db.rename_table(self.domain, rename_table)
+
    def run(self):
+        print(banner)
+        dt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        print(f'[*] Starting OneForAll @ {dt}\n')
        logger.log('INFOR', f'开始运行OneForAll')
        self.domains = utils.get_domains(self.target)
        if self.domains:
            for self.domain in self.domains:
-                collect = Collect(self.domain, export=False)
-                collect.run()
-                if self.brute:
-                    # 由于爆破会有大量dns解析请求 并发常常会导致其他任务中的网络请求超时
-                    brute = AIOBrute(self.domain)
-                    brute.run()
-                table_name = self.domain.replace('.', '_')
-                db_conn = database.connect_db()
-                self.datas = database.get_data(db_conn, table_name).as_dict()
-                loop = asyncio.get_event_loop()
-                asyncio.set_event_loop(loop)
-                self.datas = loop.run_until_complete(resolve.bulk_query_a(self.datas))
-                self.datas = loop.run_until_complete(request.bulk_get_request(self.datas, self.port))
-                loop.run_until_complete(asyncio.sleep(0.25))  # 在关闭事件循环前加入一小段延迟让底层连接得到关闭的缓冲时间
-                loop.close()
-                database.clear_table(db_conn, table_name)
-                database.save_db(db_conn, table_name, self.datas)
-                # database.deduplicate_subdomain(db_conn, table_name)
-                # database.remove_invalid(db_conn, table_name)
-                # 数据库导出
-                if not self.path:
-                    self.path = config.result_save_path.joinpath(f'{self.domain}.{self.format}')
-                dbexport.export(table_name, db_conn, self.valid, self.path, self.format, self.output)
-                db_conn.close()
+                self.main()
        else:
            logger.log('FATAL', f'获取域名失败')
        logger.log('INFOR', f'结束运行OneForAll')
@@ -95,4 +144,4 @@ class OneForAll(object):

 if __name__ == '__main__':
    fire.Fire(OneForAll)
-    # OneForAll('owasp.org').run()
+    # OneForAll('example.com').run()
Author	SHA1	Message	Date
shmilylty	e1d5515c44	v0.0.4	2019-08-11 21:34:29 +08:00
shmilylty	bf10c1a652	v0.0.4	2019-08-11 21:32:04 +08:00
shmilylty	33d7b6cc02	参数调优	2019-08-11 20:49:38 +08:00
shmilylty	903b18a058	生成请求地址中443端口的特殊处理	2019-08-11 20:39:58 +08:00
shmilylty	38117f06d1	参数调优	2019-08-11 20:20:42 +08:00
shmilylty	f9117ff795	添加resp.json()异常处理	2019-08-11 20:10:23 +08:00
shmilylty	d2219bdf8c	添加搜索退出判断	2019-08-11 20:08:06 +08:00
shmilylty	3c8627035d	设置域传送超时为30秒	2019-08-11 19:58:10 +08:00
shmilylty	4bb52f4183	参数调优	2019-08-11 15:53:49 +08:00
shmilylty	f426b8f2f6	重构	2019-08-11 03:48:36 +08:00
shmilylty	ebd81b83e9	pep8	2019-08-10 14:54:08 +08:00
shmilylty	d70ebe637e	升级tqdm版本	2019-08-10 12:02:32 +08:00
shmilylty	d10bc9c8d9	默认不使用代理	2019-08-10 03:30:52 +08:00
shmilylty	a7b189f311	调整执行finish函数位置	2019-08-10 00:48:51 +08:00
shmilylty	98642da4f3	拆分出执行域传送函数	2019-08-09 23:46:09 +08:00
shmilylty	f17aea1f44	删除无用日志输出	2019-08-09 23:20:21 +08:00
shmilylty	611580aa97	修正模块名字	2019-08-09 21:02:27 +08:00
shmilylty	b21e31a223	database.py修改为类	2019-08-09 18:47:01 +08:00
shmilylty	4c63bd672f	修正valid参数使用描述	2019-08-09 16:56:03 +08:00
shmilylty	c7f86fdefa	新增2个模块	2019-08-09 16:54:10 +08:00
shmilylty	c026dc550e	新增检查sitemap收集子域名模块	2019-08-09 16:43:09 +08:00
shmilylty	f2b08f6b02	新增检查robots.txt收集子域名模块	2019-08-09 16:42:03 +08:00
Jing Ling	f0f9f9b75e	删除无用模板	2019-08-09 15:40:04 +08:00
Jing Ling	a9347b8ef0	修改提交Bug提示	2019-08-09 15:32:46 +08:00
shmilylty	c2679a6ceb	暂时不探测https	2019-08-09 15:13:22 +08:00
shmilylty	9197aaf52b	pep8	2019-08-09 14:25:21 +08:00
shmilylty	272662fa82	修改pip3安装命令	2019-08-09 14:24:50 +08:00
shmilylty	623a21a11e	移除fake-useragent依赖	2019-08-09 11:39:21 +08:00
shmilylty	05040d3b8d	resolve_callback函数添加说明并做容错处理	2019-08-09 01:12:52 +08:00
shmilylty	e248826327	v0.0.3	2019-08-08 23:37:09 +08:00
shmilylty	07de3ecc2b	解决报WRONG_VERSION_NUMBER问题	2019-08-08 22:34:56 +08:00
shmilylty	518b27e386	Python 3.8 on Xenial Linux	2019-08-08 21:57:17 +08:00
shmilylty	8148c56fbc	不使用回调	2019-08-08 17:30:02 +08:00
shmilylty	a0269b1814	调整对齐	2019-08-08 16:14:42 +08:00
shmilylty	efaa806260	修改pipenv安装命令	2019-08-08 16:10:52 +08:00
shmilylty	d4e77aeac2	设置系统变量PYTHONIOENCODING为utf-8 去掉使用reconfigure函数设置	2019-08-08 13:59:20 +08:00
shmilylty	f1fdb32b56	添加终端输出为utf-8编码	2019-08-08 12:11:40 +08:00
shmilylty	f5ba141097	终端日志输出改为sys.stderr	2019-08-08 11:53:38 +08:00
shmilylty	3e004dd069	pip8	2019-08-08 01:23:34 +08:00
shmilylty	47cfaca9ee	pep8	2019-08-07 23:52:02 +08:00
Jing Ling	c6ddd83700	更新配置信息	2019-08-07 19:06:23 +08:00
Jing Ling	827c3f5f93	更新群链接	2019-08-07 18:38:59 +08:00
Jing Ling	de8c9a5c3b	Merge pull request #3 from Windforce17/master [README]fix wrong url format	2019-08-07 18:32:50 +08:00
shmilylty	28bfb80dc4	移除fake-useragent依赖	2019-08-07 18:28:36 +08:00
shmilylty	3e3b852830	移除Windows python支持（travisci官方不支持）	2019-08-07 16:37:57 +08:00
shmilylty	8f676e2ebe	添加Windows，linux和MacOSX系统测试	2019-08-07 16:04:00 +08:00
shmilylty	0fee65dd36	except BaseException	2019-08-07 15:50:13 +08:00
shmilylty	cfd60b366c	pip8	2019-08-07 15:49:27 +08:00
shmilylty	18851408c2	pip8格式化使用ssl=False	2019-08-07 15:17:40 +08:00
shmilylty	f74c45a67d	pep6	2019-08-07 15:16:55 +08:00
zhichen.Wang	a0163ed582	[README]fix wrong url format	2019-08-07 15:10:09 +08:00
shmilylty	85aa7d2ec4	添加字典	2019-08-07 01:44:08 +08:00
shmilylty	b628859d32	pep8	2019-08-07 01:32:49 +08:00
shmilylty	8ba3b926ce	pep8	2019-08-06 22:52:44 +08:00
shmilylty	a5cbcd08fa	修改错误模块名	2019-08-06 12:58:25 +08:00
shmilylty	7945d8024b	添加爆破模块和segment参数说明	2019-08-06 12:37:38 +08:00
shmilylty	d9c12ef9c3	deal_results函数里添加响应结果判断	2019-08-06 12:07:02 +08:00
shmilylty	23d4765e8e	Merge branch 'master' of https://github.com/shmilylty/OneForAll	2019-08-06 02:16:20 +08:00
shmilylty	f17a2636ae	修改为默认不使用代理，默认请求超时30秒。	2019-08-06 02:14:45 +08:00
shmilylty	84dd464a02	添加任务分组参数	2019-08-06 02:13:39 +08:00
Jing Ling	8429f0f015	添加Bug报告模板	2019-08-06 01:33:41 +08:00
shmilylty	62d2b1e4bd	修改默认使用字典文件	2019-08-06 01:09:29 +08:00
shmilylty	ca09ea5870	移除参数ssl=False	2019-08-06 00:43:30 +08:00
shmilylty	365162a474	修改程序入口为fire	2019-08-06 00:42:49 +08:00
shmilylty	60d8a2a88a	向TCPConnector函数添加ssl=False禁用SSL证书检查	2019-08-06 00:22:28 +08:00
shmilylty	cd22034415	修改为正确执行resp.json()的条件	2019-08-06 00:13:35 +08:00
shmilylty	e706c51b74	修改加群链接	2019-08-06 00:03:35 +08:00
shmilylty	8420258ff7	增加判断响应体不为空才执行resp.json()	2019-08-06 00:00:55 +08:00
shmilylty	31666fdd75	移除aiobrute向aiodns_query_a()函数传semaphore参数	2019-08-05 23:38:53 +08:00
shmilylty	e3f9d5e149	修复aiobrute中没有传semaphore参数问题	2019-08-05 22:28:17 +08:00
Jing Ling	a651c2148c	修改安装命令	2019-08-05 21:42:31 +08:00
shmilylty	e8781c7a29	添加响应判断	2019-08-05 19:11:40 +08:00
shmilylty	edd6b35388	添加更新说明	2019-08-05 18:33:07 +08:00
shmilylty	d8b3fe4bd8	new_table_name使用双引号包裹	2019-08-05 18:12:39 +08:00
shmilylty	5ccf57d595	表名用双引号包裹解决issues/2	2019-08-05 18:01:48 +08:00
shmilylty	11642edf6e	优化代码结构删除无用队列结果集参数	2019-08-05 18:00:01 +08:00
shmilylty	53213304e2	修改为api调用	2019-08-05 17:58:35 +08:00
shmilylty	bf20197631	添加api检查	2019-08-05 16:00:46 +08:00
shmilylty	8bad8b74d6	修改编译失败解决方法链接地址	2019-08-05 14:29:36 +08:00
shmilylty	a313c2f3e4	添加编译失败解决方法	2019-08-05 14:27:49 +08:00
shmilylty	e3312533e9	优化cdx和csp模块，这两个模块默认使用代理。	2019-08-05 12:20:58 +08:00
shmilylty	a899753db9	添加.gitignore文件	2019-08-05 11:39:05 +08:00
shmilylty	1855b4714f	解决请求响应为空resp.json()报错问题	2019-08-05 11:38:20 +08:00
Jing Ling	33e382af58	修复徽章排列问题	2019-08-04 21:29:59 +08:00
shmilylty	89e5943e6d	v0.0.2	2019-08-04 21:15:26 +08:00
shmilylty	205286bcd7	完善文档	2019-08-04 21:08:52 +08:00
shmilylty	95bfa6030a	修改自动集成化设置	2019-08-04 19:39:17 +08:00
shmilylty	84b64f8f99	添加有关集成化管理和代码覆盖率检查设置	2019-08-04 18:58:26 +08:00
shmilylty	45e955ae4e	上传新的演示svg	2019-08-04 05:55:34 +08:00
shmilylty	93f0fcd972	修复演示svg问题	2019-08-04 05:54:07 +08:00
shmilylty	ed0a1ef9cd	进一步完善文档	2019-08-04 05:15:18 +08:00
shmilylty	6a49ee22ab	修改Fire为程序入口	2019-08-04 03:36:33 +08:00
shmilylty	e76ee57c8a	添加运行输出banner	2019-08-04 03:20:59 +08:00
shmilylty	666942640e	修改日志输出格式	2019-08-04 03:20:25 +08:00
shmilylty	ecc6bd955f	修改参数说明	2019-08-04 00:57:16 +08:00
shmilylty	8eda8702ea	修改日志输出	2019-08-03 22:57:04 +08:00
shmilylty	2f1c1fed97	修改日志输出	2019-08-03 22:34:59 +08:00
shmilylty	3635f8f0a6	修改收集模块默认超时为请求超时秒	2019-08-03 22:05:47 +08:00
shmilylty	8bf9f2f0f6	修改Fire为程序入口	2019-08-03 21:28:08 +08:00
shmilylty	439fc9aa8e	修改日志输出格式，降低代码重复率。	2019-08-03 21:21:48 +08:00
shmilylty	6241dd16db	修复Typo问题	2019-08-03 18:38:13 +08:00
shmilylty	87de10478f	更改搜索地址	2019-08-03 18:20:08 +08:00
shmilylty	e5b9e30f00	统一为半角括号	2019-08-03 17:40:10 +08:00
shmilylty	98cea782b8	移除无用图片	2019-08-03 17:00:12 +08:00
shmilylty	142f1d13fb	移除使用通用字典选项	2019-08-03 16:51:11 +08:00
shmilylty	d9b6dd5be9	修改指定使用模块示例	2019-08-03 16:41:18 +08:00
shmilylty	e9553d1332	修改为标准的DocString格式	2019-08-03 16:39:57 +08:00
shmilylty	197712f1d1	完善收集模块说明文档	2019-08-03 15:17:57 +08:00
shmilylty	a76db09621	移除brotlipy依赖，更新fire到0.2.1解决报错。	2019-08-03 14:25:02 +08:00
Jing Ling	df808bcd53	Fixed Typo	2019-08-02 23:25:20 +08:00
Jing Ling	895cdb91da	添加A-Team链接地址	2019-08-02 22:32:52 +08:00
Jing Ling	125fed228a	Update README.md	2019-08-02 22:31:40 +08:00
shmilylty	ff4e413569	修改为中文格式	2019-08-02 19:07:54 +08:00
shmilylty	bfa8aecb0d	修改贡献者Black Star链接，增加了iceMatcha贡献者	2019-08-02 19:07:16 +08:00
shmilylty	0db3791f0f	入口修改为使用Fire	2019-08-02 19:05:35 +08:00
shmilylty	32c1e154de	降低tqdm版本	2019-08-02 19:04:36 +08:00
shmilylty	055e84e48d	添加说明文档	2019-08-02 19:04:04 +08:00
shmilylty	b693859d6b	移除无用的oneforall/requirements.txt	2019-08-02 18:58:31 +08:00
Jing Ling	87d30bd363	修复链接地址	2019-08-02 01:07:18 +08:00