Merge pull request #411 from Sci-Twi/patch-1

exporting utf-8 csv files

.dockerignore

@@ -0,0 +1 @@
+.git/

.github/ISSUE_TEMPLATE/bug_report.md

@@ -36,7 +36,7 @@ A clear and concise description of the actual results (optional, such as any err
 Screenshot of complete OneForAll execution process (recommended upload)
 
 **Log upload**
-Upload oneforall.log log files (it is recommended to upload logs in case of complex problems)
+Upload oneforall.log files (it is recommended to upload logs in case of complex problems)
 
 **Supplementary information**
 Some other supplementary notes about bug

.github/workflows/docker-image.yml

@@ -0,0 +1,32 @@
+name: Release Docker Image
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+
+  build:
+    name: build and push
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check out the repo
+      uses: actions/checkout@v3
+    
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+      
+    - name: Setup Docker buildx
+      uses: docker/setup-buildx-action@v2
+      
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_PASS }}
+    
+    - name: Build the Docker image
+      run: docker buildx build --push --platform linux/amd64,linux/arm64 -t shmilylty/oneforall:latest .

Dockerfile

@@ -1,5 +1,5 @@
-FROM python:3.8-alpine3.10
+FROM --platform=$TARGETPLATFORM python:3.8-alpine3.10
-MAINTAINER milktea@vmoe.info
+LABEL maintainer="milktea@vmoe.info"
 
 RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
 RUN apk update && apk --no-cache add git build-base libffi-dev libxml2-dev libxslt-dev libressl-dev
@@ -10,7 +10,7 @@ RUN git clone https://github.com/blechschmidt/massdns
 WORKDIR /massdns
 RUN make 
 ADD . /OneForAll/
-RUN mv /massdns/bin/massdns /OneForAll/thirdparty/massdns/massdns_linux_x86_64
+RUN mv /massdns/bin/massdns /OneForAll/thirdparty/massdns/massdns_linux_$(uname -m)
 RUN mkdir /OneForAll/results
 WORKDIR /OneForAll/
 

Pipfile.lock

@@ -33,17 +33,17 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
+                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
-                "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
+                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
             ],
-            "version": "==2020.6.20"
+            "version": "==2020.12.5"
         },
         "chardet": {
             "hashes": [
-                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
-                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+                "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
             ],
-            "version": "==3.0.4"
+            "version": "==4.0.0"
         },
         "colorama": {
             "hashes": [
@@ -55,11 +55,11 @@
         },
         "dnspython": {
             "hashes": [
-                "sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7",
+                "sha256:95d12f6ef0317118d2a1a6fc49aac65ffec7eb8087474158f42f26a639135216",
-                "sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d"
+                "sha256:e4a87f0b573201a0f3727fa18a516b055fd1107e0e5477cded4a2de497df1dd4"
             ],
             "index": "pypi",
-            "version": "==2.0.0"
+            "version": "==2.1.0"
         },
         "exrex": {
             "hashes": [
@@ -70,24 +70,71 @@
         },
         "fire": {
             "hashes": [
-                "sha256:9736a16227c3d469e5d2d296bce5b4d8fa8d7851e953bda327a455fc2994307f"
+                "sha256:c5e2b8763699d1142393a46d0e3e790c5eb2f0706082df8f647878842c216a62"
             ],
             "index": "pypi",
-            "version": "==0.3.1"
+            "version": "==0.4.0"
         },
         "future": {
             "hashes": [
                 "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
             ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==0.18.2"
         },
+        "greenlet": {
+            "hashes": [
+                "sha256:0a77691f0080c9da8dfc81e23f4e3cffa5accf0f5b56478951016d7cfead9196",
+                "sha256:0ddd77586553e3daf439aa88b6642c5f252f7ef79a39271c25b1d4bf1b7cbb85",
+                "sha256:111cfd92d78f2af0bc7317452bd93a477128af6327332ebf3c2be7df99566683",
+                "sha256:122c63ba795fdba4fc19c744df6277d9cfd913ed53d1a286f12189a0265316dd",
+                "sha256:181300f826625b7fd1182205b830642926f52bd8cdb08b34574c9d5b2b1813f7",
+                "sha256:1a1ada42a1fd2607d232ae11a7b3195735edaa49ea787a6d9e6a53afaf6f3476",
+                "sha256:1bb80c71de788b36cefb0c3bb6bfab306ba75073dbde2829c858dc3ad70f867c",
+                "sha256:1d1d4473ecb1c1d31ce8fd8d91e4da1b1f64d425c1dc965edc4ed2a63cfa67b2",
+                "sha256:292e801fcb3a0b3a12d8c603c7cf340659ea27fd73c98683e75800d9fd8f704c",
+                "sha256:2c65320774a8cd5fdb6e117c13afa91c4707548282464a18cf80243cf976b3e6",
+                "sha256:4365eccd68e72564c776418c53ce3c5af402bc526fe0653722bc89efd85bf12d",
+                "sha256:5352c15c1d91d22902582e891f27728d8dac3bd5e0ee565b6a9f575355e6d92f",
+                "sha256:58ca0f078d1c135ecf1879d50711f925ee238fe773dfe44e206d7d126f5bc664",
+                "sha256:5d4030b04061fdf4cbc446008e238e44936d77a04b2b32f804688ad64197953c",
+                "sha256:5d69bbd9547d3bc49f8a545db7a0bd69f407badd2ff0f6e1a163680b5841d2b0",
+                "sha256:5f297cb343114b33a13755032ecf7109b07b9a0020e841d1c3cedff6602cc139",
+                "sha256:62afad6e5fd70f34d773ffcbb7c22657e1d46d7fd7c95a43361de979f0a45aef",
+                "sha256:647ba1df86d025f5a34043451d7c4a9f05f240bee06277a524daad11f997d1e7",
+                "sha256:719e169c79255816cdcf6dccd9ed2d089a72a9f6c42273aae12d55e8d35bdcf8",
+                "sha256:7cd5a237f241f2764324396e06298b5dee0df580cf06ef4ada0ff9bff851286c",
+                "sha256:875d4c60a6299f55df1c3bb870ebe6dcb7db28c165ab9ea6cdc5d5af36bb33ce",
+                "sha256:90b6a25841488cf2cb1c8623a53e6879573010a669455046df5f029d93db51b7",
+                "sha256:94620ed996a7632723a424bccb84b07e7b861ab7bb06a5aeb041c111dd723d36",
+                "sha256:b5f1b333015d53d4b381745f5de842f19fe59728b65f0fbb662dafbe2018c3a5",
+                "sha256:c5b22b31c947ad8b6964d4ed66776bcae986f73669ba50620162ba7c832a6b6a",
+                "sha256:c93d1a71c3fe222308939b2e516c07f35a849c5047f0197442a4d6fbcb4128ee",
+                "sha256:cdb90267650c1edb54459cdb51dab865f6c6594c3a47ebd441bc493360c7af70",
+                "sha256:cfd06e0f0cc8db2a854137bd79154b61ecd940dce96fad0cba23fe31de0b793c",
+                "sha256:d3789c1c394944084b5e57c192889985a9f23bd985f6d15728c745d380318128",
+                "sha256:da7d09ad0f24270b20f77d56934e196e982af0d0a2446120cb772be4e060e1a2",
+                "sha256:df3e83323268594fa9755480a442cabfe8d82b21aba815a71acf1bb6c1776218",
+                "sha256:df8053867c831b2643b2c489fe1d62049a98566b1646b194cc815f13e27b90df",
+                "sha256:e1128e022d8dce375362e063754e129750323b67454cac5600008aad9f54139e",
+                "sha256:e6e9fdaf6c90d02b95e6b0709aeb1aba5affbbb9ccaea5502f8638e4323206be",
+                "sha256:eac8803c9ad1817ce3d8d15d1bb82c2da3feda6bee1153eec5c58fa6e5d3f770",
+                "sha256:eb333b90036358a0e2c57373f72e7648d7207b76ef0bd00a4f7daad1f79f5203",
+                "sha256:ed1d1351f05e795a527abc04a0d82e9aecd3bdf9f46662c36ff47b0b00ecaf06",
+                "sha256:f3dc68272990849132d6698f7dc6df2ab62a88b0d36e54702a8fd16c0490e44f",
+                "sha256:f59eded163d9752fd49978e0bab7a1ff21b1b8d25c05f0995d140cc08ac83379",
+                "sha256:f5e2d36c86c7b03c94b8459c3bd2c9fe2c7dab4b258b8885617d44a22e453fb7",
+                "sha256:f6f65bf54215e4ebf6b01e4bb94c49180a589573df643735107056f7a910275b",
+                "sha256:f8450d5ef759dbe59f84f2c9f77491bb3d3c44bc1a573746daf086e70b14c243",
+                "sha256:f97d83049715fd9dec7911860ecf0e17b48d8725de01e45de07d8ac0bd5bc378"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==1.0.0"
+        },
         "idna": {
             "hashes": [
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
         "loguru": {
@@ -109,79 +156,74 @@
         },
         "requests": {
             "hashes": [
-                "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
+                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
-                "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
+                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
             ],
             "index": "pypi",
-            "version": "==2.24.0"
+            "version": "==2.25.1"
         },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
                 "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.15.0"
         },
         "soupsieve": {
             "hashes": [
-                "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
+                "sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc",
-                "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
+                "sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b"
             ],
             "markers": "python_version >= '3.0'",
-            "version": "==2.0.1"
+            "version": "==2.2.1"
         },
         "sqlalchemy": {
             "hashes": [
-                "sha256:009e8388d4d551a2107632921320886650b46332f61dc935e70c8bcf37d8e0d6",
+                "sha256:02b039e0e7e6de2f15ea2d2de3995e31a170e700ec0b37b4eded662171711d19",
-                "sha256:0157c269701d88f5faf1fa0e4560e4d814f210c01a5b55df3cab95e9346a8bcc",
+                "sha256:08943201a1e3c6238e48f4d5d56c27ea1e1b39d3d9f36a9d81fc3cfb0e1b83bd",
-                "sha256:0a92745bb1ebbcb3985ed7bda379b94627f0edbc6c82e9e4bac4fb5647ae609a",
+                "sha256:0ee0054d4a598d2920cae14bcbd33e200e02c5e3b47b902627f8cf5d4c9a2a4b",
-                "sha256:0cca1844ba870e81c03633a99aa3dc62256fb96323431a5dec7d4e503c26372d",
+                "sha256:11e7a86209f69273e75d2dd64b06c0c2660e39cd942fce2170515c404ed7358a",
-                "sha256:166917a729b9226decff29416f212c516227c2eb8a9c9f920d69ced24e30109f",
+                "sha256:1294f05916c044631fd626a4866326bbfbd17f62bd37510d000afaef4b35bd74",
-                "sha256:1f5f369202912be72fdf9a8f25067a5ece31a2b38507bb869306f173336348da",
+                "sha256:2f11b5783933bff55291ca06496124347627d211ff2e509e846af1c35de0a3fb",
-                "sha256:2909dffe5c9a615b7e6c92d1ac2d31e3026dc436440a4f750f4749d114d88ceb",
+                "sha256:301d0cd6ef1dc73b607748183da857e712d6f743de8d92b1e1f8facfb0ba2aa2",
-                "sha256:2b5dafed97f778e9901b79cc01b88d39c605e0545b4541f2551a2fd785adc15b",
+                "sha256:344b58b4b4193b72e8b768a51ef6eb5a4c948ce313a0f23e2ea081e71ce8ac0e",
-                "sha256:2e9bd5b23bba8ae8ce4219c9333974ff5e103c857d9ff0e4b73dc4cb244c7d86",
+                "sha256:44e11a06168782b6d485daef197783366ce7ab0d5eea0066c899ae06cef47bbc",
-                "sha256:3aa6d45e149a16aa1f0c46816397e12313d5e37f22205c26e06975e150ffcf2a",
+                "sha256:45b091ccbf94374ed14abde17e9a04522b0493a17282eaaf4383efdd413f5243",
-                "sha256:4bdbdb8ca577c6c366d15791747c1de6ab14529115a2eb52774240c412a7b403",
+                "sha256:48540072f43b3c080159ec1f24a4b014c0ee83d3b73795399974aa358a8cf71b",
-                "sha256:53fd857c6c8ffc0aa6a5a3a2619f6a74247e42ec9e46b836a8ffa4abe7aab327",
+                "sha256:4df07161897191ed8d4a0cfc92425c81296160e5c5f76c9256716d3085172883",
-                "sha256:5cdfe54c1e37279dc70d92815464b77cd8ee30725adc9350f06074f91dbfeed2",
+                "sha256:4f7ce3bfdab6520554af4a5b1df4513d45388624d015ba4d921daf48ce1d6503",
-                "sha256:5d92c18458a4aa27497a986038d5d797b5279268a2de303cd00910658e8d149c",
+                "sha256:5361e25181b9872d6906c8c9be7dc05cb0a0951d71ee59ee5a71c1deb301b8a8",
-                "sha256:632b32183c0cb0053194a4085c304bc2320e5299f77e3024556fa2aa395c2a8b",
+                "sha256:6f8fdad2f335d2f3ca2f3ee3b01404f7abcf519b03de2c510f1f42d16e39ffb4",
-                "sha256:7c735c7a6db8ee9554a3935e741cf288f7dcbe8706320251eb38c412e6a4281d",
+                "sha256:70a1387396ea5b3022539b560c287daf79403d8b4b365f89b56d660e625a4457",
-                "sha256:7cd40cb4bc50d9e87b3540b23df6e6b24821ba7e1f305c1492b0806c33dbdbec",
+                "sha256:7481f9c2c832a3bf37c80bee44d91ac9938b815cc06f7e795b976e300914aab9",
-                "sha256:84f0ac4a09971536b38cc5d515d6add7926a7e13baa25135a1dbb6afa351a376",
+                "sha256:7c0c7bb49167ac738ca6ee6e7f94a9988a7e4e261d8da335341e8c8c8f3b2e9b",
-                "sha256:8dcbf377529a9af167cbfc5b8acec0fadd7c2357fc282a1494c222d3abfc9629",
+                "sha256:7de84feb31af3d8fdf819cac2042928d0b60d3cb16f49c4b2f48d88db46e79f6",
-                "sha256:950f0e17ffba7a7ceb0dd056567bc5ade22a11a75920b0e8298865dc28c0eff6",
+                "sha256:7f5087104c3c5af11ea59e49ae66c33ca98b14a47d3796ae97498fca53f84aef",
-                "sha256:9e379674728f43a0cd95c423ac0e95262500f9bfd81d33b999daa8ea1756d162",
+                "sha256:81badd7d3e0e6aba70a5d1b50fabe8112e9835a6fdb0684054c3fe5378ce0d01",
-                "sha256:b15002b9788ffe84e42baffc334739d3b68008a973d65fad0a410ca5d0531980",
+                "sha256:82f11b679df91275788be6734dd4a9dfa29bac67b85326992609f62b05bdab37",
-                "sha256:b6f036ecc017ec2e2cc2a40615b41850dc7aaaea6a932628c0afc73ab98ba3fb",
+                "sha256:8301ecf3e819eb5dbc171e84654ff60872807775301a55fe35b0ab2ba3742031",
-                "sha256:bad73f9888d30f9e1d57ac8829f8a12091bdee4949b91db279569774a866a18e",
+                "sha256:8d6a9feb5efd2fdab25c6d5a0a5589fed9d789f5ec57ec12263fd0e60ce1dea6",
-                "sha256:bbc58fca72ce45a64bb02b87f73df58e29848b693869e58bd890b2ddbb42d83b",
+                "sha256:915d4fa08776c0252dc5a34fa15c6490f66f411ea1ac9492022f98875d6baf20",
-                "sha256:bca4d367a725694dae3dfdc86cf1d1622b9f414e70bd19651f5ac4fb3aa96d61",
+                "sha256:94040a92b6676f9ffdab6c6b479b3554b927a635c90698c761960b266b04fc88",
-                "sha256:be41d5de7a8e241864189b7530ca4aaf56a5204332caa70555c2d96379e18079",
+                "sha256:a08027ae84efc563f0f2f341dda572eadebeca38c0ae028a009988f27e9e6230",
-                "sha256:bf53d8dddfc3e53a5bda65f7f4aa40fae306843641e3e8e701c18a5609471edf",
+                "sha256:a103294583383660d9e06dbd82037dc8e94c184bdcb27b2be44ae4457dafc6b4",
-                "sha256:c092fe282de83d48e64d306b4bce03114859cdbfe19bf8a978a78a0d44ddadb1",
+                "sha256:c22bfac8d3b955cdb13f0fcd6343156bf56d925196cf7d9ab9ce9f61d3f1e11c",
-                "sha256:c3ab23ee9674336654bf9cac30eb75ac6acb9150dc4b1391bec533a7a4126471",
+                "sha256:c3810ebcf1d42c532c8f5c3f442c705d94442a27a32f2df5344f0857306ab321",
-                "sha256:ce64a44c867d128ab8e675f587aae7f61bd2db836a3c4ba522d884cd7c298a77",
+                "sha256:ee4ddc904fb6414b5118af5b8d45e428aac2ccda01326b2ba2fe4354b0d8d1ae",
-                "sha256:d05cef4a164b44ffda58200efcb22355350979e000828479971ebca49b82ddb1",
+                "sha256:f16801795f1ffe9472360589a04301018c79e4582a85e68067275bb4f765e4e2",
-                "sha256:d2f25c7f410338d31666d7ddedfa67570900e248b940d186b48461bd4e5569a1",
+                "sha256:f62c57ceadedeb8e7b98b48ac4d684bf2b0f73b9d882fed3ca260d9aedf6403f",
-                "sha256:d3b709d64b5cf064972b3763b47139e4a0dc4ae28a36437757f7663f67b99710",
+                "sha256:fbb0fda1c574975807aceb0e2332e0ecfe9e5656c191ed482c1a5eafe7a33823"
-                "sha256:e32e3455db14602b6117f0f422f46bc297a3853ae2c322ecd1e2c4c04daf6ed5",
-                "sha256:ed53209b5f0f383acb49a927179fa51a6e2259878e164273ebc6815f3a752465",
-                "sha256:f605f348f4e6a2ba00acb3399c71d213b92f27f2383fc4abebf7a37368c12142",
-                "sha256:fcdb3755a7c355bc29df1b5e6fb8226d5c8b90551d202d69d0076a8a5649d68b"
             ],
             "index": "pypi",
-            "version": "==1.3.20"
+            "version": "==1.4.5"
         },
         "tenacity": {
             "hashes": [
-                "sha256:29ae90e7faf488a8628432154bb34ace1cca58244c6ea399fd33f066ac71339a",
+                "sha256:5bd16ef5d3b985647fe28dfa6f695d343aa26479a04e8792b9d3c8f49e361ae1",
-                "sha256:5a5d3dcd46381abe8b4f82b5736b8726fd3160c6c7161f53f8af7f1eb9b82173"
+                "sha256:a0ce48587271515db7d3a5e700df9ae69cce98c4b57c23a4886da15243603dd8"
             ],
             "index": "pypi",
-            "version": "==6.2.0"
+            "version": "==7.0.0"
         },
         "termcolor": {
             "hashes": [
@@ -191,11 +233,11 @@
         },
         "tqdm": {
             "hashes": [
-                "sha256:9ad44aaf0fc3697c06f6e05c7cf025dd66bc7bcb7613c66d85f4464c47ac8fad",
+                "sha256:9fdf349068d047d4cfbe24862c425883af1db29bcddf4b0eeb2524f6fbdb23c7",
-                "sha256:ef54779f1c09f346b2b5a8e5c61f96fbcb639929e640e59f8cf810794f406432"
+                "sha256:d666ae29164da3e517fcf125e41d4fe96e5bb375cd87ff9763f6b38b5592fe33"
             ],
             "index": "pypi",
-            "version": "==4.51.0"
+            "version": "==4.59.0"
         },
         "treelib": {
             "hashes": [
@@ -206,11 +248,10 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2",
+                "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
-                "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e"
+                "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.4"
-            "version": "==1.25.11"
         },
         "win32-setctime": {
             "hashes": [

README.md

@@ -4,8 +4,8 @@
 [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
-[![python](https://img.shields.io/badge/python-3.6|3.7|3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/python-3.6+-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.4.2-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/release-v0.4.5-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
 
 👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md)
 
@@ -80,6 +80,8 @@ git stash pop    # 释放本地修改
 ```
 config
 ├── api.py
+├── default.py
+├── __init__.py
 ├── log.py
 └── setting.py
 ```
@@ -88,7 +90,7 @@ config
 
 ```shell
 docker pull shmilylty/oneforall
-docker run -it --rm -v ~/results:/OneForAll/results -v ~/.config:/OneForAll/config oneforall --target example.com run
+docker run -it --rm -v ~/results:/OneForAll/results -v ~/.config:/OneForAll/config shmilylty/oneforall --target example.com run
 ```
 参数直接加在指令末尾，结果会输出在本地目录`~/results`，如需保存到其他位置，可以自行修改
 </details>
@@ -227,12 +229,14 @@ FLAGS
   1. 利用证书透明度收集子域（目前有6个模块：`censys_api`，`certspotter`，`crtsh`，`entrust`，`google`，`spyse_api`）
   2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，检查robots文件`robots`，检查sitemap文件`sitemap`，利用NSEC记录遍历DNS域`dnssec`，后续会添加NSEC3记录等模块）
   3. 利用网上爬虫档案收集子域（目前有2个模块：`archivecrawl`，`commoncrawl`，此模块还在调试，该模块还有待添加和完善）
-  4. 利用DNS数据集收集子域（目前有24个模块：`binaryedge_api`, `bufferover`, `cebaidu`, `chinaz`, `chinaz_api`, `circl_api`, `cloudflare`, `dnsdb_api`, `dnsdumpster`, `hackertarget`, `ip138`, `ipv4info_api`, `netcraft`, `passivedns_api`, `ptrarchive`, `qianxun`, `rapiddns`, `riddler`, `robtex`, `securitytrails_api`, `sitedossier`, `threatcrowd`, `wzpc`, `ximcx`）
+  4. 利用DNS数据集收集子域（目前有24个模块：`bevigil_api`, `binaryedge_api`, `bufferover`, `cebaidu`, `chinaz`, `chinaz_api`, `circl_api`, `cloudflare`, `dnsdb_api`, `dnsdumpster`, `hackertarget`, `ip138`, `ipv4info_api`, `netcraft`, `passivedns_api`, `ptrarchive`, `qianxun`, `rapiddns`, `riddler`, `robtex`, `securitytrails_api`, `sitedossier`, `threatcrowd`, `wzpc`, `ximcx`）
   5. 利用DNS查询收集子域（目前有5个模块：通过枚举常见的SRV记录并做查询来收集子域`srv`，以及通过查询域名的DNS记录中的MX,NS,SOA,TXT记录来收集子域）
   6. 利用威胁情报平台数据收集子域（目前有6个模块：`alienvault`, `riskiq_api`，`threatbook_api`，`threatminer`，`virustotal`，`virustotal_api`该模块还有待添加和完善）
   7. 利用搜索引擎发现子域（目前有18个模块：`ask`, `baidu`, `bing`, `bing_api`, `duckduckgo`, `exalead`, `fofa_api`, `gitee`, `github`, `github_api`, `google`, `google_api`, `shodan_api`, `so`, `sogou`, `yahoo`, `yandex`, `zoomeye_api`），在搜索模块中除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
 * **支持子域爆破**，该模块有常规的字典爆破，也有自定义的fuzz模式，支持批量爆破和递归爆破，自动判断泛解析并处理。
 * **支持子域验证**，默认开启子域验证，自动解析子域DNS，自动请求子域获取title和banner，并综合判断子域存活情况。
+* **支持子域爬取**，根据已有的子域，请求子域响应体以及响应体里的JS，从中再次发现新的子域。
+* **支持子域置换**，根据已有的子域，使用子域替换技术再次发现新的子域。
 * **支持子域接管**，默认开启子域接管风险检查，支持子域自动接管（目前只有Github，有待完善），支持批量检查。
 * **处理功能强大**，发现的子域结果支持自动去除，自动DNS解析，HTTP请求探测，自动筛选出有效子域，拓展子域的Banner信息，最终支持的导出格式有`txt`, `csv`, `json`。
 * **速度极快**，[收集模块](https://github.com/shmilylty/OneForAll/tree/master/collect.py)使用多线程调用，[爆破模块](https://github.com/shmilylty/OneForAll/tree/master/brute.py)使用[massdns](https://github.com/blechschmidt/massdns)，DNS解析速度每秒可解析350000以上个域名，子域验证中DNS解析和HTTP请求使用异步多协程，多线程检查[子域接管](https://github.com/shmilylty/OneForAll/tree/master/takeover.py)风险。
@@ -267,7 +271,6 @@ FLAGS
 ## ⌛后续计划
 
 - [ ] 各模块持续优化和完善
-- [x] 子域收集爬虫实现（包括从JS等静态资源文件中收集子域）
 - [ ] 操作强大交互人性的前端界面实现
 
 更多信息请参阅[后续开发计划](https://github.com/shmilylty/OneForAll/tree/master/docs/todo.md)。

brute.py

@@ -10,13 +10,9 @@ OneForAll subdomain brute module
 import gc
 import json
 import time
-import secrets
 
 import exrex
 import fire
-import tenacity
-from dns.exception import Timeout
-from dns.resolver import NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers
 
 import export
 from common import utils
@@ -296,7 +292,7 @@ class Brute(Module):
         brute.py --target d.com --fuzz True --place m.*.d.com --fuzzlist subnames.txt run
 
     Note:
-        --fmt csv/json (result fmt)
+        --fmt csv/json (result format)
         --path   Result path (default None, automatically generated)
 
 
@@ -346,7 +342,6 @@ class Brute(Module):
         self.ips_times = dict()  # IP集合出现次数
         self.enable_wildcard = None  # 当前域名是否使用泛解析
         self.quite = False
-        self.in_china = None
 
     def gen_brute_dict(self, domain):
         logger.log('INFOR', f'Generating dictionary for {domain}')
@@ -429,7 +424,7 @@ class Brute(Module):
 
         if self.enable_wildcard:
             wildcard_ips, wildcard_ttl = wildcard.collect_wildcard_record(domain, ns_ip_list)
-        ns_path = utils.get_ns_path(self.in_china, self.enable_wildcard, ns_ip_list)
+        ns_path = utils.get_ns_path(settings.use_china_nameservers, self.enable_wildcard, ns_ip_list)
 
         dict_set = self.gen_brute_dict(domain)
 
@@ -463,8 +458,6 @@ class Brute(Module):
 
     def run(self):
         logger.log('INFOR', f'Start running {self.source} module')
-        if self.in_china is None:
-            _, self.in_china = utils.get_net_env()
         self.domains = utils.get_domains(self.target, self.targets)
         for self.domain in self.domains:
             self.results = list()  # 置空

common/database.py

@@ -234,7 +234,7 @@ class Database(object):
         table_name = table_name.replace('.', '_')
         sql = f'select id, alive, request, resolve, url, subdomain, level,' \
               f'cname, ip, public, cdn, port, status, reason, title, banner,' \
-              f'cidr, asn, org, addr, isp, source from "{table_name}" order by subdomain'
+              f'cidr, asn, org, addr, isp, source from "{table_name}" '
         if alive and limit:
             if limit in ['resolve', 'request']:
                 where = f' where {limit} = 1'
@@ -242,6 +242,7 @@ class Database(object):
         elif alive:
             where = f' where alive = 1'
             sql += where
+        sql += ' order by subdomain'
         logger.log('TRACE', f'Get the data from {table_name} table')
         return self.query(sql)
 

common/domain.py

@@ -55,6 +55,8 @@ class Domain(object):
 
         :return: registered domain result
         """
+        if not settings.use_tld_extract:
+            return self.string
         result = self.extract()
         if result:
             return result.registered_domain

common/records.py

@@ -230,7 +230,7 @@ class RecordCollection(object):
 
         # Ensure that we don't have more than one row.
         try:
-            self[1]
+            return self[1]
         except IndexError:
             return self.first(default=default, as_dict=as_dict,
                               as_ordereddict=as_ordereddict)

common/request.py

@@ -208,7 +208,7 @@ def save(name, total, req_data, resp_queue):
             new_info = gen_new_info(old_info, resp)
             db.insert_table(name, new_info)
             resp_queue.task_done()
-        if i >= total:
+        if i >= total:  # 得存入完所有请求结果才能结束
             break
     db.close()
 

common/resolve.py

@@ -38,6 +38,7 @@ def update_data(data, infos):
     new_data = list()
     for index, items in enumerate(data):
         if items.get('ip'):
+            new_data.append(items)
             continue
         subdomain = items.get('subdomain')
         record = infos.get(subdomain)

common/search.py

@@ -69,8 +69,9 @@ class Search(Module):
     def recursive_subdomain(self):
         # 递归搜索下一层的子域
         # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+        subdomains = self.subdomains.copy()
         for layer_num in range(1, self.recursive_times):
-            for subdomain in self.subdomains:
+            for subdomain in subdomains:
                 # 进行下一层子域搜索的限制条件
                 count = subdomain.count('.') - self.domain.count('.')
                 if count == layer_num:

common/utils.py

@@ -10,10 +10,10 @@ import platform
 import subprocess
 from urllib.parse import scheme_chars
 from ipaddress import IPv4Address, ip_address
+from distutils.version import LooseVersion
 from pathlib import Path
 from stat import S_IXUSR
 
-import dns
 import requests
 import tenacity
 from dns.resolver import Resolver
@@ -54,7 +54,7 @@ def gen_fake_header():
     """
     Generate fake request headers
     """
-    headers = settings.request_default_headers
+    headers = settings.request_default_headers.copy()
     if not isinstance(headers, dict):
         headers = dict()
     if settings.enable_random_ua:
@@ -192,7 +192,7 @@ def check_path(path, name, fmt):
     else:
         path = default_path
     path = Path(path)
-    if not path.suffix:  # 输入是目录的情况
+    if path.is_dir():  # 输入是目录的情况
         path = path.joinpath(filename)
     parent_dir = path.parent
     if not parent_dir.exists():
@@ -248,11 +248,11 @@ def save_to_file(path, data):
     :return: 保存成功与否
     """
     try:
-        with open(path, 'w', errors='ignore', newline='') as file:
+        with open(path, 'w', errors='ignore', newline='', encoding='utf-8') as file:
             file.write(data)
             return True
     except TypeError:
-        with open(path, 'wb') as file:
+        with open(path, 'wb', encoding='utf-8') as file:
             file.write(data)
             return True
     except Exception as e:
@@ -330,6 +330,8 @@ def export_all_results(path, name, fmt, datas):
         row_list.append(Record(keys, values))
     rows = RecordCollection(iter(row_list))
     content = rows.export(fmt)
+    if fmt == 'csv':
+        content = '\ufeff' + content
     save_to_file(path, content)
 
 
@@ -477,7 +479,7 @@ def ip_is_public(ip_str):
 
 
 def get_request_count():
-    return 32
+    return os.cpu_count() * 16
 
 
 def uniq_dict_list(dict_list):
@@ -495,28 +497,35 @@ def delete_file(*paths):
 @tenacity.retry(stop=tenacity.stop_after_attempt(3),
                 wait=tenacity.wait_fixed(2))
 def check_net():
-    urls = ['http://ipinfo.io/json', 'http://ipconfig.io/json']
+    times = 0
+    while True:
+        times += 1
+        urls = ['https://www.baidu.com', 'https://www.bing.com',
+                'https://www.cloudflare.com', 'https://www.akamai.com/',
+                'https://www.fastly.com/', 'https://www.amazon.com/']
         url = random.choice(urls)
-    header = {'User_Agent': 'curl'}
+        logger.log('DEBUG', f'Trying to access {url}')
+        header = get_random_header()
+        proxy = get_proxy()
         timeout = settings.request_timeout_second
         verify = settings.request_ssl_verify
-    logger.log('DEBUG', f'Trying to access {url}')
+        session = requests.Session()
+        session.trust_env = False
         session = requests.Session()
         session.trust_env = False
         try:
-        rsp = session.get(url, headers=header, timeout=timeout, verify=verify)
+            rsp = session.get(url, headers=header, proxies=proxy,
+                              timeout=timeout, verify=verify)
         except Exception as e:
             logger.log('ERROR', e.args)
-        logger.log('ALERT', 'Unable to access Internet, retrying...')
+            logger.log('ALERT', f'Unable to access Internet, retrying for the {times}th time')
-        raise e
-    logger.log('DEBUG', 'Access to Internet OK')
-    country = rsp.json().get('country').lower()
-    if country in ['cn', 'china']:
-        logger.log('DEBUG', f'The computer is located in China')
-        return True, True
         else:
-        logger.log('DEBUG', f'The computer is not located in China')
+            if rsp.status_code == 200:
-        return True, False
+                logger.log('DEBUG', 'Access to Internet OK')
+                return True
+        if times >= 3:
+            logger.log('ALERT', 'Access to Internet failed')
+            return False
 
 
 def check_dep():
@@ -526,7 +535,7 @@ def check_dep():
     if implementation != 'CPython':
         logger.log('FATAL', f'OneForAll only passed the test under CPython')
         exit(1)
-    if version < '3.6':
+    if LooseVersion(version) < LooseVersion('3.6'):
         logger.log('FATAL', 'OneForAll requires Python 3.6 or higher')
         exit(1)
 
@@ -538,7 +547,7 @@ def get_net_env():
     except Exception as e:
         logger.log('DEBUG', e.args)
         logger.log('ALERT', 'Please check your network environment.')
-        return False, None
+        return False
     return result
 
 
@@ -606,11 +615,11 @@ def get_massdns_path(massdns_dir):
     machine = platform.machine().lower()
     name = f'massdns_{system}_{machine}'
     if system == 'windows':
-        name = name + '.exe'
+        name = f'massdns.exe'
         if machine == 'amd64':
             massdns_dir = massdns_dir.joinpath('windows', 'x64')
         else:
-            massdns_dir = massdns_dir.joinpath('windows', 'x84')
+            massdns_dir = massdns_dir.joinpath('windows', 'x86')
     path = massdns_dir.joinpath(name)
     path.chmod(S_IXUSR)
     if not path.exists():

config/api.py

@@ -7,6 +7,9 @@ censys_api_secret = ''
 # 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
 binaryedge_api = ''
 
+# BeVigil API: https://bevigil.com/osint-api
+bevigil_api = ''
+
 # Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
 chinaz_api = ''
 
@@ -44,8 +47,7 @@ threatbook_api_key = ''
 virustotal_api_key = ''
 
 # https://www.zoomeye.org/doc?channel=api
-zoomeye_api_usermail = ''
+zoomeye_api_key = ''
-zoomeye_api_password = ''
 
 # Spyse可以免费注册获取API: https://spyse.com/
 spyse_api_token = ''
@@ -74,3 +76,19 @@ github_api_token = ''
 
 # obtain Cloudflare API key from https://dash.cloudflare.com/profile/api-tokens
 cloudflare_api_token = ''
+
+# https://hunter.qianxin.com/home/userInfo
+hunter_api_key = ''
+
+# https://api-docs.fullhunt.io/
+fullhunt_api_key = ''
+
+
+# 登录quake之后可在个人中心获取key https://quake.360.net/quake/#/personal?tab=message
+quake_api_key = ''
+
+#https://www.racent.com/ctlog F2>Network抓包获取Token
+racent_api_token = ''
+
+# https://windvane.lichoin.com/
+windvane_api_token = ''

config/default.py

@@ -18,6 +18,7 @@ result_save_dir = relative_directory.joinpath('results')  # 结果保存目录
 temp_save_dir = result_save_dir.joinpath('temp')
 
 # OneForAll入口参数设置
+enable_check_network = True  # 开启网络环境检查
 enable_check_version = True  # 开启最新版本检查
 enable_brute_module = True  # 使用爆破模块(默认True)
 enable_dns_resolve = True  # 使用DNS解析子域(默认True)
@@ -27,12 +28,11 @@ enable_altdns_module = True  # 开启altdns模块,开启会利用置换技术重
 enable_enrich_module = True  # 开启enrich模块，开启会富化出信息，如ip的cdn，cidr，asn，org，addr和isp等信息
 enable_banner_identify = True  # 开启WEB指纹识别模块(默认True)
 enable_takeover_check = False  # 开启子域接管风险检查(默认False)
-# 参数可选值有'small', 'medium', 'large'
+# 参数可选值有 'small', 'medium', 'large'
-http_request_port = 'small'  # HTTP请求子域(默认'small'，探测80,443端口)
+http_request_port = 'small'  # HTTP请求子域(默认 'small'，探测80,443端口)
 # 参数可选值True，False分别表示导出存活，全部子域结果
 result_export_alive = False  # 只导出存活的子域结果(默认False)
-# 参数可选格式有'rst', 'csv', 'tsv', 'json', 'yaml', 'html',
+# 参数可选格式有 'csv', 'json'
-# 'jira', 'xls', 'xlsx', 'dbf', 'latex', 'ods'
 result_save_format = 'csv'  # 子域结果保存文件格式(默认csv)
 # 参数path默认None使用OneForAll结果目录自动生成路径
 result_save_path = None  # 子域结果保存文件路径(默认None)
@@ -42,8 +42,7 @@ save_module_result = False  # 保存各模块发现结果为json文件(默认Fal
 enable_all_module = True  # 启用所有收集模块(默认True)
 enable_partial_module = []  # 启用部分收集模块 必须禁用enable_all_module才能生效
 # 只使用ask和baidu搜索引擎收集子域的示例
-# enable_partial_module = [('modules.search', 'ask')
+# enable_partial_module = ['modules.search.ask', 'modules.search.baidu']
-#                          ('modules.search', 'baidu')]
 module_thread_timeout = 90.0  # 每个收集模块线程超时时间(默认90秒)
 
 # 爆破模块设置
@@ -56,6 +55,7 @@ brute_socket_num = 1  # 爆破时每个进程下的socket数量
 brute_resolve_num = 15  # 解析失败时尝试换名称服务器重查次数
 # 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径)
 brute_wordlist_path = None
+use_china_nameservers = True  # 使用中国域名服务器 如果你所在网络不在中国则建议设置False
 # 域名的权威DNS名称服务器的保存路径 当域名开启了泛解析时会使用该名称服务器来进行A记录查询
 authoritative_dns_path = data_storage_dir.joinpath('authoritative_dns.txt')
 enable_recursive_brute = False  # 是否使用递归爆破(默认False)
@@ -75,7 +75,12 @@ brute_ip_blacklist = {'0.0.0.0', '0.0.0.1'}  # IP黑名单 子域解析到IP黑
 ip_appear_maximum = 100  # 多个子域解析到同一IP次数超过100次则标记为非法(泛解析)子域
 
 # altdns模块设置
-enable_fast_alt = True  # 是否开启快速置换(默认True，只使用部分置换规则)
+altdns_increase_num = True
+altdns_decrease_num = True
+altdns_replace_word = False
+altdns_insert_word = False
+altdns_add_word = False
+
 
 # banner识别模块设置
 banner_process_number = 4  # 识别进程数量(默认4)
@@ -94,8 +99,8 @@ request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
 
 
 # 请求设置
-request_thread_count = None  # 请求线程数量(默认None，则根据内存大小设置)
+request_thread_count = None  # 请求线程数量(默认None，则根据情况自动设置)
-request_timeout_second = (3.05, 27)  # 请求超时秒数(默认connect timout推荐略大于3秒，read秒)
+request_timeout_second = (13, 27)  # 请求超时秒数(默认connect timout推荐略大于3秒)
 request_ssl_verify = False  # 请求SSL验证(默认False)
 request_allow_redirect = True  # 请求允许重定向(默认True)
 request_redirect_limit = 10  # 请求跳转限制(默认10次)
@@ -170,6 +175,9 @@ censys_api_secret = ''
 # 免费的API有效期只有1个月，到期之后可以再次生成，每月可以查询250次。
 binaryedge_api = ''
 
+# BeVigil API: https://bevigil.com/osint-api
+bevigil_api = ''
+
 # Chinaz可以免费注册获取API：http://api.chinaz.com/ApiDetails/Alexa
 chinaz_api = ''
 
@@ -207,8 +215,7 @@ threatbook_api_key = ''
 virustotal_api_key = ''
 
 # https://www.zoomeye.org/doc?channel=api
-zoomeye_api_usermail = ''
+zoomeye_api_key = ''
-zoomeye_api_password = ''
 
 # Spyse可以免费注册获取API: https://spyse.com/
 spyse_api_token = ''
@@ -237,3 +244,12 @@ github_api_token = ''
 
 # obtain Cloudflare API key from https://dash.cloudflare.com/profile/api-tokens
 cloudflare_api_token = ''
+
+# https://hunter.qianxin.com/home/userInfo
+hunter_api_key = ''
+
+# https://api-docs.fullhunt.io/
+fullhunt_api_key = ''
+
+# https://windvane.lichoin.com/
+windvane_api_token = ''

config/log.py

@@ -23,13 +23,13 @@ logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \
               '<blue>{line}</blue> - <level>{message}</level>'
 
 logger.remove()
-logger.level(name='TRACE', color='<cyan><bold>', icon='✏️')
+logger.level(name='TRACE', color='<cyan><bold>')
-logger.level(name='DEBUG', color='<blue><bold>', icon='🐞 ')
+logger.level(name='DEBUG', color='<blue><bold>')
-logger.level(name='INFOR', no=20, color='<green><bold>', icon='ℹ️')
+logger.level(name='INFOR', no=20, color='<green><bold>')
-logger.level(name='QUITE', no=25, color='<green><bold>', icon='🤫 ')
+logger.level(name='QUITE', no=25, color='<green><bold>')
-logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️')
+logger.level(name='ALERT', no=30, color='<yellow><bold>')
-logger.level(name='ERROR', color='<red><bold>', icon='❌️')
+logger.level(name='ERROR', color='<red><bold>')
-logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️')
+logger.level(name='FATAL', no=50, color='<RED><bold>')
 
 # 如果你想在命令终端静默运行OneForAll，可以将以下一行中的level设置为QUITE
 # 命令终端日志级别默认为INFOR

config/setting.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 """
-OneForAll配置
+OneForAll自定义配置
 """
 
 import pathlib
@@ -10,6 +10,7 @@ relative_directory = pathlib.Path(__file__).parent.parent  # OneForAll代码相
 data_storage_dir = relative_directory.joinpath('data')  # 数据存放目录
 
 # OneForAll入口参数设置
+enable_check_network = True  # 开启网络环境检查
 enable_check_version = True  # 开启最新版本检查
 enable_brute_module = True  # 使用爆破模块(默认True)
 enable_dns_resolve = True  # 使用DNS解析子域(默认True)
@@ -32,13 +33,13 @@ save_module_result = False  # 保存各模块发现结果为json文件(默认Fal
 enable_all_module = True  # 启用所有收集模块(默认True)
 enable_partial_module = []  # 启用部分收集模块 必须禁用enable_all_module才能生效
 # 只使用ask和baidu搜索引擎收集子域的示例
-# enable_partial_module = [('modules.search', 'ask')
+# enable_partial_module = ['modules.search.ask', 'modules.search.baidu']
-#                          ('modules.search', 'baidu')]
 
 # 爆破模块设置
 brute_concurrent_num = 2000  # 爆破时并发查询数量(默认2000，最大推荐10000)
 # 爆破所使用的字典路径(默认None则使用data/subdomains.txt，自定义字典请使用绝对路径)
 brute_wordlist_path = None
+use_china_nameservers = True  # 使用中国域名服务器 如果你所在网络不在中国则建议设置False
 enable_recursive_brute = False  # 是否使用递归爆破(默认False)
 brute_recursive_depth = 2  # 递归爆破深度(默认2层)
 # 爆破下一层子域所使用的字典路径(默认None则使用data/subnames_next.txt，自定义字典请使用绝对路径)
@@ -68,10 +69,40 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
                         'YandexSearch', 'CrossDomainXml',
                         'ContentSecurityPolicy']  # 代理自定义的模块
 request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
-                       'https': 'https://127.0.0.1:1080'}]  # 代理池
+                       'https': 'http://127.0.0.1:1080'}]  # 代理池
 # request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
 #                        'https': 'socks5h://127.0.0.1:10808'}]  # 代理池
 
+
+# 请求设置
+request_thread_count = None  # 请求线程数量(默认None，则根据情况自动设置)
+request_timeout_second = (13, 27)  # 请求超时秒数(默认connect timout推荐略大于3秒)
+request_ssl_verify = False  # 请求SSL验证(默认False)
+request_allow_redirect = True  # 请求允许重定向(默认True)
+request_redirect_limit = 10  # 请求跳转限制(默认10次)
+# 默认请求头 可以在headers里添加自定义请求头
+request_default_headers = {
+    'Accept': 'text/html,application/xhtml+xml,'
+              'application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Encoding': 'gzip, deflate',
+    'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+    'Cache-Control': 'max-age=0',
+    'DNT': '1',
+    'Referer': 'https://www.google.com/',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                  '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Upgrade-Insecure-Requests': '1',
+    'X-Forwarded-For': '127.0.0.1'
+}
+enable_random_ua = True  # 使用随机UA(默认True，开启可以覆盖request_default_headers的UA)
+
+
 # 搜索模块设置
 enable_recursive_search = False  # 递归搜索子域
 search_recursive_times = 2  # 递归搜索层数
+
+# 网络空间测绘引擎设置
+cam_records_maximum_per_domain = 1000   # 对于单个主域名，在测绘引擎中的最多查询多少条记录，防止泛解析和CDN浪费积分，对 fofa, hunter, quake, zoomeye 生效，最低为100
+
+# 是否从输入的数据中使用tldextract提取主域名。若设为 False，OneForAll会直接将输入的域名作为主域名，比如北京分公司的域名 beijing.10086.com 就不会被解析成母公司的域名 10086.com
+use_tld_extract = True

data/altdns_wordlist.txt

@@ -350,3 +350,637 @@ vi
 web1
 westeurope
 z
+previous
+new
+new1
+east
+south
+west
+north
+southeast
+northwest
+es
+nt
+tx
+tencent
+ali
+aliyun
+center
+stable
+release
+ga
+rc
+01
+02
+03
+dc
+mq
+oa
+uc
+zk
+apm
+bbs
+biw
+bot
+bus
+cat
+crm
+dc1
+doc
+ec2
+efk
+elk
+gce
+hub
+job
+jpa
+jwt
+lib
+log
+mbs
+mgt
+mvc
+nms
+pan
+pod
+srv
+sso
+svn
+tms
+wms
+www
+amqp
+apis
+app1
+avro
+blog
+boot
+cicd
+flex
+gocd
+gogs
+good
+grid
+guns
+hdfs
+hive
+ldap
+live
+logs
+mqtt
+nifi
+note
+open
+pipe
+pods
+shop
+show
+solr
+tsdb
+user
+wiki
+work
+zuul
+apiv1
+apiv2
+apiv3
+apiv4
+apiv5
+app01
+batch
+beats
+cacti
+camel
+chaos
+drone
+dubbo
+event
+feign
+flink
+flume
+geode
+gitea
+goods
+graph
+group
+habor
+hbase
+html5
+infra
+kafka
+kylin
+label
+maven
+mesos
+micro
+minio
+nacos
+neo4j
+nerve
+nexus
+node1
+oauth
+oozie
+redis
+route
+scala
+spark
+sqoop
+stack
+store
+storm
+webui
+api-v1
+api-v2
+api-v3
+api-v4
+api-v5
+consul
+falcon
+galaxy
+goblin
+gradle
+group1
+hadoop
+harbor
+influx
+kibana
+lcinga
+logapi
+logger
+nagios
+node01
+oauth2
+office
+pgraph
+ribbon
+router
+scribe
+sleuth
+spark1
+splunk
+stream
+syslog
+sysmon
+tracer
+travis
+triton
+tuning
+web-ui
+weblog
+zabbix
+zipkin
+airflow
+akumuli
+ansible
+bigdata
+breaker
+brogmon
+catalog
+circuit
+content
+datadog
+diagram
+eureka1
+fluentd
+gemfire
+grafana
+hystrix
+invoker
+jupyter
+kinesis
+library
+logging
+manager
+meeting
+monitor
+netdata
+netflix
+nodered
+pushapi
+recruit
+restapi
+restful
+rsyslog
+storage
+tracing
+turbine
+webflow
+activemq
+actuator
+cadvisor
+collectd
+contract
+dataflow
+elkstack
+exporter
+filebeat
+graphite
+heapster
+influxdb
+librenms
+logstash
+marathon
+node-red
+opentsdb
+pipeline
+platform
+push-api
+rabbitmq
+schedule
+searcher
+sentinel
+tracking
+atlassian
+cassandra
+community
+discovery
+dockerhub
+dzzoffice
+elk-stack
+hostgroup
+kubernete
+logsearch
+mapreduce
+office365
+openstack
+pagerduty
+serverset
+terraform
+websocket
+zookeeper
+clickhouse
+cloudwatch
+dashboards
+datacenter
+dispatcher
+hostgroup1
+kube-state
+openfalcon
+prometheus
+servicelog
+skywalking
+usercenter
+api-gateway
+data-center
+eureka-zuul
+kube-status
+loganalysis
+open-falcon
+opentracing
+pushgateway
+recruitment
+restful-api
+service-log
+serviceslog
+system-cube
+user-center
+alertmanager
+apis-gateway
+eureka-admin
+grafana-kong
+nacos-config
+push-gateway
+rest-gateway
+services-log
+zuul-gateway
+consul-config
+elasticsearch
+eureka-client
+eureka-server
+node-exporter
+kong-dashboard
+kylin-dashboard
+restful-gateway
+grafana-dashboard
+kylin-system-cube
+grafana-management
+apollo
+apolloconfig
+apolloadmin
+apolloservice
+ws
+lucene
+beam
+struts
+arrow
+hudi
+cloudstack
+pulsar
+commons
+subversion
+asterixdb
+superset
+mxnet
+httpd
+ignite
+sling
+shardingsphere
+axis
+spamassassin
+apisix
+openoffice
+db
+cordova
+qpid
+ofbiz
+tapestry
+impala
+ambari
+cocoon
+carbondata
+geronimo
+dolphinscheduler
+doris
+tvm
+wicket
+ant
+iceberg
+jackrabbit
+myfaces
+pdfbox
+james
+nuttx
+drill
+incubator
+tomee
+ozone
+trafficserver
+accumulo
+kudu
+druid
+pinot
+phoenix
+harmony
+directory
+perl
+thrift
+pig
+felix
+cxf
+echarts
+karaf
+servicemix
+xmlgraphics
+openwhisk
+iotdb
+tuscany
+couchdb
+jclouds
+rocketmq
+trafficcontrol
+tinkerpop
+hc
+calcite
+jmeter
+netbeans
+uima
+poi
+zeppelin
+tez
+isis
+atlas
+mahout
+mynewt
+tika
+mina
+nutch
+deltaspike
+xalan
+portals
+servicecomb
+ranger
+groovy
+jena
+brooklyn
+whimsical
+knox
+forrest
+avalon
+openmeetings
+xerces
+bookkeeper
+bigtop
+apr
+metron
+samza
+openjpa
+tajo
+lenya
+stratos
+cayenne
+airavata
+weex
+shindig
+shenyu
+velocity
+aries
+lens
+fineract
+gobblin
+parquet
+apex
+seatunnel
+syncope
+jakarta
+royale
+usergrid
+manifoldcf
+inlong
+libcloud
+hawq
+gump
+allura
+aurora
+ibatis
+ratis
+helix
+synapse
+kyuubi
+sentry
+archiva
+daffodil
+reef
+trafodion
+sis
+xml
+devlake
+hama
+openwebbeans
+guacamole
+pivot
+lucenenet
+curator
+gora
+olingo
+ode
+jspwiki
+river
+stanbol
+giraph
+streams
+slider
+dlab
+plc4x
+shiro
+juddi
+eventmesh
+roller
+taverna
+oodt
+climate
+bloodhound
+chemistry
+unomi
+yunikorn
+brpc
+orc
+rave
+beehive
+eagle
+yetus
+deltacloud
+clerezza
+datasketches
+systemml
+continuum
+tamaya
+opennlp
+buildstream
+stdcxx
+rya
+santuario
+buildr
+submarine
+linkis
+freemarker
+juneau
+any23
+ariatosca
+lucy
+marmotta
+heron
+datalab
+singa
+whirr
+celix
+vcl
+streampipes
+chukwa
+ctakes
+crunch
+madlib
+metamodel
+xmlbeans
+quickstep
+edgent
+wookie
+griffin
+hop
+uniffle
+esme
+fluo
+nlpcraft
+abdera
+creadur
+predictionio
+wink
+ace
+pagespeed
+kvrocks
+mnemonic
+tcl
+twill
+gearpump
+tubemq
+oltu
+quetz
+livy
+sdap
+johnzon
+systemds
+polygene
+labs
+pegasus
+streampark
+attic
+age
+pekko
+teaclave
+htrace
+ponymail
+tiles
+hivemall
+bahir
+click
+empire-db
+bval
+hivemind
+myriad
+hugegraph
+devicemap
+excalibur
+tuweni
+sedona
+vxquery
+datafu
+joshua
+commonsrdf
+shale
+directmemory
+mrunit
+tephra
+nemo
+senssoft
+flagon
+pirk
+toree
+omid
+distributedlog
+celeborn
+s2graph
+onami
+serf
+etch
+samoa
+milagro
+corinthia
+wayang
+diversity
+batchee
+ripple
+opendal
+gossip
+annotator
+causeway
+sirona
+spot
+baremaps
+mrql
+depot
+crail
+steve
+petri
+amaterasu
+liminal
+iota
+paimon
+kibble
+horn
+hdt
+drat
+openaz
+marvin
+provisionr
+mesatee
+cmda
+bluemarlin
+concerted
+warble
+tac
+composer
+cotton
+kie
+android
+infratest
+zabbix

data/authoritative_dns.txt

@@ -0,0 +1,27 @@
+47.254.51.88
+163.177.156.225
+161.117.97.232
+218.98.58.194
+117.91.188.195
+14.17.109.84
+58.52.135.164
+172.96.125.3
+106.38.197.52
+163.177.156.225
+117.91.188.196
+218.98.58.194
+58.52.135.165
+172.96.125.3
+106.38.197.48
+47.254.51.88
+161.117.97.232
+14.17.109.85
+172.96.125.3
+14.17.109.83
+163.177.156.225
+161.117.97.232
+218.98.58.194
+47.254.51.88
+58.52.135.163
+117.91.188.194
+106.38.197.52

data/cdn_cname_keywords.json

@@ -120,9 +120,7 @@
     "edgekey": "Akamai",
     "fastly": "Fastly",
     "chinacache": "ChinaCache",
-    "edgekey": "Akamai",
     "akamai": "Akamai",
-    "fastly": "Fastly",
     "edgecast": "EdgeCast",
     "azioncdn": "Azion",
     "cachefly": "CacheFly",
@@ -135,7 +133,6 @@
     "cloudflare": "CloudFlare",
     "hwcdn": "HighWinds",
     "kxcdn": "KeyCDN",
-    "awsdns": "KeyCDN",
     "fpbns": "Level3",
     "footprint": "Level3",
     "llnwd": "LimeLight",
@@ -155,33 +152,20 @@
     "cdnsun": "CDN SUN",
     "cdnvideo": "CDN Video",
     "clients.turbobytes.net": "TurboBytes",
-    "clients.turbobytes.net": "TurboBytes",
     "turbobytes-cdn.com": "TurboBytes",
     "afxcdn.net": "afxcdn.net",
-    "akamai.net": "Akamai",
     "akamaiedge.net": "Akamai",
-    "akadns.net": "Akamai",
     "akamaitechnologies.com": "Akamai",
     "gslb.tbcache.com": "Alimama",
-    "cloudfront.net": "Amazon Cloudfront",
-    "anankecdn.com.br": "Ananke",
     "att-dsa.net": "AT&T",
-    "azioncdn.net": "Azion",
     "belugacdn.com": "BelugaCDN",
     "bluehatnetwork.com": "Blue Hat Network",
     "systemcdn.net": "EdgeCast",
-    "cachefly.net": "Cachefly",
-    "cdn77.net": "CDN77",
-    "cdn77.org": "CDN77",
     "panthercdn.com": "CDNetworks",
     "cdngc.net": "CDNetworks",
     "gccdn.net": "CDNetworks",
     "gccdn.cn": "CDNetworks",
-    "cdnify.io": "CDNify",
-    "ccgslb.com": "ChinaCache",
-    "ccgslb.net": "ChinaCache",
     "c3cache.net": "ChinaCache",
-    "chinacache.net": "ChinaCache",
     "cncssr.chinacache.net": "ChinaCache",
     "c3cdn.net": "ChinaCache",
     "lxdns.com": "ChinaNetCenter",
@@ -189,11 +173,9 @@
     "mwcloudcdn.com": "QUANTIL/ChinaNetCenter",
     "cloudflare.com": "Cloudflare",
     "cloudflare.net": "Cloudflare",
-    "edgecastcdn.net": "EdgeCast",
     "adn.": "EdgeCast",
     "wac.": "EdgeCast",
     "wpc.": "EdgeCast",
-    "fastly.net": "Fastly",
     "fastlylb.net": "Fastly",
     "google.": "Google",
     "googlesyndication.": "Google",
@@ -201,17 +183,11 @@
     "googleusercontent.com": "Google",
     "l.doubleclick.net": "Google",
     "hiberniacdn.com": "Hibernia",
-    "hwcdn.net": "Highwinds",
-    "incapdns.net": "Incapsula",
     "inscname.net": "Instartlogic",
     "insnw.net": "Instartlogic",
-    "internapcdn.net": "Internap",
-    "kxcdn.com": "KeyCDN",
     "lswcdn.net": "LeaseWeb CDN",
-    "footprint.net": "Level3",
     "llnwd.net": "Limelight",
     "lldns.net": "Limelight",
-    "netdna-cdn.com": "MaxCDN",
     "netdna-ssl.com": "MaxCDN",
     "netdna.com": "MaxCDN",
     "stackpathdns.com": "StackPath",

data/fingerprints.json

@@ -24,11 +24,6 @@
         "cname":["myshopify.com"],
         "response":["Sorry, this shop is currently unavailable.", "Only one step left!"]
     },
-    {
-        "name":"instapage",
-        "cname":["pageserve.co", "secure.pageserve.co", "https://instapage.com/"],
-        "response":["Looks Like You're Lost","The page you're looking for is no longer available."]
-    },
     {
         "name":"desk",
         "cname":["desk.com"],
@@ -245,5 +240,10 @@
         "name":"readme",
         "cname":["readme.io"],
         "response":["Project doesnt exist... yet!"]
+    },
+    {
+        "name":"alibaba_oss",
+        "cname":["aliyuncs.com"],
+        "response":["NoSuchBucket", "The specified bucket does not exist."]
     }
 ]

docs/changes.md

@@ -8,6 +8,18 @@ OneForAll遵守[语义化版本格式](https://semver.org/)。
 # Unreleased
 
 # Released
+## [0.4.5](https://github.com/shmilylty/oneforall/releases/tag/v0.4.5) - 2022-07-10
+- 修复了#254
+
+## [0.4.4](https://github.com/shmilylty/oneforall/releases/tag/v0.4.4) - 2022-07-03
+- 修复了多个已知问题
+- 添加了多个查询接口
+- 添加对M1芯片的Mac支持
+
+## [0.4.3](https://github.com/shmilylty/oneforall/releases/tag/v0.4.3) - 2020-11-29
+- 修复了已知问题
+- 更新了文档
+
 ## [0.4.2](https://github.com/shmilylty/oneforall/releases/tag/v0.4.2) - 2020-11-23
 - 添加了数据表初始化处理流程，修复了#163中出现的问题。
 

docs/collection_modules.md

@@ -1,15 +1,15 @@
 #  收集模块说明 #
 
-如果要使用通过API收集子域的模块请先到[api.py](../oneforall/config/api.py)配置相关信息，大多平台的API都是可以注册账号免费获取的。
+如果要使用通过API收集子域的模块请先到[api.py](../config/api.py)配置相关信息，大多平台的API都是可以注册账号免费获取的。
 
-如果你指定使用某些模块可以在[api.py](../oneforall/config/api.py)中设置：
+如果你指定使用某些模块可以在[api.py](../config/api.py)中设置：
 
 ```python
 enable_all_module = False  # 不开启所有模块
 enable_partial_module = [('modules.search', 'ask'), ('modules.search', 'baidu')]  # 只使用ask和baidu搜索引擎收集子域
 ```
 
-如果你指定使用某些模块使用代理可以在[api.py](../oneforall/config/api.py)中设置：
+如果你指定使用某些模块使用代理可以在[api.py](../config/api.py)中设置：
 
 ```python
 enable_proxy = True  # 使用代理
@@ -23,12 +23,12 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
     
       | 模块名称    | 是否需要代理 | 是否需要API | 其他说明                                           |
       | ----------- | ------------ | ----------- | -------------------------------------------------- |
-      | censys_api  | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | censys_api  | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | certspotter | 否           | 否          |                                                    |
       | crtsh       | 否           | 否          |                                                    |
       | entrust     | 否           | 否          |                                                    |
       | google      | 是           | 否          |                                                    |
-      | spyse_api   | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | spyse_api   | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
 
 
   2. 常规检查收集子域（目前有4个模块：域传送漏洞利用`axfr`，检查跨域策略文件`cdx`，检查HTTPS证书`cert`，检查内容安全策略`csp`，后续会添加检查NSEC记录，NSEC3记录等模块）
@@ -52,25 +52,25 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
 
       | 模块名称           | 是否需要代理 | 是否需要API | 其他说明                                           |
       | ------------------ | ------------ | ----------- | -------------------------------------------------- |
-      | binaryedge_api     | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | binaryedge_api     | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | bufferover         | 否           | 否          |                                                    |
       | cebaidu            | 否           | 否          |                                                    |
       | chinaz             | 否           | 否          |                                                    |
-      | chinaz_api         | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | chinaz_api         | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
-      | circl_api          | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | circl_api          | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
-      | cloudflare_api     | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | cloudflare_api     | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | dnsdb              | 否           | 否          |                                                    |
-      | dnsdb_api          | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | dnsdb_api          | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | dnsdumpster        | 否           | 否          |                                                    |
       | hackertarget       | 否           | 否          |                                                    |
       | ip138              | 否           | 否          |                                                    |
-      | ipv4info           | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | ipv4info           | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | netcraft           | 否           | 否          |                                                    |
-      | passivedns_api     | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | passivedns_api     | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
-      | ptrarchive         | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | ptrarchive         | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
-      | riddler            | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | riddler            | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | robtex             | 否           | 否          |                                                    |
-      | securitytrails_api | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py) |
+      | securitytrails_api | 否           | 是          | API使用和申请见[api.py](../config/api.py) |
       | sitedossier        | 否           | 否          |                                                    |
       | threatcrowd        | 否           | 否          |                                                    |
       | ximcx              | 否           | 否          |                                                    |
@@ -84,11 +84,11 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
       | 模块名称       | 是否需要代理 | 是否需要API | 其他说明                                           |
       | -------------- | ------------ | ----------- | ------------------------------------------------- |
       | alienvault     | 否           | 否          |                                                    |
-      | riskiq_api     | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py)         |
+      | riskiq_api     | 否           | 是          | API使用和申请见[api.py](../config/api.py)         |
-      | threatbook_api | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py)         |
+      | threatbook_api | 否           | 是          | API使用和申请见[api.py](../config/api.py)         |
       | threatminer    | 否           | 否          |                                                    |
       | virustotal     | 否           | 否          |                                                    |
-      | virustotal_api | 否           | 是          | API使用和申请见[api.py](../oneforall/config/api.py)         |
+      | virustotal_api | 否           | 是          | API使用和申请见[api.py](../config/api.py)         |
   7. 利用搜索引擎发现子域（目前有16个模块：`ask`, `bing_api`, `fofa_api`, `shodan_api`, `yahoo`, `baidu`, `duckduckgo`, `github`, `google`, `so`, `yandex`, `bing`, `exalead`, `google_api`, `sogou`, `zoomeye_api`）
 
      除特殊搜索引擎，通用的搜索引擎都支持自动排除搜索，全量搜索，递归搜索。
@@ -98,17 +98,17 @@ proxy_partial_module = ['GoogleQuery', 'AskSearch']  # 只代理GoogleQuery和As
      | ask         | 是                     | 否          |                                                             |
      | baidu       | 否                     | 否          |                                                             |
      | bing        | 否                     | 否          |                                                             |
-     | bing_api    | 否                     | 是          | API使用和申请见[api.py](../oneforall/config/api.py)                  |
+     | bing_api    | 否                     | 是          | API使用和申请见[api.py](../config/api.py)                  |
      | duckduckgo  | 是                     | 否          |                                                             |
      | exalead     | 否，最好使用国外代理。    | 否          |                                                             |
-     | fofa_api    | 否                     | 是          | API使用和申请见[api.py](../oneforall/config/api.py)                  |
+     | fofa_api    | 否                     | 是          | API使用和申请见[api.py](../config/api.py)                  |
      | gitee       | 否                     | 否          |                                                             |
-     | github      | 否                     | 否          | 在[api.py](../oneforall/config/api.py)设置Github邮件名和密码。        |
+     | github      | 否                     | 否          | 在[api.py](../config/api.py)设置Github邮件名和密码。        |
      | google      | 是                     | 否          |                                                             |
-     | google_api  | 是                     | 是          | API使用和申请见[api.py](../oneforall/config/api.py)                  |
+     | google_api  | 是                     | 是          | API使用和申请见[api.py](../config/api.py)                  |
-     | shodan_api  | 否，最好使用国外代理。    | 是          | API使用和申请见[api.py](../oneforall/config/api.py)                  |
+     | shodan_api  | 否，最好使用国外代理。    | 是          | API使用和申请见[api.py](../config/api.py)                  |
      | so          | 否                     | 否          |                                                             |
      | sogou       | 否                     | 否          |                                                             |
      | yahoo       | 是                     | 否          |                                                             |
      | yandex      | 是                     | 否          |                                                             |
-     | zoomeye_api | 否                     | 是          | API使用和申请见[api.py](../oneforall/config/api.py)                  |
+     | zoomeye_api | 否                     | 是          | API使用和申请见[api.py](../config/api.py)                  |

docs/en-us/README.md

@@ -4,8 +4,8 @@
 [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
-[![python](https://img.shields.io/badge/python-3.6|3.7|3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/python-3.6+-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
-[![python](https://img.shields.io/badge/release-v0.4.2-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
+[![python](https://img.shields.io/badge/release-v0.4.5-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
 
 👊**OneForAll is a powerful subdomain integration tool**  📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
 
@@ -123,7 +123,7 @@ Let's take the command `python3 oneforall.py --target example.com run` as an exa
 
 `example_com_now_result` table stores the collection results of the current subdomains. Usually using this table is enough.
 
-For more information, please see [Field explanation](./docs/field.md).
+For more information, please see [Field explanation](../field.md).
 
 </details>
 
@@ -221,7 +221,7 @@ Problems with other tools
 
 In order to solve the above problems, OneForAll born! As its name, OneForAll is committed to becoming the only one subdomain integration tool you need. We hope that one day OneForAll can be called "probably the best subdomain tool"
 
-At present, OneForAll is under development, there must be a lot of problems and areas for improvement. Welcome to submit [Issues](https://github.com/shmilylty/OneForAll/issues) or [PR](https://github.com/shmilylty/OneForAll/pulls), If you like, star please✨. You can contact me through QQ group [**824414244**](//shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771) or twitter [tweet](https://twitter.com/shmilylty) to me: 👨‍👨‍👦‍👦.
+At present, OneForAll is under development, there must be a lot of problems and areas for improvement. Welcome to submit [Issues](https://github.com/shmilylty/OneForAll/issues) or [PR](https://github.com/shmilylty/OneForAll/pulls), If you like, star please✨. You can contact me through QQ group [**824414244**](https://shang.qq.com/wpa/qunwpa?idkey=125d3689b60445cdbb11e4ddff38036b7f6f2abbf4f7957df5dddba81aa90771) or twitter [tweet](https://twitter.com/shmilylty) to me: 👨‍👨‍👦‍👦.
 
 ## 👍Features
 
@@ -229,12 +229,14 @@ At present, OneForAll is under development, there must be a lot of problems and
 1. Use 6 certificate modules: `censys_api`, `certspotter`, `crtsh`, `entrust`, `google`, `spyse_api`.
 2. Use 6 baseline testing modules: scan domain transfer vulnerability `axfr`, cross-domain policy file `cdx`, HTTPS certificate `cert`, content security policy `csp`, robots file `robots`, and sitemap file `sitemap`, NSEC record `nsec`. NSEC3 record and other modules will be added later.
 3. Use 2 web crawler modules: `archirawl`, `commoncrawl`, which is still being debugged and needs to be added and improved). 
-4. Use 24 DNS datasets modules: `binaryedge_api`, `bufferover`, `cebaidu`, `chinaz`, `chinaz_api`, `circl_api`, `cloudflare`, `dnsdb_api`, `dnsdumpster`, `hackertarget`, `ip138`, `ipv4info_api`, `netcraft`, `passivedns_api`, `ptrarchive`, `qianxun`, `rapiddns`, `riddler`, `robtex`, `securitytrails_api`, `sitedossier`, `threatcrowd`, `wzpc`, `ximcx`.
+4. Use 24 DNS datasets modules: `bevigil`, `binaryedge_api`, `bufferover`, `cebaidu`, `chinaz`, `chinaz_api`, `circl_api`, `cloudflare`, `dnsdb_api`, `dnsdumpster`, `hackertarget`, `ip138`, `ipv4info_api`, `netcraft`, `passivedns_api`, `ptrarchive`, `qianxun`, `rapiddns`, `riddler`, `robtex`, `securitytrails_api`, `sitedossier`, `threatcrowd`, `wzpc`, `ximcx`.
 5. Use 6 DNS queries modules: enumerating SRV records `srv` and collect from `MX`, `NS`, `SOA`, `TXT`, `SPF`. 
 6. Use 6 threat intelligence modules: `alienvault`, `riskiq_ api`, `threatbook_ api`, `threatkeeper `, `virustotal`, `virustotal_ api`, which need to be added and improved.
 7. Use 16 search engines modules: `ask`, `baidu`, `bing`, `bing_api`, `fofa_api`, `gitee`, `github_api`, `google`, `google_api`, `shodan_api`, `so`, `sogou`, `yahoo`, `yandex`, `zoomeye_api`, except for special search engines. General search engines support automatic exclusion of search, full search and recursive search. 
 * **Support subdomain brute force**, can use dictionary mode or custom fuzz mode. Supports bulk brute and recursive brute, and automatically determine wildcard or not and processing.
-* **Support subdmain verification**, default enable, automatically resolve DNS, request subdomain to obtain response, and determine subdomain alive or not.
+* **Support subdomain verification**, default enable, automatically resolve DNS, request subdomain to obtain response, and determine subdomain alive or not.
+* **Support subdomain crawling**, according to the existing subdomains, the response body of the request subdomain and the JS in the response body can be found again from the new subdomain.
+* **Support subdomain replacement**, according to the existing subdomain, use subdomain replacement technology to discover new subdomains again.
 * **Support subdomain takeover**, default enable, supports bulk inspection, and automatic takeover subdomain (only Github, remains to be improved at present).
 * **Powerful processing feature**, support automatic deduplicate, DNS resolve, HTTP request, filter valid subdomains and information for subdomains. Supported export formats: `txt`, `csv`, `json`.
 * **Very fast**, [collection module](https://github.com/shmilylty/OneForAll/tree/master/collect.py) uses multi-threading, [brute module](https://github.com/shmilylty/OneForAll/tree/master/brute.py) uses [MassDNS](https://github.com/blechschmidt/massdns), MassDNS is capable of resolving over 350,000 names per second using publicly available resolvers. DNS resolve and HTTP requests use async-coroutine. [subdomain takeover](https://github.com/shmilylty/OneForAll/tree/master/takeover.py) uses multi-threading.
@@ -268,7 +270,6 @@ The project uses [SemVer](https://semver.org/) for version management, and you c
 ## ⌛Follow-up plan
 
 - [ ] Continuous optimize and improve of each module
-- [x] Subdomain collection crawler (collect subdomains from static files such as JS)
 - [ ] Implementation of front-end interface for powerful interaction
 
 For more details, read [todo.md](https://github.com/shmilylty/OneForAll/tree/master/docs/todo.md).
@@ -282,7 +283,7 @@ Very warmly welcome all people to make OneForAll better together!
 * **[Jing Ling](https://github.com/shmilylty)**
   * Core developer
 
-You can view all contributors and their contributions in the [contributor documentation](https://github.com/shmilylty/OneForAll/tree/master/docs/contributors.md)) and thank them for making OneForAll more powerful and useful.
+You can view all contributors and their contributions in the [contributor documentation](https://github.com/shmilylty/OneForAll/tree/master/docs/contributors.md) and thank them for making OneForAll more powerful and useful.
 
 ## 📄License
 
@@ -294,6 +295,8 @@ Thanks to the various subdomain collection projects of online open source!
 
 Thanks ace of [A-Team](https://github.com/QAX-A-Team) for their enthusiastic and unselfish answers!
 
+Developed with drive and [PyCharm](https://www.jetbrains.com/pycharm/)!
+
 ## 📜Disclaimer 
 
 This tool can only be used in the safety construction of enterprises with sufficient legal authorization.

docs/field.md

@@ -60,7 +60,7 @@ HTTP响应的状态码
 
 网络连接情况及详情
 
-###title
+### title
 
 网站标题
 
@@ -69,9 +69,11 @@ HTTP响应的状态码
 网站指纹信息
 
 ### history
+
 请求时URL跳转历史
 
 ### response
+
 响应体文本内容
 
 ### times

export.py

@@ -60,6 +60,8 @@ def do_export(fmt, path, rows, show, domain, target):
     if show:
         print(rows.dataset)
     data = rows.export(fmt)
+    if fmt == 'csv':
+        data = '\ufeff' + data
     utils.save_to_file(path, data)
     logger.log('ALERT', f'The subdomain result for {domain}: {path}')
     data = rows.as_dict()

modules/altdns.py

@@ -186,11 +186,15 @@ class Altdns(Module):
         for subdomain in self.now_subdomains:
             subname, parts = split_domain(subdomain)
             subnames = subname.split('.')
+            if settings.altdns_increase_num:
                 self.increase_num(subname)
+            if settings.altdns_decrease_num:
                 self.decrease_num(subname)
+            if settings.altdns_replace_word:
                 self.replace_word(subname)
-            if not settings.enable_fast_alt:
+            if settings.altdns_insert_word:
                 self.insert_word(parts)
+            if settings.altdns_add_word:
                 self.add_word(subnames)
         count = len(self.new_subdomains)
         logger.log('DEBUG', f'The altdns module generated {count} subdomains')

modules/certificates/censys_api.py

@@ -9,7 +9,7 @@ class CensysAPI(Query):
         self.domain = domain
         self.module = 'Certificate'
         self.source = "CensysAPIQuery"
-        self.addr = 'https://www.censys.io/api/v1/search/certificates'
+        self.addr = 'https://search.censys.io/api/v2/certificates/search'
         self.id = settings.censys_api_id
         self.secret = settings.censys_api_secret
         self.delay = 3.0  # Censys 接口查询速率限制 最快2.5秒查1次
@@ -20,26 +20,30 @@ class CensysAPI(Query):
         """
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
-        data = {
+        params = {
-            'query': f'parsed.names: {self.domain}',
+            'q': f'names: {self.domain}',
-            'page': 1,
+            'per_page': 100,
-            'fields': ['parsed.subject_dn', 'parsed.names'],
+        }
-            'flatten': True}
+        resp = self.get(self.addr, params=params, auth=(self.id, self.secret))
-        resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
         if not resp:
             return
         json = resp.json()
         status = json.get('status')
-        if status != 'ok':
+        if status != 'OK':
             logger.log('ALERT', f'{self.source} module {status}')
             return
         subdomains = self.match_subdomains(resp.text)
         self.subdomains.update(subdomains)
-        pages = json.get('metadata').get('pages')
+        next_cursor = json.get("result").get("links").get("next")
-        for page in range(2, pages + 1):
+        while next_cursor:
-            data['page'] = page
+            tmp_params = {
-            resp = self.post(self.addr, json=data, auth=(self.id, self.secret))
+                'q': f'names: {self.domain}',
-            self.subdomains = self.collect_subdomains(resp)
+                'per_page': 100,
+                "cursor": next_cursor
+            }
+            tmp_resp = self.get(self.addr, params=tmp_params, auth=(self.id, self.secret))
+            self.subdomains = self.collect_subdomains(tmp_resp)
+            next_cursor = tmp_resp.json().get("result").get("links").get("next")
 
     def run(self):
         """

modules/certificates/crtsh.py

@@ -1,4 +1,6 @@
 from common.query import Query
+import json
+import os
 
 
 class Crtsh(Query):
@@ -15,11 +17,44 @@ class Crtsh(Query):
         """
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
+        self.timeout = 120
         params = {'q': f'%.{self.domain}', 'output': 'json'}
         resp = self.get(self.addr, params)
         if not resp:
             return
         text = resp.text.replace(r'\n', ' ')
+        """
+        * > altdns
+        """
+        subDomains = set()
+        try:
+            jsonData = json.loads(text)
+        except Exception as e:
+            pass
+        for i in range(len(jsonData)):
+            try:
+                name_value = str(jsonData[i]['name_value'])
+            except Exception as e:
+                pass
+            if '*' in name_value:
+                try:
+                    if 'certificates' in os.path.dirname(os.path.abspath(__file__)):
+                        dictFile = open("../../data/altdns_wordlist.txt", "r", encoding='utf8')
+                    else:
+                        dictFile = open("./data/altdns_wordlist.txt", "r", encoding='utf8')
+                    for line in dictFile.readlines():
+                        altdns = line.strip()
+                        result = name_value.replace('*', altdns)
+                        if self.domain in result:
+                            subDomains.add(result)
+                except Exception as e:
+                    pass
+        if len(subDomains) > 0:
+            for x in subDomains:
+                text = text + ',' + x + ','
+        """
+        * > altdns end
+        """
         subdomains = self.match_subdomains(text)
         self.subdomains.update(subdomains)
 
@@ -46,4 +81,4 @@ def run(domain):
 
 
 if __name__ == '__main__':
-    run('example.com')
+    run('163.com')

modules/certificates/myssl.py

@@ -0,0 +1,46 @@
+from common.query import Query
+
+
+class MySSL(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Certificate'
+        self.source = 'MySSLQuery'
+        self.addr = 'https://myssl.com/api/v1/discover_sub_domain'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'domain': self.domain}
+        resp = self.get(self.addr, params)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+
+    """
+    query = MySSL(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('freebuf.com')

modules/certificates/racent.py

@@ -0,0 +1,49 @@
+from config import settings
+from common.query import Query
+
+
+class Racent(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Certificate'
+        self.source = 'RacentQuery'
+        self.addr = 'https://face.racent.com/tool/query_ctlog'
+        self.api = settings.racent_api_token
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'token': self.api, 'keyword': self.domain}
+        resp = self.get(self.addr, params)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.api):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Racent(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

modules/check/nsec.py

@@ -26,6 +26,9 @@ class NSEC(Check):
                 self.subdomains.update(subdomains)
             if subdomain == self.domain:  # 当查出子域为主域 说明完成了一个循环 不再继续查询
                 break
+            if domain != self.domain:  # 防止出现wwdmas.cn 000.000.wwdmas.cn 000.000.000.wwdmas.cn情况
+                if domain.split('.')[0] == subdomain.split('.')[0]:
+                    break
             domain = subdomain
         return self.subdomains
 

modules/collect.py

@@ -23,7 +23,7 @@ class Collect(object):
                 module_path = settings.module_dir.joinpath(module)
                 for path in module_path.rglob('*.py'):
                     import_module = f'modules.{module}.{path.stem}'
-                    self.modules.append([import_module, path.stem])
+                    self.modules.append(import_module)
         else:
             self.modules = settings.enable_partial_module
 
@@ -31,7 +31,8 @@ class Collect(object):
         """
         Import do function
         """
-        for module, name in self.modules:
+        for module in self.modules:
+            name = module.split('.')[-1]
             import_object = importlib.import_module(module)
             func = getattr(import_object, 'run')
             self.collect_funcs.append([func, name])

modules/datasets/bevigil.py

@@ -0,0 +1,50 @@
+from config import settings
+from common.query import Query
+
+
+class BeVigilAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'BeVigilOsintApi'
+        self.addr = 'http://osint.bevigil.com/api/{}/subdomains/'
+        self.api = settings.bevigil_api
+    
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+
+        self.header = self.get_header()
+        self.header.update({"X-Access-Token": self.api})
+        self.proxy = self.get_proxy(self.source)
+        url = self.addr.format(self.domain)
+        resp = self.get(url)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.api):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = BeVigilAPI(domain)
+    query.run()
+
+if __name__ == '__main__':
+    run('example.com')

modules/datasets/dnsdumpster.py

@@ -21,7 +21,8 @@ class DNSDumpster(Query):
             return
         self.cookie = resp.cookies
         data = {'csrfmiddlewaretoken': self.cookie.get('csrftoken'),
-                'targetip': self.domain}
+                'targetip': self.domain,
+                'user':'free'}
         resp = self.post(self.addr, data)
         self.subdomains = self.collect_subdomains(resp)
 

modules/datasets/dnsgrep.py

@@ -0,0 +1,44 @@
+from common.query import Query
+
+
+class Dnsgrep(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'DnsgrepQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        url = 'https://www.dnsgrep.cn/subdomain/' + self.domain
+        resp = self.get(url)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Dnsgrep(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

modules/datasets/fullhunt.py

@@ -0,0 +1,48 @@
+from config import settings
+from common.query import Query
+
+
+class FullHuntAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'FullHuntAPIQuery'
+        self.api = settings.fullhunt_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({'X-API-KEY': self.api})
+        self.proxy = self.get_proxy(self.source)
+        url = f'https://fullhunt.io/api/v1/domain/{self.domain}/subdomains'
+        resp = self.get(url)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+
+    """
+    query = FullHuntAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('qq.com')

modules/datasets/netcraft.py

@@ -12,39 +12,21 @@ class NetCraft(Query):
         self.domain = domain
         self.module = 'Dataset'
         self.source = 'NetCraftQuery'
-        self.init = 'https://searchdns.netcraft.com/'
+        self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains&position=limited'
-        self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains'
         self.page_num = 1
         self.per_page_num = 20
 
-    def bypass_verification(self):
-        """
-        绕过NetCraft的JS验证
-        """
-        self.header = self.get_header()  # Netcraft会检查User-Agent
-        resp = self.get(self.init)
-        if not resp:
-            return False
-        self.cookie = resp.cookies
-        cookie_value = self.cookie['netcraft_js_verification_challenge']
-        cookie_encode = parse.unquote(cookie_value).encode('utf-8')
-        verify_taken = hashlib.sha1(cookie_encode).hexdigest()
-        self.cookie['netcraft_js_verification_response'] = verify_taken
-        return True
-
     def query(self):
         """
         向接口查询子域并做子域匹配
         """
-        if not self.bypass_verification():
+        self.header = self.get_header()  # NetCraft会检查User-Agent
-            return
+        self.proxy = self.get_proxy(self.source)
         last = ''
         while True:
             time.sleep(self.delay)
-            self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
-            params = {'restriction': 'site ends with',
+            params = {'host': '*.' + self.domain,
-                      'host': '.' + self.domain,
                       'from': self.page_num}
             resp = self.get(self.addr + last, params)
             subdomains = self.match_subdomains(resp)

modules/datasets/urlscan.py

@@ -0,0 +1,45 @@
+from common.query import Query
+
+
+class Urlscan(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'UrlscanQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        url = 'https://urlscan.io/api/v1/search/'
+        params = {'q': 'domain:' + self.domain}
+        resp = self.get(url, params)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Urlscan(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('sangfor.com')

modules/datasets/windvane.py

@@ -0,0 +1,92 @@
+from config import settings
+from common.query import Query
+
+
+class Windvane(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = "WindvaneQuery"
+        self.addr = 'https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService/ListSubDomain'
+        self.api_key = settings.windvane_api_token
+        self.page_size = 1000
+        
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({
+            'Content-Type': 'application/json',
+            'Referer': 'https://windvane.lichoin.com'
+        })
+        
+        if self.api_key:
+            self.header.update({'X-Api-Key': self.api_key})
+            
+        self.proxy = self.get_proxy(self.source)
+        
+        page = 1
+        total_pages = 1
+        all_subdomains = []
+        
+        while page <= total_pages:
+            data = {
+                "domain": self.domain,
+                "page_request": {
+                    "page": page,
+                    "count": self.page_size
+                }
+            }
+            
+            resp = self.post(self.addr, json=data)
+            if not resp:
+                break
+                
+            try:
+                result = resp.json()
+                
+                if result.get('code') != 0:
+                    break
+                
+                data_section = result.get('data', {})
+
+                subdomains = self.match_subdomains(resp)
+                if not subdomains:
+                    break
+                self.subdomains.update(subdomains)
+                
+                page_info = data_section.get('page_response', {})
+                total_pages = int(page_info.get('total_page', 1))
+                
+                page += 1
+                
+            except:
+                break
+        
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Windvane(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('baidu.com')

modules/datasets/wzpc.py

@@ -1,68 +0,0 @@
-import time
-from config.log import logger
-from common.query import Query
-
-
-class WZPCQuery(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = domain
-        self.module = 'Dataset'
-        self.source = 'WZPCQuery'
-
-    def query(self):
-        """
-        向接口查询子域并做子域匹配
-        """
-
-        base_addr = 'http://114.55.181.28/check_web/' \
-                    'databaseInfo_mainSearch.action'
-        page_num = 1
-        while True:
-            time.sleep(self.delay)
-            self.header = self.get_header()
-            self.proxy = self.get_proxy(self.source)
-            params = {'isSearch': 'true', 'searchType': 'url',
-                      'term': self.domain, 'pageNo': page_num}
-            try:
-                resp = self.get(base_addr, params)
-            except Exception as e:
-                logger.log('ERROR', e.args)
-                break
-            if not resp:
-                break
-            subdomains = self.match_subdomains(resp.text)
-            if not subdomains:  # 没有发现子域名则停止查询
-                break
-            self.subdomains.update(subdomains)
-            if not subdomains:
-                break
-            if page_num > 10:
-                break
-            page_num += 1
-
-    def run(self):
-        """
-        类执行入口
-        """
-        self.begin()
-        self.query()
-        self.finish()
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-
-
-def run(domain):
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    """
-    query = WZPCQuery(domain)
-    query.run()
-
-
-if __name__ == '__main__':
-    run('sc.gov.cn')
-    run('bkzy.org')

modules/dnsquery/mx.py

@@ -7,7 +7,7 @@ class QueryMX(Lookup):
         self.domain = domain
         self.module = 'dnsquery'
         self.source = "QueryMX"
-        self.type = 'MX'  # 利用的DNS记录的MX记录收集子域
+        self.qtype = 'MX'  # 利用的DNS记录的MX记录收集子域
 
     def run(self):
         """

modules/finder.py

@@ -179,6 +179,7 @@ def convert_to_dict(url_list):
         url_dict.append({'url': url})
     return url_dict
 
+
 def find_subdomains(domain, data):
     subdomains = set()
     js_urls = set()

modules/intelligence/riskiq_api.py

@@ -8,7 +8,7 @@ class RiskIQ(Query):
         self.domain = domain
         self.module = 'Intelligence'
         self.source = 'RiskIQAPIQuery'
-        self.addr = 'https://api.passivetotal.org/v2/enrichment/subdomains'
+        self.addr = 'https://api.riskiq.net/pt/v2/enrichment/subdomains'
         self.user = settings.riskiq_api_username
         self.key = settings.riskiq_api_key
 
@@ -17,6 +17,7 @@ class RiskIQ(Query):
         向接口查询子域并做子域匹配
         """
         self.header = self.get_header()
+        self.header.update({'Accept': 'application/json'})
         self.proxy = self.get_proxy(self.source)
         params = {'query': self.domain}
         resp = self.get(url=self.addr,

modules/intelligence/threatminer.py

@@ -7,7 +7,7 @@ class ThreatMiner(Query):
         self.domain = domain
         self.module = 'Intelligence'
         self.source = 'ThreatMinerQuery'
-        self.addr = 'https://www.threatminer.org/getData.php'
+        self.addr = 'https://api.threatminer.org/v2/domain.php'
 
     def query(self):
         """
@@ -15,8 +15,7 @@ class ThreatMiner(Query):
         """
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
-        params = {'e': 'subdomains_container',
+        params = {'q': self.domain, 'rt': 5}
-                  'q': self.domain, 't': 0, 'rt': 10}
         resp = self.get(self.addr, params)
         self.subdomains = self.collect_subdomains(resp)
 

modules/intelligence/virustotal_api.py

@@ -28,6 +28,8 @@ class VirusTotalAPI(Query):
             self.subdomains.update(subdomains)
             data = resp.json()
             next_cursor = data.get('meta').get('cursor')
+            if not next_cursor:
+                break
 
     def run(self):
         """

modules/search/bing.py

@@ -19,9 +19,9 @@ class Bing(Search):
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
         """
-        self.page_num = 0  # 二次搜索重新置0
         self.header = self.get_header()
         self.proxy = self.get_proxy(self.source)
+        self.page_num = 0  # 二次搜索重新置0
         resp = self.get(self.init)
         if not resp:
             return

modules/search/bing_api.py

@@ -9,7 +9,7 @@ class BingAPI(Search):
         self.domain = domain
         self.module = 'Search'
         self.source = 'BingAPISearch'
-        self.addr = 'https://api.cognitive.microsoft.com/bing/v7.0/search'
+        self.addr = 'https://api.bing.microsoft.com/v7.0/search'
         self.id = settings.bing_api_id
         self.key = settings.bing_api_key
         self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数

modules/search/fofa_api.py

@@ -11,7 +11,7 @@ class FoFa(Search):
         self.domain = domain
         self.module = 'Search'
         self.source = 'FoFaAPISearch'
-        self.addr = 'https://fofa.so/api/v1/search/all'
+        self.addr = 'https://fofa.info/api/v1/search/all'
         self.delay = 1
         self.email = settings.fofa_api_email
         self.key = settings.fofa_api_key
@@ -23,7 +23,7 @@ class FoFa(Search):
         self.page_num = 1
         subdomain_encode = f'domain="{self.domain}"'.encode('utf-8')
         query_data = base64.b64encode(subdomain_encode)
-        while True:
+        while 100 * self.page_num < settings.cam_records_maximum_per_domain:
             time.sleep(self.delay)
             self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
@@ -32,7 +32,7 @@ class FoFa(Search):
                      'qbase64': query_data,
                      'page': self.page_num,
                      'full': 'true',
-                     'size': 1000}
+                     'size': min(1000, settings.cam_records_maximum_per_domain)}
             resp = self.get(self.addr, query)
             if not resp:
                 return
@@ -42,7 +42,7 @@ class FoFa(Search):
                 break
             self.subdomains.update(subdomains)
             size = resp_json.get('size')
-            if size < 1000:
+            if size < min(1000, settings.cam_records_maximum_per_domain):
                 break
             self.page_num += 1
 

modules/search/gitee.py

@@ -11,7 +11,6 @@ class Gitee(Search):
         self.module = 'Search'
         self.addr = 'https://search.gitee.com/'
         self.domain = domain
-        self.header = self.get_header()
 
     def search(self):
         """
@@ -20,6 +19,8 @@ class Gitee(Search):
         page_num = 1
         while True:
             time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
             params = {'pageno': page_num, 'q': self.domain, 'type': 'code'}
             try:
                 resp = self.get(self.addr, params=params)

modules/search/github_api.py

@@ -1,6 +1,5 @@
 import time
 
-import requests
 from config import settings
 from common.search import Search
 from config.log import logger
@@ -13,54 +12,32 @@ class GithubAPI(Search):
         self.module = 'Search'
         self.addr = 'https://api.github.com/search/code'
         self.domain = domain
-        self.session = requests.Session()
+        self.delay = 5
-        self.session.trust_env = False
-        self.auth_url = 'https://api.github.com'
         self.token = settings.github_api_token
 
-    def auth_github(self):
-        """
-        github api 认证
-
-        :return: 认证失败返回False 成功返回True
-        """
-        self.session.headers.update({'Authorization': 'token ' + self.token})
-        try:
-            resp = self.session.get(self.auth_url)
-        except Exception as e:
-            logger.log('ERROR', e.args)
-            return False
-        if resp.status_code != 200:
-            resp_json = resp.json()
-            msg = resp_json.get('message')
-            logger.log('ERROR', msg)
-            return False
-        return True
-
     def search(self):
         """
         向接口查询子域并做子域匹配
         """
-        self.session.headers = self.get_header()
+        self.header = self.get_header()
-        self.session.proxies = self.get_proxy(self.source)
+        self.proxy = self.get_proxy(self.source)
-        self.session.verify = self.verify
+        self.header.update(
-        self.session.headers.update(
             {'Accept': 'application/vnd.github.v3.text-match+json'})
+        self.header.update(
+            {'Authorization': 'token ' + self.token})
 
-        if not self.auth_github():
-            logger.log('ERROR', f'{self.source} module login failed')
-            return
         page = 1
         while True:
             time.sleep(self.delay)
             params = {'q': self.domain, 'per_page': 100,
-                      'page': page, 'sort': 'indexed'}
+                      'page': page, 'sort': 'indexed',
+                      'access_token': self.token}
             try:
-                resp = self.session.get(self.addr, params=params)
+                resp = self.get(self.addr, params=params)
             except Exception as e:
                 logger.log('ERROR', e.args)
                 break
-            if resp.status_code != 200:
+            if not resp or resp.status_code != 200:
                 logger.log('ERROR', f'{self.source} module query failed')
                 break
             subdomains = self.match_subdomains(resp)
@@ -106,4 +83,4 @@ def run(domain):
 
 
 if __name__ == '__main__':
-    run('example.com')
+    run('freebuf.com')

modules/search/hunter_api.py

@@ -0,0 +1,72 @@
+import base64
+import time
+
+from config import settings
+from common.search import Search
+
+
+class Hunter(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'HunterAPISearch'
+        self.addr = 'https://hunter.qianxin.com/openApi/search'
+        self.delay = 1
+        self.key = settings.hunter_api_key
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        self.page_num = 1
+        subdomain_encode = f'domain_suffix="{self.domain}"'.encode('utf-8')
+        query_data = base64.b64encode(subdomain_encode)
+        while 100 * self.page_num < settings.cam_records_maximum_per_domain:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = {'api-key': self.key,
+                     'search': query_data,
+                     'page': self.page_num,
+                     'page_size': 100,
+                     'is_web': 1}
+            resp = self.get(self.addr, query)
+            if not resp:
+                return
+            resp_json = resp.json()
+            subdomains = self.match_subdomains(resp)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains.update(subdomains)
+            total = resp_json.get('data').get('total')
+            if self.page_num * 100 >= int(total):
+                break
+            self.page_num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.key):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Hunter(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    run('freebuf.com')

modules/search/quake_api.py

@@ -0,0 +1,72 @@
+import time
+
+from config import settings
+from common.search import Search
+
+
+class Quake(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Quake'
+        self.source = "QuakeAPISearch"
+        self.addr = 'https://quake.360.net/api/v3/search/quake_service'
+        self.delay = 1
+        self.key = settings.quake_api_key
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        self.per_page_num = 100
+        self.page_num = 0
+        while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.header.update({'Content-Type': 'application/json'})
+            self.header.update({'X-QuakeToken': self.key})
+            self.proxy = self.get_proxy(self.source)
+
+            query = {'query': 'domain:"' + self.domain + '"',
+                     'start': self.page_num * self.per_page_num,
+                     'size': self.per_page_num,
+                     'include': ["service.http.host"]}
+            resp = self.post(self.addr, json=query)
+            if not resp:
+                return
+            resp_json = resp.json()
+            subdomains = self.match_subdomains(resp)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains.update(subdomains)
+            total = resp_json.get('meta').get('pagination').get('total')
+            self.page_num += 1
+            if self.page_num * self.per_page_num >= int(total):
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.key):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Quake(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('nosugartech.com')

modules/search/so.py

@@ -72,4 +72,4 @@ def run(domain):
 
 
 if __name__ == '__main__':
-    run('example.com')
+    run('mi.com')

modules/search/wzsearch.py

@@ -0,0 +1,68 @@
+import time
+from common.search import Search
+
+
+class WzSearch(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'WzSearch'
+        self.addr = 'https://www.wuzhuiso.com/s'
+
+    def search(self, domain, filtered_subdomain=''):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        """
+        self.page_num = 1
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'q': query, 'pn': self.page_num, 'src': 'page_www', 'fr': 'none'}
+            resp = self.get(self.addr, params)
+            subdomains = self.match_subdomains(resp, fuzzy=False)
+            if not self.check_subdomains(subdomains):
+                break
+            self.subdomains.update(subdomains)
+            self.page_num += 1
+            if 'next" href' not in resp.text:
+                print(subdomains)
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.search(self.domain)
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = WzSearch(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    run('qq.com')

modules/search/yahoo.py

@@ -21,13 +21,14 @@ class Yahoo(Search):
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
         """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
         resp = self.get(self.init)
         if not resp:
             return
         self.cookie = resp.cookies  # 获取cookie Yahoo在搜索时需要带上cookie
         while True:
             time.sleep(self.delay)
-            self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             query = 'site:.' + domain + filtered_subdomain
             params = {'p': query, 'b': self.page_num, 'pz': self.per_page_num}

modules/search/yandex.py

@@ -20,6 +20,8 @@ class Yandex(Search):
         :param str domain: 域名
         :param str filtered_subdomain: 过滤的子域
         """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
         self.page_num = 0  # 二次搜索重新置0
         resp = self.get(self.init)
         if not resp:
@@ -27,7 +29,6 @@ class Yandex(Search):
         self.cookie = resp.cookies  # 获取cookie
         while True:
             time.sleep(self.delay)
-            self.header = self.get_header()
             self.proxy = self.get_proxy(self.source)
             query = 'site:.' + domain + filtered_subdomain
             params = {'text': query, 'p': self.page_num,

modules/search/zoomeye_api.py

@@ -1,7 +1,6 @@
 import time
 from config import settings
 from common.search import Search
-from config.log import logger
 
 
 class ZoomEyeAPI(Search):
@@ -10,59 +9,47 @@ class ZoomEyeAPI(Search):
         self.domain = domain
         self.module = 'Search'
         self.source = 'ZoomEyeAPISearch'
-        self.addr = 'https://api.zoomeye.org/web/search'
+        self.addr = 'https://api.zoomeye.org/domain/search'
         self.delay = 2
-        self.user = settings.zoomeye_api_usermail
+        self.key = settings.zoomeye_api_key
-        self.pwd = settings.zoomeye_api_password
-
-    def login(self):
-        """
-        登陆获取查询taken
-        """
-        url = 'https://api.zoomeye.org/user/login'
-        data = {'username': self.user, 'password': self.pwd}
-        resp = self.post(url=url, json=data)
-        if not resp:
-            logger.log('ALERT', f'{self.source} module login failed')
-            return None
-        data = resp.json()
-        if resp.status_code == 200:
-            logger.log('DEBUG', f'{self.source} module login success')
-            return data.get('access_token')
-        else:
-            logger.log('ALERT', data.get('message'))
-            return None
 
     def search(self):
         """
         发送搜索请求并做子域匹配
         """
-        page_num = 1
+        self.per_page_num = 30
-        access_token = self.login()
+        self.page_num = 1
-        if not access_token:
+        while self.per_page_num * self.page_num < settings.cam_records_maximum_per_domain:
-            return
-        while True:
             time.sleep(self.delay)
             self.header = self.get_header()
+            self.header.update({'API-KEY': self.key})
             self.proxy = self.get_proxy(self.source)
-            self.header.update({'Authorization': 'JWT ' + access_token})
+
-            params = {'query': 'hostname:' + self.domain, 'page': page_num}
+            params = {'q': self.domain,
+                      'page': self.page_num,
+                      'type': 1}
             resp = self.get(self.addr, params)
+            if not resp:
+                return
+            if resp.status_code == 403:
+                break
+            resp_json = resp.json()
             subdomains = self.match_subdomains(resp)
             if not subdomains:  # 搜索没有发现子域名则停止搜索
                 break
             self.subdomains.update(subdomains)
-            page_num += 1
+            total = resp_json.get('total')
-            if page_num > 500:
+            self.page_num += 1
+            if self.page_num * self.per_page_num >= int(total):
                 break
-            if resp.status_code == 403:
+            if self.page_num > 400:
                 break
 
     def run(self):
         """
         类执行入口
         """
-        if not self.have_api(self.user, self.pwd):
+        if not self.have_api(self.key):
             return
         self.begin()
         self.search()
@@ -83,4 +70,4 @@ def run(domain):
 
 
 if __name__ == '__main__':
-    run('mi.com')
+    run('zhipin.com')

modules/wildcard.py

@@ -124,17 +124,16 @@ def detect_wildcard(domain):
     return is_enable
 
 
-@tenacity.retry(stop=tenacity.stop_after_attempt(2))
 def get_wildcard_record(domain, resolver):
     logger.log('INFOR', f"Query {domain} 's wildcard dns record "
                         f"in authoritative name server")
     try:
         answer = resolver.query(domain, 'A')
-    # 如果查询随机域名A记录时抛出Timeout异常则重新查询
+
     except Timeout as e:
         logger.log('ALERT', f'Query timeout, retrying')
         logger.log('DEBUG', e.args)
-        raise e
+        return None, None
     except (NXDOMAIN, YXDOMAIN, NoAnswer, NoNameservers) as e:
         logger.log('DEBUG', e.args)
         logger.log('DEBUG', f'{domain} dont have A record on authoritative name server')
@@ -166,7 +165,6 @@ def collect_wildcard_record(domain, authoritative_ns):
     resolver.cache = None  # 不使用DNS缓存
     ips = set()
     ttl = int()
-    ttls_check = list()
     ips_stat = dict()
     ips_check = list()
     while True:
@@ -178,21 +176,20 @@ def collect_wildcard_record(domain, authoritative_ns):
             logger.log('DEBUG', e.args)
             logger.log('ALERT', f'Multiple query errors,'
                                 f'try to query a new random subdomain')
-            continue
+            # 查询出错退出循环
-        # 每5次查询检查结果列表 如果都没结果则结束查询
+            break
+        # 每5次连续查询后检查结果列表
         ips_check.append(ip)
-        ttls_check.append(ttl)
+        # 如果出现50个以上的泛解析则结束查询
+        if len(ips) >= 50:
+            break
+        # 如果连续5次查询都没结果则结束查询
         if len(ips_check) == 5:
             if not any(ips_check):
                 logger.log('ALERT', 'The query ends because there are '
                                     'no results for 5 consecutive queries.')
                 break
             ips_check = list()
-        if len(ttls_check) == 5 and len(set(ttls_check)) == 5:
-            logger.log('ALERT', 'The query ends because there are '
-                                '5 different TTL results for 5 consecutive queries.')
-            ips, ttl = set(), int()
-            break
         if ip is None:
             continue
         ips.update(ip)
@@ -206,7 +203,7 @@ def collect_wildcard_record(domain, authoritative_ns):
             if times >= 2:
                 addrs.append(addr)
         # 大部分的IP地址出现次数大于2次停止收集泛解析IP记录
-        if len(addrs) / len(ips) >= 0.8:
+        if len(addrs) / len(ips) >= 0.7:
             break
     logger.log('DEBUG', f'Collected the wildcard dns record of {domain}\n{ips}\n{ttl}')
     return ips, ttl

oneforall.py

@@ -32,7 +32,7 @@ blue = '\033[01;34m'
 red = '\033[1;31m'
 end = '\033[0m'
 
-version = 'v0.4.1'
+version = 'v0.4.5'
 message = white + '{' + red + version + ' #dev' + white + '}'
 
 oneforall_banner = f"""
@@ -98,7 +98,6 @@ class OneForAll(object):
         self.domains = set()  # All domains that are to be collected
         self.data = list()  # The subdomain results of the current domain
         self.datas = list()  # All subdomain results of the domain
-        self.in_china = None
         self.access_internet = False
         self.enable_wildcard = False
 
@@ -138,7 +137,7 @@ class OneForAll(object):
         :return: exported data
         :rtype: list
         """
-        return export.export_data(self.domain, alive=self.alive, fmt=self.fmt)
+        return export.export_data(self.domain, alive=self.alive, fmt=self.fmt, path=self.path)
 
     def main(self):
         """
@@ -166,14 +165,15 @@ class OneForAll(object):
             # may cause other network tasks to be error
             brute = Brute(self.domain, word=True, export=False)
             brute.enable_wildcard = self.enable_wildcard
-            brute.in_china = self.in_china
             brute.quite = True
             brute.run()
 
         utils.deal_data(self.domain)
         # Export results without resolve
         if not self.dns:
-            return self.export_data()
+            self.data = self.export_data()
+            self.datas.extend(self.data)
+            return self.data
 
         self.data = utils.get_data(self.domain)
 
@@ -185,7 +185,9 @@ class OneForAll(object):
 
         # Export results without HTTP request
         if not self.req:
-            return self.export_data()
+            self.data = self.export_data()
+            self.datas.extend(self.data)
+            return self.data
 
         if self.enable_wildcard:
             # deal wildcard
@@ -210,8 +212,8 @@ class OneForAll(object):
             enrich = Enrich(self.domain)
             enrich.run()
 
-        # Export
+        self.data = self.export_data()
-        self.datas.extend(self.export_data())
+        self.datas.extend(self.data)
 
         # Scan subdomain takeover
         if self.takeover:
@@ -233,7 +235,7 @@ class OneForAll(object):
         logger.log('DEBUG', 'Python ' + utils.python_version())
         logger.log('DEBUG', 'OneForAll ' + version)
         utils.check_dep()
-        self.access_internet, self.in_china = utils.get_net_env()
+        self.access_internet = utils.get_net_env()
         if self.access_internet and settings.enable_check_version:
             utils.check_version(version)
         logger.log('INFOR', 'Start running OneForAll')

requirements.txt

@@ -1,23 +1,23 @@
--i https://mirrors.aliyun.com/pypi/simple/
+beautifulsoup4==4.11.1
-beautifulsoup4==4.9.3
 bs4==0.0.1
-certifi==2020.6.20
+certifi==2022.06.15
-chardet==3.0.4
+chardet==5.0.0
-colorama==0.4.4; sys_platform == 'win32'
+colorama==0.4.4
-dnspython==2.0.0
+dnspython==2.2.1
-exrex==0.10.5
+exrex==0.11.0
-fire==0.3.1
+fire==0.4.0
-future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+future==0.18.2
-idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+idna==3.3
-loguru==0.5.3
+loguru==0.6.0
-pysocks==1.7.1
+PySocks==1.7.1
-requests==2.24.0
+requests==2.28.1
-six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+six==1.16.0
-soupsieve==2.0.1; python_version >= '3.0'
+soupsieve==2.3.2
-sqlalchemy==1.3.20
+SQLAlchemy==1.3.22
-tenacity==6.2.0
+tenacity==8.0.1
 termcolor==1.1.0
-tqdm==4.51.0
+tqdm==4.64.0
 treelib==1.6.1
-urllib3==1.25.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+urllib3==1.26.9
-win32-setctime==1.0.3; sys_platform == 'win32'
+win32-setctime==1.1.0
+setuptools

takeover.py

@@ -70,7 +70,7 @@ class Takeover(Module):
         self.path = path
         self.fmt = fmt
         self.fingerprints = None
-        self.subdomainq = Queue()
+        self.queue = Queue()  # subdomain queue
         self.cnames = list()
         self.results = Dataset()
 
@@ -107,10 +107,10 @@ class Takeover(Module):
             self.compare(subdomain, cname, responses)
 
     def check(self):
-        while not self.subdomainq.empty():  # 保证域名队列遍历结束后能退出线程
+        while not self.queue.empty():  # 保证域名队列遍历结束后能退出线程
-            subdomain = self.subdomainq.get()  # 从队列中获取域名
+            subdomain = self.queue.get()  # 从队列中获取域名
             self.worker(subdomain)
-            self.subdomainq.task_done()
+            self.queue.task_done()
 
     def progress(self):
         bar = tqdm()
@@ -118,7 +118,7 @@ class Takeover(Module):
         bar.desc = 'Check Progress'
         bar.ncols = 80
         while True:
-            done = bar.total - self.subdomainq.qsize()
+            done = bar.total - self.queue.qsize()
             bar.n = done
             bar.update()
             if done == bar.total:  # 完成队列中所有子域的检查退出
@@ -141,7 +141,7 @@ class Takeover(Module):
             self.results.headers = ['subdomain', 'cname']
             # 创建待检查的子域队列
             for domain in self.subdomains:
-                self.subdomainq.put(domain)
+                self.queue.put(domain)
             # 进度线程
             progress_thread = Thread(target=self.progress, name='ProgressThread',
                                      daemon=True)
@@ -152,7 +152,7 @@ class Takeover(Module):
                                       daemon=True)
                 check_thread.start()
 
-            self.subdomainq.join()
+            self.queue.join()
             self.save()
         else:
             logger.log('FATAL', f'Failed to obtain domain')

test.py

@@ -10,10 +10,13 @@ from oneforall import OneForAll
 
 def oneforall(domain):
     test = OneForAll(target=domain)
+    test.dns = True
     test.brute = True
     test.req = True
     test.takeover = True
     test.run()
+    results = test.datas
+    print(results)
 
 
 if __name__ == '__main__':