重构子域请求模块

Pipfile

@@ -13,15 +13,12 @@ requests = "*"
 records = "*"
 tldextract = "*"
 exrex = "*"
-aiohttp = "*"
 fire = "*"
 bs4 = "*"
 pysocks = "*"
 cloudscraper = "*"
 tablib = "*"
-psutil = "*"
 tenacity = "*"
-ip2location = "*"
 treelib = "*"
 
 [requires]

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "78a5771767751025a636d651f53cce059223e1e045d351290fa5e7b413bc172b"
+            "sha256": "fde2c88929fdf63725faea24a4fc7d82d6792caaca6676c08e96a40227f47e18"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -16,38 +16,6 @@
         ]
     },
     "default": {
-        "aiohttp": {
-            "hashes": [
-                "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e",
-                "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326",
-                "sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a",
-                "sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654",
-                "sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a",
-                "sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4",
-                "sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17",
-                "sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec",
-                "sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd",
-                "sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48",
-                "sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59",
-                "sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965"
-            ],
-            "index": "pypi",
-            "version": "==3.6.2"
-        },
-        "async-timeout": {
-            "hashes": [
-                "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
-                "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
-            ],
-            "version": "==3.0.1"
-        },
-        "attrs": {
-            "hashes": [
-                "sha256:0ef97238856430dcf9228e07f316aefc17e8939fc8507e18c6501b761ef1a42a",
-                "sha256:2867b7b9f8326499ab5b0e2d12801fa5c98842d2cbd22b35112ae04bf85b4dff"
-            ],
-            "version": "==20.1.0"
-        },
         "beautifulsoup4": {
             "hashes": [
                 "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7",
@@ -140,14 +108,6 @@
             ],
             "version": "==2.10"
         },
-        "ip2location": {
-            "hashes": [
-                "sha256:4a83f32098091b6aafaef63b393331bf973bbf54547db82de38f30108b5f079c",
-                "sha256:bbddbd5e3f79c18a9710afd43be09cb21724d7324557668017c545b5c5e62bc4"
-            ],
-            "index": "pypi",
-            "version": "==8.4.1"
-        },
         "jdcal": {
             "hashes": [
                 "sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba",
@@ -163,51 +123,12 @@
             "index": "pypi",
             "version": "==0.5.1"
         },
-        "multidict": {
-            "hashes": [
-                "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a",
-                "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000",
-                "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2",
-                "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507",
-                "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5",
-                "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7",
-                "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d",
-                "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463",
-                "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19",
-                "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3",
-                "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b",
-                "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c",
-                "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87",
-                "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7",
-                "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430",
-                "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255",
-                "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d"
-            ],
-            "version": "==4.7.6"
-        },
         "openpyxl": {
             "hashes": [
                 "sha256:626d38647c063d55803ef4971c4d43226538d4e95cb6260c094e363ee33e10c7"
             ],
             "version": "==2.4.11"
         },
-        "psutil": {
-            "hashes": [
-                "sha256:0ee3c36428f160d2d8fce3c583a0353e848abb7de9732c50cf3356dd49ad63f8",
-                "sha256:10512b46c95b02842c225f58fa00385c08fa00c68bac7da2d9a58ebe2c517498",
-                "sha256:4080869ed93cce662905b029a1770fe89c98787e543fa7347f075ade761b19d6",
-                "sha256:5e9d0f26d4194479a13d5f4b3798260c20cecf9ac9a461e718eb59ea520a360c",
-                "sha256:66c18ca7680a31bf16ee22b1d21b6397869dda8059dbdb57d9f27efa6615f195",
-                "sha256:68d36986ded5dac7c2dcd42f2682af1db80d4bce3faa126a6145c1637e1b559f",
-                "sha256:90990af1c3c67195c44c9a889184f84f5b2320dce3ee3acbd054e3ba0b4a7beb",
-                "sha256:a5b120bb3c0c71dfe27551f9da2f3209a8257a178ed6c628a819037a8df487f1",
-                "sha256:d8a82162f23c53b8525cf5f14a355f5d1eea86fa8edde27287dd3a98399e4fdf",
-                "sha256:f2018461733b23f308c298653c8903d32aaad7873d25e1d228765e91ae42c3f2",
-                "sha256:ff1977ba1a5f71f89166d5145c3da1cea89a0fdb044075a12c720ee9123ec818"
-            ],
-            "index": "pypi",
-            "version": "==5.7.2"
-        },
         "pyparsing": {
             "hashes": [
                 "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
@@ -273,10 +194,13 @@
                 "sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb",
                 "sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804",
                 "sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6",
+                "sha256:26c5ca9d09f0e21b8671a32f7d83caad5be1f6ff45eef5ec2f6fd0db85fc5dc0",
                 "sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe",
                 "sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de",
                 "sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36",
                 "sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e",
+                "sha256:465c999ef30b1c7525f81330184121521418a67189053bcf585824d833c05b66",
+                "sha256:51064ee7938526bab92acd049d41a1dc797422256086b39c08bafeffb9d304c6",
                 "sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc",
                 "sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d",
                 "sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce",
@@ -291,6 +215,7 @@
                 "sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e",
                 "sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba",
                 "sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02",
+                "sha256:b6ff91356354b7ff3bd208adcf875056d3d886ed7cef90c571aef2ab8a554b12",
                 "sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86",
                 "sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d",
                 "sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7",
@@ -361,28 +286,6 @@
             ],
             "markers": "sys_platform == 'win32'",
             "version": "==1.0.1"
-        },
-        "yarl": {
-            "hashes": [
-                "sha256:040b237f58ff7d800e6e0fd89c8439b841f777dd99b4a9cca04d6935564b9409",
-                "sha256:17668ec6722b1b7a3a05cc0167659f6c95b436d25a36c2d52db0eca7d3f72593",
-                "sha256:3a584b28086bc93c888a6c2aa5c92ed1ae20932f078c46509a66dce9ea5533f2",
-                "sha256:4439be27e4eee76c7632c2427ca5e73703151b22cae23e64adb243a9c2f565d8",
-                "sha256:48e918b05850fffb070a496d2b5f97fc31d15d94ca33d3d08a4f86e26d4e7c5d",
-                "sha256:9102b59e8337f9874638fcfc9ac3734a0cfadb100e47d55c20d0dc6087fb4692",
-                "sha256:9b930776c0ae0c691776f4d2891ebc5362af86f152dd0da463a6614074cb1b02",
-                "sha256:b3b9ad80f8b68519cc3372a6ca85ae02cc5a8807723ac366b53c0f089db19e4a",
-                "sha256:bc2f976c0e918659f723401c4f834deb8a8e7798a71be4382e024bcc3f7e23a8",
-                "sha256:c22c75b5f394f3d47105045ea551e08a3e804dc7e01b37800ca35b58f856c3d6",
-                "sha256:c52ce2883dc193824989a9b97a76ca86ecd1fa7955b14f87bf367a61b6232511",
-                "sha256:ce584af5de8830d8701b8979b18fcf450cef9a382b1a3c8ef189bedc408faf1e",
-                "sha256:da456eeec17fa8aa4594d9a9f27c0b1060b6a75f2419fe0c00609587b2695f4a",
-                "sha256:db6db0f45d2c63ddb1a9d18d1b9b22f308e52c83638c26b422d520a815c4b3fb",
-                "sha256:df89642981b94e7db5596818499c4b2219028f2a528c9c37cc1de45bf2fd3a3f",
-                "sha256:f18d68f2be6bf0e89f1521af2b1bb46e66ab0018faafa81d70f358153170a317",
-                "sha256:f379b7f83f23fe12823085cd6b906edc49df969eb99757f58ff382349a3303c6"
-            ],
-            "version": "==1.5.1"
         }
     },
     "develop": {}

README.md

@@ -4,7 +4,7 @@
 [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
-[![python](https://img.shields.io/badge/python-3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/python-3.6|3.7|3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
 [![python](https://img.shields.io/badge/release-v0.3.0-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
 
 👊**OneForAll是一款功能强大的子域收集工具**  📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md)
@@ -15,21 +15,6 @@
 
 📢 请务必花一点时间阅读此文档，有助于你快速熟悉OneForAll！
 
-<details>
-<summary><b>🐍安装要求</b></summary>
-
-OneForAll基于[Python 3.8.0]( https://www.python.org/downloads/release/python-380/ )开发和测试，请使用高于Python 3.8.0的稳定发行版本，其他版本可能会出现一些问题（Windows平台必须使用3.8.0以上版本），安装Python环境可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)。运行以下命令检查Python和pip3版本：
-```bash
-python -V
-pip3 -V
-```
-如果你看到类似以下的输出便说明Python环境没有问题：
-```bash
-Python 3.8.0
-pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python38\site-packages\pip (python 3.8)
-```
-</details>
-
 <details>
 <summary><b>✔安装步骤（git 版）</b></summary>
 

brute.py

@@ -592,8 +592,6 @@ class Brute(Module):
         self.domain = str()  # 当前正在进行爆破的域名
         self.ips_times = dict()  # IP集合出现次数
         self.enable_wildcard = False  # 当前域名是否使用泛解析
-        self.wildcard_check = settings.brute_wildcard_check
-        self.wildcard_deal = settings.brute_wildcard_deal
         self.check_env = True
         self.quite = False
 

common/module.py

@@ -22,9 +22,9 @@ class Module(object):
         self.cookie = None
         self.header = dict()
         self.proxy = None
-        self.delay = settings.request_delay  # 请求睡眠时延
+        self.delay = 1  # 请求睡眠时延
-        self.timeout = settings.request_timeout  # 请求超时时间
+        self.timeout = settings.request_timeout_second  # 请求超时时间
-        self.verify = settings.request_verify  # 请求SSL验证
+        self.verify = settings.request_ssl_verify  # 请求SSL验证
         self.domain = str()  # 当前进行子域名收集的主域
         self.subdomains = set()  # 存放发现的子域
         self.infos = dict()  # 存放子域有关信息
@@ -199,7 +199,7 @@ class Module(object):
         :param str module: module name
         :return: proxy
         """
-        if not settings.enable_proxy:
+        if not settings.enable_request_proxy:
             logger.log('TRACE', f'All modules do not use proxy')
             return self.proxy
         if settings.proxy_all_module:

common/request.py

@@ -1,10 +1,10 @@
 import json
-import asyncio
+from threading import Thread
-import functools
+from queue import Queue
+
 
-import aiohttp
 import tqdm
-from aiohttp import ClientSession
+import requests
 from bs4 import BeautifulSoup
 
 from common import utils
@@ -12,17 +12,17 @@ from config.log import logger
 from config import settings
 
 
-def get_limit_conn():
+def req_thread_count():
-    count = settings.limit_open_conn
+    count = settings.request_thread_count
     if isinstance(count, int):
         count = max(16, count)
     else:
-        count = utils.get_coroutine_count()
+        count = utils.get_request_count()
-    logger.log('DEBUG', f'Request coroutine {count}')
+    logger.log('DEBUG', f'Number of request threads {count}')
     return count
 
 
-def get_ports(port):
+def get_port_seq(port):
     logger.log('DEBUG', 'Getting port range')
     ports = set()
     if isinstance(port, (set, list, tuple)):
@@ -40,81 +40,33 @@ def get_ports(port):
     return set(ports)
 
 
-def gen_req_data(data, ports):
+def gen_req_url(domain, port):
+    if str(port).endswith('443'):
+        url = f'https://{domain}:{port}'
+        if port == 443:
+            url = f'https://{domain}'
+        return url
+    url = f'http://{domain}:{port}'
+    if port == 80:
+        url = f'http://{domain}'
+    return url
+
+
+def gen_req_urls(data, ports):
     logger.log('INFOR', 'Generating request urls')
-    new_data = []
+    urls = set()
-    for data in data:
+    for info in data:
-        resolve = data.get('resolve')
+        resolve = info.get('resolve')
         # 解析不成功的子域不进行http请求探测
         if resolve != 1:
             continue
-        subdomain = data.get('subdomain')
+        subdomain = info.get('subdomain')
         for port in ports:
-            if str(port).endswith('443'):
+            urls.add(gen_req_url(subdomain, port))
-                url = f'https://{subdomain}:{port}'
+    return urls
-                if port == 443:
-                    url = f'https://{subdomain}'
-                data['id'] = None
-                data['url'] = url
-                data['port'] = port
-                new_data.append(data)
-                data = dict(data)  # 需要生成一个新的字典对象
-            else:
-                url = f'http://{subdomain}:{port}'
-                if port == 80:
-                    url = f'http://{subdomain}'
-                data['id'] = None
-                data['url'] = url
-                data['port'] = port
-                new_data.append(data)
-                data = dict(data)  # 需要生成一个新的字典对象
-    return new_data
 
 
-async def fetch(session, method, url):
+def get_html_title(markup):
-    """
-    请求
-
-    :param session: session对象
-    :param method: 请求方法
-    :param str url: url地址
-    :return: 响应对象和响应文本
-    """
-    timeout = aiohttp.ClientTimeout(total=None,
-                                    connect=None,
-                                    sock_read=settings.sockread_timeout,
-                                    sock_connect=settings.sockconn_timeout)
-    try:
-        if method == 'HEAD':
-            async with session.head(url,
-                                    ssl=settings.verify_ssl,
-                                    allow_redirects=settings.allow_redirects,
-                                    timeout=timeout,
-                                    proxy=settings.aiohttp_proxy) as resp:
-                text = await resp.text()
-        else:
-            async with session.get(url,
-                                   ssl=settings.verify_ssl,
-                                   allow_redirects=settings.allow_redirects,
-                                   timeout=timeout,
-                                   proxy=settings.aiohttp_proxy) as resp:
-
-                try:
-                    # 先尝试用utf-8解码
-                    text = await resp.text(encoding='utf-8', errors='strict')
-                except UnicodeError:
-                    try:
-                        # 再尝试用gb18030解码
-                        text = await resp.text(encoding='gb18030', errors='strict')
-                    except UnicodeError:
-                        # 最后尝试自动解码
-                        text = await resp.text(encoding=None, errors='ignore')
-        return resp, text
-    except Exception as e:
-        return e, None
-
-
-def get_title(markup):
     """
     获取标题
 
@@ -161,107 +113,107 @@ def get_jump_urls(history):
     return urls
 
 
-def request_callback(future, index, datas):
+def get(url, resp_queue, session):
-    resp, text = future.result()
+    timeout = settings.request_timeout_second
-    if isinstance(resp, BaseException):
+    redirect = settings.request_allow_redirect
-        exception = resp
+    proxy = utils.get_proxy()
-        logger.log('TRACE', exception.args)
-        name = utils.get_classname(exception)
-        datas[index]['reason'] = name + ' ' + str(exception)
-        datas[index]['request'] = 0
-        datas[index]['alive'] = 0
-    else:
-        datas[index]['reason'] = resp.reason
-        datas[index]['status'] = resp.status
-        datas[index]['request'] = 1
-        if resp.status == 400 or resp.status >= 500:
-            datas[index]['alive'] = 0
-        else:
-            datas[index]['alive'] = 1
-        headers = resp.headers
-        if settings.enable_banner_identify:
-            datas[index]['banner'] = utils.get_sample_banner(headers)
-        datas[index]['header'] = json.dumps(dict(headers))
-        history = resp.history
-        datas[index]['history'] = json.dumps(get_jump_urls(history))
-        if isinstance(text, str):
-            title = get_title(text).strip()
-            datas[index]['title'] = utils.remove_invalid_string(title)
-            datas[index]['response'] = utils.remove_invalid_string(text)
-
-
-def get_connector():
-    count = get_limit_conn()
-    return aiohttp.TCPConnector(ttl_dns_cache=300,
-                                ssl=settings.verify_ssl,
-                                limit=count,
-                                limit_per_host=settings.limit_per_host)
-
-
-async def async_request(urls):
-    results = list()
-    connector = get_connector()
-    headers = utils.get_random_header()
-    session = ClientSession(connector=connector, headers=headers)
-    tasks = []
-    for i, url in enumerate(urls):
-        task = asyncio.ensure_future(fetch(session, 'GET', url))
-        tasks.append(task)
-    if tasks:
-        futures = asyncio.as_completed(tasks)
-        for future in tqdm.tqdm(futures,
-                                total=len(tasks),
-                                desc='Request Progress',
-                                ncols=80):
-            result = await future
-            results.append(result)
-    await session.close()
-    return results
-
-
-async def bulk_request(data, port):
-    ports = get_ports(port)
-    no_req_data = utils.get_filtered_data(data)
-    to_req_data = gen_req_data(data, ports)
-    method = settings.request_method.upper()
-    logger.log('INFOR', f'Use {method} method to request')
-    logger.log('INFOR', 'Async subdomains request in progress')
-    connector = get_connector()
-    headers = utils.get_random_header()
-    session = ClientSession(connector=connector, headers=headers)
-    tasks = []
-    for num, data in enumerate(to_req_data):
-        url = data.get('url')
-        task = asyncio.ensure_future(fetch(session, method, url))
-        task.add_done_callback(functools.partial(request_callback,
-                                                 index=num,
-                                                 datas=to_req_data))
-        tasks.append(task)
-    if tasks:
-        futures = asyncio.as_completed(tasks)
-        for future in tqdm.tqdm(futures,
-                                total=len(tasks),
-                                desc='Request Progress',
-                                ncols=80):
-            await future
-    await session.close()
-    return to_req_data + no_req_data
-
-
-def set_loop_policy():
     try:
-        import uvloop
+        resp = session.get(url, timeout=timeout, allow_redirects=redirect, proxies=proxy)
-    except ImportError:
+    except Exception as e:
-        pass
+        logger.log('DEBUG', e.args)
+        return
+    resp_queue.put(resp)
+
+
+def request(urls_queue, resp_queue, session):
+    while not urls_queue.empty():
+        url = urls_queue.get()
+        get(url, resp_queue, session)
+        urls_queue.task_done()
+
+
+def progress(urls, resp_queue):
+    bar = tqdm.tqdm()
+    bar.total = len(urls)
+    bar.desc = 'Request Progress'
+    bar.ncols = 80
+    while True:
+        done = resp_queue.qsize()
+        bar.n = done
+        bar.update()
+        if done == bar.total:
+            break
+
+
+def get_session():
+    header = utils.gen_fake_header()
+    verify = settings.request_ssl_verify
+    redirect_limit = settings.request_redirect_limit
+    session = requests.Session()
+    session.headers = header
+    session.verify = verify
+    session.max_redirects = redirect_limit
+    return session
+
+
+def bulk_request(urls):
+    logger.log('INFOR', 'Requesting urls in bulk')
+    resp_list = list()
+    urls_queue = Queue()
+    resp_queue = Queue()
+    for url in urls:
+        urls_queue.put(url)
+    session = get_session()
+    thread_count = req_thread_count()
+
+    progress_thread = Thread(target=progress, args=(urls, resp_queue))
+    progress_thread.start()
+
+    for _ in range(thread_count):
+        request_thread = Thread(target=request, args=(urls_queue, resp_queue, session))
+        request_thread.start()
+
+    urls_queue.join()
+
+    while not resp_queue.empty():
+        resp = resp_queue.get()
+        resp_list.append(resp)
+    return resp_list
+
+
+def gen_new_info(info, resp):
+    port = resp.raw._pool.port
+    info['port'] = port
+    info['url'] = resp.url
+    info['reason'] = resp.reason
+    code = resp.status_code
+    info['status'] = code
+    info['request'] = 1
+    if code == 400 or code >= 500:
+        info['alive'] = 0
     else:
-        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+        info['alive'] = 1
+    headers = resp.headers
+    if settings.enable_banner_identify:
+        info['banner'] = utils.get_sample_banner(headers)
+    info['header'] = json.dumps(dict(headers))
+    history = resp.history
+    info['history'] = json.dumps(get_jump_urls(history))
+    text = utils.decode_resp_text(resp)
+    title = get_html_title(text).strip()
+    info['title'] = utils.remove_invalid_string(title)
+    info['response'] = utils.remove_invalid_string(text)
+    return info
 
 
-def set_loop():
+def gen_new_data(data, resp_list):
-    set_loop_policy()
+    new_data = list()
-    loop = asyncio.get_event_loop()
+    for resp in resp_list:
-    asyncio.set_event_loop(loop)
+        subdomain = resp.raw._pool.host
-    return loop
+        for info in data:
+            if info.get('subdomain') == subdomain:
+                new_data.append(gen_new_info(info, resp))
+    return new_data
 
 
 def run_request(domain, data, port):
@@ -274,25 +226,18 @@ def run_request(domain, data, port):
     :return list: result
     """
     logger.log('INFOR', f'Start requesting subdomains of {domain}')
-    loop = set_loop()
     data = utils.set_id_none(data)
-    request_coroutine = bulk_request(data, port)
+    ports = get_port_seq(port)
-    data = loop.run_until_complete(request_coroutine)
+    filtered_data = utils.get_filtered_data(data)
-    loop.run_until_complete(asyncio.sleep(0.25))
+    req_urls = gen_req_urls(data, ports)
+    resp_list = bulk_request(req_urls)
+    new_data = gen_new_data(data, resp_list)
+    data = new_data + filtered_data
     count = utils.count_alive(data)
     logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
     return data
 
 
-def urls_request(urls):
-    logger.log('INFOR', 'Start urls request module')
-    loop = set_loop()
-    request_coroutine = async_request(urls)
-    data = loop.run_until_complete(request_coroutine)
-    loop.run_until_complete(asyncio.sleep(0.25))
-    return data
-
-
 def save_db(name, data):
     """
     Save request results to database

common/utils.py

@@ -10,7 +10,6 @@ import subprocess
 from ipaddress import IPv4Address, ip_address
 from stat import S_IXUSR
 
-import psutil
 import tenacity
 import requests
 from pathlib import Path
@@ -49,10 +48,10 @@ def gen_fake_header():
     """
     Generate fake request headers
     """
-    headers = settings.headers
+    headers = settings.request_default_headers
     if not isinstance(headers, dict):
         headers = dict()
-    if settings.random_user_agent:
+    if settings.enable_random_ua:
         ua = random.choice(user_agents)
         headers['User-Agent'] = ua
     headers['Accept-Encoding'] = 'gzip, deflate'
@@ -74,7 +73,7 @@ def get_random_proxy():
     Get random proxy
     """
     try:
-        return random.choice(settings.proxy_pool)
+        return random.choice(settings.request_proxy_pool)
     except IndexError:
         return None
 
@@ -83,7 +82,7 @@ def get_proxy():
     """
     Get proxy
     """
-    if settings.enable_proxy:
+    if settings.enable_request_proxy:
         return get_random_proxy()
     return None
 
@@ -502,27 +501,8 @@ def get_process_num():
         return 1
 
 
-def get_coroutine_count():
+def get_request_count():
-    """
+    return os.cpu_count() * 10
-    根据内存大小获取并发数量
-    """
-    mem = psutil.virtual_memory()
-    total_mem = mem.total
-    g_size = 1024 * 1024 * 1024
-    if total_mem < 1 * g_size:
-        return 16
-    elif total_mem < 2 * g_size:
-        return 32
-    elif total_mem < 4 * g_size:
-        return 64
-    elif total_mem < 8 * g_size:
-        return 128
-    elif total_mem < 16 * g_size:
-        return 256
-    elif total_mem < 32 * g_size:
-        return 512
-    else:
-        return 1024
 
 
 def uniq_dict_list(dict_list):
@@ -562,7 +542,6 @@ def check_net():
 
 def check_pre():
     logger.log('INFOR', 'Checking dependent environment')
-    system = platform.system()
     implementation = platform.python_implementation()
     version = platform.python_version()
     if implementation != 'CPython':
@@ -571,16 +550,6 @@ def check_pre():
     if version < '3.6':
         logger.log('FATAL', 'OneForAll requires Python 3.6 or higher')
         exit(1)
-    if system == 'Windows' and implementation == 'CPython' and version < '3.8':
-        logger.log('FATAL', 'OneForAll requires Python 3.8 '
-                            'or higher when running on Windows')
-        exit(1)
-    if system in {"Linux", "Darwin"}:
-        try:
-            import uvloop
-        except ImportError:
-            logger.log('ALERT', f'Please install the uvloop library manually '
-                                f'to accelerate subdomain requests')
 
 
 def check_env():
@@ -599,8 +568,8 @@ def check_version(local):
     api = 'https://api.github.com/repos/shmilylty/OneForAll/releases/latest'
     header = get_random_header()
     proxy = get_proxy()
-    timeout = settings.request_timeout
+    timeout = settings.request_timeout_second
-    verify = settings.request_verify
+    verify = settings.request_ssl_verify
     try:
         resp = requests.get(url=api, headers=header, proxies=proxy,
                             timeout=timeout, verify=verify)
@@ -747,3 +716,14 @@ def get_url_resp(url):
         logger.log('DEBUG', e.args)
         return None
     return resp
+
+
+def decode_resp_text(resp):
+    try:
+        text = resp.text(encoding='utf-8', errors='strict')  # 先尝试用utf-8严格解码
+    except UnicodeError:
+        try:
+            text = resp.text(encoding='gb18030', errors='strict')  # 再尝试用gb18030严格解码
+        except UnicodeError:
+            text = resp.text(encoding=None, errors='ignore')  # 最后尝试自动解码
+    return text

config/default.py

@@ -6,6 +6,9 @@ OneForAll默认配置
 import pathlib
 import warnings
 
+# 禁用所有警告信息
+warnings.filterwarnings("ignore")
+
 # 路径设置
 relative_directory = pathlib.Path(__file__).parent.parent  # OneForAll代码相对路径
 module_dir = relative_directory.joinpath('modules')  # OneForAll模块目录
@@ -79,24 +82,39 @@ ip_appear_maximum = 100  # 多个子域解析到同一IP次数超过100次则标
 banner_process_number = 4  # 识别进程数量(默认4)
 
 # 代理设置
-enable_proxy = False  # 是否使用代理(全局开关)
+enable_request_proxy = False  # 是否使用代理(全局开关，默认False)
 proxy_all_module = False  # 代理所有模块
 proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
                         'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
                         'YandexSearch', 'CrossDomainXml',
                         'ContentSecurityPolicy']  # 代理自定义的模块
-proxy_pool = [{'http': 'http://127.0.0.1:1080',
+request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
-               'https': 'https://127.0.0.1:1080'}]  # 代理池
+                       'https': 'https://127.0.0.1:1080'}]  # 代理池
-# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
+# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
-#                'https': 'socks5h://127.0.0.1:10808'}]  # 代理池
+#                        'https': 'socks5h://127.0.0.1:10808'}]  # 代理池
 
 
-# 网络请求设置
+# 请求设置
-request_delay = 1  # 请求时延(默认1秒)
+request_thread_count = None  # 请求线程数量(默认None，则根据内存大小设置)
-request_timeout = 30  # 请求超时(默认30秒)
+request_timeout_second = 30  # 请求超时秒数(默认30秒)
-request_verify = False  # 请求SSL验证(默认False)
+request_ssl_verify = False  # 请求SSL验证(默认False)
-# 禁用所有警告信息
+request_allow_redirect = True  # 请求允许重定向(默认True)
-warnings.filterwarnings("ignore")
+request_redirect_limit = 10  # 请求跳转限制(默认10次)
+# 默认请求头 可以在headers里添加自定义请求头
+request_default_headers = {
+    'Accept': 'text/html,application/xhtml+xml,'
+              'application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Encoding': 'gzip, deflate',
+    'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+    'Cache-Control': 'max-age=0',
+    'DNT': '1',
+    'Referer': 'https://www.google.com/',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                  '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Upgrade-Insecure-Requests': '1',
+    'X-Forwarded-For': '127.0.0.1'
+}
+enable_random_ua = True  # 使用随机UA(默认True，开启可以覆盖request_default_headers的UA)
 
 # 搜索模块设置
 # 开启全量搜索会尽量去获取搜索引擎搜索的全部结果，不过搜索耗时可能会过长
@@ -139,42 +157,11 @@ large_ports = [80, 81, 280, 300, 443, 591, 593, 832, 888, 901, 981, 1010, 1080,
                18092, 20000, 20720, 24465, 28017, 28080, 30821, 43110, 61600]
 ports = {'small': small_ports, 'medium': medium_ports, 'large': large_ports}
 
-# aiohttp有关配置
-verify_ssl = False
-# aiohttp 支持 HTTP/HTTPS形式的代理
-aiohttp_proxy = None  # 示例 proxy="http://user:pass@some.proxy.com"
-allow_redirects = True  # 允许请求跳转
-# 为了保证请求质量 请谨慎更改以下设置
-# request_method只能是HEAD或GET,HEAD请求方法更快，但是不能获取响应体并提取从中title及banner
-request_method = 'GET'  # 使用请求方法，默认GET
-sockread_timeout = 6  # 每个请求socket读取超时时间，默认6秒
-sockconn_timeout = 3  # 每个请求socket连接超时时间，默认3秒
-limit_open_conn = None  # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置
-# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
-limit_per_host = 10  # 0表示不限制,默认10
-
 common_subnames = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
                    'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
                    'help', 'live', 'mall', 'blogs', 'files', 'forum',
                    'store', 'mobile'}
 
-# 请求头设置
-# 可以在headers里添加自定义请求头
-headers = {
-    'Accept': 'text/html,application/xhtml+xml,'
-              'application/xml;q=0.9,*/*;q=0.8',
-    'Accept-Encoding': 'gzip, deflate',
-    'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
-    'Cache-Control': 'max-age=0',
-    'DNT': '1',
-    'Referer': 'https://www.google.com/',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
-                  '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
-    'Upgrade-Insecure-Requests': '1',
-    'X-Forwarded-For': '127.0.0.1'
-}
-random_user_agent = True  # 使用随机UA(默认True，开启可以覆盖header的设置)
-
 # 模块API配置
 # Censys可以免费注册获取API：https://censys.io/api
 censys_api_id = ''

config/setting.py

@@ -37,8 +37,6 @@ enable_partial_module = []  # 启用部分收集模块 必须禁用enable_all_mo
 #                          ('modules.search', 'baidu')]
 
 # 爆破模块设置
-brute_wildcard_check = True  # 开启泛解析检测(默认True)
-brute_wildcard_deal = True  # 开启泛解析处理(默认True)
 brute_concurrent_num = 2000  # 爆破时并发查询数量(默认2000，最大推荐10000)
 # 爆破所使用的字典路径 默认data/subdomains.txt
 brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
@@ -66,25 +64,18 @@ brute_ip_blacklist = {'0.0.0.0', '0.0.0.1'}  # IP黑名单 子域解析到IP黑
 ip_appear_maximum = 100  # 多个子域解析到同一IP次数超过100次则标记为非法(泛解析)子域
 
 # 代理设置
-enable_proxy = False  # 是否使用代理(全局开关)
+enable_request_proxy = False  # 是否使用代理(全局开关)
 proxy_all_module = False  # 代理所有模块
 proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
                         'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
                         'YandexSearch', 'CrossDomainXml',
                         'ContentSecurityPolicy']  # 代理自定义的模块
-proxy_pool = [{'http': 'http://127.0.0.1:1080',
+request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
-               'https': 'https://127.0.0.1:1080'}]  # 代理池
+                       'https': 'https://127.0.0.1:1080'}]  # 代理池
-# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
+# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
-#                'https': 'socks5h://127.0.0.1:10808'}]  # 代理池
+#                        'https': 'socks5h://127.0.0.1:10808'}]  # 代理池
 
 # 搜索模块设置
 enable_recursive_search = False  # 递归搜索子域
 search_recursive_times = 2  # 递归搜索层数
 
-# aiohttp有关配置
-# aiohttp 支持 HTTP/HTTPS形式的代理
-aiohttp_proxy = None  # 示例 proxy="http://user:pass@some.proxy.com"
-# 为了保证请求质量 请谨慎更改以下设置
-sockread_timeout = 6  # 每个请求socket读取超时时间，默认6秒
-sockconn_timeout = 3  # 每个请求socket连接超时时间，默认3秒
-limit_open_conn = None  # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置

docs/en-us/README.md

@@ -4,7 +4,7 @@
 [![codecov](https://codecov.io/gh/shmilylty/OneForAll/branch/master/graph/badge.svg)](https://codecov.io/gh/shmilylty/OneForAll)
 [![Maintainability](https://api.codeclimate.com/v1/badges/1287668a6b4c72af683e/maintainability)](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
 [![License](https://img.shields.io/github/license/shmilylty/OneForAll)](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
-[![python](https://img.shields.io/badge/python-3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
+[![python](https://img.shields.io/badge/python-3.6|3.7|3.8-blue)](https://github.com/shmilylty/OneForAll/tree/master/)
 [![python](https://img.shields.io/badge/release-v0.3.0-brightgreen)](https://github.com/shmilylty/OneForAll/releases)
 
 👊**OneForAll is a powerful subdomain integration tool**  📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
@@ -15,23 +15,6 @@
 
 📢 Please read this document to help you start quickly!
 
-<details>
-<summary><b>🐍Installation requirements</b></summary>
-
-OneForAll is developed and tested based on [Python 3.8.0](https://www.python.org/downloads/release/python-380/). Recommend use release higher than Python 3.8.0 (Windows platform must use Python 3.8.0 or later). For more information on installing the Python environment, please read [Python 3 installation Guide](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3).
-
-After installation python, run the following command to check the Python and pip3 versions:
-```bash
-python -V
-pip3 -V
-```
-If you see the following output, there is no problem with the Python environment:
-```bash
-Python 3.8.0
-pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.8)
-```
-</details>
-
 <details>
 <summary><b>✔Installation steps (for Git)</b></summary>
 

modules/finder.py

@@ -190,9 +190,10 @@ def find_subdomains(domain, data):
         subdomains.update(find_in_resp(domain, req_url, rsp_html))
         js_urls.update(find_js_urls(domain, req_url, rsp_html))
 
-    resp_data = request.urls_request(js_urls)
+    resp_data = request.bulk_request(js_urls)
-    for resp, text in resp_data:
+    for resp in resp_data:
-        if not text:
+        if not resp:
             continue
+        text = utils.decode_resp_text(resp)
         subdomains.update(find_in_resp(domain, resp.url, text))
     return subdomains

requirements.txt

@@ -1,6 +1,3 @@
-aiohttp==3.6.2
-async-timeout==3.0.1
-attrs==20.1.0
 beautifulsoup4==4.9.1
 bs4==0.0.1
 certifi==2020.6.20
@@ -14,12 +11,9 @@ exrex==0.10.5
 fire==0.3.1
 future==0.18.2
 idna==2.10
-IP2Location==8.4.1
 jdcal==1.4.1
 loguru==0.5.1
-multidict==4.7.6
 openpyxl==2.4.11
-psutil==5.7.2
 pyparsing==2.4.7
 PySocks==1.7.1
 records==0.5.3
@@ -36,5 +30,4 @@ tldextract==2.2.3
 tqdm==4.48.2
 treelib==1.6.1
 urllib3==1.25.10
-win32-setctime==1.0.1
+win32-setctime==1.0.1
-yarl==1.5.1

test.py

@@ -8,7 +8,7 @@ Example
 from oneforall import OneForAll
 
 if __name__ == '__main__':
-    test = OneForAll(target='hackfun.org')
+    test = OneForAll(target='mi.com')
     test.brute = True
     test.req = True
     test.takeover = True