diff --git a/Pipfile b/Pipfile
index 8a90da0..490f42b 100644
--- a/Pipfile
+++ b/Pipfile
@@ -13,15 +13,12 @@ requests = "*"
records = "*"
tldextract = "*"
exrex = "*"
-aiohttp = "*"
fire = "*"
bs4 = "*"
pysocks = "*"
cloudscraper = "*"
tablib = "*"
-psutil = "*"
tenacity = "*"
-ip2location = "*"
treelib = "*"
[requires]
diff --git a/Pipfile.lock b/Pipfile.lock
index dc642bc..ad71cf1 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "78a5771767751025a636d651f53cce059223e1e045d351290fa5e7b413bc172b"
+ "sha256": "fde2c88929fdf63725faea24a4fc7d82d6792caaca6676c08e96a40227f47e18"
},
"pipfile-spec": 6,
"requires": {
@@ -16,38 +16,6 @@
]
},
"default": {
- "aiohttp": {
- "hashes": [
- "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e",
- "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326",
- "sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a",
- "sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654",
- "sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a",
- "sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4",
- "sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17",
- "sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec",
- "sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd",
- "sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48",
- "sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59",
- "sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965"
- ],
- "index": "pypi",
- "version": "==3.6.2"
- },
- "async-timeout": {
- "hashes": [
- "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
- "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
- ],
- "version": "==3.0.1"
- },
- "attrs": {
- "hashes": [
- "sha256:0ef97238856430dcf9228e07f316aefc17e8939fc8507e18c6501b761ef1a42a",
- "sha256:2867b7b9f8326499ab5b0e2d12801fa5c98842d2cbd22b35112ae04bf85b4dff"
- ],
- "version": "==20.1.0"
- },
"beautifulsoup4": {
"hashes": [
"sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7",
@@ -140,14 +108,6 @@
],
"version": "==2.10"
},
- "ip2location": {
- "hashes": [
- "sha256:4a83f32098091b6aafaef63b393331bf973bbf54547db82de38f30108b5f079c",
- "sha256:bbddbd5e3f79c18a9710afd43be09cb21724d7324557668017c545b5c5e62bc4"
- ],
- "index": "pypi",
- "version": "==8.4.1"
- },
"jdcal": {
"hashes": [
"sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba",
@@ -163,51 +123,12 @@
"index": "pypi",
"version": "==0.5.1"
},
- "multidict": {
- "hashes": [
- "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a",
- "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000",
- "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2",
- "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507",
- "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5",
- "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7",
- "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d",
- "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463",
- "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19",
- "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3",
- "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b",
- "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c",
- "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87",
- "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7",
- "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430",
- "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255",
- "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d"
- ],
- "version": "==4.7.6"
- },
"openpyxl": {
"hashes": [
"sha256:626d38647c063d55803ef4971c4d43226538d4e95cb6260c094e363ee33e10c7"
],
"version": "==2.4.11"
},
- "psutil": {
- "hashes": [
- "sha256:0ee3c36428f160d2d8fce3c583a0353e848abb7de9732c50cf3356dd49ad63f8",
- "sha256:10512b46c95b02842c225f58fa00385c08fa00c68bac7da2d9a58ebe2c517498",
- "sha256:4080869ed93cce662905b029a1770fe89c98787e543fa7347f075ade761b19d6",
- "sha256:5e9d0f26d4194479a13d5f4b3798260c20cecf9ac9a461e718eb59ea520a360c",
- "sha256:66c18ca7680a31bf16ee22b1d21b6397869dda8059dbdb57d9f27efa6615f195",
- "sha256:68d36986ded5dac7c2dcd42f2682af1db80d4bce3faa126a6145c1637e1b559f",
- "sha256:90990af1c3c67195c44c9a889184f84f5b2320dce3ee3acbd054e3ba0b4a7beb",
- "sha256:a5b120bb3c0c71dfe27551f9da2f3209a8257a178ed6c628a819037a8df487f1",
- "sha256:d8a82162f23c53b8525cf5f14a355f5d1eea86fa8edde27287dd3a98399e4fdf",
- "sha256:f2018461733b23f308c298653c8903d32aaad7873d25e1d228765e91ae42c3f2",
- "sha256:ff1977ba1a5f71f89166d5145c3da1cea89a0fdb044075a12c720ee9123ec818"
- ],
- "index": "pypi",
- "version": "==5.7.2"
- },
"pyparsing": {
"hashes": [
"sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
@@ -273,10 +194,13 @@
"sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb",
"sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804",
"sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6",
+ "sha256:26c5ca9d09f0e21b8671a32f7d83caad5be1f6ff45eef5ec2f6fd0db85fc5dc0",
"sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe",
"sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de",
"sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36",
"sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e",
+ "sha256:465c999ef30b1c7525f81330184121521418a67189053bcf585824d833c05b66",
+ "sha256:51064ee7938526bab92acd049d41a1dc797422256086b39c08bafeffb9d304c6",
"sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc",
"sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d",
"sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce",
@@ -291,6 +215,7 @@
"sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e",
"sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba",
"sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02",
+ "sha256:b6ff91356354b7ff3bd208adcf875056d3d886ed7cef90c571aef2ab8a554b12",
"sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86",
"sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d",
"sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7",
@@ -361,28 +286,6 @@
],
"markers": "sys_platform == 'win32'",
"version": "==1.0.1"
- },
- "yarl": {
- "hashes": [
- "sha256:040b237f58ff7d800e6e0fd89c8439b841f777dd99b4a9cca04d6935564b9409",
- "sha256:17668ec6722b1b7a3a05cc0167659f6c95b436d25a36c2d52db0eca7d3f72593",
- "sha256:3a584b28086bc93c888a6c2aa5c92ed1ae20932f078c46509a66dce9ea5533f2",
- "sha256:4439be27e4eee76c7632c2427ca5e73703151b22cae23e64adb243a9c2f565d8",
- "sha256:48e918b05850fffb070a496d2b5f97fc31d15d94ca33d3d08a4f86e26d4e7c5d",
- "sha256:9102b59e8337f9874638fcfc9ac3734a0cfadb100e47d55c20d0dc6087fb4692",
- "sha256:9b930776c0ae0c691776f4d2891ebc5362af86f152dd0da463a6614074cb1b02",
- "sha256:b3b9ad80f8b68519cc3372a6ca85ae02cc5a8807723ac366b53c0f089db19e4a",
- "sha256:bc2f976c0e918659f723401c4f834deb8a8e7798a71be4382e024bcc3f7e23a8",
- "sha256:c22c75b5f394f3d47105045ea551e08a3e804dc7e01b37800ca35b58f856c3d6",
- "sha256:c52ce2883dc193824989a9b97a76ca86ecd1fa7955b14f87bf367a61b6232511",
- "sha256:ce584af5de8830d8701b8979b18fcf450cef9a382b1a3c8ef189bedc408faf1e",
- "sha256:da456eeec17fa8aa4594d9a9f27c0b1060b6a75f2419fe0c00609587b2695f4a",
- "sha256:db6db0f45d2c63ddb1a9d18d1b9b22f308e52c83638c26b422d520a815c4b3fb",
- "sha256:df89642981b94e7db5596818499c4b2219028f2a528c9c37cc1de45bf2fd3a3f",
- "sha256:f18d68f2be6bf0e89f1521af2b1bb46e66ab0018faafa81d70f358153170a317",
- "sha256:f379b7f83f23fe12823085cd6b906edc49df969eb99757f58ff382349a3303c6"
- ],
- "version": "==1.5.1"
}
},
"develop": {}
diff --git a/README.md b/README.md
index f54c85c..913cd54 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@
[](https://codecov.io/gh/shmilylty/OneForAll)
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
-[](https://github.com/shmilylty/OneForAll/tree/master/)
+[](https://github.com/shmilylty/OneForAll/tree/master/)
[](https://github.com/shmilylty/OneForAll/releases)
👊**OneForAll是一款功能强大的子域收集工具** 📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md)
@@ -15,21 +15,6 @@
📢 请务必花一点时间阅读此文档，有助于你快速熟悉OneForAll！
-
-🐍安装要求
-
-OneForAll基于[Python 3.8.0]( https://www.python.org/downloads/release/python-380/ )开发和测试，请使用高于Python 3.8.0的稳定发行版本，其他版本可能会出现一些问题（Windows平台必须使用3.8.0以上版本），安装Python环境可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)。运行以下命令检查Python和pip3版本：
-```bash
-python -V
-pip3 -V
-```
-如果你看到类似以下的输出便说明Python环境没有问题：
-```bash
-Python 3.8.0
-pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python38\site-packages\pip (python 3.8)
-```
-
-
✔安装步骤（git 版）
diff --git a/brute.py b/brute.py
index cce90c9..06a4d4d 100644
--- a/brute.py
+++ b/brute.py
@@ -592,8 +592,6 @@ class Brute(Module):
self.domain = str() # 当前正在进行爆破的域名
self.ips_times = dict() # IP集合出现次数
self.enable_wildcard = False # 当前域名是否使用泛解析
- self.wildcard_check = settings.brute_wildcard_check
- self.wildcard_deal = settings.brute_wildcard_deal
self.check_env = True
self.quite = False
diff --git a/common/module.py b/common/module.py
index 30a40bc..c41735b 100644
--- a/common/module.py
+++ b/common/module.py
@@ -22,9 +22,9 @@ class Module(object):
self.cookie = None
self.header = dict()
self.proxy = None
- self.delay = settings.request_delay # 请求睡眠时延
- self.timeout = settings.request_timeout # 请求超时时间
- self.verify = settings.request_verify # 请求SSL验证
+ self.delay = 1 # 请求睡眠时延
+ self.timeout = settings.request_timeout_second # 请求超时时间
+ self.verify = settings.request_ssl_verify # 请求SSL验证
self.domain = str() # 当前进行子域名收集的主域
self.subdomains = set() # 存放发现的子域
self.infos = dict() # 存放子域有关信息
@@ -199,7 +199,7 @@ class Module(object):
:param str module: module name
:return: proxy
"""
- if not settings.enable_proxy:
+ if not settings.enable_request_proxy:
logger.log('TRACE', f'All modules do not use proxy')
return self.proxy
if settings.proxy_all_module:
diff --git a/common/request.py b/common/request.py
index e6caa5a..1966793 100644
--- a/common/request.py
+++ b/common/request.py
@@ -1,10 +1,10 @@
import json
-import asyncio
-import functools
+from threading import Thread
+from queue import Queue
+
-import aiohttp
import tqdm
-from aiohttp import ClientSession
+import requests
from bs4 import BeautifulSoup
from common import utils
@@ -12,17 +12,17 @@ from config.log import logger
from config import settings
-def get_limit_conn():
- count = settings.limit_open_conn
+def req_thread_count():
+ count = settings.request_thread_count
if isinstance(count, int):
count = max(16, count)
else:
- count = utils.get_coroutine_count()
- logger.log('DEBUG', f'Request coroutine {count}')
+ count = utils.get_request_count()
+ logger.log('DEBUG', f'Number of request threads {count}')
return count
-def get_ports(port):
+def get_port_seq(port):
logger.log('DEBUG', 'Getting port range')
ports = set()
if isinstance(port, (set, list, tuple)):
@@ -40,81 +40,33 @@ def get_ports(port):
return set(ports)
-def gen_req_data(data, ports):
+def gen_req_url(domain, port):
+ if str(port).endswith('443'):
+ url = f'https://{domain}:{port}'
+ if port == 443:
+ url = f'https://{domain}'
+ return url
+ url = f'http://{domain}:{port}'
+ if port == 80:
+ url = f'http://{domain}'
+ return url
+
+
+def gen_req_urls(data, ports):
logger.log('INFOR', 'Generating request urls')
- new_data = []
- for data in data:
- resolve = data.get('resolve')
+ urls = set()
+ for info in data:
+ resolve = info.get('resolve')
# 解析不成功的子域不进行http请求探测
if resolve != 1:
continue
- subdomain = data.get('subdomain')
+ subdomain = info.get('subdomain')
for port in ports:
- if str(port).endswith('443'):
- url = f'https://{subdomain}:{port}'
- if port == 443:
- url = f'https://{subdomain}'
- data['id'] = None
- data['url'] = url
- data['port'] = port
- new_data.append(data)
- data = dict(data) # 需要生成一个新的字典对象
- else:
- url = f'http://{subdomain}:{port}'
- if port == 80:
- url = f'http://{subdomain}'
- data['id'] = None
- data['url'] = url
- data['port'] = port
- new_data.append(data)
- data = dict(data) # 需要生成一个新的字典对象
- return new_data
+ urls.add(gen_req_url(subdomain, port))
+ return urls
-async def fetch(session, method, url):
- """
- 请求
-
- :param session: session对象
- :param method: 请求方法
- :param str url: url地址
- :return: 响应对象和响应文本
- """
- timeout = aiohttp.ClientTimeout(total=None,
- connect=None,
- sock_read=settings.sockread_timeout,
- sock_connect=settings.sockconn_timeout)
- try:
- if method == 'HEAD':
- async with session.head(url,
- ssl=settings.verify_ssl,
- allow_redirects=settings.allow_redirects,
- timeout=timeout,
- proxy=settings.aiohttp_proxy) as resp:
- text = await resp.text()
- else:
- async with session.get(url,
- ssl=settings.verify_ssl,
- allow_redirects=settings.allow_redirects,
- timeout=timeout,
- proxy=settings.aiohttp_proxy) as resp:
-
- try:
- # 先尝试用utf-8解码
- text = await resp.text(encoding='utf-8', errors='strict')
- except UnicodeError:
- try:
- # 再尝试用gb18030解码
- text = await resp.text(encoding='gb18030', errors='strict')
- except UnicodeError:
- # 最后尝试自动解码
- text = await resp.text(encoding=None, errors='ignore')
- return resp, text
- except Exception as e:
- return e, None
-
-
-def get_title(markup):
+def get_html_title(markup):
"""
获取标题
@@ -161,107 +113,107 @@ def get_jump_urls(history):
return urls
-def request_callback(future, index, datas):
- resp, text = future.result()
- if isinstance(resp, BaseException):
- exception = resp
- logger.log('TRACE', exception.args)
- name = utils.get_classname(exception)
- datas[index]['reason'] = name + ' ' + str(exception)
- datas[index]['request'] = 0
- datas[index]['alive'] = 0
- else:
- datas[index]['reason'] = resp.reason
- datas[index]['status'] = resp.status
- datas[index]['request'] = 1
- if resp.status == 400 or resp.status >= 500:
- datas[index]['alive'] = 0
- else:
- datas[index]['alive'] = 1
- headers = resp.headers
- if settings.enable_banner_identify:
- datas[index]['banner'] = utils.get_sample_banner(headers)
- datas[index]['header'] = json.dumps(dict(headers))
- history = resp.history
- datas[index]['history'] = json.dumps(get_jump_urls(history))
- if isinstance(text, str):
- title = get_title(text).strip()
- datas[index]['title'] = utils.remove_invalid_string(title)
- datas[index]['response'] = utils.remove_invalid_string(text)
-
-
-def get_connector():
- count = get_limit_conn()
- return aiohttp.TCPConnector(ttl_dns_cache=300,
- ssl=settings.verify_ssl,
- limit=count,
- limit_per_host=settings.limit_per_host)
-
-
-async def async_request(urls):
- results = list()
- connector = get_connector()
- headers = utils.get_random_header()
- session = ClientSession(connector=connector, headers=headers)
- tasks = []
- for i, url in enumerate(urls):
- task = asyncio.ensure_future(fetch(session, 'GET', url))
- tasks.append(task)
- if tasks:
- futures = asyncio.as_completed(tasks)
- for future in tqdm.tqdm(futures,
- total=len(tasks),
- desc='Request Progress',
- ncols=80):
- result = await future
- results.append(result)
- await session.close()
- return results
-
-
-async def bulk_request(data, port):
- ports = get_ports(port)
- no_req_data = utils.get_filtered_data(data)
- to_req_data = gen_req_data(data, ports)
- method = settings.request_method.upper()
- logger.log('INFOR', f'Use {method} method to request')
- logger.log('INFOR', 'Async subdomains request in progress')
- connector = get_connector()
- headers = utils.get_random_header()
- session = ClientSession(connector=connector, headers=headers)
- tasks = []
- for num, data in enumerate(to_req_data):
- url = data.get('url')
- task = asyncio.ensure_future(fetch(session, method, url))
- task.add_done_callback(functools.partial(request_callback,
- index=num,
- datas=to_req_data))
- tasks.append(task)
- if tasks:
- futures = asyncio.as_completed(tasks)
- for future in tqdm.tqdm(futures,
- total=len(tasks),
- desc='Request Progress',
- ncols=80):
- await future
- await session.close()
- return to_req_data + no_req_data
-
-
-def set_loop_policy():
+def get(url, resp_queue, session):
+ timeout = settings.request_timeout_second
+ redirect = settings.request_allow_redirect
+ proxy = utils.get_proxy()
try:
- import uvloop
- except ImportError:
- pass
+ resp = session.get(url, timeout=timeout, allow_redirects=redirect, proxies=proxy)
+ except Exception as e:
+ logger.log('DEBUG', e.args)
+ return
+ resp_queue.put(resp)
+
+
+def request(urls_queue, resp_queue, session):
+ while not urls_queue.empty():
+ url = urls_queue.get()
+ get(url, resp_queue, session)
+ urls_queue.task_done()
+
+
+def progress(urls, resp_queue):
+ bar = tqdm.tqdm()
+ bar.total = len(urls)
+ bar.desc = 'Request Progress'
+ bar.ncols = 80
+ while True:
+ done = resp_queue.qsize()
+ bar.n = done
+ bar.update()
+ if done == bar.total:
+ break
+
+
+def get_session():
+ header = utils.gen_fake_header()
+ verify = settings.request_ssl_verify
+ redirect_limit = settings.request_redirect_limit
+ session = requests.Session()
+ session.headers = header
+ session.verify = verify
+ session.max_redirects = redirect_limit
+ return session
+
+
+def bulk_request(urls):
+ logger.log('INFOR', 'Requesting urls in bulk')
+ resp_list = list()
+ urls_queue = Queue()
+ resp_queue = Queue()
+ for url in urls:
+ urls_queue.put(url)
+ session = get_session()
+ thread_count = req_thread_count()
+
+ progress_thread = Thread(target=progress, args=(urls, resp_queue))
+ progress_thread.start()
+
+ for _ in range(thread_count):
+ request_thread = Thread(target=request, args=(urls_queue, resp_queue, session))
+ request_thread.start()
+
+ urls_queue.join()
+
+ while not resp_queue.empty():
+ resp = resp_queue.get()
+ resp_list.append(resp)
+ return resp_list
+
+
+def gen_new_info(info, resp):
+ port = resp.raw._pool.port
+ info['port'] = port
+ info['url'] = resp.url
+ info['reason'] = resp.reason
+ code = resp.status_code
+ info['status'] = code
+ info['request'] = 1
+ if code == 400 or code >= 500:
+ info['alive'] = 0
else:
- asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+ info['alive'] = 1
+ headers = resp.headers
+ if settings.enable_banner_identify:
+ info['banner'] = utils.get_sample_banner(headers)
+ info['header'] = json.dumps(dict(headers))
+ history = resp.history
+ info['history'] = json.dumps(get_jump_urls(history))
+ text = utils.decode_resp_text(resp)
+ title = get_html_title(text).strip()
+ info['title'] = utils.remove_invalid_string(title)
+ info['response'] = utils.remove_invalid_string(text)
+ return info
-def set_loop():
- set_loop_policy()
- loop = asyncio.get_event_loop()
- asyncio.set_event_loop(loop)
- return loop
+def gen_new_data(data, resp_list):
+ new_data = list()
+ for resp in resp_list:
+ subdomain = resp.raw._pool.host
+ for info in data:
+ if info.get('subdomain') == subdomain:
+ new_data.append(gen_new_info(info, resp))
+ return new_data
def run_request(domain, data, port):
@@ -274,25 +226,18 @@ def run_request(domain, data, port):
:return list: result
"""
logger.log('INFOR', f'Start requesting subdomains of {domain}')
- loop = set_loop()
data = utils.set_id_none(data)
- request_coroutine = bulk_request(data, port)
- data = loop.run_until_complete(request_coroutine)
- loop.run_until_complete(asyncio.sleep(0.25))
+ ports = get_port_seq(port)
+ filtered_data = utils.get_filtered_data(data)
+ req_urls = gen_req_urls(data, ports)
+ resp_list = bulk_request(req_urls)
+ new_data = gen_new_data(data, resp_list)
+ data = new_data + filtered_data
count = utils.count_alive(data)
logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
return data
-def urls_request(urls):
- logger.log('INFOR', 'Start urls request module')
- loop = set_loop()
- request_coroutine = async_request(urls)
- data = loop.run_until_complete(request_coroutine)
- loop.run_until_complete(asyncio.sleep(0.25))
- return data
-
-
def save_db(name, data):
"""
Save request results to database
diff --git a/common/utils.py b/common/utils.py
index e1290d1..a8f3f90 100644
--- a/common/utils.py
+++ b/common/utils.py
@@ -10,7 +10,6 @@ import subprocess
from ipaddress import IPv4Address, ip_address
from stat import S_IXUSR
-import psutil
import tenacity
import requests
from pathlib import Path
@@ -49,10 +48,10 @@ def gen_fake_header():
"""
Generate fake request headers
"""
- headers = settings.headers
+ headers = settings.request_default_headers
if not isinstance(headers, dict):
headers = dict()
- if settings.random_user_agent:
+ if settings.enable_random_ua:
ua = random.choice(user_agents)
headers['User-Agent'] = ua
headers['Accept-Encoding'] = 'gzip, deflate'
@@ -74,7 +73,7 @@ def get_random_proxy():
Get random proxy
"""
try:
- return random.choice(settings.proxy_pool)
+ return random.choice(settings.request_proxy_pool)
except IndexError:
return None
@@ -83,7 +82,7 @@ def get_proxy():
"""
Get proxy
"""
- if settings.enable_proxy:
+ if settings.enable_request_proxy:
return get_random_proxy()
return None
@@ -502,27 +501,8 @@ def get_process_num():
return 1
-def get_coroutine_count():
- """
- 根据内存大小获取并发数量
- """
- mem = psutil.virtual_memory()
- total_mem = mem.total
- g_size = 1024 * 1024 * 1024
- if total_mem < 1 * g_size:
- return 16
- elif total_mem < 2 * g_size:
- return 32
- elif total_mem < 4 * g_size:
- return 64
- elif total_mem < 8 * g_size:
- return 128
- elif total_mem < 16 * g_size:
- return 256
- elif total_mem < 32 * g_size:
- return 512
- else:
- return 1024
+def get_request_count():
+ return os.cpu_count() * 10
def uniq_dict_list(dict_list):
@@ -562,7 +542,6 @@ def check_net():
def check_pre():
logger.log('INFOR', 'Checking dependent environment')
- system = platform.system()
implementation = platform.python_implementation()
version = platform.python_version()
if implementation != 'CPython':
@@ -571,16 +550,6 @@ def check_pre():
if version < '3.6':
logger.log('FATAL', 'OneForAll requires Python 3.6 or higher')
exit(1)
- if system == 'Windows' and implementation == 'CPython' and version < '3.8':
- logger.log('FATAL', 'OneForAll requires Python 3.8 '
- 'or higher when running on Windows')
- exit(1)
- if system in {"Linux", "Darwin"}:
- try:
- import uvloop
- except ImportError:
- logger.log('ALERT', f'Please install the uvloop library manually '
- f'to accelerate subdomain requests')
def check_env():
@@ -599,8 +568,8 @@ def check_version(local):
api = 'https://api.github.com/repos/shmilylty/OneForAll/releases/latest'
header = get_random_header()
proxy = get_proxy()
- timeout = settings.request_timeout
- verify = settings.request_verify
+ timeout = settings.request_timeout_second
+ verify = settings.request_ssl_verify
try:
resp = requests.get(url=api, headers=header, proxies=proxy,
timeout=timeout, verify=verify)
@@ -747,3 +716,14 @@ def get_url_resp(url):
logger.log('DEBUG', e.args)
return None
return resp
+
+
+def decode_resp_text(resp):
+ try:
+ text = resp.text(encoding='utf-8', errors='strict') # 先尝试用utf-8严格解码
+ except UnicodeError:
+ try:
+ text = resp.text(encoding='gb18030', errors='strict') # 再尝试用gb18030严格解码
+ except UnicodeError:
+ text = resp.text(encoding=None, errors='ignore') # 最后尝试自动解码
+ return text
diff --git a/config/default.py b/config/default.py
index 069ac2a..976f167 100644
--- a/config/default.py
+++ b/config/default.py
@@ -6,6 +6,9 @@ OneForAll默认配置
import pathlib
import warnings
+# 禁用所有警告信息
+warnings.filterwarnings("ignore")
+
# 路径设置
relative_directory = pathlib.Path(__file__).parent.parent # OneForAll代码相对路径
module_dir = relative_directory.joinpath('modules') # OneForAll模块目录
@@ -79,24 +82,39 @@ ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标
banner_process_number = 4 # 识别进程数量(默认4)
# 代理设置
-enable_proxy = False # 是否使用代理(全局开关)
+enable_request_proxy = False # 是否使用代理(全局开关，默认False)
proxy_all_module = False # 代理所有模块
proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
'YandexSearch', 'CrossDomainXml',
'ContentSecurityPolicy'] # 代理自定义的模块
-proxy_pool = [{'http': 'http://127.0.0.1:1080',
- 'https': 'https://127.0.0.1:1080'}] # 代理池
-# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
-# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
+request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
+ 'https': 'https://127.0.0.1:1080'}] # 代理池
+# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
+# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
-# 网络请求设置
-request_delay = 1 # 请求时延(默认1秒)
-request_timeout = 30 # 请求超时(默认30秒)
-request_verify = False # 请求SSL验证(默认False)
-# 禁用所有警告信息
-warnings.filterwarnings("ignore")
+# 请求设置
+request_thread_count = None # 请求线程数量(默认None，则根据内存大小设置)
+request_timeout_second = 30 # 请求超时秒数(默认30秒)
+request_ssl_verify = False # 请求SSL验证(默认False)
+request_allow_redirect = True # 请求允许重定向(默认True)
+request_redirect_limit = 10 # 请求跳转限制(默认10次)
+# 默认请求头 可以在headers里添加自定义请求头
+request_default_headers = {
+ 'Accept': 'text/html,application/xhtml+xml,'
+ 'application/xml;q=0.9,*/*;q=0.8',
+ 'Accept-Encoding': 'gzip, deflate',
+ 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+ 'Cache-Control': 'max-age=0',
+ 'DNT': '1',
+ 'Referer': 'https://www.google.com/',
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+ '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+ 'Upgrade-Insecure-Requests': '1',
+ 'X-Forwarded-For': '127.0.0.1'
+}
+enable_random_ua = True # 使用随机UA(默认True，开启可以覆盖request_default_headers的UA)
# 搜索模块设置
# 开启全量搜索会尽量去获取搜索引擎搜索的全部结果，不过搜索耗时可能会过长
@@ -139,42 +157,11 @@ large_ports = [80, 81, 280, 300, 443, 591, 593, 832, 888, 901, 981, 1010, 1080,
18092, 20000, 20720, 24465, 28017, 28080, 30821, 43110, 61600]
ports = {'small': small_ports, 'medium': medium_ports, 'large': large_ports}
-# aiohttp有关配置
-verify_ssl = False
-# aiohttp 支持 HTTP/HTTPS形式的代理
-aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com"
-allow_redirects = True # 允许请求跳转
-# 为了保证请求质量 请谨慎更改以下设置
-# request_method只能是HEAD或GET,HEAD请求方法更快，但是不能获取响应体并提取从中title及banner
-request_method = 'GET' # 使用请求方法，默认GET
-sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒
-sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒
-limit_open_conn = None # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置
-# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
-limit_per_host = 10 # 0表示不限制,默认10
-
common_subnames = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
'help', 'live', 'mall', 'blogs', 'files', 'forum',
'store', 'mobile'}
-# 请求头设置
-# 可以在headers里添加自定义请求头
-headers = {
- 'Accept': 'text/html,application/xhtml+xml,'
- 'application/xml;q=0.9,*/*;q=0.8',
- 'Accept-Encoding': 'gzip, deflate',
- 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
- 'Cache-Control': 'max-age=0',
- 'DNT': '1',
- 'Referer': 'https://www.google.com/',
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
- 'Upgrade-Insecure-Requests': '1',
- 'X-Forwarded-For': '127.0.0.1'
-}
-random_user_agent = True # 使用随机UA(默认True，开启可以覆盖header的设置)
-
# 模块API配置
# Censys可以免费注册获取API：https://censys.io/api
censys_api_id = ''
diff --git a/config/setting.py b/config/setting.py
index f855702..07fb2d2 100644
--- a/config/setting.py
+++ b/config/setting.py
@@ -37,8 +37,6 @@ enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_mo
# ('modules.search', 'baidu')]
# 爆破模块设置
-brute_wildcard_check = True # 开启泛解析检测(默认True)
-brute_wildcard_deal = True # 开启泛解析处理(默认True)
brute_concurrent_num = 2000 # 爆破时并发查询数量(默认2000，最大推荐10000)
# 爆破所使用的字典路径 默认data/subdomains.txt
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
@@ -66,25 +64,18 @@ brute_ip_blacklist = {'0.0.0.0', '0.0.0.1'} # IP黑名单 子域解析到IP黑
ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标记为非法(泛解析)子域
# 代理设置
-enable_proxy = False # 是否使用代理(全局开关)
+enable_request_proxy = False # 是否使用代理(全局开关)
proxy_all_module = False # 代理所有模块
proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
'YandexSearch', 'CrossDomainXml',
'ContentSecurityPolicy'] # 代理自定义的模块
-proxy_pool = [{'http': 'http://127.0.0.1:1080',
- 'https': 'https://127.0.0.1:1080'}] # 代理池
-# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
-# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
+request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
+ 'https': 'https://127.0.0.1:1080'}] # 代理池
+# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
+# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
# 搜索模块设置
enable_recursive_search = False # 递归搜索子域
search_recursive_times = 2 # 递归搜索层数
-# aiohttp有关配置
-# aiohttp 支持 HTTP/HTTPS形式的代理
-aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com"
-# 为了保证请求质量 请谨慎更改以下设置
-sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒
-sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒
-limit_open_conn = None # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置
diff --git a/docs/en-us/README.md b/docs/en-us/README.md
index 8e96ec4..5223e47 100644
--- a/docs/en-us/README.md
+++ b/docs/en-us/README.md
@@ -4,7 +4,7 @@
[](https://codecov.io/gh/shmilylty/OneForAll)
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
-[](https://github.com/shmilylty/OneForAll/tree/master/)
+[](https://github.com/shmilylty/OneForAll/tree/master/)
[](https://github.com/shmilylty/OneForAll/releases)
👊**OneForAll is a powerful subdomain integration tool** 📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
@@ -15,23 +15,6 @@
📢 Please read this document to help you start quickly!
-
-🐍Installation requirements
-
-OneForAll is developed and tested based on [Python 3.8.0](https://www.python.org/downloads/release/python-380/). Recommend use release higher than Python 3.8.0 (Windows platform must use Python 3.8.0 or later). For more information on installing the Python environment, please read [Python 3 installation Guide](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3).
-
-After installation python, run the following command to check the Python and pip3 versions:
-```bash
-python -V
-pip3 -V
-```
-If you see the following output, there is no problem with the Python environment:
-```bash
-Python 3.8.0
-pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.8)
-```
-
-
✔Installation steps (for Git)
diff --git a/modules/finder.py b/modules/finder.py
index 12c9867..b21de6b 100644
--- a/modules/finder.py
+++ b/modules/finder.py
@@ -190,9 +190,10 @@ def find_subdomains(domain, data):
subdomains.update(find_in_resp(domain, req_url, rsp_html))
js_urls.update(find_js_urls(domain, req_url, rsp_html))
- resp_data = request.urls_request(js_urls)
- for resp, text in resp_data:
- if not text:
+ resp_data = request.bulk_request(js_urls)
+ for resp in resp_data:
+ if not resp:
continue
+ text = utils.decode_resp_text(resp)
subdomains.update(find_in_resp(domain, resp.url, text))
return subdomains
diff --git a/requirements.txt b/requirements.txt
index 0ffe773..c9c188e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,3 @@
-aiohttp==3.6.2
-async-timeout==3.0.1
-attrs==20.1.0
beautifulsoup4==4.9.1
bs4==0.0.1
certifi==2020.6.20
@@ -14,12 +11,9 @@ exrex==0.10.5
fire==0.3.1
future==0.18.2
idna==2.10
-IP2Location==8.4.1
jdcal==1.4.1
loguru==0.5.1
-multidict==4.7.6
openpyxl==2.4.11
-psutil==5.7.2
pyparsing==2.4.7
PySocks==1.7.1
records==0.5.3
@@ -36,5 +30,4 @@ tldextract==2.2.3
tqdm==4.48.2
treelib==1.6.1
urllib3==1.25.10
-win32-setctime==1.0.1
-yarl==1.5.1
\ No newline at end of file
+win32-setctime==1.0.1
\ No newline at end of file
diff --git a/test.py b/test.py
index 1b62fc9..e59c64f 100644
--- a/test.py
+++ b/test.py
@@ -8,7 +8,7 @@ Example
from oneforall import OneForAll
if __name__ == '__main__':
- test = OneForAll(target='hackfun.org')
+ test = OneForAll(target='mi.com')
test.brute = True
test.req = True
test.takeover = True