mirror of
https://github.com/shmilylty/OneForAll.git
synced 2026-08-26 04:47:48 +08:00
重构子域请求模块
This commit is contained in:
Jing Ling
@@ -13,15 +13,12 @@ requests = "*"
|
||||
records = "*"
|
||||
tldextract = "*"
|
||||
exrex = "*"
|
||||
aiohttp = "*"
|
||||
fire = "*"
|
||||
bs4 = "*"
|
||||
pysocks = "*"
|
||||
cloudscraper = "*"
|
||||
tablib = "*"
|
||||
psutil = "*"
|
||||
tenacity = "*"
|
||||
ip2location = "*"
|
||||
treelib = "*"
|
||||
|
||||
[requires]
|
||||
|
||||
Generated
+5
-102
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "78a5771767751025a636d651f53cce059223e1e045d351290fa5e7b413bc172b"
|
||||
"sha256": "fde2c88929fdf63725faea24a4fc7d82d6792caaca6676c08e96a40227f47e18"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
@@ -16,38 +16,6 @@
|
||||
]
|
||||
},
|
||||
"default": {
|
||||
"aiohttp": {
|
||||
"hashes": [
|
||||
"sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e",
|
||||
"sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326",
|
||||
"sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a",
|
||||
"sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654",
|
||||
"sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a",
|
||||
"sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4",
|
||||
"sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17",
|
||||
"sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec",
|
||||
"sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd",
|
||||
"sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48",
|
||||
"sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59",
|
||||
"sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.6.2"
|
||||
},
|
||||
"async-timeout": {
|
||||
"hashes": [
|
||||
"sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
|
||||
"sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
|
||||
],
|
||||
"version": "==3.0.1"
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:0ef97238856430dcf9228e07f316aefc17e8939fc8507e18c6501b761ef1a42a",
|
||||
"sha256:2867b7b9f8326499ab5b0e2d12801fa5c98842d2cbd22b35112ae04bf85b4dff"
|
||||
],
|
||||
"version": "==20.1.0"
|
||||
},
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7",
|
||||
@@ -140,14 +108,6 @@
|
||||
],
|
||||
"version": "==2.10"
|
||||
},
|
||||
"ip2location": {
|
||||
"hashes": [
|
||||
"sha256:4a83f32098091b6aafaef63b393331bf973bbf54547db82de38f30108b5f079c",
|
||||
"sha256:bbddbd5e3f79c18a9710afd43be09cb21724d7324557668017c545b5c5e62bc4"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==8.4.1"
|
||||
},
|
||||
"jdcal": {
|
||||
"hashes": [
|
||||
"sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba",
|
||||
@@ -163,51 +123,12 @@
|
||||
"index": "pypi",
|
||||
"version": "==0.5.1"
|
||||
},
|
||||
"multidict": {
|
||||
"hashes": [
|
||||
"sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a",
|
||||
"sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000",
|
||||
"sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2",
|
||||
"sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507",
|
||||
"sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5",
|
||||
"sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7",
|
||||
"sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d",
|
||||
"sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463",
|
||||
"sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19",
|
||||
"sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3",
|
||||
"sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b",
|
||||
"sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c",
|
||||
"sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87",
|
||||
"sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7",
|
||||
"sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430",
|
||||
"sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255",
|
||||
"sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d"
|
||||
],
|
||||
"version": "==4.7.6"
|
||||
},
|
||||
"openpyxl": {
|
||||
"hashes": [
|
||||
"sha256:626d38647c063d55803ef4971c4d43226538d4e95cb6260c094e363ee33e10c7"
|
||||
],
|
||||
"version": "==2.4.11"
|
||||
},
|
||||
"psutil": {
|
||||
"hashes": [
|
||||
"sha256:0ee3c36428f160d2d8fce3c583a0353e848abb7de9732c50cf3356dd49ad63f8",
|
||||
"sha256:10512b46c95b02842c225f58fa00385c08fa00c68bac7da2d9a58ebe2c517498",
|
||||
"sha256:4080869ed93cce662905b029a1770fe89c98787e543fa7347f075ade761b19d6",
|
||||
"sha256:5e9d0f26d4194479a13d5f4b3798260c20cecf9ac9a461e718eb59ea520a360c",
|
||||
"sha256:66c18ca7680a31bf16ee22b1d21b6397869dda8059dbdb57d9f27efa6615f195",
|
||||
"sha256:68d36986ded5dac7c2dcd42f2682af1db80d4bce3faa126a6145c1637e1b559f",
|
||||
"sha256:90990af1c3c67195c44c9a889184f84f5b2320dce3ee3acbd054e3ba0b4a7beb",
|
||||
"sha256:a5b120bb3c0c71dfe27551f9da2f3209a8257a178ed6c628a819037a8df487f1",
|
||||
"sha256:d8a82162f23c53b8525cf5f14a355f5d1eea86fa8edde27287dd3a98399e4fdf",
|
||||
"sha256:f2018461733b23f308c298653c8903d32aaad7873d25e1d228765e91ae42c3f2",
|
||||
"sha256:ff1977ba1a5f71f89166d5145c3da1cea89a0fdb044075a12c720ee9123ec818"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==5.7.2"
|
||||
},
|
||||
"pyparsing": {
|
||||
"hashes": [
|
||||
"sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
|
||||
@@ -273,10 +194,13 @@
|
||||
"sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb",
|
||||
"sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804",
|
||||
"sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6",
|
||||
"sha256:26c5ca9d09f0e21b8671a32f7d83caad5be1f6ff45eef5ec2f6fd0db85fc5dc0",
|
||||
"sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe",
|
||||
"sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de",
|
||||
"sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36",
|
||||
"sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e",
|
||||
"sha256:465c999ef30b1c7525f81330184121521418a67189053bcf585824d833c05b66",
|
||||
"sha256:51064ee7938526bab92acd049d41a1dc797422256086b39c08bafeffb9d304c6",
|
||||
"sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc",
|
||||
"sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d",
|
||||
"sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce",
|
||||
@@ -291,6 +215,7 @@
|
||||
"sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e",
|
||||
"sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba",
|
||||
"sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02",
|
||||
"sha256:b6ff91356354b7ff3bd208adcf875056d3d886ed7cef90c571aef2ab8a554b12",
|
||||
"sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86",
|
||||
"sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d",
|
||||
"sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7",
|
||||
@@ -361,28 +286,6 @@
|
||||
],
|
||||
"markers": "sys_platform == 'win32'",
|
||||
"version": "==1.0.1"
|
||||
},
|
||||
"yarl": {
|
||||
"hashes": [
|
||||
"sha256:040b237f58ff7d800e6e0fd89c8439b841f777dd99b4a9cca04d6935564b9409",
|
||||
"sha256:17668ec6722b1b7a3a05cc0167659f6c95b436d25a36c2d52db0eca7d3f72593",
|
||||
"sha256:3a584b28086bc93c888a6c2aa5c92ed1ae20932f078c46509a66dce9ea5533f2",
|
||||
"sha256:4439be27e4eee76c7632c2427ca5e73703151b22cae23e64adb243a9c2f565d8",
|
||||
"sha256:48e918b05850fffb070a496d2b5f97fc31d15d94ca33d3d08a4f86e26d4e7c5d",
|
||||
"sha256:9102b59e8337f9874638fcfc9ac3734a0cfadb100e47d55c20d0dc6087fb4692",
|
||||
"sha256:9b930776c0ae0c691776f4d2891ebc5362af86f152dd0da463a6614074cb1b02",
|
||||
"sha256:b3b9ad80f8b68519cc3372a6ca85ae02cc5a8807723ac366b53c0f089db19e4a",
|
||||
"sha256:bc2f976c0e918659f723401c4f834deb8a8e7798a71be4382e024bcc3f7e23a8",
|
||||
"sha256:c22c75b5f394f3d47105045ea551e08a3e804dc7e01b37800ca35b58f856c3d6",
|
||||
"sha256:c52ce2883dc193824989a9b97a76ca86ecd1fa7955b14f87bf367a61b6232511",
|
||||
"sha256:ce584af5de8830d8701b8979b18fcf450cef9a382b1a3c8ef189bedc408faf1e",
|
||||
"sha256:da456eeec17fa8aa4594d9a9f27c0b1060b6a75f2419fe0c00609587b2695f4a",
|
||||
"sha256:db6db0f45d2c63ddb1a9d18d1b9b22f308e52c83638c26b422d520a815c4b3fb",
|
||||
"sha256:df89642981b94e7db5596818499c4b2219028f2a528c9c37cc1de45bf2fd3a3f",
|
||||
"sha256:f18d68f2be6bf0e89f1521af2b1bb46e66ab0018faafa81d70f358153170a317",
|
||||
"sha256:f379b7f83f23fe12823085cd6b906edc49df969eb99757f58ff382349a3303c6"
|
||||
],
|
||||
"version": "==1.5.1"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
[](https://codecov.io/gh/shmilylty/OneForAll)
|
||||
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/releases)
|
||||
|
||||
👊**OneForAll是一款功能强大的子域收集工具** 📝[English Document](https://github.com/shmilylty/OneForAll/tree/master/docs/en-us/README.md)
|
||||
@@ -15,21 +15,6 @@
|
||||
|
||||
📢 请务必花一点时间阅读此文档，有助于你快速熟悉OneForAll！
|
||||
|
||||
<details>
|
||||
<summary><b>🐍安装要求</b></summary>
|
||||
|
||||
OneForAll基于[Python 3.8.0]( https://www.python.org/downloads/release/python-380/ )开发和测试，请使用高于Python 3.8.0的稳定发行版本，其他版本可能会出现一些问题（Windows平台必须使用3.8.0以上版本），安装Python环境可以参考[Python 3 安装指南](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3)。运行以下命令检查Python和pip3版本：
|
||||
```bash
|
||||
python -V
|
||||
pip3 -V
|
||||
```
|
||||
如果你看到类似以下的输出便说明Python环境没有问题：
|
||||
```bash
|
||||
Python 3.8.0
|
||||
pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python38\site-packages\pip (python 3.8)
|
||||
```
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>✔安装步骤（git 版）</b></summary>
|
||||
|
||||
|
||||
@@ -592,8 +592,6 @@ class Brute(Module):
|
||||
self.domain = str() # 当前正在进行爆破的域名
|
||||
self.ips_times = dict() # IP集合出现次数
|
||||
self.enable_wildcard = False # 当前域名是否使用泛解析
|
||||
self.wildcard_check = settings.brute_wildcard_check
|
||||
self.wildcard_deal = settings.brute_wildcard_deal
|
||||
self.check_env = True
|
||||
self.quite = False
|
||||
|
||||
|
||||
+4
-4
@@ -22,9 +22,9 @@ class Module(object):
|
||||
self.cookie = None
|
||||
self.header = dict()
|
||||
self.proxy = None
|
||||
self.delay = settings.request_delay # 请求睡眠时延
|
||||
self.timeout = settings.request_timeout # 请求超时时间
|
||||
self.verify = settings.request_verify # 请求SSL验证
|
||||
self.delay = 1 # 请求睡眠时延
|
||||
self.timeout = settings.request_timeout_second # 请求超时时间
|
||||
self.verify = settings.request_ssl_verify # 请求SSL验证
|
||||
self.domain = str() # 当前进行子域名收集的主域
|
||||
self.subdomains = set() # 存放发现的子域
|
||||
self.infos = dict() # 存放子域有关信息
|
||||
@@ -199,7 +199,7 @@ class Module(object):
|
||||
:param str module: module name
|
||||
:return: proxy
|
||||
"""
|
||||
if not settings.enable_proxy:
|
||||
if not settings.enable_request_proxy:
|
||||
logger.log('TRACE', f'All modules do not use proxy')
|
||||
return self.proxy
|
||||
if settings.proxy_all_module:
|
||||
|
||||
+132
-187
@@ -1,10 +1,10 @@
|
||||
import json
|
||||
import asyncio
|
||||
import functools
|
||||
from threading import Thread
|
||||
from queue import Queue
|
||||
|
||||
|
||||
import aiohttp
|
||||
import tqdm
|
||||
from aiohttp import ClientSession
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from common import utils
|
||||
@@ -12,17 +12,17 @@ from config.log import logger
|
||||
from config import settings
|
||||
|
||||
|
||||
def get_limit_conn():
|
||||
count = settings.limit_open_conn
|
||||
def req_thread_count():
|
||||
count = settings.request_thread_count
|
||||
if isinstance(count, int):
|
||||
count = max(16, count)
|
||||
else:
|
||||
count = utils.get_coroutine_count()
|
||||
logger.log('DEBUG', f'Request coroutine {count}')
|
||||
count = utils.get_request_count()
|
||||
logger.log('DEBUG', f'Number of request threads {count}')
|
||||
return count
|
||||
|
||||
|
||||
def get_ports(port):
|
||||
def get_port_seq(port):
|
||||
logger.log('DEBUG', 'Getting port range')
|
||||
ports = set()
|
||||
if isinstance(port, (set, list, tuple)):
|
||||
@@ -40,81 +40,33 @@ def get_ports(port):
|
||||
return set(ports)
|
||||
|
||||
|
||||
def gen_req_data(data, ports):
|
||||
def gen_req_url(domain, port):
|
||||
if str(port).endswith('443'):
|
||||
url = f'https://{domain}:{port}'
|
||||
if port == 443:
|
||||
url = f'https://{domain}'
|
||||
return url
|
||||
url = f'http://{domain}:{port}'
|
||||
if port == 80:
|
||||
url = f'http://{domain}'
|
||||
return url
|
||||
|
||||
|
||||
def gen_req_urls(data, ports):
|
||||
logger.log('INFOR', 'Generating request urls')
|
||||
new_data = []
|
||||
for data in data:
|
||||
resolve = data.get('resolve')
|
||||
urls = set()
|
||||
for info in data:
|
||||
resolve = info.get('resolve')
|
||||
# 解析不成功的子域不进行http请求探测
|
||||
if resolve != 1:
|
||||
continue
|
||||
subdomain = data.get('subdomain')
|
||||
subdomain = info.get('subdomain')
|
||||
for port in ports:
|
||||
if str(port).endswith('443'):
|
||||
url = f'https://{subdomain}:{port}'
|
||||
if port == 443:
|
||||
url = f'https://{subdomain}'
|
||||
data['id'] = None
|
||||
data['url'] = url
|
||||
data['port'] = port
|
||||
new_data.append(data)
|
||||
data = dict(data) # 需要生成一个新的字典对象
|
||||
else:
|
||||
url = f'http://{subdomain}:{port}'
|
||||
if port == 80:
|
||||
url = f'http://{subdomain}'
|
||||
data['id'] = None
|
||||
data['url'] = url
|
||||
data['port'] = port
|
||||
new_data.append(data)
|
||||
data = dict(data) # 需要生成一个新的字典对象
|
||||
return new_data
|
||||
urls.add(gen_req_url(subdomain, port))
|
||||
return urls
|
||||
|
||||
|
||||
async def fetch(session, method, url):
|
||||
"""
|
||||
请求
|
||||
|
||||
:param session: session对象
|
||||
:param method: 请求方法
|
||||
:param str url: url地址
|
||||
:return: 响应对象和响应文本
|
||||
"""
|
||||
timeout = aiohttp.ClientTimeout(total=None,
|
||||
connect=None,
|
||||
sock_read=settings.sockread_timeout,
|
||||
sock_connect=settings.sockconn_timeout)
|
||||
try:
|
||||
if method == 'HEAD':
|
||||
async with session.head(url,
|
||||
ssl=settings.verify_ssl,
|
||||
allow_redirects=settings.allow_redirects,
|
||||
timeout=timeout,
|
||||
proxy=settings.aiohttp_proxy) as resp:
|
||||
text = await resp.text()
|
||||
else:
|
||||
async with session.get(url,
|
||||
ssl=settings.verify_ssl,
|
||||
allow_redirects=settings.allow_redirects,
|
||||
timeout=timeout,
|
||||
proxy=settings.aiohttp_proxy) as resp:
|
||||
|
||||
try:
|
||||
# 先尝试用utf-8解码
|
||||
text = await resp.text(encoding='utf-8', errors='strict')
|
||||
except UnicodeError:
|
||||
try:
|
||||
# 再尝试用gb18030解码
|
||||
text = await resp.text(encoding='gb18030', errors='strict')
|
||||
except UnicodeError:
|
||||
# 最后尝试自动解码
|
||||
text = await resp.text(encoding=None, errors='ignore')
|
||||
return resp, text
|
||||
except Exception as e:
|
||||
return e, None
|
||||
|
||||
|
||||
def get_title(markup):
|
||||
def get_html_title(markup):
|
||||
"""
|
||||
获取标题
|
||||
|
||||
@@ -161,107 +113,107 @@ def get_jump_urls(history):
|
||||
return urls
|
||||
|
||||
|
||||
def request_callback(future, index, datas):
|
||||
resp, text = future.result()
|
||||
if isinstance(resp, BaseException):
|
||||
exception = resp
|
||||
logger.log('TRACE', exception.args)
|
||||
name = utils.get_classname(exception)
|
||||
datas[index]['reason'] = name + ' ' + str(exception)
|
||||
datas[index]['request'] = 0
|
||||
datas[index]['alive'] = 0
|
||||
else:
|
||||
datas[index]['reason'] = resp.reason
|
||||
datas[index]['status'] = resp.status
|
||||
datas[index]['request'] = 1
|
||||
if resp.status == 400 or resp.status >= 500:
|
||||
datas[index]['alive'] = 0
|
||||
else:
|
||||
datas[index]['alive'] = 1
|
||||
headers = resp.headers
|
||||
if settings.enable_banner_identify:
|
||||
datas[index]['banner'] = utils.get_sample_banner(headers)
|
||||
datas[index]['header'] = json.dumps(dict(headers))
|
||||
history = resp.history
|
||||
datas[index]['history'] = json.dumps(get_jump_urls(history))
|
||||
if isinstance(text, str):
|
||||
title = get_title(text).strip()
|
||||
datas[index]['title'] = utils.remove_invalid_string(title)
|
||||
datas[index]['response'] = utils.remove_invalid_string(text)
|
||||
|
||||
|
||||
def get_connector():
|
||||
count = get_limit_conn()
|
||||
return aiohttp.TCPConnector(ttl_dns_cache=300,
|
||||
ssl=settings.verify_ssl,
|
||||
limit=count,
|
||||
limit_per_host=settings.limit_per_host)
|
||||
|
||||
|
||||
async def async_request(urls):
|
||||
results = list()
|
||||
connector = get_connector()
|
||||
headers = utils.get_random_header()
|
||||
session = ClientSession(connector=connector, headers=headers)
|
||||
tasks = []
|
||||
for i, url in enumerate(urls):
|
||||
task = asyncio.ensure_future(fetch(session, 'GET', url))
|
||||
tasks.append(task)
|
||||
if tasks:
|
||||
futures = asyncio.as_completed(tasks)
|
||||
for future in tqdm.tqdm(futures,
|
||||
total=len(tasks),
|
||||
desc='Request Progress',
|
||||
ncols=80):
|
||||
result = await future
|
||||
results.append(result)
|
||||
await session.close()
|
||||
return results
|
||||
|
||||
|
||||
async def bulk_request(data, port):
|
||||
ports = get_ports(port)
|
||||
no_req_data = utils.get_filtered_data(data)
|
||||
to_req_data = gen_req_data(data, ports)
|
||||
method = settings.request_method.upper()
|
||||
logger.log('INFOR', f'Use {method} method to request')
|
||||
logger.log('INFOR', 'Async subdomains request in progress')
|
||||
connector = get_connector()
|
||||
headers = utils.get_random_header()
|
||||
session = ClientSession(connector=connector, headers=headers)
|
||||
tasks = []
|
||||
for num, data in enumerate(to_req_data):
|
||||
url = data.get('url')
|
||||
task = asyncio.ensure_future(fetch(session, method, url))
|
||||
task.add_done_callback(functools.partial(request_callback,
|
||||
index=num,
|
||||
datas=to_req_data))
|
||||
tasks.append(task)
|
||||
if tasks:
|
||||
futures = asyncio.as_completed(tasks)
|
||||
for future in tqdm.tqdm(futures,
|
||||
total=len(tasks),
|
||||
desc='Request Progress',
|
||||
ncols=80):
|
||||
await future
|
||||
await session.close()
|
||||
return to_req_data + no_req_data
|
||||
|
||||
|
||||
def set_loop_policy():
|
||||
def get(url, resp_queue, session):
|
||||
timeout = settings.request_timeout_second
|
||||
redirect = settings.request_allow_redirect
|
||||
proxy = utils.get_proxy()
|
||||
try:
|
||||
import uvloop
|
||||
except ImportError:
|
||||
pass
|
||||
resp = session.get(url, timeout=timeout, allow_redirects=redirect, proxies=proxy)
|
||||
except Exception as e:
|
||||
logger.log('DEBUG', e.args)
|
||||
return
|
||||
resp_queue.put(resp)
|
||||
|
||||
|
||||
def request(urls_queue, resp_queue, session):
|
||||
while not urls_queue.empty():
|
||||
url = urls_queue.get()
|
||||
get(url, resp_queue, session)
|
||||
urls_queue.task_done()
|
||||
|
||||
|
||||
def progress(urls, resp_queue):
|
||||
bar = tqdm.tqdm()
|
||||
bar.total = len(urls)
|
||||
bar.desc = 'Request Progress'
|
||||
bar.ncols = 80
|
||||
while True:
|
||||
done = resp_queue.qsize()
|
||||
bar.n = done
|
||||
bar.update()
|
||||
if done == bar.total:
|
||||
break
|
||||
|
||||
|
||||
def get_session():
|
||||
header = utils.gen_fake_header()
|
||||
verify = settings.request_ssl_verify
|
||||
redirect_limit = settings.request_redirect_limit
|
||||
session = requests.Session()
|
||||
session.headers = header
|
||||
session.verify = verify
|
||||
session.max_redirects = redirect_limit
|
||||
return session
|
||||
|
||||
|
||||
def bulk_request(urls):
|
||||
logger.log('INFOR', 'Requesting urls in bulk')
|
||||
resp_list = list()
|
||||
urls_queue = Queue()
|
||||
resp_queue = Queue()
|
||||
for url in urls:
|
||||
urls_queue.put(url)
|
||||
session = get_session()
|
||||
thread_count = req_thread_count()
|
||||
|
||||
progress_thread = Thread(target=progress, args=(urls, resp_queue))
|
||||
progress_thread.start()
|
||||
|
||||
for _ in range(thread_count):
|
||||
request_thread = Thread(target=request, args=(urls_queue, resp_queue, session))
|
||||
request_thread.start()
|
||||
|
||||
urls_queue.join()
|
||||
|
||||
while not resp_queue.empty():
|
||||
resp = resp_queue.get()
|
||||
resp_list.append(resp)
|
||||
return resp_list
|
||||
|
||||
|
||||
def gen_new_info(info, resp):
|
||||
port = resp.raw._pool.port
|
||||
info['port'] = port
|
||||
info['url'] = resp.url
|
||||
info['reason'] = resp.reason
|
||||
code = resp.status_code
|
||||
info['status'] = code
|
||||
info['request'] = 1
|
||||
if code == 400 or code >= 500:
|
||||
info['alive'] = 0
|
||||
else:
|
||||
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
|
||||
info['alive'] = 1
|
||||
headers = resp.headers
|
||||
if settings.enable_banner_identify:
|
||||
info['banner'] = utils.get_sample_banner(headers)
|
||||
info['header'] = json.dumps(dict(headers))
|
||||
history = resp.history
|
||||
info['history'] = json.dumps(get_jump_urls(history))
|
||||
text = utils.decode_resp_text(resp)
|
||||
title = get_html_title(text).strip()
|
||||
info['title'] = utils.remove_invalid_string(title)
|
||||
info['response'] = utils.remove_invalid_string(text)
|
||||
return info
|
||||
|
||||
|
||||
def set_loop():
|
||||
set_loop_policy()
|
||||
loop = asyncio.get_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
return loop
|
||||
def gen_new_data(data, resp_list):
|
||||
new_data = list()
|
||||
for resp in resp_list:
|
||||
subdomain = resp.raw._pool.host
|
||||
for info in data:
|
||||
if info.get('subdomain') == subdomain:
|
||||
new_data.append(gen_new_info(info, resp))
|
||||
return new_data
|
||||
|
||||
|
||||
def run_request(domain, data, port):
|
||||
@@ -274,25 +226,18 @@ def run_request(domain, data, port):
|
||||
:return list: result
|
||||
"""
|
||||
logger.log('INFOR', f'Start requesting subdomains of {domain}')
|
||||
loop = set_loop()
|
||||
data = utils.set_id_none(data)
|
||||
request_coroutine = bulk_request(data, port)
|
||||
data = loop.run_until_complete(request_coroutine)
|
||||
loop.run_until_complete(asyncio.sleep(0.25))
|
||||
ports = get_port_seq(port)
|
||||
filtered_data = utils.get_filtered_data(data)
|
||||
req_urls = gen_req_urls(data, ports)
|
||||
resp_list = bulk_request(req_urls)
|
||||
new_data = gen_new_data(data, resp_list)
|
||||
data = new_data + filtered_data
|
||||
count = utils.count_alive(data)
|
||||
logger.log('INFOR', f'Found that {domain} has {count} alive subdomains')
|
||||
return data
|
||||
|
||||
|
||||
def urls_request(urls):
|
||||
logger.log('INFOR', 'Start urls request module')
|
||||
loop = set_loop()
|
||||
request_coroutine = async_request(urls)
|
||||
data = loop.run_until_complete(request_coroutine)
|
||||
loop.run_until_complete(asyncio.sleep(0.25))
|
||||
return data
|
||||
|
||||
|
||||
def save_db(name, data):
|
||||
"""
|
||||
Save request results to database
|
||||
|
||||
+19
-39
@@ -10,7 +10,6 @@ import subprocess
|
||||
from ipaddress import IPv4Address, ip_address
|
||||
from stat import S_IXUSR
|
||||
|
||||
import psutil
|
||||
import tenacity
|
||||
import requests
|
||||
from pathlib import Path
|
||||
@@ -49,10 +48,10 @@ def gen_fake_header():
|
||||
"""
|
||||
Generate fake request headers
|
||||
"""
|
||||
headers = settings.headers
|
||||
headers = settings.request_default_headers
|
||||
if not isinstance(headers, dict):
|
||||
headers = dict()
|
||||
if settings.random_user_agent:
|
||||
if settings.enable_random_ua:
|
||||
ua = random.choice(user_agents)
|
||||
headers['User-Agent'] = ua
|
||||
headers['Accept-Encoding'] = 'gzip, deflate'
|
||||
@@ -74,7 +73,7 @@ def get_random_proxy():
|
||||
Get random proxy
|
||||
"""
|
||||
try:
|
||||
return random.choice(settings.proxy_pool)
|
||||
return random.choice(settings.request_proxy_pool)
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
@@ -83,7 +82,7 @@ def get_proxy():
|
||||
"""
|
||||
Get proxy
|
||||
"""
|
||||
if settings.enable_proxy:
|
||||
if settings.enable_request_proxy:
|
||||
return get_random_proxy()
|
||||
return None
|
||||
|
||||
@@ -502,27 +501,8 @@ def get_process_num():
|
||||
return 1
|
||||
|
||||
|
||||
def get_coroutine_count():
|
||||
"""
|
||||
根据内存大小获取并发数量
|
||||
"""
|
||||
mem = psutil.virtual_memory()
|
||||
total_mem = mem.total
|
||||
g_size = 1024 * 1024 * 1024
|
||||
if total_mem < 1 * g_size:
|
||||
return 16
|
||||
elif total_mem < 2 * g_size:
|
||||
return 32
|
||||
elif total_mem < 4 * g_size:
|
||||
return 64
|
||||
elif total_mem < 8 * g_size:
|
||||
return 128
|
||||
elif total_mem < 16 * g_size:
|
||||
return 256
|
||||
elif total_mem < 32 * g_size:
|
||||
return 512
|
||||
else:
|
||||
return 1024
|
||||
def get_request_count():
|
||||
return os.cpu_count() * 10
|
||||
|
||||
|
||||
def uniq_dict_list(dict_list):
|
||||
@@ -562,7 +542,6 @@ def check_net():
|
||||
|
||||
def check_pre():
|
||||
logger.log('INFOR', 'Checking dependent environment')
|
||||
system = platform.system()
|
||||
implementation = platform.python_implementation()
|
||||
version = platform.python_version()
|
||||
if implementation != 'CPython':
|
||||
@@ -571,16 +550,6 @@ def check_pre():
|
||||
if version < '3.6':
|
||||
logger.log('FATAL', 'OneForAll requires Python 3.6 or higher')
|
||||
exit(1)
|
||||
if system == 'Windows' and implementation == 'CPython' and version < '3.8':
|
||||
logger.log('FATAL', 'OneForAll requires Python 3.8 '
|
||||
'or higher when running on Windows')
|
||||
exit(1)
|
||||
if system in {"Linux", "Darwin"}:
|
||||
try:
|
||||
import uvloop
|
||||
except ImportError:
|
||||
logger.log('ALERT', f'Please install the uvloop library manually '
|
||||
f'to accelerate subdomain requests')
|
||||
|
||||
|
||||
def check_env():
|
||||
@@ -599,8 +568,8 @@ def check_version(local):
|
||||
api = 'https://api.github.com/repos/shmilylty/OneForAll/releases/latest'
|
||||
header = get_random_header()
|
||||
proxy = get_proxy()
|
||||
timeout = settings.request_timeout
|
||||
verify = settings.request_verify
|
||||
timeout = settings.request_timeout_second
|
||||
verify = settings.request_ssl_verify
|
||||
try:
|
||||
resp = requests.get(url=api, headers=header, proxies=proxy,
|
||||
timeout=timeout, verify=verify)
|
||||
@@ -747,3 +716,14 @@ def get_url_resp(url):
|
||||
logger.log('DEBUG', e.args)
|
||||
return None
|
||||
return resp
|
||||
|
||||
|
||||
def decode_resp_text(resp):
|
||||
try:
|
||||
text = resp.text(encoding='utf-8', errors='strict') # 先尝试用utf-8严格解码
|
||||
except UnicodeError:
|
||||
try:
|
||||
text = resp.text(encoding='gb18030', errors='strict') # 再尝试用gb18030严格解码
|
||||
except UnicodeError:
|
||||
text = resp.text(encoding=None, errors='ignore') # 最后尝试自动解码
|
||||
return text
|
||||
|
||||
+29
-42
@@ -6,6 +6,9 @@ OneForAll默认配置
|
||||
import pathlib
|
||||
import warnings
|
||||
|
||||
# 禁用所有警告信息
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
# 路径设置
|
||||
relative_directory = pathlib.Path(__file__).parent.parent # OneForAll代码相对路径
|
||||
module_dir = relative_directory.joinpath('modules') # OneForAll模块目录
|
||||
@@ -79,24 +82,39 @@ ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标
|
||||
banner_process_number = 4 # 识别进程数量(默认4)
|
||||
|
||||
# 代理设置
|
||||
enable_proxy = False # 是否使用代理(全局开关)
|
||||
enable_request_proxy = False # 是否使用代理(全局开关，默认False)
|
||||
proxy_all_module = False # 代理所有模块
|
||||
proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
|
||||
'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
|
||||
'YandexSearch', 'CrossDomainXml',
|
||||
'ContentSecurityPolicy'] # 代理自定义的模块
|
||||
proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
||||
'https': 'https://127.0.0.1:1080'}] # 代理池
|
||||
# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
||||
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
||||
request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
||||
'https': 'https://127.0.0.1:1080'}] # 代理池
|
||||
# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
||||
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
||||
|
||||
|
||||
# 网络请求设置
|
||||
request_delay = 1 # 请求时延(默认1秒)
|
||||
request_timeout = 30 # 请求超时(默认30秒)
|
||||
request_verify = False # 请求SSL验证(默认False)
|
||||
# 禁用所有警告信息
|
||||
warnings.filterwarnings("ignore")
|
||||
# 请求设置
|
||||
request_thread_count = None # 请求线程数量(默认None，则根据内存大小设置)
|
||||
request_timeout_second = 30 # 请求超时秒数(默认30秒)
|
||||
request_ssl_verify = False # 请求SSL验证(默认False)
|
||||
request_allow_redirect = True # 请求允许重定向(默认True)
|
||||
request_redirect_limit = 10 # 请求跳转限制(默认10次)
|
||||
# 默认请求头 可以在headers里添加自定义请求头
|
||||
request_default_headers = {
|
||||
'Accept': 'text/html,application/xhtml+xml,'
|
||||
'application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
|
||||
'Cache-Control': 'max-age=0',
|
||||
'DNT': '1',
|
||||
'Referer': 'https://www.google.com/',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||||
'(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'X-Forwarded-For': '127.0.0.1'
|
||||
}
|
||||
enable_random_ua = True # 使用随机UA(默认True，开启可以覆盖request_default_headers的UA)
|
||||
|
||||
# 搜索模块设置
|
||||
# 开启全量搜索会尽量去获取搜索引擎搜索的全部结果，不过搜索耗时可能会过长
|
||||
@@ -139,42 +157,11 @@ large_ports = [80, 81, 280, 300, 443, 591, 593, 832, 888, 901, 981, 1010, 1080,
|
||||
18092, 20000, 20720, 24465, 28017, 28080, 30821, 43110, 61600]
|
||||
ports = {'small': small_ports, 'medium': medium_ports, 'large': large_ports}
|
||||
|
||||
# aiohttp有关配置
|
||||
verify_ssl = False
|
||||
# aiohttp 支持 HTTP/HTTPS形式的代理
|
||||
aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com"
|
||||
allow_redirects = True # 允许请求跳转
|
||||
# 为了保证请求质量 请谨慎更改以下设置
|
||||
# request_method只能是HEAD或GET,HEAD请求方法更快，但是不能获取响应体并提取从中title及banner
|
||||
request_method = 'GET' # 使用请求方法，默认GET
|
||||
sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒
|
||||
sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒
|
||||
limit_open_conn = None # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置
|
||||
# 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数
|
||||
limit_per_host = 10 # 0表示不限制,默认10
|
||||
|
||||
common_subnames = {'i', 'w', 'm', 'en', 'us', 'zh', 'w3', 'app', 'bbs',
|
||||
'web', 'www', 'job', 'docs', 'news', 'blog', 'data',
|
||||
'help', 'live', 'mall', 'blogs', 'files', 'forum',
|
||||
'store', 'mobile'}
|
||||
|
||||
# 请求头设置
|
||||
# 可以在headers里添加自定义请求头
|
||||
headers = {
|
||||
'Accept': 'text/html,application/xhtml+xml,'
|
||||
'application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
|
||||
'Cache-Control': 'max-age=0',
|
||||
'DNT': '1',
|
||||
'Referer': 'https://www.google.com/',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||||
'(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'X-Forwarded-For': '127.0.0.1'
|
||||
}
|
||||
random_user_agent = True # 使用随机UA(默认True，开启可以覆盖header的设置)
|
||||
|
||||
# 模块API配置
|
||||
# Censys可以免费注册获取API：https://censys.io/api
|
||||
censys_api_id = ''
|
||||
|
||||
+5
-14
@@ -37,8 +37,6 @@ enable_partial_module = [] # 启用部分收集模块 必须禁用enable_all_mo
|
||||
# ('modules.search', 'baidu')]
|
||||
|
||||
# 爆破模块设置
|
||||
brute_wildcard_check = True # 开启泛解析检测(默认True)
|
||||
brute_wildcard_deal = True # 开启泛解析处理(默认True)
|
||||
brute_concurrent_num = 2000 # 爆破时并发查询数量(默认2000，最大推荐10000)
|
||||
# 爆破所使用的字典路径 默认data/subdomains.txt
|
||||
brute_wordlist_path = data_storage_dir.joinpath('subnames.txt')
|
||||
@@ -66,25 +64,18 @@ brute_ip_blacklist = {'0.0.0.0', '0.0.0.1'} # IP黑名单 子域解析到IP黑
|
||||
ip_appear_maximum = 100 # 多个子域解析到同一IP次数超过100次则标记为非法(泛解析)子域
|
||||
|
||||
# 代理设置
|
||||
enable_proxy = False # 是否使用代理(全局开关)
|
||||
enable_request_proxy = False # 是否使用代理(全局开关)
|
||||
proxy_all_module = False # 代理所有模块
|
||||
proxy_partial_module = ['GoogleQuery', 'AskSearch', 'DuckDuckGoSearch',
|
||||
'GoogleAPISearch', 'GoogleSearch', 'YahooSearch',
|
||||
'YandexSearch', 'CrossDomainXml',
|
||||
'ContentSecurityPolicy'] # 代理自定义的模块
|
||||
proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
||||
'https': 'https://127.0.0.1:1080'}] # 代理池
|
||||
# proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
||||
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
||||
request_proxy_pool = [{'http': 'http://127.0.0.1:1080',
|
||||
'https': 'https://127.0.0.1:1080'}] # 代理池
|
||||
# request_proxy_pool = [{'http': 'socks5h://127.0.0.1:10808',
|
||||
# 'https': 'socks5h://127.0.0.1:10808'}] # 代理池
|
||||
|
||||
# 搜索模块设置
|
||||
enable_recursive_search = False # 递归搜索子域
|
||||
search_recursive_times = 2 # 递归搜索层数
|
||||
|
||||
# aiohttp有关配置
|
||||
# aiohttp 支持 HTTP/HTTPS形式的代理
|
||||
aiohttp_proxy = None # 示例 proxy="http://user:pass@some.proxy.com"
|
||||
# 为了保证请求质量 请谨慎更改以下设置
|
||||
sockread_timeout = 6 # 每个请求socket读取超时时间，默认6秒
|
||||
sockconn_timeout = 3 # 每个请求socket连接超时时间，默认3秒
|
||||
limit_open_conn = None # 限制同一时间打开的连接总数，默认None将根据系统内存大小自动设置
|
||||
|
||||
+1
-18
@@ -4,7 +4,7 @@
|
||||
[](https://codecov.io/gh/shmilylty/OneForAll)
|
||||
[](https://codeclimate.com/github/shmilylty/OneForAll/maintainability)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/LICENSE)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/tree/master/)
|
||||
[](https://github.com/shmilylty/OneForAll/releases)
|
||||
|
||||
👊**OneForAll is a powerful subdomain integration tool** 📝[中文文档](https://github.com/shmilylty/OneForAll/tree/master/README.md)
|
||||
@@ -15,23 +15,6 @@
|
||||
|
||||
📢 Please read this document to help you start quickly!
|
||||
|
||||
<details>
|
||||
<summary><b>🐍Installation requirements</b></summary>
|
||||
|
||||
OneForAll is developed and tested based on [Python 3.8.0](https://www.python.org/downloads/release/python-380/). Recommend use release higher than Python 3.8.0 (Windows platform must use Python 3.8.0 or later). For more information on installing the Python environment, please read [Python 3 installation Guide](https://pythonguidecn.readthedocs.io/zh/latest/starting/installation.html#python-3).
|
||||
|
||||
After installation python, run the following command to check the Python and pip3 versions:
|
||||
```bash
|
||||
python -V
|
||||
pip3 -V
|
||||
```
|
||||
If you see the following output, there is no problem with the Python environment:
|
||||
```bash
|
||||
Python 3.8.0
|
||||
pip 19.2.2 from C:\Users\shmilylty\AppData\Roaming\Python\Python37\site-packages\pip (python 3.8)
|
||||
```
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>✔Installation steps (for Git)</b></summary>
|
||||
|
||||
|
||||
+4
-3
@@ -190,9 +190,10 @@ def find_subdomains(domain, data):
|
||||
subdomains.update(find_in_resp(domain, req_url, rsp_html))
|
||||
js_urls.update(find_js_urls(domain, req_url, rsp_html))
|
||||
|
||||
resp_data = request.urls_request(js_urls)
|
||||
for resp, text in resp_data:
|
||||
if not text:
|
||||
resp_data = request.bulk_request(js_urls)
|
||||
for resp in resp_data:
|
||||
if not resp:
|
||||
continue
|
||||
text = utils.decode_resp_text(resp)
|
||||
subdomains.update(find_in_resp(domain, resp.url, text))
|
||||
return subdomains
|
||||
|
||||
+1
-8
@@ -1,6 +1,3 @@
|
||||
aiohttp==3.6.2
|
||||
async-timeout==3.0.1
|
||||
attrs==20.1.0
|
||||
beautifulsoup4==4.9.1
|
||||
bs4==0.0.1
|
||||
certifi==2020.6.20
|
||||
@@ -14,12 +11,9 @@ exrex==0.10.5
|
||||
fire==0.3.1
|
||||
future==0.18.2
|
||||
idna==2.10
|
||||
IP2Location==8.4.1
|
||||
jdcal==1.4.1
|
||||
loguru==0.5.1
|
||||
multidict==4.7.6
|
||||
openpyxl==2.4.11
|
||||
psutil==5.7.2
|
||||
pyparsing==2.4.7
|
||||
PySocks==1.7.1
|
||||
records==0.5.3
|
||||
@@ -36,5 +30,4 @@ tldextract==2.2.3
|
||||
tqdm==4.48.2
|
||||
treelib==1.6.1
|
||||
urllib3==1.25.10
|
||||
win32-setctime==1.0.1
|
||||
yarl==1.5.1
|
||||
win32-setctime==1.0.1
|
||||
Reference in New Issue
Block a user