From 2dbb5a1097a58f877a36b136b6f2712072037b18 Mon Sep 17 00:00:00 2001 From: shmilylty Date: Sun, 20 Oct 2019 23:42:24 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=8D=E4=BD=BF=E7=94=A8aiodns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- oneforall/common/request.py | 30 +++++++++++++----------------- oneforall/common/resolve.py | 27 +++++++++++++++------------ oneforall/common/utils.py | 2 +- oneforall/config.py | 6 +++--- oneforall/modules/dnsquery/srv.py | 11 ++++++----- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/oneforall/common/request.py b/oneforall/common/request.py index c6cab42..4ff8061 100644 --- a/oneforall/common/request.py +++ b/oneforall/common/request.py @@ -1,5 +1,3 @@ -# coding=utf-8 - import asyncio import functools @@ -62,28 +60,26 @@ def gen_new_datas(datas, ports): return new_datas -async def fetch(session, url, semaphore): +async def fetch(session, url): """ 请求 :param session: session对象 :param url: url地址 - :param semaphore: 并发信号量 :return: 响应对象和响应文本 """ timeout = aiohttp.ClientTimeout(total=config.get_timeout) - async with semaphore: - async with session.get(url, - ssl=config.verify_ssl, - allow_redirects=config.get_redirects, - timeout=timeout, - proxy=config.get_proxy) as resp: + async with session.get(url, + ssl=config.verify_ssl, + allow_redirects=config.get_redirects, + timeout=timeout, + proxy=config.get_proxy) as resp: - try: - text = await resp.text(encoding='gb2312') # 先尝试用fb2312解码 - except UnicodeDecodeError: - text = await resp.text(errors='ignore') - return resp, text + try: + text = await resp.text(encoding='gb2312') # 先尝试用fb2312解码 + except UnicodeDecodeError: + text = await resp.text(errors='ignore') + return resp, text def get_title(markup): @@ -159,7 +155,7 @@ async def bulk_get_request(datas, port): limit=limit_open_conn, limit_per_host=config.limit_per_host) - semaphore = asyncio.Semaphore(limit_open_conn) + # semaphore = asyncio.Semaphore(limit_open_conn) header = None if config.fake_header: header = utils.gen_fake_header() @@ -167,7 +163,7 @@ async def bulk_get_request(datas, port): tasks = [] for i, data in enumerate(new_datas): url = data.get('url') - task = asyncio.ensure_future(fetch(session, url, semaphore)) + task = asyncio.ensure_future(fetch(session, url)) task.add_done_callback(functools.partial(request_callback, index=i, datas=new_datas)) diff --git a/oneforall/common/resolve.py b/oneforall/common/resolve.py index 647d587..a320cf9 100644 --- a/oneforall/common/resolve.py +++ b/oneforall/common/resolve.py @@ -22,7 +22,7 @@ def dns_resolver(): return resolver -def dns_query_a(hostname): +async def dns_query_a(hostname): """ 查询A记录 @@ -30,7 +30,12 @@ def dns_query_a(hostname): :return: 查询结果 """ resolver = dns_resolver() - return resolver.query(hostname, 'A') + try: + answer = resolver.query(hostname, 'A') + except BaseException as e: + logger.log('DEBUG', e.args) + answer = None + return answer def aiodns_resolver(): @@ -76,18 +81,16 @@ def resolve_callback(future, index, datas): :param datas: 结果集 """ try: - result = future.result() + answer = future.result() except BaseException as e: datas[index]['ips'] = str(e.args) datas[index]['valid'] = 0 else: - if isinstance(result, tuple): - _, answers = result - if answers: - ips = {record.host for record in answers} - datas[index]['ips'] = str(ips) - else: - datas[index]['ips'] = 'No answers' + if answer: + ips = {item.address for item in answer} + datas[index]['ips'] = str(ips) + else: + datas[index]['ips'] = 'No answers' async def bulk_query_a(datas): @@ -99,11 +102,11 @@ async def bulk_query_a(datas): """ logger.log('INFOR', '正在异步查询子域的A记录') tasks = [] - semaphore = asyncio.Semaphore(config.limit_resolve_conn) + # semaphore = asyncio.Semaphore(config.limit_resolve_conn) for i, data in enumerate(datas): if not data.get('ips'): subdomain = data.get('subdomain') - task = asyncio.ensure_future(aiodns_query_a(subdomain, semaphore)) + task = asyncio.ensure_future(dns_query_a(subdomain)) task.add_done_callback(functools.partial(resolve_callback, index=i, datas=datas)) # 回调 diff --git a/oneforall/common/utils.py b/oneforall/common/utils.py index 8706b0b..d46f95e 100644 --- a/oneforall/common/utils.py +++ b/oneforall/common/utils.py @@ -137,7 +137,7 @@ def get_semaphore(): """ system = platform.system() if system == 'Windows': - return 300 + return 800 elif system == 'Linux': return 800 elif system == 'Darwin': diff --git a/oneforall/config.py b/oneforall/config.py index 60aaf5f..2492721 100644 --- a/oneforall/config.py +++ b/oneforall/config.py @@ -31,7 +31,7 @@ enable_verify_subdomain = True # 验证子域有效性(默认True) enable_wildcard_check = True # 开启泛解析检测 会去掉泛解析的子域 # 爆破时使用的进程数(根据系统中CPU数量情况设置 不宜大于CPU数量 默认为系统中的CPU数量) brute_process_num = os.cpu_count() -brute_coroutine_num = 128 # 爆破时每个进程下的协程数(不宜大于1000) +brute_coroutine_num = 64 # 爆破时每个进程下的协程数(不宜大于500) # 爆破所使用的字典路径 默认data/subdomains.txt brute_wordlist_path = data_storage_path.joinpath('subnames.txt') brute_task_segment = 500 @@ -82,7 +82,7 @@ resolver_nameservers = [ ] # 指定查询的DNS域名服务器 resolver_timeout = 5.0 # 解析超时时间 resolver_lifetime = 30.0 # 解析存活时间 -limit_resolve_conn = 50 # 限制同一时间解析的数量(默认50) +limit_resolve_conn = 500 # 限制同一时间解析的数量(默认500) # http探测设置 small_ports = {80, 443} @@ -106,7 +106,7 @@ get_timeout = 120 # http请求探测总超时时间 None或者0则表示不检 get_redirects = True # 允许请求跳转 fake_header = True # 使用伪造请求头 # 限制同一时间打开的连接数(默认None，根据系统不同设置，Windows系统400 其他系统800) -limit_open_conn = None +limit_open_conn = 0 # 限制同一时间在同一个端点((host, port, is_ssl) 3者都一样的情况)打开的连接数 limit_per_host = 0 # 默认0表示不限制 diff --git a/oneforall/modules/dnsquery/srv.py b/oneforall/modules/dnsquery/srv.py index a032e18..b623e24 100644 --- a/oneforall/modules/dnsquery/srv.py +++ b/oneforall/modules/dnsquery/srv.py @@ -50,14 +50,15 @@ class BruteSRV(Module): tasks = [] for name in names: tasks.append(self.query(name)) - loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) group = asyncio.gather(*tasks) results = loop.run_until_complete(group) - for result in results: - if result is None: + for answer in results: + if answer is None: continue - for answer in result: - subdomains = utils.match_subdomain(self.domain, str(answer)) + for item in answer: + subdomains = utils.match_subdomain(self.domain, str(item)) self.subdomains = self.subdomains.union(subdomains) if not len(self.subdomains): logger.log('DEBUG', f'没有找到{self.domain}的SRV记录')