更新依赖

Pipfile.lock

@@ -57,11 +57,11 @@
         },
         "beautifulsoup4": {
             "hashes": [
-                "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169",
+                "sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a",
-                "sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931",
+                "sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887",
-                "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57"
+                "sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae"
             ],
-            "version": "==4.8.1"
+            "version": "==4.8.2"
         },
         "brotli": {
             "hashes": [
@@ -125,11 +125,11 @@
         },
         "cloudscraper": {
             "hashes": [
-                "sha256:5d809b88f163859b035fa5e0b75956eec31c40b5d187afa44ab02afd99c8422a",
+                "sha256:28da1c49fefe4626debd145b1900d8fdd04de3f6e0474c15101647bc37537563",
-                "sha256:90cb7f59e3f0e7e49064a00307c54ea6e0319afdd95c9d1e76f8c531c85b5e91"
+                "sha256:a3d1e2fb4a57d2041fa59fc5db9b9c9481c5cbae132d54019ec838ccb33ff1cc"
             ],
             "index": "pypi",
-            "version": "==1.2.16"
+            "version": "==1.2.18"
         },
         "colorama": {
             "hashes": [
@@ -196,11 +196,11 @@
         },
         "js2py": {
             "hashes": [
-                "sha256:6e5628abfff2fb4051e8e77a353e44831f474e2ceb865278271897f7f326aeb6",
+                "sha256:168952e3827198f68eacbf84a7b23c80a55794415291f821ec6c96f3fd9c3253",
-                "sha256:bf87cb4432944470f11fed9c1cb8d0312dd505e7b867362f55102f24379ab94f"
+                "sha256:6c45cce8eb66b16f8dc692c6e6a2d509be9544b63cfb386eadf7309a931feb62"
             ],
             "index": "pypi",
-            "version": "==0.66"
+            "version": "==0.67"
         },
         "loguru": {
             "hashes": [
@@ -250,25 +250,25 @@
         },
         "multidict": {
             "hashes": [
-                "sha256:20081b14c923d2c5122c13e060d0ee334e078e1802c36006b20c8d7a59ee6a52",
+                "sha256:0f04bf4c15d8417401a10a650c349ccc0285943681bfd87d3690587d7714a9b4",
-                "sha256:335344a3c3b19845c73a7826f359c51c4a12a1ccd2392b5f572a63b452bfc771",
+                "sha256:15a61c0df2d32487e06f6084eabb48fd9e8b848315e397781a70caf9670c9d78",
-                "sha256:49e80c53659c7ac50ec1c4b5fa50f045b67fffeb5b735dccb6205e4ff122e8b6",
+                "sha256:3c5e2dcbe6b04cbb4303e47a896757a77b676c5e5db5528be7ff92f97ba7ab95",
-                "sha256:615a282acd530a1bc1b01f069a8c5874cb7c2780c287a2895ad5ab7407540e9d",
+                "sha256:5d2b32b890d9e933d3ced417924261802a857abdee9507b68c75014482145c03",
-                "sha256:63d9a3d93a514549760cb68c82864966bddb6ab53a3326931c8db9ad29414603",
+                "sha256:5e5fb8bfebf87f2e210306bf9dd8de2f1af6782b8b78e814060ae9254ab1f297",
-                "sha256:77264002c184538df5dcb4fc1de5df6803587fa30bbe12203a7a3436b8aafc0f",
+                "sha256:63ba2be08d82ea2aa8b0f7942a74af4908664d26cb4ff60c58eadb1e33e7da00",
-                "sha256:7dd6f6a51b64d0a6473bc30c53e1d73fcb461c437f43662b7d6d701bd90db253",
+                "sha256:73740fcdb38f0adcec85e97db7557615b50ec4e5a3e73e35878720bcee963382",
-                "sha256:7f4e591ec80619e74c50b7800f9db9b0e01d2099094767050dfe2e78e1c41839",
+                "sha256:78bed18e7f1eb21f3d10ff3acde900b4d630098648fe1d65bb4abfb3e22c4900",
-                "sha256:824716bba5a4efd74ddd36a3830efb9e49860149514ef7c41aac0144757ebb5d",
+                "sha256:a02fade7b5476c4f88efe9593ff2f3286698d8c6d715ba4f426954f73f382026",
-                "sha256:8f30ead697c2e37147d82ba8019952b5ea99bd3d1052f1f1ebff951eaa953209",
+                "sha256:aacbde3a8875352a640efa2d1b96e5244a29b0f8df79cbf1ec6470e86fd84697",
-                "sha256:a03efe8b7591c77d9ad4b9d81dcfb9c96e538ae25eb114385f35f4d7ffa3bac2",
+                "sha256:be813fb9e5ce41a5a99a29cdb857144a1bd6670883586f995b940a4878dc5238",
-                "sha256:b86e8e33a0a24240b293e7fad233a7e886bae6e51ca6923d39f4e313dd1d5578",
+                "sha256:bfcad6da0b8839f01a819602aaa5c5a5b4c85ecbfae9b261a31df3d9262fb31e",
-                "sha256:c1c64c93b8754a5cebd495d136f47a5ca93cbfceba532e306a768c27a0c1292b",
+                "sha256:c2bfc0db3166e68515bc4a2b9164f4f75ae9c793e9635f8651f2c9ffc65c8dad",
-                "sha256:d4dafdcfbf0ac80fc5f00603f0ce43e487c654ae34a656e4749f175d9832b1b5",
+                "sha256:c66d11870ae066499a3541963e6ce18512ca827c2aaeaa2f4e37501cee39ac5d",
-                "sha256:daf6d89e47418e38af98e1f2beb3fe0c8aa34806f681d04df314c0f131dcf01d",
+                "sha256:cc7f2202b753f880c2e4123f9aacfdb94560ba893e692d24af271dac41f8b8d9",
-                "sha256:e03b7addca96b9eb24d6eabbdc3041e8f71fd47b316e0f3c0fa993fc7b99002c",
+                "sha256:d1f45e5bb126662ba66ee579831ce8837b1fd978115c9657e32eb3c75b92973d",
-                "sha256:ff53a434890a16356bc45c0b90557efd89d0e5a820dbab37015d7ee630c6707a"
+                "sha256:ed5f3378c102257df9e2dc9ce6468dabf68bee9ec34969cfdc472631aba00316"
             ],
-            "version": "==4.7.2"
+            "version": "==4.7.3"
         },
         "odfpy": {
             "hashes": [
@@ -396,11 +396,11 @@
         },
         "tqdm": {
             "hashes": [
-                "sha256:166a82cdea964ae45528e0cc89436255ff2be73dc848bdf239f13c501cae5dc7",
+                "sha256:4789ccbb6fc122b5a6a85d512e4e41fc5acad77216533a6f2b8ce51e0f265c23",
-                "sha256:9036904496bd2afacf836a6f206c5a766ce11d3e9319d54a4e794c0f34b111dc"
+                "sha256:efab950cf7cc1e4d8ee50b2bb9c8e4a89f8307b49e0b2c9cfef3ec4ca26655eb"
             ],
             "index": "pypi",
-            "version": "==4.41.0"
+            "version": "==4.41.1"
         },
         "tzlocal": {
             "hashes": [

oneforall/aiobrute.py

@@ -13,7 +13,8 @@ import queue
 import signal
 import asyncio
 import secrets
-
+import functools
+from multiprocessing import Manager
 import aiomultiprocess as aiomp
 import exrex
 import fire
@@ -60,7 +61,7 @@ def detect_wildcard(domain):
 
 def wildcard_by_compare(ips, ttl, wildcard_ips, wildcard_ttl):
     """
-    通过与泛解析返回的IP集合和判TTL值进行对比判断发现的子域是否是泛解析子域
+    通过与泛解析返回的IP集合和返回的TTL值进行对比判断发现的子域是否是泛解析子域
 
     :param set ips: 子域A记录查询出的IP集合
     :param int ttl: 子域A记录查询出的TTL整型值
@@ -130,6 +131,25 @@ def gen_brute_domains(domain, path):
     return domains
 
 
+def progress(pr_queue, total):
+    bar = tqdm.tqdm()
+    bar.total = total
+    bar.desc = 'Progress'
+    bar.ncols = 60
+    while True:
+        done = pr_queue.qsize()
+        bar.n = done
+        bar.update()
+        if done == total:
+            return
+
+
+async def aiodns_query_a(pr_queue, hostname):
+    results = await resolve.aiodns_query_a(hostname)
+    pr_queue.put(1)
+    return results
+
+
 class AIOBrute(Module):
     """
     OneForAll多进程多协程异步子域爆破模块
@@ -210,42 +230,38 @@ class AIOBrute(Module):
             logger.log('INFOR', f'使用{self.wordlist}字典')
             domains = gen_brute_domains(domain, self.wordlist)
         domains = list(domains)
-        return utils.split_list(domains, self.segment)  # 分割任务组
+        return domains
+        # return utils.split_list(domains, self.segment)  # 分割任务组
 
     def deal_results(self, results):
-        for answer in results:
+        for result in results:
+            hostname, answer = result
             if answer is None:
                 continue
             if isinstance(answer, Exception):
                 # logger.log('DEBUG', f'爆破{subdomain}时出错 {str(answers)}')
                 continue
-            ips = {item.address for item in answer}
+            name, alias, ips = answer
             # 取值 如果是首次出现的IP集合 出现次数先赋值0
             value = self.ips_times.setdefault(str(ips), 0)
             self.ips_times[str(ips)] = value + 1
-            ttl = answer.rrset.ttl
-            subdomain = str(answer.rrset.name)
             # 目前域名开启了泛解析
             if self.enable_wildcard:
                 # 通过对比查询的子域和响应的子域来判断真实子域
                 # 去掉解析到CDN的情况
-                if not subdomain.endswith(self.domain + '.'):
+                if 'cdn' in name:
                     continue
-                # 通过对比解析到的IP集合和TTL确定子域来判断真实子域
+                if not name.endswith(self.domain + '.'):
-                if wildcard_by_compare(ips,
-                                       ttl,
-                                       self.wildcard_ips,
-                                       self.wildcard_ttl):
                     continue
                 # 通过对比解析到的IP集合的次数来判断真实子域
                 if wildcard_by_times(ips, self.ips_times):
                     continue
             # 只添加没有出现过的子域
-            if subdomain not in self.subdomains:
+            if hostname not in self.subdomains:
-                logger.log('INFOR', f'发现{self.domain}的子域: {subdomain} '
+                logger.log('INFOR', f'发现{self.domain}的子域: {hostname} '
-                                    f'解析IP: {ips} TTL: {ttl}')
+                                    f'解析到: {name} IP: {ips}')
-                self.subdomains.add(subdomain)
+                self.subdomains.add(hostname)
-                self.records[subdomain] = str(ips)
+                self.records[hostname] = str(ips)
 
     async def main(self, domain, rx_queue):
         if not self.fuzz:  # fuzz模式不探测域名是否使用泛解析
@@ -253,13 +269,18 @@ class AIOBrute(Module):
                 = detect_wildcard(domain)
         tasks = self.gen_tasks(domain)
         logger.log('INFOR', f'正在爆破{domain}的域名')
-        for task in tqdm.tqdm(tasks, total=len(tasks),
+        # for task in tqdm.tqdm(tasks, total=len(tasks),
-                              desc='Progress'):
+        #                       desc='Progress'):
+        m = Manager()
+        pr_queue = m.Queue()
+        loop = asyncio.get_event_loop()
+        loop.run_in_executor(None, progress, pr_queue, len(tasks))
+        wrapped_query = functools.partial(aiodns_query_a, pr_queue)
         async with aiomp.Pool(processes=self.process,
                               initializer=init_worker,
                               childconcurrency=self.coroutine) as pool:
             try:
-                    results = await pool.map(resolve.dns_query_a, task)
+                results = await pool.map(wrapped_query, tasks)
             except KeyboardInterrupt:
                 logger.log('ALERT', '爆破终止正在退出')
                 pool.terminate()  # 关闭pool，结束工作进程，不在处理未完成的任务。

requirements.txt

@@ -2,12 +2,12 @@ aiohttp==3.6.2
 aiomultiprocess==0.7.0
 async-timeout==3.0.1
 attrs==19.3.0
-beautifulsoup4==4.8.1
+beautifulsoup4==4.8.2
 Brotli==1.0.7
 bs4==0.0.1
 certifi==2019.11.28
 chardet==3.0.4
-cloudscraper==1.2.16
+cloudscraper==1.2.18
 colorama==0.4.3
 defusedxml==0.6.0
 dnspython==1.16.0
@@ -17,11 +17,11 @@ exrex==0.10.5
 fire==0.2.1
 idna==2.8
 jdcal==1.4.1
-Js2Py==0.66
+Js2Py==0.67
 loguru==0.4.0
 lxml==4.4.2
 MarkupPy==1.14
-multidict==4.7.2
+multidict==4.7.3
 odfpy==1.4.0
 openpyxl==2.4.11
 pyjsparser==2.7.1
@@ -38,7 +38,7 @@ SQLAlchemy==1.3.12
 tablib==0.14.0
 termcolor==1.1.0
 tldextract==2.2.2
-tqdm==4.41.0
+tqdm==4.41.1
 tzlocal==2.0.0
 urllib3==1.25.7
 win32-setctime==1.0.1