Merge remote-tracking branch 'origin/master'

# Conflicts:
#	common/request.py

README.md

@@ -95,9 +95,9 @@ pipenv run python oneforall.py --target example.com run
 ```
 3. 开启爆破模块运行（使用massdns进行爆破，网络占用极大，可能会阻塞网络）
 ```bash
-python3 run python oneforall.py --target example.com --burte True run
+python3 run python oneforall.py --target example.com --brute True run
 # or
-pipenv run python oneforall.py --target example.com --burte True run
+pipenv run python oneforall.py --target example.com --brute True run
 ```
 </details>
 

common/request.py

@@ -113,6 +113,14 @@ def get_jump_urls(history):
     return urls
 
 
+def get_progress_bar(total):
+    bar = tqdm.tqdm()
+    bar.total = total
+    bar.desc = 'Request Progress'
+    bar.ncols = 80
+    return bar
+
+
 def get(url, resp_queue, session):
     timeout = settings.request_timeout_second
     redirect = settings.request_allow_redirect
@@ -132,16 +140,13 @@ def request(urls_queue, resp_queue, session):
         urls_queue.task_done()
 
 
-def progress(urls, resp_queue):
-    bar = tqdm.tqdm()
-    bar.total = len(urls)
-    bar.desc = 'Request Progress'
-    bar.ncols = 80
+def progress(bar, total, urls_queue):
     while True:
-        done = resp_queue.qsize()
+        remaining = urls_queue.qsize()
+        done = total - remaining
         bar.n = done
         bar.update()
-        if done == bar.total:
+        if remaining == 0:
             break
 
 
@@ -163,17 +168,16 @@ def bulk_request(urls):
     resp_queue = Queue()
     for url in urls:
         urls_queue.put(url)
+    total = len(urls)
     session = get_session()
     thread_count = req_thread_count()
+    bar = get_progress_bar(total)
 
-    progress_thread = Thread(target=progress, name='ProgressThread',
-                             args=(urls, resp_queue), daemon=True)
+    progress_thread = Thread(target=progress, args=(bar, total, urls_queue))
     progress_thread.start()
 
-    for i in range(thread_count):
-        request_thread = Thread(target=request, name=f'RequestThread-{i}',
-                                args=(urls_queue, resp_queue, session),
-                                daemon=True)
+    for _ in range(thread_count):
+        request_thread = Thread(target=request, args=(urls_queue, resp_queue, session))
         request_thread.start()
 
     urls_queue.join()

common/utils.py

@@ -719,11 +719,17 @@ def get_url_resp(url):
 
 
 def decode_resp_text(resp):
+    content = resp.content
+    if not content:
+        return str('')
     try:
-        text = resp.text(encoding='utf-8', errors='strict')  # 先尝试用utf-8严格解码
-    except UnicodeError:
+        # 先尝试用utf-8严格解码
+        content = str(content, encoding='utf-8', errors='strict')
+    except (LookupError, TypeError, UnicodeError):
         try:
-            text = resp.text(encoding='gb18030', errors='strict')  # 再尝试用gb18030严格解码
-        except UnicodeError:
-            text = resp.text(encoding=None, errors='ignore')  # 最后尝试自动解码
-    return text
+            # 再尝试用gb18030严格解码
+            content = str(content, encoding='gb18030', errors='strict')
+        except (LookupError, TypeError, UnicodeError):
+            # 最后尝试自动解码
+            content = str(content,  errors='replace')
+    return content

takeover.py

@@ -83,14 +83,14 @@ class Takeover(Module):
         utils.save_data(self.path, data)
 
     def compare(self, subdomain, cname, responses):
-        domain_resp = self.get('http://' + subdomain, check=False)
-        cname_resp = self.get('http://' + cname, check=False)
+        domain_resp = self.get('http://' + subdomain, check=False, ignore=True)
+        cname_resp = self.get('http://' + cname, check=False, ignore=True)
         if domain_resp is None or cname_resp is None:
             return
 
         for resp in responses:
             if resp in domain_resp.text and resp in cname_resp.text:
-                logger.log('ALERT', f'{subdomain}Subdomain takeover threat found')
+                logger.log('ALERT', f'{subdomain} takeover threat found')
                 self.results.append([subdomain, cname])
                 break
 
@@ -142,13 +142,13 @@ class Takeover(Module):
             # 创建待检查的子域队列
             for domain in self.subdomains:
                 self.subdomainq.put(domain)
+            # 进度线程
+            progress_thread = Thread(target=self.progress)
+            progress_thread.start()
             # 检查线程
             for _ in range(self.thread):
-                check_thread = Thread(target=self.check, daemon=True)
+                check_thread = Thread(target=self.check)
                 check_thread.start()
-            # 进度线程
-            progress_thread = Thread(target=self.progress, daemon=True)
-            progress_thread.start()
 
             self.subdomainq.join()
             self.save()
@@ -164,4 +164,3 @@ class Takeover(Module):
 
 if __name__ == '__main__':
     fire.Fire(Takeover)
-

test.py

@@ -8,7 +8,7 @@ Example
 from oneforall import OneForAll
 
 if __name__ == '__main__':
-    test = OneForAll(target='mi.com')
+    test = OneForAll(target='freebuf.com')
     test.brute = True
     test.req = True
     test.takeover = True