优化代码结构 删除无用队列结果集参数

oneforall/modules/crawl/archivecrawl.py

@@ -1,6 +1,6 @@
 # coding=utf-8
-import queue
 import cdx_toolkit
+
 from common.crawl import Crawl
 from config import logger
 
@@ -31,7 +31,7 @@ class ArchiveCrawl(Crawl):
                 subdomains_find = self.match(self.register(domain), url + resp.text)
                 self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
 
-    def run(self, rx_queue):
+    def run(self):
         """
         类执行入口
         """
@@ -44,23 +44,21 @@ class ArchiveCrawl(Crawl):
         self.save_json()
         self.gen_result()
         self.save_db()
-        rx_queue.put(self.results)
         self.finish()
 
 
-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
     """
     类统一调用入口
 
     :param str domain: 域名
-    :param rx_queue: 结果集队列
     """
     crawl = ArchiveCrawl(domain)
-    crawl.run(rx_queue)
+    crawl.run()
     logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
     logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
 
 
 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+
+    do('example.com')

oneforall/modules/crawl/commoncrawl.py

@@ -1,5 +1,4 @@
 # coding=utf-8
-import queue
 
 import cdx_toolkit
 from tqdm import tqdm
@@ -33,7 +32,7 @@ class CommonCrawl(Crawl):
                 subdomains_find = self.match(self.register(domain), resp.text)
                 self.subdomains = self.subdomains.union(subdomains_find)  # 合并搜索子域名搜索结果
 
-    def run(self, rx_queue):
+    def run(self):
         """
         类执行入口
         """
@@ -46,23 +45,21 @@ class CommonCrawl(Crawl):
         self.save_json()
         self.gen_result()
         self.save_db()
-        rx_queue.put(self.results)
         self.finish()
 
 
-def do(domain, rx_queue):  # 统一入口名字 方便多线程调用
+def do(domain):  # 统一入口名字 方便多线程调用
     """
     类统一调用入口
 
     :param str domain: 域名
-    :param rx_queue: 结果集队列
     """
     crawl = CommonCrawl(domain)
-    crawl.run(rx_queue)
+    crawl.run()
     logger.log('INFOR', f'{crawl.source}模块耗时{crawl.elapsed}秒发现{crawl.domain}的子域{len(crawl.subdomains)}个')
     logger.log('DEBUG', f'{crawl.source}模块发现{crawl.domain}的子域 {crawl.subdomains}')
 
 
 if __name__ == '__main__':
-    result_queue = queue.Queue()
-    do('example.com', result_queue)
+
+    do('example.com')