移除不稳定的bufferover和threatcrowd模块

移除cloudscraper依赖

Pipfile

@@ -14,7 +14,6 @@ tldextract = "*"
 exrex = "*"
 fire = "*"
 bs4 = "*"
-cloudscraper = "*"
 tenacity = "*"
 treelib = "*"
 sqlalchemy = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "b73543264fb05b4cf1e331e782b66e43171c7b0cb9157796308c14aa62f11356"
+            "sha256": "7ad607a807779ea8674fd9d46eb4474f11349f766d94d721c23ad2b29a033105"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -45,14 +45,6 @@
             ],
             "version": "==3.0.4"
         },
-        "cloudscraper": {
-            "hashes": [
-                "sha256:793095bbc37aae84a5c38d9f66b56f3c83a00f0b1ff2d334f75a2b8f88b924af",
-                "sha256:a7ae28b5dc4e4ef0789ef608c42a4382bf6d3e3dd6b8117c2e3633103131ae43"
-            ],
-            "index": "pypi",
-            "version": "==1.2.46"
-        },
         "colorama": {
             "hashes": [
                 "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff",
@@ -98,18 +90,11 @@
         },
         "loguru": {
             "hashes": [
-                "sha256:70201d5fce26da89b7a5f168caa2bb674e06b969829f56737db1d6472e53e7c3",
-                "sha256:e5d362a43cd2fc2da63551d79a6830619c4d5b3a8b976515748026f92f351b61"
+                "sha256:5aecbf13bc8e2f6e5a5d0475460a345b44e2885464095ea7de44e8795857ad33",
+                "sha256:a5e5e196b9958feaf534ac2050171d16576bae633074ce3e73af7dda7e9a58ae"
             ],
             "index": "pypi",
-            "version": "==0.5.1"
-        },
-        "pyparsing": {
-            "hashes": [
-                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
-                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
-            ],
-            "version": "==2.4.7"
+            "version": "==0.5.2"
         },
         "pysocks": {
             "hashes": [
@@ -137,13 +122,6 @@
             ],
             "version": "==1.5.1"
         },
-        "requests-toolbelt": {
-            "hashes": [
-                "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f",
-                "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"
-            ],
-            "version": "==0.9.1"
-        },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
@@ -242,11 +220,11 @@
         },
         "win32-setctime": {
             "hashes": [
-                "sha256:568fd636c68350bcc54755213fe01966fe0a6c90b386c0776425944a0382abef",
-                "sha256:b47e5023ec7f0b4962950902b15bc56464a380d869f59d27dbf9ab423b23e8f9"
+                "sha256:02b4c5959ca0b195f45c98115826c6e8a630b7cf648e724feaab1a5aa6250640",
+                "sha256:47aa7c43548c1fc0a4f026d1944b748b37036df116c7c4cf908e82638d854313"
             ],
             "markers": "sys_platform == 'win32'",
-            "version": "==1.0.1"
+            "version": "==1.0.2"
         }
     },
     "develop": {}

modules/datasets/bufferover.py

@@ -1,54 +0,0 @@
-import cloudscraper
-from common.query import Query
-from config.log import logger
-
-
-class BufferOver(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = domain
-        self.module = 'Dataset'
-        self.source = 'BufferOverQuery'
-        self.addr = 'https://dns.bufferover.run/dns?q='
-
-    def query(self):
-        """
-        向接口查询子域并做子域匹配
-        """
-        # 绕过cloudFlare验证
-        scraper = cloudscraper.create_scraper()
-        scraper.proxies = self.get_proxy(self.source)
-        url = self.addr + self.domain
-        try:
-            resp = scraper.get(url, timeout=self.timeout)
-        except Exception as e:
-            logger.log('ERROR', e.args)
-            return
-        if resp.status_code != 200:
-            return
-        self.subdomains = self.collect_subdomains(resp)
-
-    def run(self):
-        """
-        类执行入口
-        """
-        self.begin()
-        self.query()
-        self.finish()
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-
-
-def run(domain):
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    """
-    query = BufferOver(domain)
-    query.run()
-
-
-if __name__ == '__main__':
-    run('example.com')

modules/datasets/threatcrowd.py

@@ -1,54 +0,0 @@
-import cloudscraper
-
-from common.query import Query
-from config.log import logger
-
-
-class ThreatCrowd(Query):
-    def __init__(self, domain):
-        Query.__init__(self)
-        self.domain = domain
-        self.module = 'Dataset'
-        self.source = 'ThreatCrowdQuery'
-        self.addr = 'https://www.threatcrowd.org/searchApi' \
-                    '/v2/domain/report?domain='
-
-    def query(self):
-        # 绕过cloudFlare验证
-        scraper = cloudscraper.create_scraper()
-        scraper.proxies = self.get_proxy(self.source)
-        url = self.addr + self.domain
-        try:
-            resp = scraper.get(url, timeout=self.timeout)
-        except Exception as e:
-            logger.log('ERROR', e.args)
-            return
-        if resp.status_code != 200:
-            return
-        subdomains = self.match_subdomains(resp.text)
-        self.subdomains.update(subdomains)
-
-    def run(self):
-        """
-        类执行入口
-        """
-        self.begin()
-        self.query()
-        self.finish()
-        self.save_json()
-        self.gen_result()
-        self.save_db()
-
-
-def run(domain):
-    """
-    类统一调用入口
-
-    :param str domain: 域名
-    """
-    query = ThreatCrowd(domain)
-    query.run()
-
-
-if __name__ == '__main__':
-    run('mi.com')

requirements.txt

@@ -3,18 +3,15 @@ beautifulsoup4==4.9.1
 bs4==0.0.1
 certifi==2020.6.20
 chardet==3.0.4
-cloudscraper==1.2.46
 colorama==0.4.3 ; sys_platform == 'win32'
 dnspython==2.0.0
 exrex==0.10.5
 fire==0.3.1
 future==0.18.2
 idna==2.10
-loguru==0.5.1
-pyparsing==2.4.7
+loguru==0.5.2
 pysocks==1.7.1
 requests-file==1.5.1
-requests-toolbelt==0.9.1
 requests[socks]==2.24.0
 six==1.15.0
 soupsieve==2.0.1
@@ -25,4 +22,4 @@ tldextract==2.2.3
 tqdm==4.48.2
 treelib==1.6.1
 urllib3==1.25.10
-win32-setctime==1.0.1 ; sys_platform == 'win32'
+win32-setctime==1.0.2 ; sys_platform == 'win32'