改用自带的html.parser

Pipfile

@@ -17,7 +17,6 @@ exrex = "*"
 aiohttp = "*"
 fire = "*"
 bs4 = "*"
-lxml = "*"
 pysocks = "*"
 cloudscraper = "*"
 js2py = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "80401fb48ed5ae18b53fd9283b236ccf9f40f324624b0d191a5e2eebb80c23db"
+            "sha256": "d61abc7e1e2904735bf756b2efa14cce255f158d0f961a5ef00f42f775e1fb43"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -83,11 +83,11 @@
                 "sha256:9d1c2dd27a1083fefd05b1b2f8df4a6bc2aaa6c21dd82cd41c8ae5e7c23a87f8",
                 "sha256:a13ce9b419fe9f277c63f700efb0e444331509d1881b5610d2ba7e9080606967",
                 "sha256:a19ef0952b9d2803df88dff07f45a6c92d5676afb9b8d69cf32232d684036d11",
+                "sha256:a44d41e7a80cc832f04f4c1577a67bd311e28a6e40702a258590d8949a204d9d",
                 "sha256:ad766ca8b8c1419b71a22756b45264f45725c86133dc80a7cbe30b6b78c75620",
                 "sha256:ad7963f261988ee0883816b6b9f206f11461c9b3cb5cfbca0c9ab5adc406d395",
                 "sha256:af0451e23016631a2f52925a10d738ac4a0f794ac315c30380b22efc0c90cbc6",
                 "sha256:c16201060c5a3f8742e3deae759014251ac92f382f82bc2a41dc079ff18c3f24",
-                "sha256:c43b202f65891861a9a336984a103de25de235f756de69e32db893156f767013",
                 "sha256:c675c6cce4295cb1a692f3de7416aacace7314e064b94bc86e93aceefce7fd3e",
                 "sha256:d17cec0b992b1434f5f9df9986563605a4d1b1acd5574c87fc2ac014bcbd3316",
                 "sha256:dc91f6129953861a73d9a65c52a8dd682b561a9ebaf65283541645cab6489917",
@@ -125,11 +125,11 @@
         },
         "cloudscraper": {
             "hashes": [
-                "sha256:2bcd017740c7dc0b8d64b55742638a50960ba82b9fd75fc38b9e94c5cd641aa5",
-                "sha256:74618cfd55841dd0df959a164ee30b9456ecaddcbc7215cc13f89e7a0e3cadf0"
+                "sha256:478d3cf2044d6b70a8e4f436afff4a8f74cb730cc34841cc9411c80d41d5e9e3",
+                "sha256:50a131a9bfc1969b01d48eecb4ecc1acd92ee992be31c0bee4f1f31ac3cbae8f"
             ],
             "index": "pypi",
-            "version": "==1.2.33"
+            "version": "==1.2.34"
         },
         "colorama": {
             "hashes": [
@@ -203,39 +203,6 @@
             "index": "pypi",
             "version": "==0.4.1"
         },
-        "lxml": {
-            "hashes": [
-                "sha256:06d4e0bbb1d62e38ae6118406d7cdb4693a3fa34ee3762238bcb96c9e36a93cd",
-                "sha256:0701f7965903a1c3f6f09328c1278ac0eee8f56f244e66af79cb224b7ef3801c",
-                "sha256:1f2c4ec372bf1c4a2c7e4bb20845e8bcf8050365189d86806bad1e3ae473d081",
-                "sha256:4235bc124fdcf611d02047d7034164897ade13046bda967768836629bc62784f",
-                "sha256:5828c7f3e615f3975d48f40d4fe66e8a7b25f16b5e5705ffe1d22e43fb1f6261",
-                "sha256:585c0869f75577ac7a8ff38d08f7aac9033da2c41c11352ebf86a04652758b7a",
-                "sha256:5d467ce9c5d35b3bcc7172c06320dddb275fea6ac2037f72f0a4d7472035cea9",
-                "sha256:63dbc21efd7e822c11d5ddbedbbb08cd11a41e0032e382a0fd59b0b08e405a3a",
-                "sha256:7bc1b221e7867f2e7ff1933165c0cec7153dce93d0cdba6554b42a8beb687bdb",
-                "sha256:8620ce80f50d023d414183bf90cc2576c2837b88e00bea3f33ad2630133bbb60",
-                "sha256:8a0ebda56ebca1a83eb2d1ac266649b80af8dd4b4a3502b2c1e09ac2f88fe128",
-                "sha256:90ed0e36455a81b25b7034038e40880189169c308a3df360861ad74da7b68c1a",
-                "sha256:95e67224815ef86924fbc2b71a9dbd1f7262384bca4bc4793645794ac4200717",
-                "sha256:afdb34b715daf814d1abea0317b6d672476b498472f1e5aacbadc34ebbc26e89",
-                "sha256:b4b2c63cc7963aedd08a5f5a454c9f67251b1ac9e22fd9d72836206c42dc2a72",
-                "sha256:d068f55bda3c2c3fcaec24bd083d9e2eede32c583faf084d6e4b9daaea77dde8",
-                "sha256:d5b3c4b7edd2e770375a01139be11307f04341ec709cf724e0f26ebb1eef12c3",
-                "sha256:deadf4df349d1dcd7b2853a2c8796593cc346600726eff680ed8ed11812382a7",
-                "sha256:df533af6f88080419c5a604d0d63b2c33b1c0c4409aba7d0cb6de305147ea8c8",
-                "sha256:e4aa948eb15018a657702fee0b9db47e908491c64d36b4a90f59a64741516e77",
-                "sha256:e5d842c73e4ef6ed8c1bd77806bf84a7cb535f9c0cf9b2c74d02ebda310070e1",
-                "sha256:ebec08091a22c2be870890913bdadd86fcd8e9f0f22bcb398abd3af914690c15",
-                "sha256:edc15fcfd77395e24543be48871c251f38132bb834d9fdfdad756adb6ea37679",
-                "sha256:f2b74784ed7e0bc2d02bd53e48ad6ba523c9b36c194260b7a5045071abbb1012",
-                "sha256:fa071559f14bd1e92077b1b5f6c22cf09756c6de7139370249eb372854ce51e6",
-                "sha256:fd52e796fee7171c4361d441796b64df1acfceb51f29e545e812f16d023c4bbc",
-                "sha256:fe976a0f1ef09b3638778024ab9fb8cde3118f203364212c198f71341c0715ca"
-            ],
-            "index": "pypi",
-            "version": "==4.5.0"
-        },
         "multidict": {
             "hashes": [
                 "sha256:317f96bc0950d249e96d8d29ab556d01dd38888fbe68324f46fd834b430169f1",

oneforall/common/request.py

@@ -122,7 +122,7 @@ def get_title(markup):
     :param markup: html标签
     :return: 标题
     """
-    soup = BeautifulSoup(markup, 'lxml')
+    soup = BeautifulSoup(markup, 'html.parser')
 
     title = soup.title
     if title:

oneforall/modules/search/baidu.py

@@ -21,7 +21,7 @@ class Baidu(Search):
         :param html: 响应体
         :return: 子域
         """
-        bs = BeautifulSoup(html, features='lxml')
+        bs = BeautifulSoup(html, 'html.parser')
         subdomains_all = set()
         # 获取搜索结果中所有的跳转URL地址
         for find_res in bs.find_all('a', {'class': 'c-showurl'}):

oneforall/modules/search/gitee.py

@@ -33,7 +33,7 @@ class Gitee(Search):
                 break
             if 'class="empty-box"' in resp.text:
                 break
-            soup = BeautifulSoup(resp.text, 'lxml')
+            soup = BeautifulSoup(resp.text, 'html.parser')
             subdomains = self.match(self.domain, soup.text)
             self.subdomains = self.subdomains.union(subdomains)
             if not subdomains:

requirements.txt

@@ -18,7 +18,6 @@ idna==2.9
 jdcal==1.4.1
 Js2Py==0.70
 loguru==0.4.1
-lxml==4.5.0
 multidict==4.7.5
 openpyxl==2.4.11
 psutil==5.7.0