移除fake-useragent依赖

Pipfile

@@ -17,7 +17,6 @@ tldextract = "*"
 exrex = "*"
 aiohttp = "*"
 fire = "==0.2.1"
-fake-useragent = "*"
 bs4 = "*"
 cchardet = "*"
 lxml = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "cf68a50128345e403c4d2c25c59cf22490b83db80fc98da10dc41f515972e37e"
+            "sha256": "d95e37615ca48fe39a66e8c24eff76910c66b01b0739c0ab00ffa0f4df022107"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -213,13 +213,6 @@
             "index": "pypi",
             "version": "==0.10.5"
         },
-        "fake-useragent": {
-            "hashes": [
-                "sha256:c104998b750eb097eefc28ae28e92d66397598d2cf41a31aa45d5559ef1adf35"
-            ],
-            "index": "pypi",
-            "version": "==0.1.11"
-        },
         "fire": {
             "hashes": [
                 "sha256:6865fefc6981a713d2ce56a2a2c92c56c729269f74a6cddd6f4b94d16ae084c9"

oneforall/common/request.py

@@ -57,7 +57,8 @@ async def fetch(session, url, semaphore):
     """
     timeout = aiohttp.ClientTimeout(total=config.get_timeout)
     async with semaphore:
-        async with session.get(url, allow_redirects=config.get_redirects,
+        async with session.get(url,
+                               allow_redirects=config.get_redirects,
                                timeout=timeout,
                                proxy=config.get_proxy) as resp:
             text = await resp.text()

oneforall/common/utils.py

@@ -5,10 +5,21 @@ import random
 import ipaddress
 import platform
 import config
-from fake_useragent import UserAgent
 from common.domain import Domain
 from config import logger
 
+user_agents = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/68.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:61.0) '
+    'Gecko/20100101 Firefox/68.0',
+    'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0']
+
 
 def match_subdomain(domain, text, distinct=True):
     """
@@ -46,7 +57,7 @@ def gen_fake_header():
     """
     生成伪造请求头
     """
-    ua = UserAgent()
+    ua = random.choice(user_agents)
     ip = gen_random_ip()
     headers = {
         'Accept': 'text/html,application/xhtml+xml,'
@@ -58,7 +69,7 @@ def gen_fake_header():
         'DNT': '1',
         'Referer': 'https://www.google.com/',
         'Upgrade-Insecure-Requests': '1',
-        'User-Agent': ua.random,
+        'User-Agent': ua,
         'X-Forwarded-For': ip,
         'X-Real-IP': ip
     }
@@ -129,4 +140,3 @@ def get_semaphore():
         return 1000
     elif system == 'Darwin':
         return 1000
-