tldextract本地化

Pipfile

@@ -9,14 +9,14 @@ verify_ssl = true
 tqdm = "*"
 loguru = "*"
 dnspython = "*"
-requests = {extras = ["socks"],version = "*"}
-tldextract = "*"
 exrex = "*"
 fire = "*"
 bs4 = "*"
 tenacity = "*"
 treelib = "*"
 sqlalchemy = "*"
+requests = "*"
+pysocks = "*"
 
 [requires]
 python_version = "3.8"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "7ad607a807779ea8674fd9d46eb4474f11349f766d94d721c23ad2b29a033105"
+            "sha256": "b593ca54685d3e36cb21706fd3ef3ee747c0d87fa7043fa483fcc30d316960eb"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -90,11 +90,11 @@
         },
         "loguru": {
             "hashes": [
-                "sha256:5aecbf13bc8e2f6e5a5d0475460a345b44e2885464095ea7de44e8795857ad33",
-                "sha256:a5e5e196b9958feaf534ac2050171d16576bae633074ce3e73af7dda7e9a58ae"
+                "sha256:b28e72ac7a98be3d28ad28570299a393dfcd32e5e3f6a353dec94675767b6319",
+                "sha256:f8087ac396b5ee5f67c963b495d615ebbceac2796379599820e324419d53667c"
             ],
             "index": "pypi",
-            "version": "==0.5.2"
+            "version": "==0.5.3"
         },
         "pysocks": {
             "hashes": [
@@ -102,12 +102,10 @@
                 "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5",
                 "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0"
             ],
+            "index": "pypi",
             "version": "==1.7.1"
         },
         "requests": {
-            "extras": [
-                "socks"
-            ],
             "hashes": [
                 "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
                 "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
@@ -115,13 +113,6 @@
             "index": "pypi",
             "version": "==2.24.0"
         },
-        "requests-file": {
-            "hashes": [
-                "sha256:07d74208d3389d01c38ab89ef403af0cfec63957d53a0081d8eca738d0247d8e",
-                "sha256:dfe5dae75c12481f68ba353183c53a65e6044c923e64c24b2209f6c7570ca953"
-            ],
-            "version": "==1.5.1"
-        },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
@@ -188,21 +179,13 @@
             ],
             "version": "==1.1.0"
         },
-        "tldextract": {
-            "hashes": [
-                "sha256:ab0e38977a129c72729476d5f8c85a8e1f8e49e9202e1db8dca76e95da7be9a8",
-                "sha256:c2a8a392edf3ea6fa8be80930f04c3ac29e91fa604cb2139bdf6a37fc1e1ac6d"
-            ],
-            "index": "pypi",
-            "version": "==2.2.3"
-        },
         "tqdm": {
             "hashes": [
-                "sha256:1a336d2b829be50e46b84668691e0a2719f26c97c62846298dd5ae2937e4d5cf",
-                "sha256:564d632ea2b9cb52979f7956e093e831c28d441c11751682f84c86fc46e4fd21"
+                "sha256:8f3c5815e3b5e20bc40463fa6b42a352178859692a68ffaa469706e6d38342a5",
+                "sha256:faf9c671bd3fad5ebaeee366949d969dca2b2be32c872a7092a1e1a9048d105b"
             ],
             "index": "pypi",
-            "version": "==4.48.2"
+            "version": "==4.49.0"
         },
         "treelib": {
             "hashes": [

common/domain.py

@@ -1,5 +1,5 @@
 import re
-import tldextract
+from common import tldextract
 from config import settings
 
 
@@ -38,7 +38,7 @@ class Domain(object):
         """
         data_storage_dir = settings.data_storage_dir
         extract_cache_file = data_storage_dir.joinpath('public_suffix_list.dat')
-        ext = tldextract.TLDExtract(extract_cache_file, None)
+        ext = tldextract.TLDExtract(extract_cache_file)
         result = self.match()
         if result:
             return ext(result)

common/tldextract.py

@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+"""`tldextract` accurately separates the gTLD or ccTLD (generic or country code
+top-level domain) from the registered domain and subdomains of a URL.
+
+    >>> import tldextract
+
+    >>> tldextract.extract('http://forums.news.cnn.com/')
+    ExtractResult(subdomain='forums.news', domain='cnn', suffix='com')
+
+    >>> tldextract.extract('http://forums.bbc.co.uk/') # United Kingdom
+    ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk')
+
+    >>> tldextract.extract('http://www.worldbank.org.kg/') # Kyrgyzstan
+    ExtractResult(subdomain='www', domain='worldbank', suffix='org.kg')
+
+`ExtractResult` is a namedtuple, so it's simple to access the parts you want.
+
+    >>> ext = tldextract.extract('http://forums.bbc.co.uk')
+    >>> (ext.subdomain, ext.domain, ext.suffix)
+    ('forums', 'bbc', 'co.uk')
+    >>> # rejoin subdomain and domain
+    >>> '.'.join(ext[:2])
+    'forums.bbc'
+    >>> # a common alias
+    >>> ext.registered_domain
+    'bbc.co.uk'
+
+Note subdomain and suffix are _optional_. Not all URL-like inputs have a
+subdomain or a valid suffix.
+
+    >>> tldextract.extract('google.com')
+    ExtractResult(subdomain='', domain='google', suffix='com')
+
+    >>> tldextract.extract('google.notavalidsuffix')
+    ExtractResult(subdomain='google', domain='notavalidsuffix', suffix='')
+
+    >>> tldextract.extract('http://127.0.0.1:8080/deployed/')
+    ExtractResult(subdomain='', domain='127.0.0.1', suffix='')
+
+If you want to rejoin the whole namedtuple, regardless of whether a subdomain
+or suffix were found:
+
+    >>> ext = tldextract.extract('http://127.0.0.1:8080/deployed/')
+    >>> # this has unwanted dots
+    >>> '.'.join(ext)
+    '.127.0.0.1.'
+"""
+
+
+import os
+import re
+import json
+import collections
+from urllib.parse import scheme_chars
+from functools import wraps
+
+import idna
+
+from common import utils
+
+IP_RE = re.compile(r'^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$')  # pylint: disable=line-too-long
+
+SCHEME_RE = re.compile(r'^([' + scheme_chars + ']+:)?//')
+
+
+class ExtractResult(collections.namedtuple('ExtractResult', 'subdomain domain suffix')):
+    """namedtuple of a URL's subdomain, domain, and suffix."""
+
+    # Necessary for __dict__ member to get populated in Python 3+
+    __slots__ = ()
+
+    @property
+    def registered_domain(self):
+        """
+        Joins the domain and suffix fields with a dot, if they're both set.
+
+        >>> extract('http://forums.bbc.co.uk').registered_domain
+        'bbc.co.uk'
+        >>> extract('http://localhost:8080').registered_domain
+        ''
+        """
+        if self.domain and self.suffix:
+            return self.domain + '.' + self.suffix
+        return ''
+
+    @property
+    def fqdn(self):
+        """
+        Returns a Fully Qualified Domain Name, if there is a proper domain/suffix.
+
+        >>> extract('http://forums.bbc.co.uk/path/to/file').fqdn
+        'forums.bbc.co.uk'
+        >>> extract('http://localhost:8080').fqdn
+        ''
+        """
+        if self.domain and self.suffix:
+            # self is the namedtuple (subdomain domain suffix)
+            return '.'.join(i for i in self if i)
+        return ''
+
+    @property
+    def ipv4(self):
+        """
+        Returns the ipv4 if that is what the presented domain/url is
+
+        >>> extract('http://127.0.0.1/path/to/file').ipv4
+        '127.0.0.1'
+        >>> extract('http://127.0.0.1.1/path/to/file').ipv4
+        ''
+        >>> extract('http://256.1.1.1').ipv4
+        ''
+        """
+        if not (self.suffix or self.subdomain) and IP_RE.match(self.domain):
+            return self.domain
+        return ''
+
+
+class TLDExtract(object):
+    """A callable for extracting, subdomain, domain, and suffix components from a URL."""
+
+    def __init__(self, cache_file=None):
+        """
+        Constructs a callable for extracting subdomain, domain, and suffix
+        components from a URL.
+        """
+
+        self.cache_file = os.path.expanduser(cache_file or '')
+        self._extractor = None
+
+    def __call__(self, url):
+        """
+        Takes a string URL and splits it into its subdomain, domain, and
+        suffix (effective TLD, gTLD, ccTLD, etc.) component.
+
+        >>> ext = TLDExtract()
+        >>> ext('http://forums.news.cnn.com/')
+        ExtractResult(subdomain='forums.news', domain='cnn', suffix='com')
+        >>> ext('http://forums.bbc.co.uk/')
+        ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk')
+        """
+        netloc = SCHEME_RE.sub("", url) \
+            .partition("/")[0] \
+            .partition("?")[0] \
+            .partition("#")[0] \
+            .split("@")[-1] \
+            .partition(":")[0] \
+            .strip() \
+            .rstrip(".")
+
+        labels = netloc.split(".")
+
+        translations = [_decode_punycode(label) for label in labels]
+        suffix_index = self._get_tld_extractor().suffix_index(translations)
+
+        suffix = ".".join(labels[suffix_index:])
+        if not suffix and netloc and utils.looks_like_ip(netloc):
+            return ExtractResult('', netloc, '')
+
+        subdomain = ".".join(labels[:suffix_index - 1]) if suffix_index else ""
+        domain = labels[suffix_index - 1] if suffix_index else ""
+        return ExtractResult(subdomain, domain, suffix)
+
+    @property
+    def tlds(self):
+        return self._get_tld_extractor().tlds
+
+    def _get_tld_extractor(self):
+        """Get or compute this object's TLDExtractor. Looks up the TLDExtractor
+        in roughly the following order, based on the settings passed to
+        __init__:
+
+        1. Memoized on `self`
+        2. Local system cache file"""
+        # pylint: disable=no-else-return
+
+        if self._extractor:
+            return self._extractor
+        tlds = self._get_cached_tlds()
+        if tlds:
+            self._extractor = _PublicSuffixListTLDExtractor(tlds)
+            return self._extractor
+        else:
+            raise Exception("tlds is empty, cannot proceed without tlds.")
+
+    def _get_cached_tlds(self):
+        """Read the local TLD cache file. Returns None on IOError or other
+        error, or if this object is not set to use the cache
+        file."""
+        if not self.cache_file:
+            return None
+
+        with open(self.cache_file) as cache_file:
+            return json.loads(cache_file.read())
+
+
+TLD_EXTRACTOR = TLDExtract()
+
+
+@wraps(TLD_EXTRACTOR.__call__)
+def extract(url):
+    return TLD_EXTRACTOR(url)
+
+
+class _PublicSuffixListTLDExtractor(object):
+    """Wrapper around this project's main algo for PSL
+    lookups.
+    """
+    def __init__(self, tlds):
+        self.tlds = frozenset(tlds)
+
+    def suffix_index(self, lower_spl):
+        """Returns the index of the first suffix label.
+        Returns len(spl) if no suffix is found
+        """
+        length = len(lower_spl)
+        for i in range(length):
+            maybe_tld = '.'.join(lower_spl[i:])
+            exception_tld = '!' + maybe_tld
+            if exception_tld in self.tlds:
+                return i + 1
+
+            if maybe_tld in self.tlds:
+                return i
+
+            wildcard_tld = '*.' + '.'.join(lower_spl[i + 1:])
+            if wildcard_tld in self.tlds:
+                return i
+
+        return length
+
+
+def _decode_punycode(label):
+    lowered = label.lower()
+    looks_like_puny = lowered.startswith('xn--')
+    if looks_like_puny:
+        try:
+            return idna.decode(label.encode('ascii')).lower()
+        except (UnicodeError, IndexError):
+            pass
+    return lowered

common/utils.py

@@ -1,12 +1,14 @@
-import json
 import os
-import platform
-import random
 import re
-import string
-import subprocess
 import sys
 import time
+import json
+import socket
+import random
+import string
+import platform
+import subprocess
+from urllib.parse import scheme_chars
 from ipaddress import IPv4Address, ip_address
 from pathlib import Path
 from stat import S_IXUSR
@@ -34,6 +36,9 @@ user_agents = [
     'Gecko/20100101 Firefox/68.0',
     'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0']
 
+IP_RE = re.compile(r'^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$')  # pylint: disable=line-too-long
+SCHEME_RE = re.compile(r'^([' + scheme_chars + ']+:)?//')
+
 
 def gen_random_ip():
     """
@@ -748,18 +753,12 @@ def ping_avg_time(nameserver):
             logger.log('ALERT', f'100.0% packet loss, ping {nameserver} failed.')
             return None
         elif platform.system() in ('Darwin', 'Linux'):
-            try:
-                avg_time = re.findall(r'(?:min/avg/max/.+ )(?:\d+\.\d+)/(\d+\.\d+)/', text)[0]
-                logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.')
-            except IndexError:
-                return None
+            avg_time = re.findall(r'(?:min/avg/max/.+ )(?:\d+\.\d+)/(\d+\.\d+)/', text)[0]
+            logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.')
             return avg_time
         elif platform.system() == 'Windows':
-            try:
-                avg_time = re.findall(r'(?:Average|平均).+(\d.?)ms', text)[0]
-                logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.')
-            except IndexError:
-                return None
+            avg_time = re.findall(r'(?:Average|平均).+(\d.)ms', text)[0]
+            logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.')
             return avg_time
         else:
             logger.log('ALERT', f'{text}')
@@ -813,3 +812,18 @@ def default_nameserver():
         logger.log('ERROR', 'Resolver configuration could not be read '
                             'or specified no nameservers.')
         exit(1)
+
+
+def looks_like_ip(maybe_ip):
+    """Does the given str look like an IP address?"""
+    if not maybe_ip[0].isdigit():
+        return False
+
+    try:
+        socket.inet_aton(maybe_ip)
+        return True
+    except (AttributeError, UnicodeError):
+        if IP_RE.match(maybe_ip):
+            return True
+    except socket.error:
+        return False

requirements.txt

@@ -9,17 +9,15 @@ exrex==0.10.5
 fire==0.3.1
 future==0.18.2
 idna==2.10
-loguru==0.5.2
+loguru==0.5.3
 pysocks==1.7.1
-requests-file==1.5.1
-requests[socks]==2.24.0
+requests==2.24.0
 six==1.15.0
 soupsieve==2.0.1
 sqlalchemy==1.3.19
 tenacity==6.2.0
 termcolor==1.1.0
-tldextract==2.2.3
-tqdm==4.48.2
+tqdm==4.49.0
 treelib==1.6.1
 urllib3==1.25.10
 win32-setctime==1.0.2 ; sys_platform == 'win32'