From 8f7ea692b287d1b2fc14298865065ca4cd2082ca Mon Sep 17 00:00:00 2001 From: Jing Ling Date: Fri, 25 Sep 2020 03:41:33 +0800 Subject: [PATCH] =?UTF-8?q?tldextract=E6=9C=AC=E5=9C=B0=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Pipfile | 4 +- Pipfile.lock | 33 ++---- common/domain.py | 4 +- common/tldextract.py | 240 +++++++++++++++++++++++++++++++++++++++++++ common/utils.py | 44 +++++--- requirements.txt | 8 +- 6 files changed, 284 insertions(+), 49 deletions(-) create mode 100644 common/tldextract.py diff --git a/Pipfile b/Pipfile index 85f8a1b..faa48f7 100644 --- a/Pipfile +++ b/Pipfile @@ -9,14 +9,14 @@ verify_ssl = true tqdm = "*" loguru = "*" dnspython = "*" -requests = {extras = ["socks"],version = "*"} -tldextract = "*" exrex = "*" fire = "*" bs4 = "*" tenacity = "*" treelib = "*" sqlalchemy = "*" +requests = "*" +pysocks = "*" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index 1d3f135..dd1c207 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7ad607a807779ea8674fd9d46eb4474f11349f766d94d721c23ad2b29a033105" + "sha256": "b593ca54685d3e36cb21706fd3ef3ee747c0d87fa7043fa483fcc30d316960eb" }, "pipfile-spec": 6, "requires": { @@ -90,11 +90,11 @@ }, "loguru": { "hashes": [ - "sha256:5aecbf13bc8e2f6e5a5d0475460a345b44e2885464095ea7de44e8795857ad33", - "sha256:a5e5e196b9958feaf534ac2050171d16576bae633074ce3e73af7dda7e9a58ae" + "sha256:b28e72ac7a98be3d28ad28570299a393dfcd32e5e3f6a353dec94675767b6319", + "sha256:f8087ac396b5ee5f67c963b495d615ebbceac2796379599820e324419d53667c" ], "index": "pypi", - "version": "==0.5.2" + "version": "==0.5.3" }, "pysocks": { "hashes": [ @@ -102,12 +102,10 @@ "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0" ], + "index": "pypi", "version": "==1.7.1" }, "requests": { - "extras": [ - "socks" - ], "hashes": [ "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" @@ -115,13 +113,6 @@ "index": "pypi", "version": "==2.24.0" }, - "requests-file": { - "hashes": [ - "sha256:07d74208d3389d01c38ab89ef403af0cfec63957d53a0081d8eca738d0247d8e", - "sha256:dfe5dae75c12481f68ba353183c53a65e6044c923e64c24b2209f6c7570ca953" - ], - "version": "==1.5.1" - }, "six": { "hashes": [ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", @@ -188,21 +179,13 @@ ], "version": "==1.1.0" }, - "tldextract": { - "hashes": [ - "sha256:ab0e38977a129c72729476d5f8c85a8e1f8e49e9202e1db8dca76e95da7be9a8", - "sha256:c2a8a392edf3ea6fa8be80930f04c3ac29e91fa604cb2139bdf6a37fc1e1ac6d" - ], - "index": "pypi", - "version": "==2.2.3" - }, "tqdm": { "hashes": [ - "sha256:1a336d2b829be50e46b84668691e0a2719f26c97c62846298dd5ae2937e4d5cf", - "sha256:564d632ea2b9cb52979f7956e093e831c28d441c11751682f84c86fc46e4fd21" + "sha256:8f3c5815e3b5e20bc40463fa6b42a352178859692a68ffaa469706e6d38342a5", + "sha256:faf9c671bd3fad5ebaeee366949d969dca2b2be32c872a7092a1e1a9048d105b" ], "index": "pypi", - "version": "==4.48.2" + "version": "==4.49.0" }, "treelib": { "hashes": [ diff --git a/common/domain.py b/common/domain.py index 49cfab8..cd1a68b 100644 --- a/common/domain.py +++ b/common/domain.py @@ -1,5 +1,5 @@ import re -import tldextract +from common import tldextract from config import settings @@ -38,7 +38,7 @@ class Domain(object): """ data_storage_dir = settings.data_storage_dir extract_cache_file = data_storage_dir.joinpath('public_suffix_list.dat') - ext = tldextract.TLDExtract(extract_cache_file, None) + ext = tldextract.TLDExtract(extract_cache_file) result = self.match() if result: return ext(result) diff --git a/common/tldextract.py b/common/tldextract.py new file mode 100644 index 0000000..68a8e6f --- /dev/null +++ b/common/tldextract.py @@ -0,0 +1,240 @@ +# -*- coding: utf-8 -*- +"""`tldextract` accurately separates the gTLD or ccTLD (generic or country code +top-level domain) from the registered domain and subdomains of a URL. + + >>> import tldextract + + >>> tldextract.extract('http://forums.news.cnn.com/') + ExtractResult(subdomain='forums.news', domain='cnn', suffix='com') + + >>> tldextract.extract('http://forums.bbc.co.uk/') # United Kingdom + ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk') + + >>> tldextract.extract('http://www.worldbank.org.kg/') # Kyrgyzstan + ExtractResult(subdomain='www', domain='worldbank', suffix='org.kg') + +`ExtractResult` is a namedtuple, so it's simple to access the parts you want. + + >>> ext = tldextract.extract('http://forums.bbc.co.uk') + >>> (ext.subdomain, ext.domain, ext.suffix) + ('forums', 'bbc', 'co.uk') + >>> # rejoin subdomain and domain + >>> '.'.join(ext[:2]) + 'forums.bbc' + >>> # a common alias + >>> ext.registered_domain + 'bbc.co.uk' + +Note subdomain and suffix are _optional_. Not all URL-like inputs have a +subdomain or a valid suffix. + + >>> tldextract.extract('google.com') + ExtractResult(subdomain='', domain='google', suffix='com') + + >>> tldextract.extract('google.notavalidsuffix') + ExtractResult(subdomain='google', domain='notavalidsuffix', suffix='') + + >>> tldextract.extract('http://127.0.0.1:8080/deployed/') + ExtractResult(subdomain='', domain='127.0.0.1', suffix='') + +If you want to rejoin the whole namedtuple, regardless of whether a subdomain +or suffix were found: + + >>> ext = tldextract.extract('http://127.0.0.1:8080/deployed/') + >>> # this has unwanted dots + >>> '.'.join(ext) + '.127.0.0.1.' +""" + + +import os +import re +import json +import collections +from urllib.parse import scheme_chars +from functools import wraps + +import idna + +from common import utils + +IP_RE = re.compile(r'^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$') # pylint: disable=line-too-long + +SCHEME_RE = re.compile(r'^([' + scheme_chars + ']+:)?//') + + +class ExtractResult(collections.namedtuple('ExtractResult', 'subdomain domain suffix')): + """namedtuple of a URL's subdomain, domain, and suffix.""" + + # Necessary for __dict__ member to get populated in Python 3+ + __slots__ = () + + @property + def registered_domain(self): + """ + Joins the domain and suffix fields with a dot, if they're both set. + + >>> extract('http://forums.bbc.co.uk').registered_domain + 'bbc.co.uk' + >>> extract('http://localhost:8080').registered_domain + '' + """ + if self.domain and self.suffix: + return self.domain + '.' + self.suffix + return '' + + @property + def fqdn(self): + """ + Returns a Fully Qualified Domain Name, if there is a proper domain/suffix. + + >>> extract('http://forums.bbc.co.uk/path/to/file').fqdn + 'forums.bbc.co.uk' + >>> extract('http://localhost:8080').fqdn + '' + """ + if self.domain and self.suffix: + # self is the namedtuple (subdomain domain suffix) + return '.'.join(i for i in self if i) + return '' + + @property + def ipv4(self): + """ + Returns the ipv4 if that is what the presented domain/url is + + >>> extract('http://127.0.0.1/path/to/file').ipv4 + '127.0.0.1' + >>> extract('http://127.0.0.1.1/path/to/file').ipv4 + '' + >>> extract('http://256.1.1.1').ipv4 + '' + """ + if not (self.suffix or self.subdomain) and IP_RE.match(self.domain): + return self.domain + return '' + + +class TLDExtract(object): + """A callable for extracting, subdomain, domain, and suffix components from a URL.""" + + def __init__(self, cache_file=None): + """ + Constructs a callable for extracting subdomain, domain, and suffix + components from a URL. + """ + + self.cache_file = os.path.expanduser(cache_file or '') + self._extractor = None + + def __call__(self, url): + """ + Takes a string URL and splits it into its subdomain, domain, and + suffix (effective TLD, gTLD, ccTLD, etc.) component. + + >>> ext = TLDExtract() + >>> ext('http://forums.news.cnn.com/') + ExtractResult(subdomain='forums.news', domain='cnn', suffix='com') + >>> ext('http://forums.bbc.co.uk/') + ExtractResult(subdomain='forums', domain='bbc', suffix='co.uk') + """ + netloc = SCHEME_RE.sub("", url) \ + .partition("/")[0] \ + .partition("?")[0] \ + .partition("#")[0] \ + .split("@")[-1] \ + .partition(":")[0] \ + .strip() \ + .rstrip(".") + + labels = netloc.split(".") + + translations = [_decode_punycode(label) for label in labels] + suffix_index = self._get_tld_extractor().suffix_index(translations) + + suffix = ".".join(labels[suffix_index:]) + if not suffix and netloc and utils.looks_like_ip(netloc): + return ExtractResult('', netloc, '') + + subdomain = ".".join(labels[:suffix_index - 1]) if suffix_index else "" + domain = labels[suffix_index - 1] if suffix_index else "" + return ExtractResult(subdomain, domain, suffix) + + @property + def tlds(self): + return self._get_tld_extractor().tlds + + def _get_tld_extractor(self): + """Get or compute this object's TLDExtractor. Looks up the TLDExtractor + in roughly the following order, based on the settings passed to + __init__: + + 1. Memoized on `self` + 2. Local system cache file""" + # pylint: disable=no-else-return + + if self._extractor: + return self._extractor + tlds = self._get_cached_tlds() + if tlds: + self._extractor = _PublicSuffixListTLDExtractor(tlds) + return self._extractor + else: + raise Exception("tlds is empty, cannot proceed without tlds.") + + def _get_cached_tlds(self): + """Read the local TLD cache file. Returns None on IOError or other + error, or if this object is not set to use the cache + file.""" + if not self.cache_file: + return None + + with open(self.cache_file) as cache_file: + return json.loads(cache_file.read()) + + +TLD_EXTRACTOR = TLDExtract() + + +@wraps(TLD_EXTRACTOR.__call__) +def extract(url): + return TLD_EXTRACTOR(url) + + +class _PublicSuffixListTLDExtractor(object): + """Wrapper around this project's main algo for PSL + lookups. + """ + def __init__(self, tlds): + self.tlds = frozenset(tlds) + + def suffix_index(self, lower_spl): + """Returns the index of the first suffix label. + Returns len(spl) if no suffix is found + """ + length = len(lower_spl) + for i in range(length): + maybe_tld = '.'.join(lower_spl[i:]) + exception_tld = '!' + maybe_tld + if exception_tld in self.tlds: + return i + 1 + + if maybe_tld in self.tlds: + return i + + wildcard_tld = '*.' + '.'.join(lower_spl[i + 1:]) + if wildcard_tld in self.tlds: + return i + + return length + + +def _decode_punycode(label): + lowered = label.lower() + looks_like_puny = lowered.startswith('xn--') + if looks_like_puny: + try: + return idna.decode(label.encode('ascii')).lower() + except (UnicodeError, IndexError): + pass + return lowered diff --git a/common/utils.py b/common/utils.py index 1c10a47..59723fa 100644 --- a/common/utils.py +++ b/common/utils.py @@ -1,12 +1,14 @@ -import json import os -import platform -import random import re -import string -import subprocess import sys import time +import json +import socket +import random +import string +import platform +import subprocess +from urllib.parse import scheme_chars from ipaddress import IPv4Address, ip_address from pathlib import Path from stat import S_IXUSR @@ -34,6 +36,9 @@ user_agents = [ 'Gecko/20100101 Firefox/68.0', 'Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/68.0'] +IP_RE = re.compile(r'^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$') # pylint: disable=line-too-long +SCHEME_RE = re.compile(r'^([' + scheme_chars + ']+:)?//') + def gen_random_ip(): """ @@ -748,18 +753,12 @@ def ping_avg_time(nameserver): logger.log('ALERT', f'100.0% packet loss, ping {nameserver} failed.') return None elif platform.system() in ('Darwin', 'Linux'): - try: - avg_time = re.findall(r'(?:min/avg/max/.+ )(?:\d+\.\d+)/(\d+\.\d+)/', text)[0] - logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.') - except IndexError: - return None + avg_time = re.findall(r'(?:min/avg/max/.+ )(?:\d+\.\d+)/(\d+\.\d+)/', text)[0] + logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.') return avg_time elif platform.system() == 'Windows': - try: - avg_time = re.findall(r'(?:Average|平均).+(\d.?)ms', text)[0] - logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.') - except IndexError: - return None + avg_time = re.findall(r'(?:Average|平均).+(\d.)ms', text)[0] + logger.log('INFOR', f'ping {nameserver} average time {avg_time} ms.') return avg_time else: logger.log('ALERT', f'{text}') @@ -813,3 +812,18 @@ def default_nameserver(): logger.log('ERROR', 'Resolver configuration could not be read ' 'or specified no nameservers.') exit(1) + + +def looks_like_ip(maybe_ip): + """Does the given str look like an IP address?""" + if not maybe_ip[0].isdigit(): + return False + + try: + socket.inet_aton(maybe_ip) + return True + except (AttributeError, UnicodeError): + if IP_RE.match(maybe_ip): + return True + except socket.error: + return False \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ce88e6a..9aba14b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,17 +9,15 @@ exrex==0.10.5 fire==0.3.1 future==0.18.2 idna==2.10 -loguru==0.5.2 +loguru==0.5.3 pysocks==1.7.1 -requests-file==1.5.1 -requests[socks]==2.24.0 +requests==2.24.0 six==1.15.0 soupsieve==2.0.1 sqlalchemy==1.3.19 tenacity==6.2.0 termcolor==1.1.0 -tldextract==2.2.3 -tqdm==4.48.2 +tqdm==4.49.0 treelib==1.6.1 urllib3==1.25.10 win32-setctime==1.0.2 ; sys_platform == 'win32' \ No newline at end of file