添加cidr，asn，addr字段

Pipfile

@@ -21,6 +21,7 @@ cloudscraper = "*"
 tablib = "*"
 psutil = "*"
 tenacity = "*"
+ip2location = "*"
 
 [requires]
 python_version = "3.8"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "0501b2b68bd52ab7ba0d40722bc8a84bddcd2af023babc9f986645384e381462"
+            "sha256": "6556f720e7fb052c3f923f3e7986cc00c1cc230db729ad79685a127c02568c96"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -65,10 +65,10 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304",
+                "sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1",
-                "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"
+                "sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc"
             ],
-            "version": "==2020.4.5.1"
+            "version": "==2020.4.5.2"
         },
         "chardet": {
             "hashes": [
@@ -79,11 +79,11 @@
         },
         "cloudscraper": {
             "hashes": [
-                "sha256:3893be3c281ddb7a39bf35e558da19247a39a408ef48078fdabf5058b7659d6c",
+                "sha256:30289d0f98413fe3c3b74c9f7547b8e6fff6aca698c23445996cf6bf80a8bad0",
-                "sha256:db295c5ca33f22ae058f317b07c6842a2b16d75c9e11e38d21395363d089692f"
+                "sha256:e711f8a41393d9755178d6290fccbfc84f0b9db20bbdeababcf8cd9232f380cd"
             ],
             "index": "pypi",
-            "version": "==1.2.38"
+            "version": "==1.2.40"
         },
         "colorama": {
             "hashes": [
@@ -134,6 +134,14 @@
             ],
             "version": "==2.9"
         },
+        "ip2location": {
+            "hashes": [
+                "sha256:4a83f32098091b6aafaef63b393331bf973bbf54547db82de38f30108b5f079c",
+                "sha256:bbddbd5e3f79c18a9710afd43be09cb21724d7324557668017c545b5c5e62bc4"
+            ],
+            "index": "pypi",
+            "version": "==8.4.1"
+        },
         "jdcal": {
             "hashes": [
                 "sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba",
@@ -143,11 +151,11 @@
         },
         "loguru": {
             "hashes": [
-                "sha256:1e0e6ff59be5e22f863d909ca989e34bb14c21b374f6af45281e603d003dbb96",
+                "sha256:70201d5fce26da89b7a5f168caa2bb674e06b969829f56737db1d6472e53e7c3",
-                "sha256:4688d9e1f31d70e1ec7ccce5305967bc28f377eb1048d009108c11faebe05bcf"
+                "sha256:e5d362a43cd2fc2da63551d79a6830619c4d5b3a8b976515748026f92f351b61"
             ],
             "index": "pypi",
-            "version": "==0.5.0"
+            "version": "==0.5.1"
         },
         "multidict": {
             "hashes": [
@@ -320,11 +328,11 @@
         },
         "tqdm": {
             "hashes": [
-                "sha256:4733c4a10d0f2a4d098d801464bdaf5240c7dadd2a7fde4ee93b0a0efd9fb25e",
+                "sha256:07c06493f1403c1380b630ae3dcbe5ae62abcf369a93bbc052502279f189ab8c",
-                "sha256:acdafb20f51637ca3954150d0405ff1a7edde0ff19e38fb99a80a66210d2a28f"
+                "sha256:cd140979c2bebd2311dfb14781d8f19bd5a9debb92dcab9f6ef899c987fcf71f"
             ],
             "index": "pypi",
-            "version": "==4.46.0"
+            "version": "==4.46.1"
         },
         "urllib3": {
             "hashes": [

common/database.py

@@ -77,6 +77,9 @@ class Database(object):
                    f'response text,'
                    f'times text,'
                    f'ttl text,'
+                   f'cidr text,'
+                   f'asn text,'
+                   f'addr text,'
                    f'resolver text,'
                    f'module text,'
                    f'source text,'
@@ -102,13 +105,13 @@ class Database(object):
                     f'insert into "{table_name}" ('
                     f'id, type, alive, resolve, request, new, url, subdomain,'
                     f'port, level, cname, content, public, status, reason,'
-                    f'title, banner, header, response, times, ttl, resolver,'
+                    f'title, banner, header, response, times, ttl, cidr, asn,'
-                    f'module, source, elapse, find, brute, valid) '
+                    f'addr, resolver, module, source, elapse, find, brute, valid) '
                     f'values (:id, :type, :alive, :resolve, :request, :new,'
                     f':url, :subdomain, :port, :level, :cname, :content,'
                     f':public, :status, :reason, :title, :banner, :header,'
-                    f':response, :times, :ttl, :resolver, :module, :source,'
+                    f':response, :times, :ttl, :cidr, :asn, :addr, :resolver,'
-                    f':elapse, :find, :brute, :valid)', results)
+                    f':module, :source, :elapse, :find, :brute, :valid)', results)
             except Exception as e:
                 logger.log('ERROR', e)
 
@@ -231,8 +234,8 @@ class Database(object):
         table_name = table_name.replace('.', '_')
         query = f'select id, type, new, alive, request, resolve, url, ' \
                 f'subdomain, level, cname, content, public, port, status, ' \
-                f'reason, title, banner, times, ttl, resolver, module, ' \
+                f'reason, title, banner, times, ttl, cidr, asn, addr, resolver, ' \
-                f'source, elapse, find, brute, valid from "{table_name}"'
+                f'module, source, elapse, find, brute, valid from "{table_name}"'
         if alive and limit:
             if limit in ['resolve', 'request']:
                 where = f' where {limit} = 1'

common/ipasn.py

@@ -0,0 +1,74 @@
+import csv
+import zipfile
+
+from common.utils import ip_to_int
+from config.setting import data_storage_dir
+
+
+class Entry(object):
+    def __init__(self, start, end, value):
+        self.start = int(start)
+        self.end = int(end)
+        self.value = value
+
+
+class IPAsnData(object):
+    def __init__(self):
+        zip_path = data_storage_dir.joinpath("IP2LOCATION-LITE-ASN.CSV.ZIP")
+        csv_path = data_storage_dir.joinpath("IP2LOCATION-LITE-ASN.CSV")
+        if csv_path.exists():
+            asn_fp = open(csv_path)
+        else:
+            zf = zipfile.ZipFile(zip_path)
+            zf.extract('IP2LOCATION-LITE-ASN.CSV', data_storage_dir)
+            asn_fp = open(csv_path)
+        self.data = []
+        reader = csv.reader(asn_fp, delimiter=',', quotechar='"')
+        for row in reader:
+            e = Entry(row[0], row[1], row)
+            self.data.append(e)
+        asn_fp.close()
+
+    def __iter__(self):
+        return self.data.__iter__()
+
+    def find_i(self, ip, start, end):
+        if end - start < 100:
+            for i in range(start, end):
+                obj = self.data[i]
+                if obj.start <= ip <= obj.end:
+                    return obj.value
+            return None
+        else:
+            mid = start + (end - start) // 2
+            val = self.data[mid].start
+            if ip < val:
+                return self.find_i(ip, start, mid)
+            elif ip > val:
+                return self.find_i(ip, mid, end)
+            else:
+                return self.data[mid].value
+
+    def find_int(self, ip):
+        return self.find_i(ip, 0, len(self.data) - 1)
+
+    def find(self, ip):
+        return self.find_i(ip_to_int(ip), 0, len(self.data) - 1)
+
+
+class IPAsnInfo(object):
+    def __init__(self):
+        self.asn = IPAsnData()
+
+    def find(self, ip):
+        asn = self.asn.find(ip)
+        if asn:
+            result = {"cidr": asn[2], "asn": f'ASN{asn[3]} {asn[4]}'}
+            return result
+        else:
+            return None
+
+
+if __name__ == "__main__":
+    asn_info = IPAsnInfo()
+    print(asn_info.find("188.81.94.77"))

common/ipgeo.py

@@ -0,0 +1,11 @@
+import zipfile
+import IP2Location
+from config.setting import data_storage_dir
+
+
+zip_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN.ZIP")
+bin_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN")
+if not bin_path.exists():
+    zf = zipfile.ZipFile(zip_path)
+    zf.extract('IP2LOCATION-LITE-DB3.BIN', data_storage_dir)
+IpGeoInfo = IP2Location.IP2Location(bin_path)

common/module.py

@@ -302,6 +302,9 @@ class Module(object):
                       'response': None,
                       'times': None,
                       'ttl': None,
+                      'cidr': None,
+                      'asn': None,
+                      'addr': None,
                       'resolver': None,
                       'module': self.module,
                       'source': self.source,
@@ -331,6 +334,9 @@ class Module(object):
                 times = record.get('times')
                 ttl = record.get('ttl')
                 public = record.get('public')
+                cidr = record.get('cidr')
+                asn = record.get('asn')
+                addr = record.get('addr')
                 if isinstance(cname, list):
                     cname = ','.join(cname)
                     content = ','.join(content)
@@ -358,6 +364,9 @@ class Module(object):
                           'response': None,
                           'times': times,
                           'ttl': ttl,
+                          'cidr': cidr,
+                          'asn': asn,
+                          'addr': addr,
                           'resolver': resolver,
                           'module': self.module,
                           'source': self.source,

common/resolve.py

@@ -5,6 +5,8 @@ from config.log import logger
 from config import setting
 from common import utils
 from common.database import Database
+from common.ipasn import IPAsnInfo
+from common.ipgeo import IpGeoInfo
 
 
 def filter_subdomain(data):
@@ -71,6 +73,8 @@ def save_subdomains(save_path, subdomain_list):
 def deal_output(output_path):
     logger.log('INFOR', f'Processing resolved results')
     records = dict()  # 用来记录所有域名解析数据
+    ip_asn = IPAsnInfo()
+    ip_geo = IpGeoInfo
     with open(output_path) as fd:
         for line in fd:
             line = line.strip()
@@ -102,6 +106,9 @@ def deal_output(output_path):
             ips = list()
             public = list()
             ttls = list()
+            cidrs = list()
+            asns = list()
+            addrs = list()
             answers = data.get('answers')
             for answer in answers:
                 if answer.get('type') == 'A':
@@ -113,12 +120,22 @@ def deal_output(output_path):
                     ttls.append(str(ttl))
                     is_public = utils.ip_is_public(ip)
                     public.append(str(is_public))
+                    asn_info = ip_asn.find(ip)
+                    cidrs.append(asn_info.get('cidr'))
+                    asns.append(asn_info.get('asn'))
+                    addr = f'{ip_geo.get_country_long(ip)} ' \
+                           f'{ip_geo.get_region(ip)} ' \
+                           f'{ip_geo.get_city(ip)}'
+                    addrs.append(addr)
                     record['resolve'] = 1
                     record['reason'] = status
                     record['cname'] = ','.join(cname)
                     record['content'] = ','.join(ips)
                     record['public'] = ','.join(public)
                     record['ttl'] = ','.join(ttls)
+                    record['cidr'] = ','.join(cidrs)
+                    record['asn'] = ','.join(asns)
+                    record['addr'] = ','.join(addrs)
                     records[qname] = record
             if not flag:
                 record['alive'] = 0

common/utils.py

@@ -9,6 +9,7 @@ import subprocess
 from ipaddress import IPv4Address, ip_address
 from stat import S_IXUSR
 
+import IP2Location
 import psutil
 import tenacity
 import requests
@@ -638,3 +639,12 @@ def is_subname(name):
         if char not in chars:
             return False
     return True
+
+
+def ip_to_int(ip):
+    try:
+        ipv4 = IPv4Address(ip)
+    except Exception as e:
+        logger.log('ERROR', e.args)
+        return None
+    return int(ipv4)

requirements.txt

@@ -1,11 +1,13 @@
 aiohttp==3.6.2
+aiomultiprocess==0.7.0
 async-timeout==3.0.1
 attrs==19.3.0
 beautifulsoup4==4.9.1
+Brotli==1.0.7
 bs4==0.0.1
-certifi==2020.4.5.1
+certifi==2020.4.5.2
 chardet==3.0.4
-cloudscraper==1.2.38
+cloudscraper==1.2.40
 colorama==0.4.3
 dnspython==1.16.0
 docopt==0.6.2
@@ -13,13 +15,17 @@ et-xmlfile==1.0.1
 exrex==0.10.5
 fire==0.3.1
 idna==2.9
+IP2Location==8.4.1
 jdcal==1.4.1
-loguru==0.5.0
+Js2Py==0.70
+loguru==0.5.1
 multidict==4.7.6
 openpyxl==2.4.11
 psutil==5.7.0
+pyjsparser==2.7.1
 pyparsing==2.4.7
 PySocks==1.7.1
+pytz==2019.3
 records==0.5.3
 requests==2.23.0
 requests-file==1.5.1
@@ -31,7 +37,8 @@ tablib==2.0.0
 tenacity==6.2.0
 termcolor==1.1.0
 tldextract==2.2.2
-tqdm==4.46.0
+tqdm==4.46.1
+tzlocal==2.0.0
 urllib3==1.25.9
 win32-setctime==1.0.1
 yarl==1.4.2