diff --git a/Pipfile b/Pipfile index 9096828..3f7c4a8 100644 --- a/Pipfile +++ b/Pipfile @@ -21,6 +21,7 @@ cloudscraper = "*" tablib = "*" psutil = "*" tenacity = "*" +ip2location = "*" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index c073f07..754beb1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "0501b2b68bd52ab7ba0d40722bc8a84bddcd2af023babc9f986645384e381462" + "sha256": "6556f720e7fb052c3f923f3e7986cc00c1cc230db729ad79685a127c02568c96" }, "pipfile-spec": 6, "requires": { @@ -65,10 +65,10 @@ }, "certifi": { "hashes": [ - "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", - "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519" + "sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1", + "sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc" ], - "version": "==2020.4.5.1" + "version": "==2020.4.5.2" }, "chardet": { "hashes": [ @@ -79,11 +79,11 @@ }, "cloudscraper": { "hashes": [ - "sha256:3893be3c281ddb7a39bf35e558da19247a39a408ef48078fdabf5058b7659d6c", - "sha256:db295c5ca33f22ae058f317b07c6842a2b16d75c9e11e38d21395363d089692f" + "sha256:30289d0f98413fe3c3b74c9f7547b8e6fff6aca698c23445996cf6bf80a8bad0", + "sha256:e711f8a41393d9755178d6290fccbfc84f0b9db20bbdeababcf8cd9232f380cd" ], "index": "pypi", - "version": "==1.2.38" + "version": "==1.2.40" }, "colorama": { "hashes": [ @@ -134,6 +134,14 @@ ], "version": "==2.9" }, + "ip2location": { + "hashes": [ + "sha256:4a83f32098091b6aafaef63b393331bf973bbf54547db82de38f30108b5f079c", + "sha256:bbddbd5e3f79c18a9710afd43be09cb21724d7324557668017c545b5c5e62bc4" + ], + "index": "pypi", + "version": "==8.4.1" + }, "jdcal": { "hashes": [ "sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba", @@ -143,11 +151,11 @@ }, "loguru": { "hashes": [ - "sha256:1e0e6ff59be5e22f863d909ca989e34bb14c21b374f6af45281e603d003dbb96", - "sha256:4688d9e1f31d70e1ec7ccce5305967bc28f377eb1048d009108c11faebe05bcf" + "sha256:70201d5fce26da89b7a5f168caa2bb674e06b969829f56737db1d6472e53e7c3", + "sha256:e5d362a43cd2fc2da63551d79a6830619c4d5b3a8b976515748026f92f351b61" ], "index": "pypi", - "version": "==0.5.0" + "version": "==0.5.1" }, "multidict": { "hashes": [ @@ -320,11 +328,11 @@ }, "tqdm": { "hashes": [ - "sha256:4733c4a10d0f2a4d098d801464bdaf5240c7dadd2a7fde4ee93b0a0efd9fb25e", - "sha256:acdafb20f51637ca3954150d0405ff1a7edde0ff19e38fb99a80a66210d2a28f" + "sha256:07c06493f1403c1380b630ae3dcbe5ae62abcf369a93bbc052502279f189ab8c", + "sha256:cd140979c2bebd2311dfb14781d8f19bd5a9debb92dcab9f6ef899c987fcf71f" ], "index": "pypi", - "version": "==4.46.0" + "version": "==4.46.1" }, "urllib3": { "hashes": [ diff --git a/common/database.py b/common/database.py index b01c3fd..eb62b01 100644 --- a/common/database.py +++ b/common/database.py @@ -77,6 +77,9 @@ class Database(object): f'response text,' f'times text,' f'ttl text,' + f'cidr text,' + f'asn text,' + f'addr text,' f'resolver text,' f'module text,' f'source text,' @@ -102,13 +105,13 @@ class Database(object): f'insert into "{table_name}" (' f'id, type, alive, resolve, request, new, url, subdomain,' f'port, level, cname, content, public, status, reason,' - f'title, banner, header, response, times, ttl, resolver,' - f'module, source, elapse, find, brute, valid) ' + f'title, banner, header, response, times, ttl, cidr, asn,' + f'addr, resolver, module, source, elapse, find, brute, valid) ' f'values (:id, :type, :alive, :resolve, :request, :new,' f':url, :subdomain, :port, :level, :cname, :content,' f':public, :status, :reason, :title, :banner, :header,' - f':response, :times, :ttl, :resolver, :module, :source,' - f':elapse, :find, :brute, :valid)', results) + f':response, :times, :ttl, :cidr, :asn, :addr, :resolver,' + f':module, :source, :elapse, :find, :brute, :valid)', results) except Exception as e: logger.log('ERROR', e) @@ -231,8 +234,8 @@ class Database(object): table_name = table_name.replace('.', '_') query = f'select id, type, new, alive, request, resolve, url, ' \ f'subdomain, level, cname, content, public, port, status, ' \ - f'reason, title, banner, times, ttl, resolver, module, ' \ - f'source, elapse, find, brute, valid from "{table_name}"' + f'reason, title, banner, times, ttl, cidr, asn, addr, resolver, ' \ + f'module, source, elapse, find, brute, valid from "{table_name}"' if alive and limit: if limit in ['resolve', 'request']: where = f' where {limit} = 1' diff --git a/common/ipasn.py b/common/ipasn.py new file mode 100644 index 0000000..96e4d8d --- /dev/null +++ b/common/ipasn.py @@ -0,0 +1,74 @@ +import csv +import zipfile + +from common.utils import ip_to_int +from config.setting import data_storage_dir + + +class Entry(object): + def __init__(self, start, end, value): + self.start = int(start) + self.end = int(end) + self.value = value + + +class IPAsnData(object): + def __init__(self): + zip_path = data_storage_dir.joinpath("IP2LOCATION-LITE-ASN.CSV.ZIP") + csv_path = data_storage_dir.joinpath("IP2LOCATION-LITE-ASN.CSV") + if csv_path.exists(): + asn_fp = open(csv_path) + else: + zf = zipfile.ZipFile(zip_path) + zf.extract('IP2LOCATION-LITE-ASN.CSV', data_storage_dir) + asn_fp = open(csv_path) + self.data = [] + reader = csv.reader(asn_fp, delimiter=',', quotechar='"') + for row in reader: + e = Entry(row[0], row[1], row) + self.data.append(e) + asn_fp.close() + + def __iter__(self): + return self.data.__iter__() + + def find_i(self, ip, start, end): + if end - start < 100: + for i in range(start, end): + obj = self.data[i] + if obj.start <= ip <= obj.end: + return obj.value + return None + else: + mid = start + (end - start) // 2 + val = self.data[mid].start + if ip < val: + return self.find_i(ip, start, mid) + elif ip > val: + return self.find_i(ip, mid, end) + else: + return self.data[mid].value + + def find_int(self, ip): + return self.find_i(ip, 0, len(self.data) - 1) + + def find(self, ip): + return self.find_i(ip_to_int(ip), 0, len(self.data) - 1) + + +class IPAsnInfo(object): + def __init__(self): + self.asn = IPAsnData() + + def find(self, ip): + asn = self.asn.find(ip) + if asn: + result = {"cidr": asn[2], "asn": f'ASN{asn[3]} {asn[4]}'} + return result + else: + return None + + +if __name__ == "__main__": + asn_info = IPAsnInfo() + print(asn_info.find("188.81.94.77")) diff --git a/common/ipgeo.py b/common/ipgeo.py new file mode 100644 index 0000000..31e1060 --- /dev/null +++ b/common/ipgeo.py @@ -0,0 +1,11 @@ +import zipfile +import IP2Location +from config.setting import data_storage_dir + + +zip_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN.ZIP") +bin_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN") +if not bin_path.exists(): + zf = zipfile.ZipFile(zip_path) + zf.extract('IP2LOCATION-LITE-DB3.BIN', data_storage_dir) +IpGeoInfo = IP2Location.IP2Location(bin_path) diff --git a/common/module.py b/common/module.py index 082d2df..a0a4711 100644 --- a/common/module.py +++ b/common/module.py @@ -302,6 +302,9 @@ class Module(object): 'response': None, 'times': None, 'ttl': None, + 'cidr': None, + 'asn': None, + 'addr': None, 'resolver': None, 'module': self.module, 'source': self.source, @@ -331,6 +334,9 @@ class Module(object): times = record.get('times') ttl = record.get('ttl') public = record.get('public') + cidr = record.get('cidr') + asn = record.get('asn') + addr = record.get('addr') if isinstance(cname, list): cname = ','.join(cname) content = ','.join(content) @@ -358,6 +364,9 @@ class Module(object): 'response': None, 'times': times, 'ttl': ttl, + 'cidr': cidr, + 'asn': asn, + 'addr': addr, 'resolver': resolver, 'module': self.module, 'source': self.source, diff --git a/common/resolve.py b/common/resolve.py index 1958ce6..7fd6903 100644 --- a/common/resolve.py +++ b/common/resolve.py @@ -5,6 +5,8 @@ from config.log import logger from config import setting from common import utils from common.database import Database +from common.ipasn import IPAsnInfo +from common.ipgeo import IpGeoInfo def filter_subdomain(data): @@ -71,6 +73,8 @@ def save_subdomains(save_path, subdomain_list): def deal_output(output_path): logger.log('INFOR', f'Processing resolved results') records = dict() # 用来记录所有域名解析数据 + ip_asn = IPAsnInfo() + ip_geo = IpGeoInfo with open(output_path) as fd: for line in fd: line = line.strip() @@ -102,6 +106,9 @@ def deal_output(output_path): ips = list() public = list() ttls = list() + cidrs = list() + asns = list() + addrs = list() answers = data.get('answers') for answer in answers: if answer.get('type') == 'A': @@ -113,12 +120,22 @@ def deal_output(output_path): ttls.append(str(ttl)) is_public = utils.ip_is_public(ip) public.append(str(is_public)) + asn_info = ip_asn.find(ip) + cidrs.append(asn_info.get('cidr')) + asns.append(asn_info.get('asn')) + addr = f'{ip_geo.get_country_long(ip)} ' \ + f'{ip_geo.get_region(ip)} ' \ + f'{ip_geo.get_city(ip)}' + addrs.append(addr) record['resolve'] = 1 record['reason'] = status record['cname'] = ','.join(cname) record['content'] = ','.join(ips) record['public'] = ','.join(public) record['ttl'] = ','.join(ttls) + record['cidr'] = ','.join(cidrs) + record['asn'] = ','.join(asns) + record['addr'] = ','.join(addrs) records[qname] = record if not flag: record['alive'] = 0 diff --git a/common/utils.py b/common/utils.py index 13c368a..d4857eb 100644 --- a/common/utils.py +++ b/common/utils.py @@ -9,6 +9,7 @@ import subprocess from ipaddress import IPv4Address, ip_address from stat import S_IXUSR +import IP2Location import psutil import tenacity import requests @@ -638,3 +639,12 @@ def is_subname(name): if char not in chars: return False return True + + +def ip_to_int(ip): + try: + ipv4 = IPv4Address(ip) + except Exception as e: + logger.log('ERROR', e.args) + return None + return int(ipv4) diff --git a/data/IP2LOCATION-LITE-ASN.CSV.ZIP b/data/IP2LOCATION-LITE-ASN.CSV.ZIP new file mode 100644 index 0000000..af6c7ed Binary files /dev/null and b/data/IP2LOCATION-LITE-ASN.CSV.ZIP differ diff --git a/data/IP2LOCATION-LITE-DB3.BIN.ZIP b/data/IP2LOCATION-LITE-DB3.BIN.ZIP new file mode 100644 index 0000000..9c939a1 Binary files /dev/null and b/data/IP2LOCATION-LITE-DB3.BIN.ZIP differ diff --git a/requirements.txt b/requirements.txt index 1136b87..4b6f304 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,13 @@ aiohttp==3.6.2 +aiomultiprocess==0.7.0 async-timeout==3.0.1 attrs==19.3.0 beautifulsoup4==4.9.1 +Brotli==1.0.7 bs4==0.0.1 -certifi==2020.4.5.1 +certifi==2020.4.5.2 chardet==3.0.4 -cloudscraper==1.2.38 +cloudscraper==1.2.40 colorama==0.4.3 dnspython==1.16.0 docopt==0.6.2 @@ -13,13 +15,17 @@ et-xmlfile==1.0.1 exrex==0.10.5 fire==0.3.1 idna==2.9 +IP2Location==8.4.1 jdcal==1.4.1 -loguru==0.5.0 +Js2Py==0.70 +loguru==0.5.1 multidict==4.7.6 openpyxl==2.4.11 psutil==5.7.0 +pyjsparser==2.7.1 pyparsing==2.4.7 PySocks==1.7.1 +pytz==2019.3 records==0.5.3 requests==2.23.0 requests-file==1.5.1 @@ -31,7 +37,8 @@ tablib==2.0.0 tenacity==6.2.0 termcolor==1.1.0 tldextract==2.2.2 -tqdm==4.46.0 +tqdm==4.46.1 +tzlocal==2.0.0 urllib3==1.25.9 win32-setctime==1.0.1 yarl==1.4.2 \ No newline at end of file