river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-26 04:47:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

添加cidr，asn，addr字段

Browse Source
This commit is contained in:
Jing Ling
2020-06-18 10:38:41 +08:00
parent 771502cc2c
commit 171440ae59
11 changed files with 163 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/database.py
+9 -6
View File
@@ -77,6 +77,9 @@ class Database(object):
f'response text,'
f'times text,'
f'ttl text,'
f'cidr text,'
f'asn text,'
f'addr text,'
f'resolver text,'
f'module text,'
f'source text,'
@@ -102,13 +105,13 @@ class Database(object):
f'insert into "{table_name}" ('
f'id, type, alive, resolve, request, new, url, subdomain,'
f'port, level, cname, content, public, status, reason,'
f'title, banner, header, response, times, ttl, resolver,'
f'module, source, elapse, find, brute, valid) '
f'title, banner, header, response, times, ttl, cidr, asn,'
f'addr, resolver, module, source, elapse, find, brute, valid) '
f'values (:id, :type, :alive, :resolve, :request, :new,'
f':url, :subdomain, :port, :level, :cname, :content,'
f':public, :status, :reason, :title, :banner, :header,'
f':response, :times, :ttl, :resolver, :module, :source,'
f':elapse, :find, :brute, :valid)', results)
f':response, :times, :ttl, :cidr, :asn, :addr, :resolver,'
f':module, :source, :elapse, :find, :brute, :valid)', results)
except Exception as e:
logger.log('ERROR', e)
@@ -231,8 +234,8 @@ class Database(object):
table_name = table_name.replace('.', '_')
query = f'select id, type, new, alive, request, resolve, url, ' \
f'subdomain, level, cname, content, public, port, status, ' \
f'reason, title, banner, times, ttl, resolver, module, ' \
f'source, elapse, find, brute, valid from "{table_name}"'
f'reason, title, banner, times, ttl, cidr, asn, addr, resolver, ' \
f'module, source, elapse, find, brute, valid from "{table_name}"'
if alive and limit:
if limit in ['resolve', 'request']:
where = f' where {limit} = 1'
common/ipasn.py
+74
View File
@@ -0,0 +1,74 @@
import csv
import zipfile
from common.utils import ip_to_int
from config.setting import data_storage_dir
class Entry(object):
def __init__(self, start, end, value):
self.start = int(start)
self.end = int(end)
self.value = value
class IPAsnData(object):
def __init__(self):
zip_path = data_storage_dir.joinpath("IP2LOCATION-LITE-ASN.CSV.ZIP")
csv_path = data_storage_dir.joinpath("IP2LOCATION-LITE-ASN.CSV")
if csv_path.exists():
asn_fp = open(csv_path)
else:
zf = zipfile.ZipFile(zip_path)
zf.extract('IP2LOCATION-LITE-ASN.CSV', data_storage_dir)
asn_fp = open(csv_path)
self.data = []
reader = csv.reader(asn_fp, delimiter=',', quotechar='"')
for row in reader:
e = Entry(row[0], row[1], row)
self.data.append(e)
asn_fp.close()
def __iter__(self):
return self.data.__iter__()
def find_i(self, ip, start, end):
if end - start < 100:
for i in range(start, end):
obj = self.data[i]
if obj.start <= ip <= obj.end:
return obj.value
return None
else:
mid = start + (end - start) // 2
val = self.data[mid].start
if ip < val:
return self.find_i(ip, start, mid)
elif ip > val:
return self.find_i(ip, mid, end)
else:
return self.data[mid].value
def find_int(self, ip):
return self.find_i(ip, 0, len(self.data) - 1)
def find(self, ip):
return self.find_i(ip_to_int(ip), 0, len(self.data) - 1)
class IPAsnInfo(object):
def __init__(self):
self.asn = IPAsnData()
def find(self, ip):
asn = self.asn.find(ip)
if asn:
result = {"cidr": asn[2], "asn": f'ASN{asn[3]} {asn[4]}'}
return result
else:
return None
if __name__ == "__main__":
asn_info = IPAsnInfo()
print(asn_info.find("188.81.94.77"))
common/ipgeo.py
+11
View File
@@ -0,0 +1,11 @@
import zipfile
import IP2Location
from config.setting import data_storage_dir
zip_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN.ZIP")
bin_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN")
if not bin_path.exists():
zf = zipfile.ZipFile(zip_path)
zf.extract('IP2LOCATION-LITE-DB3.BIN', data_storage_dir)
IpGeoInfo = IP2Location.IP2Location(bin_path)
common/module.py
+9
View File
@@ -302,6 +302,9 @@ class Module(object):
'response': None,
'times': None,
'ttl': None,
'cidr': None,
'asn': None,
'addr': None,
'resolver': None,
'module': self.module,
'source': self.source,
@@ -331,6 +334,9 @@ class Module(object):
times = record.get('times')
ttl = record.get('ttl')
public = record.get('public')
cidr = record.get('cidr')
asn = record.get('asn')
addr = record.get('addr')
if isinstance(cname, list):
cname = ','.join(cname)
content = ','.join(content)
@@ -358,6 +364,9 @@ class Module(object):
'response': None,
'times': times,
'ttl': ttl,
'cidr': cidr,
'asn': asn,
'addr': addr,
'resolver': resolver,
'module': self.module,
'source': self.source,
common/resolve.py
+17
View File
@@ -5,6 +5,8 @@ from config.log import logger
from config import setting
from common import utils
from common.database import Database
from common.ipasn import IPAsnInfo
from common.ipgeo import IpGeoInfo
def filter_subdomain(data):
@@ -71,6 +73,8 @@ def save_subdomains(save_path, subdomain_list):
def deal_output(output_path):
logger.log('INFOR', f'Processing resolved results')
records = dict() # 用来记录所有域名解析数据
ip_asn = IPAsnInfo()
ip_geo = IpGeoInfo
with open(output_path) as fd:
for line in fd:
line = line.strip()
@@ -102,6 +106,9 @@ def deal_output(output_path):
ips = list()
public = list()
ttls = list()
cidrs = list()
asns = list()
addrs = list()
answers = data.get('answers')
for answer in answers:
if answer.get('type') == 'A':
@@ -113,12 +120,22 @@ def deal_output(output_path):
ttls.append(str(ttl))
is_public = utils.ip_is_public(ip)
public.append(str(is_public))
asn_info = ip_asn.find(ip)
cidrs.append(asn_info.get('cidr'))
asns.append(asn_info.get('asn'))
addr = f'{ip_geo.get_country_long(ip)} ' \
f'{ip_geo.get_region(ip)} ' \
f'{ip_geo.get_city(ip)}'
addrs.append(addr)
record['resolve'] = 1
record['reason'] = status
record['cname'] = ','.join(cname)
record['content'] = ','.join(ips)
record['public'] = ','.join(public)
record['ttl'] = ','.join(ttls)
record['cidr'] = ','.join(cidrs)
record['asn'] = ','.join(asns)
record['addr'] = ','.join(addrs)
records[qname] = record
if not flag:
record['alive'] = 0
common/utils.py
+10
View File
@@ -9,6 +9,7 @@ import subprocess
from ipaddress import IPv4Address, ip_address
from stat import S_IXUSR
import IP2Location
import psutil
import tenacity
import requests
@@ -638,3 +639,12 @@ def is_subname(name):
if char not in chars:
return False
return True
def ip_to_int(ip):
try:
ipv4 = IPv4Address(ip)
except Exception as e:
logger.log('ERROR', e.args)
return None
return int(ipv4)