river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-26 04:47:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

添加cidr，asn，addr字段

Browse Source
This commit is contained in:
Jing Ling
2020-06-18 10:38:41 +08:00
parent 771502cc2c
commit 171440ae59
11 changed files with 163 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Pipfile
+1
View File
@@ -21,6 +21,7 @@ cloudscraper = "*"
tablib = "*"
psutil = "*"
tenacity = "*"
ip2location = "*"
[requires]
python_version = "3.8"
Pipfile.lock
Generated
+21 -13
View File
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "0501b2b68bd52ab7ba0d40722bc8a84bddcd2af023babc9f986645384e381462"
"sha256": "6556f720e7fb052c3f923f3e7986cc00c1cc230db729ad79685a127c02568c96"
},
"pipfile-spec": 6,
"requires": {
@@ -65,10 +65,10 @@
},
"certifi": {
"hashes": [
"sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304",
"sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"
"sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1",
"sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc"
],
"version": "==2020.4.5.1"
"version": "==2020.4.5.2"
},
"chardet": {
"hashes": [
@@ -79,11 +79,11 @@
},
"cloudscraper": {
"hashes": [
"sha256:3893be3c281ddb7a39bf35e558da19247a39a408ef48078fdabf5058b7659d6c",
"sha256:db295c5ca33f22ae058f317b07c6842a2b16d75c9e11e38d21395363d089692f"
"sha256:30289d0f98413fe3c3b74c9f7547b8e6fff6aca698c23445996cf6bf80a8bad0",
"sha256:e711f8a41393d9755178d6290fccbfc84f0b9db20bbdeababcf8cd9232f380cd"
],
"index": "pypi",
"version": "==1.2.38"
"version": "==1.2.40"
},
"colorama": {
"hashes": [
@@ -134,6 +134,14 @@
],
"version": "==2.9"
},
"ip2location": {
"hashes": [
"sha256:4a83f32098091b6aafaef63b393331bf973bbf54547db82de38f30108b5f079c",
"sha256:bbddbd5e3f79c18a9710afd43be09cb21724d7324557668017c545b5c5e62bc4"
],
"index": "pypi",
"version": "==8.4.1"
},
"jdcal": {
"hashes": [
"sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba",
@@ -143,11 +151,11 @@
},
"loguru": {
"hashes": [
"sha256:1e0e6ff59be5e22f863d909ca989e34bb14c21b374f6af45281e603d003dbb96",
"sha256:4688d9e1f31d70e1ec7ccce5305967bc28f377eb1048d009108c11faebe05bcf"
"sha256:70201d5fce26da89b7a5f168caa2bb674e06b969829f56737db1d6472e53e7c3",
"sha256:e5d362a43cd2fc2da63551d79a6830619c4d5b3a8b976515748026f92f351b61"
],
"index": "pypi",
"version": "==0.5.0"
"version": "==0.5.1"
},
"multidict": {
"hashes": [
@@ -320,11 +328,11 @@
},
"tqdm": {
"hashes": [
"sha256:4733c4a10d0f2a4d098d801464bdaf5240c7dadd2a7fde4ee93b0a0efd9fb25e",
"sha256:acdafb20f51637ca3954150d0405ff1a7edde0ff19e38fb99a80a66210d2a28f"
"sha256:07c06493f1403c1380b630ae3dcbe5ae62abcf369a93bbc052502279f189ab8c",
"sha256:cd140979c2bebd2311dfb14781d8f19bd5a9debb92dcab9f6ef899c987fcf71f"
],
"index": "pypi",
"version": "==4.46.0"
"version": "==4.46.1"
},
"urllib3": {
"hashes": [
common/database.py
+9 -6
View File
@@ -77,6 +77,9 @@ class Database(object):
f'response text,'
f'times text,'
f'ttl text,'
f'cidr text,'
f'asn text,'
f'addr text,'
f'resolver text,'
f'module text,'
f'source text,'
@@ -102,13 +105,13 @@ class Database(object):
f'insert into "{table_name}" ('
f'id, type, alive, resolve, request, new, url, subdomain,'
f'port, level, cname, content, public, status, reason,'
f'title, banner, header, response, times, ttl, resolver,'
f'module, source, elapse, find, brute, valid) '
f'title, banner, header, response, times, ttl, cidr, asn,'
f'addr, resolver, module, source, elapse, find, brute, valid) '
f'values (:id, :type, :alive, :resolve, :request, :new,'
f':url, :subdomain, :port, :level, :cname, :content,'
f':public, :status, :reason, :title, :banner, :header,'
f':response, :times, :ttl, :resolver, :module, :source,'
f':elapse, :find, :brute, :valid)', results)
f':response, :times, :ttl, :cidr, :asn, :addr, :resolver,'
f':module, :source, :elapse, :find, :brute, :valid)', results)
except Exception as e:
logger.log('ERROR', e)
@@ -231,8 +234,8 @@ class Database(object):
table_name = table_name.replace('.', '_')
query = f'select id, type, new, alive, request, resolve, url, ' \
f'subdomain, level, cname, content, public, port, status, ' \
f'reason, title, banner, times, ttl, resolver, module, ' \
f'source, elapse, find, brute, valid from "{table_name}"'
f'reason, title, banner, times, ttl, cidr, asn, addr, resolver, ' \
f'module, source, elapse, find, brute, valid from "{table_name}"'
if alive and limit:
if limit in ['resolve', 'request']:
where = f' where {limit} = 1'
common/ipasn.py
+74
View File
@@ -0,0 +1,74 @@
import csv
import zipfile
from common.utils import ip_to_int
from config.setting import data_storage_dir
class Entry(object):
def __init__(self, start, end, value):
self.start = int(start)
self.end = int(end)
self.value = value
class IPAsnData(object):
def __init__(self):
zip_path = data_storage_dir.joinpath("IP2LOCATION-LITE-ASN.CSV.ZIP")
csv_path = data_storage_dir.joinpath("IP2LOCATION-LITE-ASN.CSV")
if csv_path.exists():
asn_fp = open(csv_path)
else:
zf = zipfile.ZipFile(zip_path)
zf.extract('IP2LOCATION-LITE-ASN.CSV', data_storage_dir)
asn_fp = open(csv_path)
self.data = []
reader = csv.reader(asn_fp, delimiter=',', quotechar='"')
for row in reader:
e = Entry(row[0], row[1], row)
self.data.append(e)
asn_fp.close()
def __iter__(self):
return self.data.__iter__()
def find_i(self, ip, start, end):
if end - start < 100:
for i in range(start, end):
obj = self.data[i]
if obj.start <= ip <= obj.end:
return obj.value
return None
else:
mid = start + (end - start) // 2
val = self.data[mid].start
if ip < val:
return self.find_i(ip, start, mid)
elif ip > val:
return self.find_i(ip, mid, end)
else:
return self.data[mid].value
def find_int(self, ip):
return self.find_i(ip, 0, len(self.data) - 1)
def find(self, ip):
return self.find_i(ip_to_int(ip), 0, len(self.data) - 1)
class IPAsnInfo(object):
def __init__(self):
self.asn = IPAsnData()
def find(self, ip):
asn = self.asn.find(ip)
if asn:
result = {"cidr": asn[2], "asn": f'ASN{asn[3]} {asn[4]}'}
return result
else:
return None
if __name__ == "__main__":
asn_info = IPAsnInfo()
print(asn_info.find("188.81.94.77"))
common/ipgeo.py
+11
View File
@@ -0,0 +1,11 @@
import zipfile
import IP2Location
from config.setting import data_storage_dir
zip_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN.ZIP")
bin_path = data_storage_dir.joinpath("IP2LOCATION-LITE-DB3.BIN")
if not bin_path.exists():
zf = zipfile.ZipFile(zip_path)
zf.extract('IP2LOCATION-LITE-DB3.BIN', data_storage_dir)
IpGeoInfo = IP2Location.IP2Location(bin_path)
common/module.py
+9
View File
@@ -302,6 +302,9 @@ class Module(object):
'response': None,
'times': None,
'ttl': None,
'cidr': None,
'asn': None,
'addr': None,
'resolver': None,
'module': self.module,
'source': self.source,
@@ -331,6 +334,9 @@ class Module(object):
times = record.get('times')
ttl = record.get('ttl')
public = record.get('public')
cidr = record.get('cidr')
asn = record.get('asn')
addr = record.get('addr')
if isinstance(cname, list):
cname = ','.join(cname)
content = ','.join(content)
@@ -358,6 +364,9 @@ class Module(object):
'response': None,
'times': times,
'ttl': ttl,
'cidr': cidr,
'asn': asn,
'addr': addr,
'resolver': resolver,
'module': self.module,
'source': self.source,
common/resolve.py
+17
View File
@@ -5,6 +5,8 @@ from config.log import logger
from config import setting
from common import utils
from common.database import Database
from common.ipasn import IPAsnInfo
from common.ipgeo import IpGeoInfo
def filter_subdomain(data):
@@ -71,6 +73,8 @@ def save_subdomains(save_path, subdomain_list):
def deal_output(output_path):
logger.log('INFOR', f'Processing resolved results')
records = dict() # 用来记录所有域名解析数据
ip_asn = IPAsnInfo()
ip_geo = IpGeoInfo
with open(output_path) as fd:
for line in fd:
line = line.strip()
@@ -102,6 +106,9 @@ def deal_output(output_path):
ips = list()
public = list()
ttls = list()
cidrs = list()
asns = list()
addrs = list()
answers = data.get('answers')
for answer in answers:
if answer.get('type') == 'A':
@@ -113,12 +120,22 @@ def deal_output(output_path):
ttls.append(str(ttl))
is_public = utils.ip_is_public(ip)
public.append(str(is_public))
asn_info = ip_asn.find(ip)
cidrs.append(asn_info.get('cidr'))
asns.append(asn_info.get('asn'))
addr = f'{ip_geo.get_country_long(ip)} ' \
f'{ip_geo.get_region(ip)} ' \
f'{ip_geo.get_city(ip)}'
addrs.append(addr)
record['resolve'] = 1
record['reason'] = status
record['cname'] = ','.join(cname)
record['content'] = ','.join(ips)
record['public'] = ','.join(public)
record['ttl'] = ','.join(ttls)
record['cidr'] = ','.join(cidrs)
record['asn'] = ','.join(asns)
record['addr'] = ','.join(addrs)
records[qname] = record
if not flag:
record['alive'] = 0
common/utils.py
+10
View File
@@ -9,6 +9,7 @@ import subprocess
from ipaddress import IPv4Address, ip_address
from stat import S_IXUSR
import IP2Location
import psutil
import tenacity
import requests
@@ -638,3 +639,12 @@ def is_subname(name):
if char not in chars:
return False
return True
def ip_to_int(ip):
try:
ipv4 = IPv4Address(ip)
except Exception as e:
logger.log('ERROR', e.args)
return None
return int(ipv4)
data/IP2LOCATION-LITE-ASN.CSV.ZIP
BIN
View File
Binary file not shown.
data/IP2LOCATION-LITE-DB3.BIN.ZIP
BIN
View File
Binary file not shown.
requirements.txt
+11 -4
View File
@@ -1,11 +1,13 @@
aiohttp==3.6.2
aiomultiprocess==0.7.0
async-timeout==3.0.1
attrs==19.3.0
beautifulsoup4==4.9.1
Brotli==1.0.7
bs4==0.0.1
certifi==2020.4.5.1
certifi==2020.4.5.2
chardet==3.0.4
cloudscraper==1.2.38
cloudscraper==1.2.40
colorama==0.4.3
dnspython==1.16.0
docopt==0.6.2
@@ -13,13 +15,17 @@ et-xmlfile==1.0.1
exrex==0.10.5
fire==0.3.1
idna==2.9
IP2Location==8.4.1
jdcal==1.4.1
loguru==0.5.0
Js2Py==0.70
loguru==0.5.1
multidict==4.7.6
openpyxl==2.4.11
psutil==5.7.0
pyjsparser==2.7.1
pyparsing==2.4.7
PySocks==1.7.1
pytz==2019.3
records==0.5.3
requests==2.23.0
requests-file==1.5.1
@@ -31,7 +37,8 @@ tablib==2.0.0
tenacity==6.2.0
termcolor==1.1.0
tldextract==2.2.2
tqdm==4.46.0
tqdm==4.46.1
tzlocal==2.0.0
urllib3==1.25.9
win32-setctime==1.0.1
yarl==1.4.2