river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-26 04:47:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

添加cdn判断模块

Browse Source
This commit is contained in:
Jing Ling
2020-07-26 03:23:15 +08:00
parent ca8d34ee8d
commit 91f272f4b6
10 changed files with 797 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/database.py
+14 -13
View File
@@ -69,6 +69,7 @@ class Database(object):
f'cname text,'
f'content text,'
f'public int,'
f'cdn int,'
f'status int,'
f'reason text,'
f'title text,'
@@ -79,6 +80,7 @@ class Database(object):
f'ttl text,'
f'cidr text,'
f'asn text,'
f'org text,'
f'ip2region text,'
f'ip2location text,'
f'resolver text,'
@@ -104,13 +106,14 @@ class Database(object):
try:
self.conn.bulk_query(
f'insert into "{table_name}" (id, type, alive, resolve, request, new,'
f'url, subdomain, port, level, cname, content, public, status, reason,'
f'title, banner, header, response, times, ttl, cidr, asn, ip2region,'
f'ip2location, resolver, module, source, elapse, find, brute, valid) '
f'url, subdomain, port, level, cname, content, public, cdn, status,'
f'reason, title, banner, header, response, times, ttl, cidr, asn, org,'
f' ip2region, ip2location, resolver, module, source, elapse, find,'
f'brute, valid) '
f'values (:id, :type, :alive, :resolve, :request, :new, :url, '
f':subdomain, :port, :level, :cname, :content, :public, :status,'
f':subdomain, :port, :level, :cname, :content, :public, :cdn, :status,'
f':reason, :title, :banner, :header, :response, :times, :ttl, :cidr,'
f':asn, :ip2region, :ip2location, :resolver, :module, :source,'
f':asn, :org, :ip2region, :ip2location, :resolver, :module, :source,'
f':elapse, :find, :brute, :valid)', results)
except Exception as e:
logger.log('ERROR', e)
@@ -124,9 +127,8 @@ class Database(object):
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'Determining whether the {table_name} table exists')
results = self.query(f'select count() from sqlite_master '
f'where type = "table" and '
f'name = "{table_name}"')
results = self.query(f'select count() from sqlite_master where type = "table" and'
f' name = "{table_name}"')
if results.scalar() == 0:
return False
else:
@@ -232,11 +234,10 @@ class Database(object):
:param str limit: limit value
"""
table_name = table_name.replace('.', '_')
query = f'select id, type, new, alive, request, resolve, url, ' \
f'subdomain, level, cname, content, public, port, status, ' \
f'reason, title, banner, times, ttl, cidr, asn, ip2region, ' \
f'ip2location, resolver, module, source, elapse, find, brute, valid ' \
f'from "{table_name}"'
query = f'select id, type, new, alive, request, resolve, url, subdomain, level,' \
f'cname, content, public, cdn, port, status, reason, title, banner,' \
f'times, ttl, cidr, asn, org, ip2region, ip2location, resolver, module,' \
f'source, elapse, find, brute, valid from "{table_name}"'
if alive and limit:
if limit in ['resolve', 'request']:
where = f' where {limit} = 1'
common/ipasn.py
+2 -2
View File
@@ -63,10 +63,10 @@ class IPAsnInfo(object):
def find(self, ip):
asn = self.asn.find(ip)
if asn:
result = {'cidr': asn[2], 'asn': f'AS{asn[3]} {asn[4]}'}
result = {'cidr': asn[2], 'asn': f'AS{asn[3]}', 'org': asn[4]}
return result
else:
return {'cidr': '', 'asn': ''}
return {'cidr': '', 'asn': '', 'org': ''}
if __name__ == "__main__":
common/module.py
+6 -1
View File
@@ -4,7 +4,6 @@ Module base class
"""
import json
import re
import threading
import time
@@ -278,6 +277,7 @@ class Module(object):
'cname': None,
'content': None,
'public': None,
'cdn': None,
'status': None,
'reason': None,
'title': None,
@@ -288,6 +288,7 @@ class Module(object):
'ttl': None,
'cidr': None,
'asn': None,
'org': None,
'ip2region': None,
'ip2location': None,
'resolver': None,
@@ -319,8 +320,10 @@ class Module(object):
times = record.get('times')
ttl = record.get('ttl')
public = record.get('public')
cdn = record.get('cdn')
cidr = record.get('cidr')
asn = record.get('asn')
org = record.get('org')
ip2region = record.get('ip2region')
ip2location = record.get('ip2location')
if isinstance(cname, list):
@@ -342,6 +345,7 @@ class Module(object):
'cname': cname,
'content': content,
'public': public,
'cdn': cdn,
'status': None,
'reason': reason,
'title': None,
@@ -352,6 +356,7 @@ class Module(object):
'ttl': ttl,
'cidr': cidr,
'asn': asn,
'org': org,
'ip2region': ip2region,
'ip2location': ip2location,
'resolver': resolver,
common/request.py
+2 -1
View File
@@ -1,3 +1,4 @@
import json
import asyncio
import functools
@@ -171,7 +172,7 @@ def request_callback(future, index, datas):
datas[index]['alive'] = 1
headers = resp.headers
datas[index]['banner'] = utils.get_sample_banner(headers)
datas[index]['header'] = str(dict(headers))[1:-1]
datas[index]['header'] = json.dumps(dict(headers))
if isinstance(text, str):
title = get_title(text).strip()
datas[index]['title'] = utils.remove_invalid_string(title)
common/resolve.py
+3
View File
@@ -107,6 +107,7 @@ def deal_output(output_path):
ttls = list()
cidrs = list()
asns = list()
orgs = list()
locs = list()
regs = list()
answers = data.get('answers')
@@ -123,6 +124,7 @@ def deal_output(output_path):
asn_info = ip_asn.find(ip)
cidrs.append(asn_info.get('cidr'))
asns.append(asn_info.get('asn'))
orgs.append(asn_info.get('org'))
loc = f'{ip_geo.get_country_long(ip)} ' \
f'{ip_geo.get_region(ip)} ' \
f'{ip_geo.get_city(ip)}'
@@ -137,6 +139,7 @@ def deal_output(output_path):
record['ttl'] = ','.join(ttls)
record['cidr'] = ','.join(cidrs)
record['asn'] = ','.join(asns)
record['org'] = ','.join(orgs)
record['ip2location'] = ','.join(locs)
record['ip2region'] = ','.join(regs)
records[qname] = record
common/utils.py
+6
View File
@@ -1,3 +1,4 @@
import json
import os
import re
import sys
@@ -215,6 +216,11 @@ def check_format(format, count):
return 'csv'
def load_json(path):
with open(path) as fp:
return json.load(fp)
def save_db(name, data, module):
"""
Save request results to database