river_tao/OneForAll-mirror
1
0
Fork 0
mirror of https://github.com/shmilylty/OneForAll.git synced 2026-08-25 20:37:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

添加cdn判断模块

Browse Source
This commit is contained in:
Jing Ling
2020-07-26 03:23:15 +08:00
parent ca8d34ee8d
commit 91f272f4b6
10 changed files with 797 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/database.py
+14 -13
View File
@@ -69,6 +69,7 @@ class Database(object):
f'cname text,'
f'content text,'
f'public int,'
f'cdn int,'
f'status int,'
f'reason text,'
f'title text,'
@@ -79,6 +80,7 @@ class Database(object):
f'ttl text,'
f'cidr text,'
f'asn text,'
f'org text,'
f'ip2region text,'
f'ip2location text,'
f'resolver text,'
@@ -104,13 +106,14 @@ class Database(object):
try:
self.conn.bulk_query(
f'insert into "{table_name}" (id, type, alive, resolve, request, new,'
f'url, subdomain, port, level, cname, content, public, status, reason,'
f'title, banner, header, response, times, ttl, cidr, asn, ip2region,'
f'ip2location, resolver, module, source, elapse, find, brute, valid) '
f'url, subdomain, port, level, cname, content, public, cdn, status,'
f'reason, title, banner, header, response, times, ttl, cidr, asn, org,'
f' ip2region, ip2location, resolver, module, source, elapse, find,'
f'brute, valid) '
f'values (:id, :type, :alive, :resolve, :request, :new, :url, '
f':subdomain, :port, :level, :cname, :content, :public, :status,'
f':subdomain, :port, :level, :cname, :content, :public, :cdn, :status,'
f':reason, :title, :banner, :header, :response, :times, :ttl, :cidr,'
f':asn, :ip2region, :ip2location, :resolver, :module, :source,'
f':asn, :org, :ip2region, :ip2location, :resolver, :module, :source,'
f':elapse, :find, :brute, :valid)', results)
except Exception as e:
logger.log('ERROR', e)
@@ -124,9 +127,8 @@ class Database(object):
"""
table_name = table_name.replace('.', '_')
logger.log('TRACE', f'Determining whether the {table_name} table exists')
results = self.query(f'select count() from sqlite_master '
f'where type = "table" and '
f'name = "{table_name}"')
results = self.query(f'select count() from sqlite_master where type = "table" and'
f' name = "{table_name}"')
if results.scalar() == 0:
return False
else:
@@ -232,11 +234,10 @@ class Database(object):
:param str limit: limit value
"""
table_name = table_name.replace('.', '_')
query = f'select id, type, new, alive, request, resolve, url, ' \
f'subdomain, level, cname, content, public, port, status, ' \
f'reason, title, banner, times, ttl, cidr, asn, ip2region, ' \
f'ip2location, resolver, module, source, elapse, find, brute, valid ' \
f'from "{table_name}"'
query = f'select id, type, new, alive, request, resolve, url, subdomain, level,' \
f'cname, content, public, cdn, port, status, reason, title, banner,' \
f'times, ttl, cidr, asn, org, ip2region, ip2location, resolver, module,' \
f'source, elapse, find, brute, valid from "{table_name}"'
if alive and limit:
if limit in ['resolve', 'request']:
where = f' where {limit} = 1'
common/ipasn.py
+2 -2
View File
@@ -63,10 +63,10 @@ class IPAsnInfo(object):
def find(self, ip):
asn = self.asn.find(ip)
if asn:
result = {'cidr': asn[2], 'asn': f'AS{asn[3]} {asn[4]}'}
result = {'cidr': asn[2], 'asn': f'AS{asn[3]}', 'org': asn[4]}
return result
else:
return {'cidr': '', 'asn': ''}
return {'cidr': '', 'asn': '', 'org': ''}
if __name__ == "__main__":
common/module.py
+6 -1
View File
@@ -4,7 +4,6 @@ Module base class
"""
import json
import re
import threading
import time
@@ -278,6 +277,7 @@ class Module(object):
'cname': None,
'content': None,
'public': None,
'cdn': None,
'status': None,
'reason': None,
'title': None,
@@ -288,6 +288,7 @@ class Module(object):
'ttl': None,
'cidr': None,
'asn': None,
'org': None,
'ip2region': None,
'ip2location': None,
'resolver': None,
@@ -319,8 +320,10 @@ class Module(object):
times = record.get('times')
ttl = record.get('ttl')
public = record.get('public')
cdn = record.get('cdn')
cidr = record.get('cidr')
asn = record.get('asn')
org = record.get('org')
ip2region = record.get('ip2region')
ip2location = record.get('ip2location')
if isinstance(cname, list):
@@ -342,6 +345,7 @@ class Module(object):
'cname': cname,
'content': content,
'public': public,
'cdn': cdn,
'status': None,
'reason': reason,
'title': None,
@@ -352,6 +356,7 @@ class Module(object):
'ttl': ttl,
'cidr': cidr,
'asn': asn,
'org': org,
'ip2region': ip2region,
'ip2location': ip2location,
'resolver': resolver,
common/request.py
+2 -1
View File
@@ -1,3 +1,4 @@
import json
import asyncio
import functools
@@ -171,7 +172,7 @@ def request_callback(future, index, datas):
datas[index]['alive'] = 1
headers = resp.headers
datas[index]['banner'] = utils.get_sample_banner(headers)
datas[index]['header'] = str(dict(headers))[1:-1]
datas[index]['header'] = json.dumps(dict(headers))
if isinstance(text, str):
title = get_title(text).strip()
datas[index]['title'] = utils.remove_invalid_string(title)
common/resolve.py
+3
View File
@@ -107,6 +107,7 @@ def deal_output(output_path):
ttls = list()
cidrs = list()
asns = list()
orgs = list()
locs = list()
regs = list()
answers = data.get('answers')
@@ -123,6 +124,7 @@ def deal_output(output_path):
asn_info = ip_asn.find(ip)
cidrs.append(asn_info.get('cidr'))
asns.append(asn_info.get('asn'))
orgs.append(asn_info.get('org'))
loc = f'{ip_geo.get_country_long(ip)} ' \
f'{ip_geo.get_region(ip)} ' \
f'{ip_geo.get_city(ip)}'
@@ -137,6 +139,7 @@ def deal_output(output_path):
record['ttl'] = ','.join(ttls)
record['cidr'] = ','.join(cidrs)
record['asn'] = ','.join(asns)
record['org'] = ','.join(orgs)
record['ip2location'] = ','.join(locs)
record['ip2region'] = ','.join(regs)
records[qname] = record
common/utils.py
+6
View File
@@ -1,3 +1,4 @@
import json
import os
import re
import sys
@@ -215,6 +216,11 @@ def check_format(format, count):
return 'csv'
def load_json(path):
with open(path) as fp:
return json.load(fp)
def save_db(name, data, module):
"""
Save request results to database
data/cdn_asn_list.json
+134
View File
@@ -0,0 +1,134 @@
[
"10576",
"10762",
"11748",
"131099",
"132601",
"133496",
"134409",
"135295",
"136764",
"137187",
"13777",
"13890",
"14103",
"14520",
"17132",
"199251",
"200013",
"200325",
"200856",
"201263",
"202294",
"203075",
"203139",
"204248",
"204286",
"204545",
"206227",
"206734",
"206848",
"206986",
"207158",
"208559",
"209403",
"21030",
"21257",
"23327",
"23393",
"23637",
"23794",
"24997",
"26492",
"268843",
"28709",
"29264",
"30282",
"30637",
"328126",
"36408",
"38107",
"397192",
"40366",
"43303",
"44907",
"46071",
"46177",
"47542",
"49287",
"49689",
"51286",
"55082",
"55254",
"56636",
"57363",
"58127",
"59730",
"59776",
"60068",
"60626",
"60922",
"61107",
"61159",
"62026",
"62229",
"63062",
"64232",
"8868",
"9053",
"55770",
"49846",
"49249",
"48163",
"45700",
"43639",
"39836",
"393560",
"393234",
"36183",
"35994",
"35993",
"35204",
"34850",
"34164",
"33905",
"32787",
"31377",
"31110",
"31109",
"31108",
"31107",
"30675",
"24319",
"23903",
"23455",
"23454",
"22207",
"21399",
"21357",
"21342",
"20940",
"20189",
"18717",
"18680",
"17334",
"16702",
"16625",
"12222",
"209101",
"201585",
"135429",
"395747",
"394536",
"209242",
"203898",
"202623",
"14789",
"133877",
"13335",
"132892",
"21859",
"6185",
"47823",
"4134"
]
data/cdn_ip_cidr.json
+542
View File
@@ -0,0 +1,542 @@
[
"223.99.255.0/24",
"71.152.0.0/17",
"219.153.73.0/24",
"125.39.46.0/24",
"190.93.240.0/20",
"14.0.113.0/24",
"14.0.47.0/24",
"113.20.148.0/22",
"103.75.201.0/24",
"1.32.239.0/24",
"101.79.239.0/24",
"52.46.0.0/18",
"125.88.189.0/24",
"150.138.248.0/24",
"180.153.235.0/24",
"205.251.252.0/23",
"103.1.65.0/24",
"115.127.227.0/24",
"14.0.42.0/24",
"109.199.58.0/24",
"116.211.155.0/24",
"112.253.3.0/24",
"14.0.58.0/24",
"223.112.227.0/24",
"113.20.150.0/23",
"61.182.141.0/24",
"34.216.51.0/25",
"124.95.188.0/24",
"42.51.25.0/24",
"183.136.133.0/24",
"52.220.191.0/26",
"119.84.93.0/24",
"182.118.38.0/24",
"13.59.250.0/26",
"54.178.75.0/24",
"119.84.92.0/24",
"183.131.62.0/24",
"111.32.136.0/24",
"13.124.199.0/24",
"111.47.227.0/24",
"104.37.177.0/24",
"14.0.50.0/24",
"183.230.70.0/24",
"114.111.59.0/24",
"220.181.135.0/24",
"112.140.32.0/19",
"101.79.230.0/24",
"14.0.115.0/24",
"103.28.248.0/22",
"117.34.72.0/24",
"109.199.57.0/24",
"101.79.149.0/24",
"116.128.128.0/24",
"115.231.186.0/24",
"103.22.200.0/22",
"61.155.165.0/24",
"113.20.148.0/23",
"185.254.242.0/24",
"59.36.120.0/24",
"70.132.0.0/18",
"116.31.126.0/24",
"119.147.134.0/24",
"115.127.246.0/24",
"52.47.139.0/24",
"118.107.175.0/24",
"52.78.247.128/26",
"110.93.176.0/20",
"54.240.128.0/18",
"46.51.216.0/21",
"119.31.251.0/24",
"125.39.18.0/24",
"108.175.33.0/24",
"1.31.128.0/24",
"61.151.163.0/24",
"103.95.132.0/24",
"58.215.118.0/24",
"54.233.255.128/26",
"120.52.113.0/24",
"118.107.174.0/24",
"1.32.242.0/24",
"221.195.34.0/24",
"101.79.228.0/24",
"205.251.249.0/24",
"113.200.91.0/24",
"101.79.146.0/24",
"221.238.22.0/24",
"134.19.183.0/24",
"110.93.160.0/20",
"180.97.158.0/24",
"115.127.251.0/24",
"119.167.147.0/24",
"115.127.238.0/24",
"115.127.240.0/22",
"14.0.48.0/24",
"115.127.240.0/24",
"113.7.183.0/24",
"112.140.128.0/20",
"115.127.255.0/24",
"114.31.36.0/22",
"101.79.232.0/24",
"218.98.44.0/24",
"106.119.182.0/24",
"101.79.167.0/24",
"125.39.5.0/24",
"58.49.105.0/24",
"124.202.164.0/24",
"111.177.6.0/24",
"61.133.127.0/24",
"185.11.124.0/22",
"150.138.150.0/24",
"115.127.248.0/24",
"103.74.80.0/22",
"101.79.166.0/24",
"101.71.55.0/24",
"198.41.128.0/17",
"117.21.219.0/24",
"103.231.170.0/24",
"221.204.202.0/24",
"101.79.224.0/24",
"112.25.16.0/24",
"111.177.3.0/24",
"204.246.168.0/22",
"103.40.7.0/24",
"134.226.0.0/16",
"52.15.127.128/26",
"122.190.2.0/24",
"101.203.192.0/18",
"1.32.238.0/24",
"101.79.144.0/24",
"176.34.28.0/24",
"119.84.15.0/24",
"18.216.170.128/25",
"222.88.94.0/24",
"101.79.150.0/24",
"114.111.48.0/21",
"124.95.168.0/24",
"114.111.48.0/20",
"110.93.176.0/21",
"223.111.127.0/24",
"117.23.61.0/24",
"140.207.120.0/24",
"157.255.26.0/24",
"221.204.14.0/24",
"183.222.96.0/24",
"104.37.180.0/24",
"42.236.93.0/24",
"111.63.51.0/24",
"114.31.32.0/20",
"118.180.50.0/24",
"222.240.184.0/24",
"205.251.192.0/19",
"101.79.225.0/24",
"115.127.228.0/24",
"113.20.148.0/24",
"61.213.176.0/24",
"112.65.75.0/24",
"111.13.147.0/24",
"113.20.145.0/24",
"103.253.132.0/24",
"52.222.128.0/17",
"183.203.7.0/24",
"27.221.27.0/24",
"103.79.134.0/24",
"123.150.187.0/24",
"103.15.194.0/24",
"162.158.0.0/15",
"61.163.30.0/24",
"182.140.227.0/24",
"112.25.60.0/24",
"117.148.161.0/24",
"61.182.136.0/24",
"114.31.56.0/22",
"64.252.128.0/18",
"183.61.185.0/24",
"115.127.250.0/24",
"150.138.138.0/24",
"13.210.67.128/26",
"211.162.64.0/24",
"61.174.9.0/24",
"14.0.112.0/24",
"52.52.191.128/26",
"27.221.124.0/24",
"103.4.203.0/24",
"103.14.10.0/24",
"34.232.163.208/29",
"114.31.48.0/20",
"59.51.81.0/24",
"183.60.235.0/24",
"101.227.206.0/24",
"125.39.174.0/24",
"119.167.246.0/24",
"118.107.160.0/21",
"223.166.151.0/24",
"110.93.160.0/19",
"204.246.172.0/23",
"119.31.253.0/24",
"143.204.0.0/16",
"14.0.60.0/24",
"123.151.76.0/24",
"116.193.80.0/24",
"120.241.102.0/24",
"180.96.20.0/24",
"216.137.32.0/19",
"223.94.95.0/24",
"103.4.201.0/24",
"14.0.56.0/24",
"115.127.234.0/24",
"113.20.144.0/23",
"103.248.104.0/24",
"122.143.15.0/24",
"101.79.229.0/24",
"101.79.163.0/24",
"104.37.112.0/22",
"115.127.253.0/24",
"141.101.64.0/18",
"113.20.144.0/22",
"101.79.155.0/24",
"117.148.160.0/24",
"124.193.166.0/24",
"109.94.168.0/24",
"203.90.247.0/24",
"101.79.208.0/21",
"182.118.12.0/24",
"114.31.58.0/23",
"202.162.109.0/24",
"101.79.164.0/24",
"58.216.2.0/24",
"222.216.190.0/24",
"101.79.165.0/24",
"111.6.191.0/24",
"1.255.100.0/24",
"52.84.0.0/15",
"112.65.74.0/24",
"183.250.179.0/24",
"101.79.236.0/24",
"119.31.252.0/24",
"113.20.150.0/24",
"60.12.166.0/24",
"101.79.234.0/24",
"113.17.174.0/24",
"101.79.237.0/24",
"61.54.46.0/24",
"118.212.233.0/24",
"183.110.242.0/24",
"150.138.149.0/24",
"117.34.13.0/24",
"115.127.245.0/24",
"14.0.102.0/24",
"14.0.109.0/24",
"61.130.28.0/24",
"113.20.151.0/24",
"219.159.84.0/24",
"114.111.62.0/24",
"172.64.0.0/13",
"61.155.222.0/24",
"120.52.29.0/24",
"115.127.231.0/24",
"14.0.49.0/24",
"113.202.0.0/16",
"103.248.104.0/22",
"205.251.250.0/23",
"103.216.136.0/22",
"118.107.160.0/20",
"109.87.0.0/21",
"54.239.128.0/18",
"115.127.224.0/19",
"111.202.98.0/24",
"109.94.169.0/24",
"59.38.112.0/24",
"204.246.176.0/20",
"123.133.84.0/24",
"103.4.200.0/24",
"111.161.109.0/24",
"112.84.34.0/24",
"103.82.129.0/24",
"183.3.254.0/24",
"112.137.184.0/21",
"122.227.237.0/24",
"36.42.75.0/24",
"13.35.0.0/16",
"101.226.4.0/24",
"116.140.35.0/24",
"58.250.143.0/24",
"13.54.63.128/26",
"205.251.254.0/24",
"173.245.48.0/20",
"183.61.177.0/24",
"113.20.144.0/24",
"104.37.183.0/24",
"35.158.136.0/24",
"116.211.121.0/24",
"42.236.94.0/24",
"117.34.91.0/24",
"123.6.13.0/24",
"13.224.0.0/14",
"113.20.146.0/24",
"58.58.81.0/24",
"52.124.128.0/17",
"122.228.198.0/24",
"197.234.240.0/22",
"99.86.0.0/16",
"144.220.0.0/16",
"119.188.97.0/24",
"36.27.212.0/24",
"104.37.178.0/24",
"114.31.52.0/22",
"218.65.212.0/24",
"1.255.41.0/24",
"14.0.45.0/24",
"1.32.243.0/24",
"220.170.185.0/24",
"122.190.3.0/24",
"103.79.133.0/24",
"220.181.55.0/24",
"125.39.191.0/24",
"115.127.226.0/24",
"125.39.32.0/24",
"61.120.154.0/24",
"103.4.202.0/24",
"103.79.134.0/23",
"115.127.224.0/24",
"113.20.147.0/24",
"61.156.149.0/24",
"210.209.122.0/24",
"115.127.249.0/24",
"104.37.179.0/24",
"120.52.18.0/24",
"54.192.0.0/16",
"14.0.55.0/24",
"61.160.224.0/24",
"113.207.101.0/24",
"101.79.157.0/24",
"110.93.128.0/20",
"58.251.121.0/24",
"61.240.149.0/24",
"130.176.0.0/16",
"113.107.238.0/24",
"112.65.73.0/24",
"103.75.200.0/23",
"199.83.128.0/21",
"123.129.220.0/24",
"54.230.0.0/16",
"114.111.60.0/24",
"199.27.128.0/21",
"14.0.118.0/24",
"101.79.158.0/24",
"119.31.248.0/21",
"54.182.0.0/16",
"113.31.27.0/24",
"14.17.69.0/24",
"101.79.145.0/24",
"113.20.144.0/21",
"180.163.22.0/24",
"104.37.176.0/21",
"117.25.156.0/24",
"115.127.252.0/24",
"115.127.244.0/23",
"14.0.46.0/24",
"113.207.102.0/24",
"52.199.127.192/26",
"13.113.203.0/24",
"64.252.64.0/18",
"1.32.240.0/24",
"123.129.232.0/24",
"1.32.241.0/24",
"180.163.189.0/24",
"157.255.25.0/24",
"1.32.244.0/24",
"103.248.106.0/24",
"121.48.95.0/24",
"54.239.192.0/19",
"113.20.146.0/23",
"61.136.173.0/24",
"35.162.63.192/26",
"117.34.14.0/24",
"183.232.29.0/24",
"42.81.93.0/24",
"122.228.238.0/24",
"183.61.190.0/24",
"125.39.239.0/24",
"115.127.230.0/24",
"103.140.200.0/23",
"202.102.85.0/24",
"14.0.32.0/21",
"14.0.57.0/24",
"112.25.90.0/24",
"58.211.137.0/24",
"210.22.63.0/24",
"34.226.14.0/24",
"13.32.0.0/15",
"101.79.156.0/24",
"103.89.176.0/24",
"14.0.116.0/24",
"106.42.25.0/24",
"101.79.233.0/24",
"101.79.231.0/24",
"103.75.200.0/24",
"119.188.9.0/24",
"183.232.51.0/24",
"149.126.72.0/21",
"103.21.244.0/22",
"115.127.233.0/24",
"27.221.20.0/24",
"198.143.32.0/19",
"103.248.107.0/24",
"101.79.227.0/24",
"115.127.242.0/24",
"119.31.250.0/24",
"103.82.130.0/24",
"99.84.0.0/16",
"222.73.144.0/24",
"103.79.132.0/22",
"101.79.208.0/20",
"104.37.182.0/24",
"101.79.152.0/24",
"36.99.18.0/24",
"101.71.56.0/24",
"36.250.5.0/24",
"61.158.240.0/24",
"119.188.14.0/24",
"13.249.0.0/16",
"183.214.156.0/24",
"60.221.236.0/24",
"58.30.212.0/24",
"115.127.254.0/24",
"188.114.96.0/20",
"115.127.241.0/24",
"103.4.200.0/22",
"115.127.239.0/24",
"115.127.243.0/24",
"111.32.135.0/24",
"120.221.29.0/24",
"115.127.232.0/24",
"14.0.43.0/24",
"14.0.59.0/24",
"183.61.236.0/24",
"34.223.12.224/27",
"103.24.120.0/24",
"52.57.254.0/24",
"113.207.100.0/24",
"222.186.19.0/24",
"113.20.149.0/24",
"150.138.151.0/24",
"115.231.110.0/24",
"52.56.127.0/25",
"104.37.176.0/24",
"163.177.8.0/24",
"163.53.89.0/24",
"52.82.128.0/19",
"114.111.63.0/24",
"108.162.192.0/18",
"14.136.130.0/24",
"115.127.229.0/24",
"14.17.71.0/24",
"52.212.248.0/26",
"180.163.188.0/24",
"61.182.137.0/24",
"119.161.224.0/21",
"14.0.41.0/24",
"202.162.108.0/24",
"106.122.248.0/24",
"52.66.194.128/26",
"115.127.237.0/24",
"220.170.186.0/24",
"14.0.32.0/19",
"14.0.114.0/24",
"112.90.216.0/24",
"115.127.236.0/24",
"116.193.84.0/24",
"113.207.76.0/24",
"101.79.235.0/24",
"101.79.224.0/20",
"61.155.149.0/24",
"101.79.148.0/24",
"180.163.224.0/24",
"204.246.174.0/23",
"183.60.136.0/24",
"101.227.207.0/24",
"103.248.105.0/24",
"119.188.35.0/24",
"42.236.7.0/24",
"116.193.88.0/21",
"116.193.83.0/24",
"120.199.69.0/24",
"122.226.182.0/24",
"58.20.204.0/24",
"110.93.128.0/21",
"115.231.187.0/24",
"69.28.58.0/24",
"114.31.32.0/19",
"112.25.91.0/24",
"59.52.28.0/24",
"117.27.149.0/24",
"61.147.92.0/24",
"14.0.117.0/24",
"14.0.40.0/24",
"119.97.151.0/24",
"103.199.228.0/22",
"122.70.134.0/24",
"115.127.244.0/24",
"223.112.198.0/24",
"115.127.225.0/24",
"104.16.0.0/12",
"121.12.98.0/24",
"103.31.4.0/22",
"204.246.164.0/22",
"223.94.66.0/24",
"35.167.191.128/26",
"116.31.127.0/24",
"101.79.226.0/24",
"34.195.252.0/24",
"115.127.247.0/24",
"61.240.144.0/24",
"108.175.32.0/20",
"120.197.85.0/24",
"183.232.53.0/24",
"111.161.66.0/24",
"117.34.28.0/24",
"45.64.64.0/22",
"14.0.44.0/24",
"109.86.0.0/15",
"182.23.211.0/24",
"58.211.2.0/24",
"119.36.164.0/24",
"116.55.250.0/24",
"101.227.163.0/24",
"13.228.69.0/24",
"120.221.136.0/24",
"119.188.132.0/24",
"115.127.235.0/24",
"42.236.6.0/24",
"125.88.190.0/24",
"61.54.47.0/24",
"103.27.12.0/22",
"116.193.80.0/21",
"101.79.159.0/24",
"123.155.158.0/24",
"111.47.226.0/24",
"131.0.72.0/22",
"192.230.64.0/18"
]
modules/iscdn.py
+83
View File
@@ -0,0 +1,83 @@
import json
import ipaddress
from config import setting
from common import utils
from config.log import logger
data_dir = setting.data_storage_dir
# from https://github.com/al0ne/Vxscan/blob/master/lib/iscdn.py
cdn_ip_cidr = utils.load_json(data_dir.joinpath('cdn_ip_cidr.json'))
cdn_asn_list = utils.load_json(data_dir.joinpath('cdn_asn_list.json'))
# from https://github.com/Qclover/CDNCheck/blob/master/checkCDN/cdn3_check.py
cdn_cname_keyword = utils.load_json(data_dir.joinpath('cdn_cname_keywords.json'))
cdn_header_key = utils.load_json(data_dir.joinpath('cdn_header_keys.json'))
def check_cname_count(cnames):
if len(cnames) > 1:
return True
def check_cname_keyword(cname):
for keyword in cdn_cname_keyword.keys():
if keyword in cname:
return True
def check_header_key(header):
for key in cdn_header_key:
if key in header:
return True
def check_cdn_cidr(ip):
ip = ipaddress.ip_address(ip)
for cidr in cdn_ip_cidr:
if ip in ipaddress.ip_network(cidr):
return True
def check_cdn_asn(asn):
if str(asn) in cdn_asn_list:
return True
def check_cdn(data):
for index, item in enumerate(data):
cname = item.get('cname')
if cname:
cnames = cname.split(',')
if check_cname_count(cnames):
data[index]['cdn'] = 1
continue
if check_cname_keyword(cname):
data[index]['cdn'] = 1
continue
header = item.get('header')
if header:
header = json.loads(header)
if check_header_key(header):
data[index]['cdn'] = 1
continue
ip = item.get('content')
if ip:
if check_cdn_cidr(ip):
data[index]['cdn'] = 1
continue
asn = item.get('asn')
if asn:
asn = asn[2:] # 去除AS
if check_cdn_asn(asn):
data[index]['cdn'] = 1
continue
data[index]['cdn'] = 0
return data
def save_db(name, data):
logger.log('INFOR', f'Saving cdn check results')
utils.save_db(name, data, 'cdn')
oneforall.py
+5
View File
@@ -18,6 +18,7 @@ from common import utils, resolve, request
from common.database import Database
from modules.collect import Collect
from modules.finder import Finder
from modules import iscdn
from config import setting
from config.log import logger
from takeover import Takeover
@@ -215,6 +216,10 @@ class OneForAll(object):
finder = Finder()
self.data = finder.run(self.domain, self.data, self.port)
# check cdn
self.data = iscdn.check_cdn(self.data)
iscdn.save_db(self.domain, self.data)
# Add the final result list to the total data list
self.datas.extend(self.data)