1.重构请求，边请求边存入数据库，解决内存占用过大问题。

2.format参数改为fmt。 3.添加信息富化模块。 4.优化iscdn模块代码。 5.移除数据库new字段。
2026-08-26 04:47:48 +08:00 · 2020-11-09 09:01:06 +08:00
parent 2428017d29
commit 3fbbd1dc25
17 changed files with 473 additions and 424 deletions
@@ -0,0 +1,72 @@
+from modules import iscdn
+from common import utils
+from common.database import Database
+from common.ipasn import IPAsnInfo
+from common.ipreg import IpRegData
+
+
+def get_ips(info):
+    ip = info.get('ip')
+    if not ip:
+        return None
+    ips = ip.split(',')
+    return ips
+
+
+def enrich_info(data):
+    ip_asn = IPAsnInfo()
+    ip_reg = IpRegData()
+    for index, info in enumerate(data):
+        ips = get_ips(info)
+        if not ips:
+            continue
+        public = list()
+        cidr = list()
+        asn = list()
+        org = list()
+        addr = list()
+        isp = list()
+        for ip in ips:
+            public.append(str(utils.ip_is_public(ip)))
+            asn_info = ip_asn.find(ip)
+            cidr.append(asn_info.get('cidr'))
+            asn.append(asn_info.get('asn'))
+            org.append(asn_info.get('org'))
+            ip_info = ip_reg.query(ip)
+            addr.append(ip_info.get('addr'))
+            isp.append(ip_info.get('isp'))
+        data[index]['public'] = ','.join(public)
+        data[index]['cidr'] = ','.join(cidr)
+        data[index]['asn'] = ','.join(asn)
+        data[index]['org'] = ','.join(org)
+        data[index]['addr'] = ','.join(addr)
+        data[index]['isp'] = ','.join(isp)
+        return data
+
+
+class Enrich(object):
+    def __init__(self, domain):
+        self.domain = domain
+
+    def get_data(self):
+        db = Database()
+        fields = ['url', 'cname', 'ip', 'public', 'cdn', 'header',
+                  'cidr', 'asn', 'org', 'addr', 'isp']
+        results = db.get_data_by_fields(self.domain, fields)
+        return results.as_dict()
+
+    def save_db(self, data):
+        db = Database()
+        for info in data:
+            url = info.pop('url')
+            info.pop('cname')
+            info.pop('ip')
+            info.pop('header')
+            db.update_data_by_url(self.domain, info, url)
+        db.close()
+
+    def run(self):
+        data = self.get_data()
+        data = enrich_info(data)
+        data = iscdn.do_check(data)
+        self.save_db(data)