重构泛解析探测

common/similarity.py

@@ -0,0 +1,138 @@
+"""
+根据网页结构判断页面相似性(Determine page similarity based on HTML page structure)
+判断方法：根据网页的DOM树确定网页的模板特征向量，对模板特征向量计算网页结构相似性。
+来源地址：https://github.com/SPuerBRead/HTMLSimilarity
+计算参考: https://patents.google.com/patent/CN101694668B/zh
+"""
+from treelib import Tree
+from bs4 import BeautifulSoup
+import bs4
+
+
+class DOMTree(object):
+    def __init__(self, label, attrs):
+        self.label = label
+        self.attrs = attrs
+
+
+class HTMLParser(object):
+    def __init__(self, html):
+        self.dom_id = 1
+        self.dom_tree = Tree()
+        self.bs_html = BeautifulSoup(html, 'html.parser')
+
+    def get_dom_structure_tree(self):
+        for content in self.bs_html.contents:
+            if isinstance(content, bs4.element.Tag):
+                self.bs_html = content
+        self.recursive_descendants(self.bs_html, 1)
+        return self.dom_tree
+
+    def recursive_descendants(self, descendants, parent_id):
+        if self.dom_id == 1:
+            self.dom_tree.create_node(descendants.name, self.dom_id,
+                                      data=DOMTree(descendants.name, descendants.attrs))
+            self.dom_id = self.dom_id + 1
+        for child in descendants.contents:
+            if isinstance(child, bs4.element.Tag):
+                self.dom_tree.create_node(child.name, self.dom_id, parent_id,
+                                          data=DOMTree(child.name, child.attrs))
+                self.dom_id = self.dom_id + 1
+                self.recursive_descendants(child, self.dom_id - 1)
+
+
+class Converter(object):
+    def __init__(self, dom_tree, dimension):
+        self.dom_tree = dom_tree
+        self.node_info_list = []
+        self.dimension = dimension
+        self.initial_weight = 1
+        self.attenuation_ratio = 0.6
+        self.dom_eigenvector = {}.fromkeys(range(0, dimension), 0)
+
+    def get_eigenvector(self):
+        for node_id in range(1, self.dom_tree.size() + 1):
+            node = self.dom_tree.get_node(node_id)
+            node_feature = self.create_feature(node)
+            feature_hash = self.feature_hash(node_feature)
+            node_weight = self.calculate_weight(node, node_id, feature_hash)
+            self.construct_eigenvector(feature_hash, node_weight)
+        return self.dom_eigenvector
+
+    @staticmethod
+    def create_feature(node):
+        node_attr_list = []
+        node_feature = node.data.label + '|'
+        for attr in node.data.attrs.keys():
+            node_attr_list.append(attr + ':' + str(node.data.attrs[attr]))
+        node_feature += '|'.join(node_attr_list)
+        return node_feature
+
+    @staticmethod
+    def feature_hash(node_feature):
+        return abs(hash(node_feature)) % (10 ** 8)
+
+    def calculate_weight(self, node, node_id, feature_hash):
+        brother_node_count = 0
+        depth = self.dom_tree.depth(node)
+        for brother_node in self.dom_tree.siblings(node_id):
+            brother_node_feature = self.create_feature(brother_node)
+            brother_node_feature_hash = self.feature_hash(brother_node_feature)
+            if brother_node_feature_hash == feature_hash:
+                brother_node_count = brother_node_count + 1
+        if brother_node_count:
+            node_weight = self.initial_weight * self.attenuation_ratio ** depth \
+                          * self.attenuation_ratio ** brother_node_count
+        else:
+            node_weight = self.initial_weight * self.attenuation_ratio ** depth
+        return node_weight
+
+    def construct_eigenvector(self, feature_hash, node_weight):
+        feature_hash = feature_hash % self.dimension
+        self.dom_eigenvector[feature_hash] += node_weight
+
+
+def calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension):
+    a, b = 0, 0
+    for i in range(dimension):
+        a += dom1_eigenvector[i]-dom2_eigenvector[i]
+        if dom1_eigenvector[i] and dom2_eigenvector[i]:
+            b += dom1_eigenvector[i] + dom2_eigenvector[i]
+    pseudodistance = abs(a)/b
+    return pseudodistance
+
+
+def get_pseudodistance(html_doc1, html_doc2, dimension=5000):
+    """
+    获取html文档结构相似度
+
+    :param str html_doc1: html文档
+    :param str html_doc2: html文档
+    :param int dimension: 降维后的维数
+    :return 伪距离
+    """
+    hp1 = HTMLParser(html_doc1)
+    html_doc1_dom_tree = hp1.get_dom_structure_tree()
+    hp2 = HTMLParser(html_doc2)
+    html_doc2_dom_tree = hp2.get_dom_structure_tree()
+    converter = Converter(html_doc1_dom_tree, dimension)
+    dom1_eigenvector = converter.get_eigenvector()
+    converter = Converter(html_doc2_dom_tree, dimension)
+    dom2_eigenvector = converter.get_eigenvector()
+    return calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension)
+
+
+def is_similar(html_doc1, html_doc2, dimension=5000):
+    """
+    根据计算出的伪距离来判断是否html页面结构相似
+
+    :param str html_doc1: html文档
+    :param str html_doc2: html文档
+    :param int dimension: 降维后的维数
+    :return 是否相似（伪距离value<0.2时相似，value>0.2时不相似）
+    """
+    value = get_pseudodistance(html_doc1, html_doc2, dimension)
+    if value > 0.2:
+        return False
+    else:
+        return True

common/utils.py

@@ -741,3 +741,15 @@ def check_random_subdomain(subdomains):
         if subdomain:
             logger.log('ALERT', f'Please check whether {subdomain} is correct or not')
             return True
+
+
+def get_url_resp(url):
+    timeout = settings.request_timeout
+    verify = settings.request_verify
+    try:
+        resp = requests.get(url, params=None, timeout=timeout, verify=verify)
+    except Exception as e:
+        logger.log('DEBUG', f'Error request {url}')
+        logger.log('DEBUG', e.args)
+        return None
+    return resp