From a332d3bdc41118e7f970eb543e5ff9ddc59ef295 Mon Sep 17 00:00:00 2001
From: shmilylty <admin@hackfun.org>
Date: Tue, 27 Aug 2019 17:36:19 +0800
Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E5=86=99=E6=A0=87=E9=A2=98=E8=8E=B7?=
 =?UTF-8?q?=E5=8F=96?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 oneforall/common/request.py | 50 ++++++++++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 14 deletions(-)

diff --git a/oneforall/common/request.py b/oneforall/common/request.py
index 6583a5d..59ce25c 100644
--- a/oneforall/common/request.py
+++ b/oneforall/common/request.py
@@ -78,6 +78,41 @@ async def fetch(session, url, semaphore):
         return resp, text
 
 
+def get_title(markup):
+    """
+    获取标题
+
+    :param markup: html标签
+    :return: 标题
+    """
+    soup = BeautifulSoup(markup, 'lxml')
+
+    title = soup.title
+    if title:
+        return title.text.strip()
+
+    h1 = soup.h1
+    if h1:
+        return h1.text.strip()
+
+    h2 = soup.h2
+    if h2:
+        return h2.text.strip()
+
+    desc = soup.find('meta', attrs={'name': 'description'})
+    if desc:
+        return desc['content'].strip()
+
+    word = soup.find('meta', attrs={'name': 'keywords'})
+    if word:
+        return word['content'].strip()
+
+    if len(markup) <= 200:
+        return markup.strip()
+
+    return soup.text.strip()
+
+
 def request_callback(future, index, datas):
     try:
         result = future.result()
@@ -98,20 +133,7 @@ def request_callback(future, index, datas):
                           'Via': headers.get('Via'),
                           'X-Powered-By': headers.get('X-Powered-By')})
             datas[index]['banner'] = banner
-            soup = BeautifulSoup(text, 'lxml')
-            title = soup.title
-            desc = soup.find('meta', attrs={'name': 'description'})
-            word = soup.find('meta', attrs={'name': 'keywords'})
-            if title:
-                datas[index]['title'] = title.text.strip()
-            elif desc:
-                datas[index]['title'] = desc['content'].strip()
-            elif word:
-                datas[index]['title'] = word['content'].strip()
-            elif len(text) <= 200:
-                datas[index]['title'] = text.strip()
-            else:
-                datas[index]['title'] = soup.text.strip()
+            datas[index]['title'] = get_title(text)
 
 
 async def bulk_get_request(datas, port):