add cleaning process by GLMs.

app.py

@@ -1,9 +1,10 @@
-import math, torch, gradio as gr
+import torch, gradio as gr
 
 from lex_rank import LexRank
 from lex_rank_text2vec_v1 import LexRankText2VecV1
 from lex_rank_L12 import LexRankL12
 from sentence_transformers import SentenceTransformer, util
+from ask_glm_4_help import GlmHelper
 
 
 # ---===--- instances ---===---
@@ -11,6 +12,7 @@ embedder = SentenceTransformer('paraphrase-multilingual-mpnet-base-v2')
 lex = LexRank()
 lex_distiluse_v1 = LexRankText2VecV1()
 lex_l12 = LexRankL12()
+glm_helper = GlmHelper()
 
 
 # 摘要方法1
@@ -23,9 +25,10 @@ def extract_handler(content, siblings, num):
     siblings = int(siblings)
     num = int(num)
 
-    summary_length = math.ceil(len(content) / 10)
-    sentences = lex.find_central(content, siblings=siblings, num=num)
-    output = ""
+    glm_summarized_content = GlmHelper.clean_raw_content(content)
+
+    sentences = lex.find_central(glm_summarized_content, siblings=siblings, num=num)
+    output = f""">>>>>经过大模型清洗之后的文章为:\n{glm_summarized_content}\n\t>>>>>摘要为:\n"""
     for index, sentence in enumerate(sentences):
         output += f"{index}: {sentence}\n"
     return output
@@ -41,9 +44,10 @@ def extract_handler_distiluse_v1(content, siblings, num):
     siblings = int(siblings)
     num = int(num)
 
-    summary_length = math.ceil(len(content) / 10)
-    sentences = lex_distiluse_v1.find_central(content, siblings=siblings, num=num)
-    output = ""
+    glm_summarized_content = GlmHelper.clean_raw_content(content)
+
+    sentences = lex.find_central(glm_summarized_content, siblings=siblings, num=num)
+    output = f""">>>>>经过大模型清洗之后的文章为:\n{glm_summarized_content}\n\t>>>>>摘要为:\n"""
     for index, sentence in enumerate(sentences):
         output += f"{index}: {sentence}\n"
     return output
@@ -59,9 +63,10 @@ def extract_handler_l12(content, siblings, num):
     siblings = int(siblings)
     num = int(num)
 
-    summary_length = math.ceil(len(content) / 10)
-    sentences = lex_l12.find_central(content, siblings=siblings, num=num)
-    output = ""
+    glm_summarized_content = GlmHelper.clean_raw_content(content)
+
+    sentences = lex.find_central(glm_summarized_content, siblings=siblings, num=num)
+    output = f""">>>>>经过大模型清洗之后的文章为:\n{glm_summarized_content}\n\t>>>>>摘要为:\n"""
     for index, sentence in enumerate(sentences):
         output += f"{index}: {sentence}\n"
     return output
@@ -103,7 +108,7 @@ with gr.Blocks() as app:
         with gr.Row():
             text_button_2 = gr.Button("生成摘要")
             siblings_input_2 = gr.Textbox(label="请输入摘要的宽度半径, 默认为0, 即显示摘要本身.")
-            num_input_2 = gr.Textbox(label="label=摘要的条数, 默认10条")
+            num_input_2 = gr.Textbox(label="摘要的条数, 默认10条")
         text_output_2 = gr.Textbox(label="摘要文本", lines=10)
     with gr.Tab("LexRank-MiniLM-L12-v2"):
         text_input_3 = gr.Textbox(label="请输入长文本:", lines=10, max_lines=1000)

ask_glm_4_help.py

@@ -0,0 +1,29 @@
+import requests
+import json
+
+SYS_MSG_4_CLEANING = "你是一个AI助手, 能将我给你的文章去除与主题无关的句子, 并尽量保留所有与主题相关的句子."
+
+class GlmHelper(object):
+    def clean_raw_content(self, content: str):
+        history = []
+        rply = self.bot_message_handler(message=content, history=history, sys_msg=SYS_MSG_4_CLEANING)
+        return rply
+
+    # 携带知识库文本询问LLM
+    def bot_message_handler(self, message: str, history: [list], sys_msg: str):
+        request_body = {
+            "prompt": f"""
+            <s>[INST] <<SYS>>\n{sys_msg}\n<</SYS>>\n\n{message} [/INST]
+            """,
+            "knowledge": """
+            """,
+            "history": history,
+            "max_length": 2048 * 4,
+        }
+        rply = requests.post("http://region-9.autodl.pro:19567/gradio", data=json.dumps(request_body))
+        try:
+            reply_from_GLM = rply.json()["response"]
+        except:
+            reply_from_GLM = "GLM Api返回了坏的请求..."
+
+        return reply_from_GLM