Update app.py

app.py

@@ -8,7 +8,9 @@ import validators, re
 from fake_useragent import UserAgent
 from bs4 import BeautifulSoup   
 import streamlit as st
-from transformers import pipeline
+from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer
+from sentence_transformers import SentenceTransformer
+import en_core_web_lg
 import time
 import base64 
 import requests
@@ -158,19 +160,235 @@ def summary_downloader(raw_text):
 	st.markdown("#### Download Summary as a File ###")
 	href = f'<a href="data:file/txt;base64,{b64}" download="{new_filename}">Click to Download!!</a>'
 	st.markdown(href,unsafe_allow_html=True)
+	
+def get_all_entities_per_sentence(text):
+    doc = nlp(text)
 
-@st.cache(allow_output_mutation=True)
+    sentences = list(doc.sents)
+
+    entities_all_sentences = []
+    for sentence in sentences:
+        entities_this_sentence = []
+
+        # SPACY ENTITIES
+        for entity in sentence.ents:
+            entities_this_sentence.append(str(entity))
+
+        # FLAIR ENTITIES (CURRENTLY NOT USED)
+        # sentence_entities = Sentence(str(sentence))
+        # tagger.predict(sentence_entities)
+        # for entity in sentence_entities.get_spans('ner'):
+        #     entities_this_sentence.append(entity.text)
+
+        # XLM ENTITIES
+        entities_xlm = [entity["word"] for entity in ner_model(str(sentence))]
+        for entity in entities_xlm:
+            entities_this_sentence.append(str(entity))
+
+        entities_all_sentences.append(entities_this_sentence)
+
+    return entities_all_sentences
+
+def get_all_entities(text):
+    all_entities_per_sentence = get_all_entities_per_sentence(text)
+    return list(itertools.chain.from_iterable(all_entities_per_sentence))
+    
+def get_and_compare_entities(article_content,summary_output):
+    
+    all_entities_per_sentence = get_all_entities_per_sentence(article_content)
+    entities_article = list(itertools.chain.from_iterable(all_entities_per_sentence))
+
+    all_entities_per_sentence = get_all_entities_per_sentence(summary_output)
+    entities_summary = list(itertools.chain.from_iterable(all_entities_per_sentence))
+
+    matched_entities = []
+    unmatched_entities = []
+    for entity in entities_summary:
+        if any(entity.lower() in substring_entity.lower() for substring_entity in entities_article):
+            matched_entities.append(entity)
+        elif any(
+                np.inner(sentence_embedding_model.encode(entity, show_progress_bar=False),
+                         sentence_embedding_model.encode(art_entity, show_progress_bar=False)) > 0.9 for
+                art_entity in entities_article):
+            matched_entities.append(entity)
+        else:
+            unmatched_entities.append(entity)
+
+    matched_entities = list(dict.fromkeys(matched_entities))
+    unmatched_entities = list(dict.fromkeys(unmatched_entities))
+
+    matched_entities_to_remove = []
+    unmatched_entities_to_remove = []
+
+    for entity in matched_entities:
+        for substring_entity in matched_entities:
+            if entity != substring_entity and entity.lower() in substring_entity.lower():
+                matched_entities_to_remove.append(entity)
+
+    for entity in unmatched_entities:
+        for substring_entity in unmatched_entities:
+            if entity != substring_entity and entity.lower() in substring_entity.lower():
+                unmatched_entities_to_remove.append(entity)
+
+    matched_entities_to_remove = list(dict.fromkeys(matched_entities_to_remove))
+    unmatched_entities_to_remove = list(dict.fromkeys(unmatched_entities_to_remove))
+
+    for entity in matched_entities_to_remove:
+        matched_entities.remove(entity)
+    for entity in unmatched_entities_to_remove:
+        unmatched_entities.remove(entity)
+
+    return matched_entities, unmatched_entities
+
+def highlight_entities(article_content,summary_output):
+   
+    markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"
+    markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"
+    markdown_end = "</mark>"
+
+    matched_entities, unmatched_entities = get_and_compare_entities(article_content,summary_output)
+
+    for entity in matched_entities:
+        summary_content = summary_output.replace(entity, markdown_start_green + entity + markdown_end)
+
+    for entity in unmatched_entities:
+        summary_content = summary_output.replace(entity, markdown_start_red + entity + markdown_end)
+    soup = BeautifulSoup(summary_content, features="html.parser")
+    return HTML_WRAPPER.format(soup)
+
+
+def render_dependency_parsing(text: Dict):
+    html = render_sentence_custom(text, nlp)
+    html = html.replace("\n\n", "\n")
+    st.write(get_svg(html), unsafe_allow_html=True)
+
+
+def check_dependency(article: bool):
+    if article:
+        text = st.session_state.article_text
+        all_entities = get_all_entities_per_sentence(text)
+    else:
+        text = st.session_state.summary_output
+        all_entities = get_all_entities_per_sentence(text)
+    doc = nlp(text)
+    tok_l = doc.to_json()['tokens']
+    test_list_dict_output = []
+
+    sentences = list(doc.sents)
+    for i, sentence in enumerate(sentences):
+        start_id = sentence.start
+        end_id = sentence.end
+        for t in tok_l:
+            if t["id"] < start_id or t["id"] > end_id:
+                continue
+            head = tok_l[t['head']]
+            if t['dep'] == 'amod' or t['dep'] == "pobj":
+                object_here = text[t['start']:t['end']]
+                object_target = text[head['start']:head['end']]
+                if t['dep'] == "pobj" and str.lower(object_target) != "in":
+                    continue
+                # ONE NEEDS TO BE ENTITY
+                if object_here in all_entities[i]:
+                    identifier = object_here + t['dep'] + object_target
+                    test_list_dict_output.append({"dep": t['dep'], "cur_word_index": (t['id'] - sentence.start),
+                                                  "target_word_index": (t['head'] - sentence.start),
+                                                  "identifier": identifier, "sentence": str(sentence)})
+                elif object_target in all_entities[i]:
+                    identifier = object_here + t['dep'] + object_target
+                    test_list_dict_output.append({"dep": t['dep'], "cur_word_index": (t['id'] - sentence.start),
+                                                  "target_word_index": (t['head'] - sentence.start),
+                                                  "identifier": identifier, "sentence": str(sentence)})
+                else:
+                    continue
+    return test_list_dict_output
+
+
+def render_svg(svg_file):
+    with open(svg_file, "r") as f:
+        lines = f.readlines()
+        svg = "".join(lines)
+
+        # """Renders the given svg string."""
+        b64 = base64.b64encode(svg.encode("utf-8")).decode("utf-8")
+        html = r'<img src="data:image/svg+xml;base64,%s"/>' % b64
+        return html
+
+
+def generate_abstractive_summary(text, type, min_len=120, max_len=512, **kwargs):
+    text = text.strip().replace("\n", " ")
+    if type == "top_p":
+        text = summarization_model(text, min_length=min_len,
+                                   max_length=max_len,
+                                   top_k=50, top_p=0.95, clean_up_tokenization_spaces=True, truncation=True, **kwargs)
+    elif type == "greedy":
+        text = summarization_model(text, min_length=min_len,
+                                   max_length=max_len, clean_up_tokenization_spaces=True, truncation=True, **kwargs)
+    elif type == "top_k":
+        text = summarization_model(text, min_length=min_len, max_length=max_len, top_k=50,
+                                   clean_up_tokenization_spaces=True, truncation=True, **kwargs)
+    elif type == "beam":
+        text = summarization_model(text, min_length=min_len,
+                                   max_length=max_len,
+                                   clean_up_tokenization_spaces=True, truncation=True, **kwargs)
+    summary = text[0]['summary_text'].replace("<n>", " ")
+    return summary
+
+def clean_text(text,doc=False,plain_text=False,url=False):
+"""Return clean text from the various input sources"""
+
+    if url:
+        is_url = validators.url(text)
+        
+        if is_url:
+            # complete text, chunks to summarize (list of sentences for long docs)
+            article_title,chunks = article_text_extractor(url=url_text)
+        
+            return article_title, chunks
+        
+    elif doc:
+        
+       clean_text = chunk_clean_text(preprocess_plain_text(extract_text_from_file(text)))
+       
+       return None, clean_text
+    
+    elif plain_text:
+        
+        clean_text = chunk_clean_text(preprocess_plain_text(text))
+        
+        return None, clean_text
+        
+# Load all different models (cached) at start time of the hugginface space
+sentence_embedding_model = get_sentence_embedding_model()
+ner_model = get_transformer_pipeline()
+nlp = get_spacy()
+  
+@st.experimental_singleton
+def get_spacy():
+    nlp = en_core_web_lg.load()
+    return nlp
+    
+@st.experimental_singleton
 def facebook_model():
     
     summarizer = pipeline('summarization',model='facebook/bart-large-cnn')
     return summarizer
     
-@st.cache(allow_output_mutation=True)
+@st.experimental_singleton
 def schleifer_model():
     
     summarizer = pipeline('summarization',model='sshleifer/distilbart-cnn-12-6')
     return summarizer
     
+@st.experimental_singleton
+def get_sentence_embedding_model():
+    return SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+    
+@st.experimental_singleton
+def get_ner_pipeline():
+    tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-large-finetuned-conll03-english")
+    model = AutoModelForTokenClassification.from_pretrained("xlm-roberta-large-finetuned-conll03-english")
+    return pipeline("ner", model=model, tokenizer=tokenizer, grouped_entities=True)
+    
 #Streamlit App
     
 st.title("Article Text and Link Extractive Summarizer 📝")
@@ -211,6 +429,14 @@ st.markdown("---")
 
 url_text = st.text_input("Please Enter a url here")
 
+if url_text:
+    article_title, clean_text = clean_text(url_text, url=True)
+    
+article_text = st.text_area(
+    label='Full Article Text',
+    value= clean_text,
+    height=250
+)
 
 st.markdown(
     "<h3 style='text-align: center; color: red;'>OR</h3>",
@@ -228,27 +454,19 @@ upload_doc = st.file_uploader(
     "Upload a .txt, .pdf, .docx file for summarization"
 )
 
-is_url = validators.url(url_text)
-
-if is_url:
-    # complete text, chunks to summarize (list of sentences for long docs)
-    article_title,chunks = article_text_extractor(url=url_text)
+if plain_text:
+    None, clean_text = clean_text(plain_text,plain_text=True)
     
 elif upload_doc:
+    None, clean_text = clean_text(plain_text,doc=True)
     
-    clean_text = chunk_clean_text(preprocess_plain_text(extract_text_from_file(upload_doc)))
-
-else:
-    
-    clean_text = chunk_clean_text(preprocess_plain_text(plain_text))
-
 summarize = st.button("Summarize")
 
 # called on toggle button [summarize]
 if summarize:
     if model_type == "Facebook-Bart":
-        if is_url:
-            text_to_summarize = chunks
+        if url_text:
+            text_to_summarize = url_clean_text
         else:
             text_to_summarize = clean_text
 
@@ -260,8 +478,8 @@ if summarize:
             summarized_text = ' '.join([summ['summary_text'] for summ in summarized_text])
     
     elif model_type == "Sshleifer-DistilBart":
-        if is_url:
-            text_to_summarize = chunks
+        if url_text:
+            text_to_summarize = url_clean_text
         else:
             text_to_summarize = clean_text
 
@@ -270,19 +488,25 @@ if summarize:
         ):
             summarizer_model = schleifer_model()
             summarized_text = summarizer_model(text_to_summarize, max_length=max_len, min_length=min_len)
-            summarized_text = ' '.join([summ['summary_text'] for summ in summarized_text])       
-    
-    # final summarized output
-    st.subheader("Summarized text")
+            summarized_text = ' '.join([summ['summary_text'] for summ in summarized_text])
     
-    if is_url:
+    with st.spinner("Calculating and matching entities, this takes a few seconds..."):
     
-        # view summarized text (expander)
-        st.markdown(f"Article title: {article_title}")
+        entity_match_html = highlight_entities(clean_text,summarized_text)
+        st.subheader("Summarized text with matched entities in Green and mismatched entities in Red relative to the original text")
+        st.markdown("####")
+        
+        if article_title:
+
+            # view summarized text (expander)
+            st.markdown(f"Article title: {article_title}")
+            
+        st.markdown("####")    
+        st.write(entity_match_html, unsafe_allow_html=True)
         
-    st.write(summarized_text)
+        st.markdown("####")     
     
-    summary_downloader(summarized_text)
+        summary_downloader(summarized_text)
 
 
 st.markdown("""