Update app.py

app.py

@@ -20,8 +20,10 @@ from typing import Dict
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from collections import Counter
 
+
 import torch
 
+
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 print(f"Device: {device}...")
 if device.type == "cuda":
@@ -65,7 +67,9 @@ examples = [
 models_List = ["Babelscape/wikineural-multilingual-ner",  "urchade/gliner_large-v2.1", "NCBO/BioPortal" ] # "urchade/gliner_large-v2.1",  "knowledgator/gliner-multitask-large-v0.5"
 #models_List = ["NCBO/BioPortal" ]
 
-categories_List = ["MED","LOC","PER","ORG","DATE","MISC"]
+#categories_List = ["MED","LOC","PER","ORG","DATE","MISC"]
+categories_List = ["MED","LOC","PER","ORG","DATE","MISC", "CONC", "BIOP", "ACTI", "ANAT", "CHEM",  "DEVI", "DISO", "GENE", "GEOG", "LIVB", "OBJC", "OCCU", "ORGA", "PHEN", "PHYS" , "PROC"]
+
 
 modelGliner=None
 modelGlinerBio=None
@@ -366,6 +370,12 @@ def process_row_BioPortal_api(args, key_bioportal, row):
         url = f"https://services.data.bioontology.org/annotatorplus/?text={context_to_annotate}&ontologies={onto_clauses}&longest_only=true&exclude_numbers=true&whole_word_only=true&exclude_synonyms=false&negation=false&experiencer=false&temporality=false&score_threshold=0&confidence_threshold=0&display_links=false&display_context=false&score=cvalue&apikey={key_bioportal}"
 
     else:
+
+        kg_restriction = getattr(args, 'KG_restriction', None)
+        if kg_restriction is not None and len(kg_restriction) == 0:
+            print("KG_restriction is provided but empty")
+            return pd.DataFrame()
+
         # args.KG_restriction does not exist or is empty
         if strtobool(args.debug):
             print("--- BIOPORTAL: " + context_to_annotate)
@@ -1148,6 +1158,12 @@ def getUrlBioAndAllOtherBioConcepts(word, args, key_virtuoso, cache_map_virtuoso
 
     else:
         # args.KG_restriction does not exist or is empty
+
+        kg_restriction = getattr(args, 'KG_restriction', None)
+        if kg_restriction is not None and len(kg_restriction) == 0:
+            print("KG_restriction is provided but empty")
+            return None, None, cache_map_virtuoso
+
         if strtobool(args.debug):
             print("--- " + word.lower())
             print("KG_restriction is not provided or empty - Consider all the KGs in the virtuoso endpoint")
@@ -1228,7 +1244,8 @@ def getUrlBioAndAllOtherBioConcepts(word, args, key_virtuoso, cache_map_virtuoso
 
                 if not data:
                     # nothing found from Bioportal
-                    return None, None, None, None, None, cache_map_virtuoso
+                    #return None, None, None, None, None, cache_map_virtuoso
+                    return None, None, cache_map_virtuoso
 
                 dff = pd.DataFrame(data)
                 dff = dff.drop(columns=['hierarchy', 'mappings'])
@@ -1276,7 +1293,8 @@ def getUrlBioAndAllOtherBioConcepts(word, args, key_virtuoso, cache_map_virtuoso
 
                     if df_expanded.empty:
                         # nothing found from Bioportal
-                        return None, None, None, None, None, cache_map_virtuoso
+                        #return None, None, None, None, None, cache_map_virtuoso
+                        return None, None, cache_map_virtuoso
 
                     # Specify the columns you want to keep
                     columns_to_keep = ["score", "from", "to", "prefLabel", "text", "@id"]
@@ -1335,13 +1353,15 @@ def getUrlBioAndAllOtherBioConcepts(word, args, key_virtuoso, cache_map_virtuoso
 
                 else:
                     #nothing found from Bioportal
-                    return None, None, None, None, None, cache_map_virtuoso
+                    #return None, None, None, None, None, cache_map_virtuoso
+                    return None, None, cache_map_virtuoso
 
 
             except Exception as err:
                 logging.error(
                     f'ERROR ON BioPortal Annotator API Call\n\tError: {err}\n TextToAnnotate: {word.lower()}\n Have a check...')
-                return None, None, None, None, None, cache_map_virtuoso
+                #return None, None, None, None, None, cache_map_virtuoso
+                return None, None, cache_map_virtuoso
 
     except Exception as err:
 
@@ -1350,7 +1370,8 @@ def getUrlBioAndAllOtherBioConcepts(word, args, key_virtuoso, cache_map_virtuoso
         #         cache_map_virtuoso[word] = {}
         #     cache_map_virtuoso[word][contextWordVirtuoso] = None
 
-        return None, None, None, None, None, cache_map_virtuoso
+        #return None, None, None, None, None, cache_map_virtuoso
+        return None, None, cache_map_virtuoso
 
 
     return entityBioeUrl, ALLURIScontext, cache_map_virtuoso
@@ -1566,18 +1587,18 @@ def getLinearTextualContextFromTriples(word,labelTriplesLIST, text_splitter, arg
         try:
 
             contextText = ""
-            if args.service_provider == "gptjrc":
-                contextText = call_model(input_text=labelTriples, prompt=myPromt, model=args.model_name,
-                                  temperature=args.temperature, delimiter=myDelimiter,
-                                  InContextExamples=[],
-                                  handler=api_call_gptjrc,
-                                  verbose=True, args=args)
-            elif args.service_provider == "HFonPremises":
-                contextText = call_model(input_text=labelTriples, prompt=myPromt, model=args.model_name,
-                                         temperature=args.temperature, delimiter=myDelimiter,
-                                         InContextExamples=[],
-                                         handler=api_call_HFonPremises,
-                                         verbose=True, args=args)
+            # if args.service_provider == "gptjrc":
+            #     contextText = call_model(input_text=labelTriples, prompt=myPromt, model=args.model_name,
+            #                       temperature=args.temperature, delimiter=myDelimiter,
+            #                       InContextExamples=[],
+            #                       handler=api_call_gptjrc,
+            #                       verbose=True, args=args)
+            # elif args.service_provider == "HFonPremises":
+            #     contextText = call_model(input_text=labelTriples, prompt=myPromt, model=args.model_name,
+            #                              temperature=args.temperature, delimiter=myDelimiter,
+            #                              InContextExamples=[],
+            #                              handler=api_call_HFonPremises,
+            #                              verbose=True, args=args)
 
 
 
@@ -2257,7 +2278,7 @@ def elinking(df_annotated_combined, text_splitter, args, key_geonames, cache_map
 
 
 
-def nerBio(text, ModelsSelection, CategoriesSelection, ScoreFilt, EntityLinking, KGchoices, history_dict: dict):
+def nerBio(text, ModelsSelection, CategoriesSelection, ScoreFilt, EntityLinking, KGchoices, state: dict):
 
     if EntityLinking:
         EnableNEL="True"
@@ -2266,7 +2287,12 @@ def nerBio(text, ModelsSelection, CategoriesSelection, ScoreFilt, EntityLinking,
 
     if not text:
         html_output = f"<div class='gr-textbox' style='white-space: pre-wrap; overflow-wrap: break-word; padding: 10px; border: 1px solid #ddd; border-radius: 5px; font-family: monospace; font-size: 12px; line-height: 24px;'>{text}</div>"
-        return {"text": text, "entities": []}, html_output, dict()
+        state = {
+            "text": "",
+            "df_annotated_dict": dict(),
+            "KGchoices": KGchoices
+        }
+        return {"text": text, "entities": []}, html_output, state
 
     df_annotated = pd.DataFrame()
 
@@ -2326,9 +2352,18 @@ def nerBio(text, ModelsSelection, CategoriesSelection, ScoreFilt, EntityLinking,
 
     df_ToAnnotate = pd.DataFrame()
 
-    #print("Are all models in any row of the 'model' column, case-insensitively?", all_models_in_any_row)
-    #if (not history_dict) or (history_dict[args.source_column][0] != text) or (all_models_in_any_row == False):
-    if (not history_dict) or (history_dict[args.source_column][0] != text):
+    previous_text = ""
+    previous_df_annotated_dict = dict()
+    previous_kg_choices = []
+    if state:
+        previous_text = state.get("text", "")
+        previous_df_annotated_dict = state.get("df_annotated_dict", {})
+        previous_kg_choices = state.get("KGchoices", [])
+
+    # print("Are all models in any row of the 'model' column, case-insensitively?", all_models_in_any_row)
+    # if (not history_dict) or (history_dict[args.source_column][0] != text) or (all_models_in_any_row == False):
+    # if (not history_dict) or (history_dict[args.source_column][0] != text):
+    if (not previous_df_annotated_dict) or (previous_text != text) or (sorted(previous_kg_choices) != sorted(KGchoices)):
 
         for model_id in models_List:  # always do all the annotations, only filter them afterwards
         #for model_id in ModelsSelection:
@@ -2377,7 +2412,11 @@ def nerBio(text, ModelsSelection, CategoriesSelection, ScoreFilt, EntityLinking,
                     # If df_annotated is not empty, concatenate new_annotations to it
                     df_annotated = pd.concat([df_annotated, new_annotations], ignore_index=True)
 
-        history = df_annotated.copy()
+        state = {
+            "text": text,
+            "df_annotated_dict": df_annotated.to_dict(),
+            "KGchoices": KGchoices
+        }
 
     else:
 
@@ -2392,9 +2431,16 @@ def nerBio(text, ModelsSelection, CategoriesSelection, ScoreFilt, EntityLinking,
         torch.cuda.manual_seed_all(args.SEED)
         ###
 
-        history = pd.DataFrame(history_dict)
+        history = pd.DataFrame(previous_df_annotated_dict)
         df_annotated = history.copy()
 
+        state = {
+            "text": text,
+            "df_annotated_dict": df_annotated.to_dict(),
+            "KGchoices": KGchoices
+        }
+
+
 
     quoted_text = text.startswith('"') & text.endswith('"')
     if (not df_annotated.empty) or quoted_text:
@@ -2404,14 +2450,14 @@ def nerBio(text, ModelsSelection, CategoriesSelection, ScoreFilt, EntityLinking,
             df_annotated = df_annotated[df_annotated['model'].str.lower().isin([model.lower() for model in ModelsSelection])]
             if df_annotated.empty and quoted_text==False:
                 html_output = f"<div class='gr-textbox' style='white-space: pre-wrap; overflow-wrap: break-word; padding: 10px; border: 1px solid #ddd; border-radius: 5px; font-family: monospace; font-size: 12px; line-height: 24px;'>{text}</div>"
-                return {"text": text, "entities": []}, html_output, history.to_dict()
+                return {"text": text, "entities": []}, html_output, state
 
         df_annotated_combined = pd.DataFrame()
         if (not df_annotated.empty):
             df_annotated_combined = entitiesFusion(df_annotated,args)
             if df_annotated_combined.empty and quoted_text==False:
                 html_output = f"<div class='gr-textbox' style='white-space: pre-wrap; overflow-wrap: break-word; padding: 10px; border: 1px solid #ddd; border-radius: 5px; font-family: monospace; font-size: 12px; line-height: 24px;'>{text}</div>"
-                return {"text": text, "entities": []}, html_output, history.to_dict()
+                return {"text": text, "entities": []}, html_output, state
             else:
                 df_annotated_combined = is_cross_inside(df_annotated_combined, args, 0.999)  #I cut all the cross inside with the 0.99. to avoid the linking
 
@@ -2566,15 +2612,19 @@ def nerBio(text, ModelsSelection, CategoriesSelection, ScoreFilt, EntityLinking,
             [cat.lower() for cat in CategoriesSelection])
         if "MED" in CategoriesSelection:
             filter_mask |= df_annotated_combined['entity_group'].str.lower().isin(
-                [cat.lower() for cat in CategoriesSelection]) | (df_annotated_combined['IsBio'] == 1)
+                [cat.lower() for cat in CategoriesSelection]) & (df_annotated_combined['IsBio'] == 1)
         if "MISC" in CategoriesSelection:
-            #filter_mask |= ~(df_annotated_combined['entity_group'].str.lower().isin([cat.lower() for cat in categories_List]))
-            filter_mask |= ~(df_annotated_combined['entity_group'].str.lower().isin([cat.lower() for cat in categories_List])) & ~(df_annotated_combined['IsBio'] == 1)  # with this cluase, I'm including not only the categories labelled as MISC, but also the other that are not MED, PER, ORG, LOC
+            # filter_mask |= ~(df_annotated_combined['entity_group'].str.lower().isin([cat.lower() for cat in categories_List]))
+            # filter_mask |= ~(df_annotated_combined['entity_group'].str.lower().isin([cat.lower() for cat in categories_List])) & ~(df_annotated_combined['IsBio'] == 1)  # with this cluase, I'm including not only the categories labelled as MISC, but also the other that are not MED, PER, ORG, LOC
+            filter_mask |= ~(
+                df_annotated_combined['entity_group'].str.lower().isin([cat.lower() for cat in categories_List])) & ~(
+                    df_annotated_combined[
+                        'IsBio'] == 1)  # with this cluase, I'm including not only the categories labelled as MISC, but also the other that are not MED, PER, ORG, LOC
 
         df_annotated_combined = df_annotated_combined[filter_mask]
         if df_annotated_combined.empty:
             html_output = f"<div class='gr-textbox' style='white-space: pre-wrap; overflow-wrap: break-word; padding: 10px; border: 1px solid #ddd; border-radius: 5px; font-family: monospace; font-size: 12px; line-height: 24px;'>{text}</div>"
-            return {"text": text, "entities": []}, html_output, history.to_dict()
+            return {"text": text, "entities": []}, html_output, state
 
         ###
 
@@ -2584,7 +2634,7 @@ def nerBio(text, ModelsSelection, CategoriesSelection, ScoreFilt, EntityLinking,
             df_annotated_combined = df_annotated_combined[df_annotated_combined['IsCrossInside'] != 1]
             if df_annotated_combined.empty:
                 html_output = f"<div class='gr-textbox' style='white-space: pre-wrap; overflow-wrap: break-word; padding: 10px; border: 1px solid #ddd; border-radius: 5px; font-family: monospace; font-size: 12px; line-height: 24px;'>{text}</div>"
-                return {"text": text, "entities": []}, html_output, history.to_dict()
+                return {"text": text, "entities": []}, html_output, state
 
         dict_annotated_combined_NER = df_annotated_combined[["end", "entity_group", "score", "start", "word"]].to_dict(orient="records")
 
@@ -2608,9 +2658,8 @@ def nerBio(text, ModelsSelection, CategoriesSelection, ScoreFilt, EntityLinking,
             #         'word'], axis=1)
             df_annotated_combined['entity_with_link'] = df_annotated_combined.apply(
                 lambda
-                    row: f"<a href='https://api-vast.jrc.service.ec.europa.eu/describe//?url={row['namedEntity']}' target='_blank'>{row['word']}</a>" if row['namedEntity'] not in [None, '', 'NaN', 'nan'] and pd.notnull(row['namedEntity']) else row[
-                        'word'], axis=1) 
-            
+                    row: f"<a href='https://api-vast.jrc.service.ec.europa.eu/describe//?url={row['namedEntity']}' target='_blank'>{row['word']}</a>"  if row['namedEntity'] not in [None, '', 'NaN', 'nan'] and pd.notnull(row['namedEntity']) else row[
+                        'word'], axis=1)
 
             # Create a new dictionary with the entity information and the link
             dict_annotated_combined_NEL = df_annotated_combined[
@@ -2676,16 +2725,16 @@ def nerBio(text, ModelsSelection, CategoriesSelection, ScoreFilt, EntityLinking,
             #         #     f'<span style="background-color: {color}">{entity_text}</span>') - (end - start)
 
             html_output = f"<div class='gr-textbox' style='white-space: pre-wrap; overflow-wrap: break-word; padding: 10px; border: 1px solid #ddd; border-radius: 5px; font-family: monospace; font-size: 12px; line-height: 24px;'>{text_with_links}</div>"
-            return {"text": text, "entities": dict_annotated_combined_NER}, html_output, history.to_dict()
+            return {"text": text, "entities": dict_annotated_combined_NER}, html_output, state
 
         else:
             html_output = f"<div class='gr-textbox' style='white-space: pre-wrap; overflow-wrap: break-word; padding: 10px; border: 1px solid #ddd; border-radius: 5px; font-family: monospace; font-size: 12px; line-height: 24px;'>{text}</div>"
-            return {"text": text, "entities": dict_annotated_combined_NER}, html_output, history.to_dict()
+            return {"text": text, "entities": dict_annotated_combined_NER}, html_output, state
 
     else:
 
         html_output = f"<div class='gr-textbox' style='white-space: pre-wrap; overflow-wrap: break-word; padding: 10px; border: 1px solid #ddd; border-radius: 5px; font-family: monospace; font-size: 12px; line-height: 24px;'>{text}</div>"
-        return {"text": text, "entities": []}, html_output, history.to_dict()
+        return {"text": text, "entities": []}, html_output, state
 
 
 
@@ -2709,13 +2758,37 @@ demo = gr.Interface(
     live=True,
     title="BioAnnotator: Biomedical Named-Entity Recognition (NER) and Linking (NEL)",
     description="""Interoperability – the capability of systems and organisations to cooperate across functional, sectoral and physical borders – is key for successful digital transformation. 
-    The [Interoperable Europe Act](https://interoperable-europe.ec.europa.eu/interoperable-europe/interoperable-europe-act) is an EU regulation that aims to strengthen public sector interoperability and will serve as a main EC policy framework for the years to come. 
-    Data exchange is vital for digital government policies, and semantic interoperability ensures systems understand each other despite different legacies and architectures.
+        The [Interoperable Europe Act](https://interoperable-europe.ec.europa.eu/interoperable-europe/interoperable-europe-act) is an EU regulation that aims to strengthen public sector interoperability and will serve as a main EC policy framework for the years to come. 
+        Data exchange is vital for digital government policies, and semantic interoperability ensures systems understand each other despite different legacies and architectures.
 
-    In this demo we show in particular the *BioAnnotator*, a prototype tool performing Biomedical Named-Entity Recognition (NER) and Linking (NEL). To give it a try, please select one or more NER models and enter some text to get it processed. Please select also the entity categories you want to extract, as well as the score to use as a threshold for the NER extraction. Finally, select whether you want to perform Named-Entity Linking (NEL) and if you want to enable the filtering to some specific biomedical ontologies only (acronyms description at: https://bioportal.bioontology.org/ontologies. See also: https://citnet.tech.ec.europa.eu/CITnet/confluence/display/DIGHEALTH/Inventory+of+existing+KGs+related+to+the+Digital+Health+domain).
-    """,
+        In this demo we show in particular the *BioAnnotator*, a prototype tool performing Biomedical Named-Entity Recognition (NER) and Linking (NEL). To give it a try, please select one or more NER models and enter some text to get it processed. Please select also the entity categories you want to extract, as well as the score to use as a threshold for the NER extraction. Finally, select whether you want to perform Named-Entity Linking (NEL) and if you want to enable the filtering to some specific biomedical ontologies only (acronyms description at: https://bioportal.bioontology.org/ontologies). See also: [InventoryHealthKGs.pdf](https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/ETOHA/KGs/InventoryHealthKGs.pdf).
+        """,
     examples=examples,
-    cache_examples=False
+    cache_examples=False,
+    article="""
+    **Categories Legend:**
+    - MED  | Medical
+    - LOC  | Locations
+    - PER  | Persons
+    - ORG  | Organizations
+    - MISC | Miscellanea
+    - CONC | Concepts & Ideas 
+    - BIOP | Biological 
+    - ACTI | Activities & Behaviors
+    - ANAT | Anatomy
+    - CHEM | Chemicals & Drugs
+    - DEVI | Devices
+    - DISO | Disorders
+    - GENE | Genes & Molecular Sequences
+    - GEOG | Geographic Areas
+    - LIVB | Living Beings
+    - OBJC | Objects
+    - OCCU | Occupations
+    - ORGA | Organizations
+    - PHEN | Phenomena
+    - PHYS | Physiology
+    - PROC | Procedures
+    """
 )