nerBio.py

import os

os.environ["CUDA_VISIBLE_DEVICES"] = "1,6"  #GPUs to use

os.environ["HF_HUB_CACHE"] = "/eos/jeodpp/home/users/consose/cache/huggingface/hub"
os.environ["HUGGINGFACE_HUB_CACHE"] = "/eos/jeodpp/home/users/consose/cache/huggingface/hub"
os.environ["HF_HOME"] = "/eos/jeodpp/home/users/consose/cache/huggingface/hub"

from transformers import file_utils

print(file_utils.default_cache_path)

import sys, re
import time
import pandas as pd
from tqdm import tqdm

from concurrent.futures import ThreadPoolExecutor, as_completed
from collections import Counter

from gliner import GLiNER, GLiNERConfig, data_processing

#os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "max_split_size_mb:512"
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'

import torch
torch.cuda.empty_cache()  # Clear cache ot torch

import logging

import tiktoken
from langchain.text_splitter import TokenTextSplitter

import requests

import re

from common import strtobool, split_camel_case, chunk_tokens, update_nested_dict, cleanInputText, token_counter, encoding_getter, extract_words, all_words_in_list, row_to_dict_string, strip_quotes, rescale_exponential_to_linear, rescale_exponential_to_logarithmic

from accelerate import Accelerator

# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# print(f"Device: {device}...")
# if device.type == "cuda":
#     print("GPU number:", torch.cuda.current_device())

accelerator = Accelerator()
#accelerator = Accelerator(mixed_precision='fp16', device_placement=False)  #fp16 bf16
device = accelerator.device
print("Using accelerator device = "+ str(device))


from transformers import AutoTokenizer, AutoModelForCausalLM
from transformers import pipeline
from transformers.pipelines.pt_utils import KeyDataset

from virtuosoQueryRest import sparqlQuery
from llmqueryNer import call_model, call_model_with_caching, process_list, setup_gptjrc, api_call_gptjrc, model_list_gptjrc #api_call_HFonPremises

import string
import datasets

import argparse
import json
import random
import numpy as np

#from retrieverRAG_testing import RAG_retrieval_Base, RAG_retrieval_Z_scores, RAG_retrieval_Percentile, RAG_retrieval_TopK, retrievePassageSimilarities
from retrieverRAG_SF import RAG_retrieval_Base

from joblib import Memory

cachedir = 'cached'
mem = Memory(cachedir, verbose=False)

# this is to completely delete the cache:
# mem.clear(warn=False)


# aa = data_processing.tokenizer.WordsSplitter('whitespace' )
# bb=aa("this is what i wanted to say")
# count = 0
# for item in bb:
#     count = count+1
#     print(item)
# print(count)

POSSIBLE_KGchoices_List = ["AI", "AIO", "AEO", "BFO", "BIM", "BCGO", "CL", "CHIRO", "CHEBI", "DCM", "FMA", "GO", "GENO",
             "GeoSPARQL", "HL7", "DOID", "HP", "HP_O", "IDO", "IAO", "ICD10", "LOINC", "MESH",
             "MONDO", "NCIT", "NCBITAXON", "NCBITaxon_", "NIFCELL", "NIFSTD", "GML", "OBCS", "OCHV", "OHPI",
             "OPB", "TRANS", "PLOSTHES", "RADLEX", "RO", "STY", "SO", "SNOMED", "STATO",
             "SYMP", "FoodOn", "UBERON", "ORDO", "HOOM", "VO", "OGMS", "EuroSciVoc"]

ONLY_Ontologies_OnBIOPORTAL = ["AI", "AIO", "AEO", "BCGO", "BFO", "BIM", "CHEBI", "CHIRO", "CL", "DCM", "DOID", "FMA", "FOODON", "GENO", "GML", "GO", "GEOSPARQL", "HL7", "HP", "HP_O", "IAO", "ICD10", "IDO", "LOINC", "MESH", "MONDO", "NCBITAXON", "NCIT", "NIFCELL", "NIFSTD", "OBCS", "OCHV", "OHPI", "OPB", "PLOSTHES", "RADLEX", "OBOREL", "SNOMEDCT", "SO", "STATO", "STY", "SYMP", "PTRANS", "UBERON", "ORDO", "HOOM", "VO", "OGMS"]






# query = 'A man is eating a piece of bread'
#
# # Define the passages list
# passages = [
#     "A man is eating food.",
#     "A man is eating pasta.",
#     "The girl is carrying a baby.",
#     "A man is riding a horse.",
# ]
#
# df_retr = RAG_retrieval_Base(questionText, passages, min_threshold=0.5, max_num_passages=3)
# print(df_retr)



# Define a handler for non-serializable objects
def default_serializer(obj):
    if isinstance(obj, np.floating):
        return float(obj)
    raise TypeError(f"Object of type {obj.__class__.__name__} is not JSON serializable")


def is_json(myjson):
    try:
        # json.loads(myjson, default=default_serializer)
        json.dumps(myjson, default=default_serializer)
    except ValueError as e:
        return False
    return True


def get_filtered_entities(txt_ents):
    filtered_ent_list = []

    for ent_dct in txt_ents:
        if ent_dct["score"] > 0.0:  # 0.5
            if ent_dct["entity_group"]:  # in ["LOC", "MISC"]:
                filtered_ent_list.append(ent_dct)

    return filtered_ent_list


def process_row_Gliner(args, tokenizerGliner, modelGlinerBio, modelGliner, glinerlabels, row):
    context_to_annotate = row[args.source_column]
    tokens = tokenizerGliner.tokenize(context_to_annotate)

    entities = []
    offset = 0

    if "gliner_large_bio" in args.model_id:
        max_chunk_length = modelGlinerBio.config.max_len
    else:
        max_chunk_length = modelGliner.config.max_len

    for chunk in chunk_tokens(tokens, (max_chunk_length - 1)):
        chunk_text = tokenizerGliner.convert_tokens_to_string(chunk)
        if "gliner_large_bio" in args.model_id:
            chunk_entities = modelGlinerBio.predict_entities(chunk_text, glinerlabels,
                                                             threshold=args.entities_filter_threshold)
        else:
            chunk_entities = modelGliner.predict_entities(chunk_text, glinerlabels,
                                                          threshold=args.entities_filter_threshold)

        adjusted_entities = []
        for entity in chunk_entities:
            adjusted_entity = {
                'text': entity['text'],
                'score': entity['score'],
                'start': entity['start'] + offset,
                'end': entity['end'] + offset,
                'label': entity['label']
            }
            adjusted_entities.append(adjusted_entity)

        entities.extend(adjusted_entities)
        offset += len(chunk_text)

    if entities and isinstance(entities, list):
        for d in entities:
            d['entity_group'] = d.pop('label')
            d['word'] = d.pop('text')
            d['entity_group'] = d['entity_group'].upper()

    return row.name, entities



def process_row_BioPortal_api(args, key_bioportal, row):

    #context_to_annotate = row[args.source_column]
    if isinstance(row, list) or isinstance(row, pd.Series):
        context_to_annotate = row[args.source_column]
    elif isinstance(row, str):
        context_to_annotate = row
    else:
        raise ValueError("Unsupported type for row. Expected list or string.")

    url=""
    if getattr(args, 'KG_restriction', None):

        # api call
        if strtobool(args.debug):
            print("--- BIOPORTAL: " + context_to_annotate)

        # args.KG_restriction exists and is not empty
        if strtobool(args.debug):
            print("KG_restriction is provided and not empty:", args.KG_restriction)

        onto_clauses = ""
        for choice in args.KG_restriction:
            if choice == "SNOMED":
                choice="SNOMEDCT"
            elif choice == "RO":
                choice = "OBOREL"
            elif choice == "TRANS":
                choice = "PTRANS"
            elif choice == "FoodOn":
                choice = "FOODON"
            elif choice == "GeoSPARQL":
                choice = "GEOSPARQL"
            # elif choice == "NCBITAXON":
            #     choice = "NCBITAXON,NCBITaxon_"
            elif choice == "NCBITaxon_":
                choice = "NCBITAXON"
            if choice in ONLY_Ontologies_OnBIOPORTAL:
                onto_clauses=onto_clauses+choice+","

        if onto_clauses and onto_clauses[-1] == ",":
            onto_clauses=onto_clauses[:-1]

        url = f"https://services.data.bioontology.org/annotatorplus/?text={context_to_annotate}&ontologies={onto_clauses}&longest_only=true&exclude_numbers=true&whole_word_only=true&exclude_synonyms=false&negation=false&experiencer=false&temporality=false&score_threshold=0&confidence_threshold=0&display_links=false&display_context=false&score=cvalue&apikey={key_bioportal}"

    else:

        kg_restriction = getattr(args, 'KG_restriction', None)
        if kg_restriction is not None and len(kg_restriction) == 0:
            print("KG_restriction is provided but empty")
            return pd.DataFrame()

        # args.KG_restriction does not exist or is empty
        if strtobool(args.debug):
            print("--- BIOPORTAL: " + context_to_annotate)
            print("KG_restriction is not provided or empty - Consider all the KGs")

        url = f"https://services.data.bioontology.org/annotatorplus/?text={context_to_annotate}&ontologies=AEO,BFO,BIM,BCGO,CL,CHIRO,CHEBI,DCM,FMA,GO,GENO,GEOSPARQL,HL7,DOID,HP,HP_O,IDO,IAO,ICD10,LOINC,MESH,MONDO,NCIT,NCBITAXON,NIFCELL,NIFSTD,GML,OBCS,OCHV,OHPI,OPB,PTRANS,PLOSTHES,RADLEX,OBOREL,STY,SO,SNOMEDCT,STATO,SYMP,FOODON,UBERON,VO&longest_only=true&exclude_numbers=true&whole_word_only=true&exclude_synonyms=false&negation=false&experiencer=false&temporality=false&score_threshold=0&confidence_threshold=0&display_links=false&display_context=false&score=cvalue&apikey={key_bioportal}"


    response = requests.get(url)

    try:
        data = response.json()

        if not data:
            # nothing found from Bioportal
            return pd.DataFrame()

        dff = pd.DataFrame(data)
        dff = dff.drop(columns=['hierarchy', 'mappings'])

        # If the columns are dictionary-like, use pd.json_normalize:
        expanded_annotated_class = pd.json_normalize(dff['annotatedClass'])

        expanded_annotations = pd.DataFrame(dff['annotations'].tolist(), index=dff.index)
        expanded_annotations = pd.json_normalize(expanded_annotations[0])

        # Join the expanded columns back to the original DataFrame
        df_expanded = dff.drop(columns=['annotatedClass', 'annotations']).join(expanded_annotated_class).join(
            expanded_annotations)

        # Snomed id replacement because in our internal knolwedgebase we have this base uri
        df_expanded['@id'] = df_expanded['@id'].str.replace(
            "http://purl.bioontology.org/ontology/SNOMEDCT/",
            "http://snomed.info/id/"
        )

        return df_expanded


    except Exception as err:
        logging.error(
            f'ERROR ON BioPortal Annotator API Call\n\tError: {err}\n TextToAnnotate: {context_to_annotate}\n Have a check...')
        return pd.DataFrame()  # empty dataframe


def parallel_process_df_Gliner(args, df, tokenizerGliner, modelGlinerBio, modelGliner, glinerlabels):
    results = []

    if args.num_cores_Gliner > 0:
        with ThreadPoolExecutor(max_workers=args.num_cores_Gliner) as executor:
            futures = [
                executor.submit(
                    process_row_Gliner, args, tokenizerGliner, modelGlinerBio, modelGliner, glinerlabels, row
                )
                for _, row in df.iterrows()
            ]

            for future in tqdm(futures):
                drm_idx, entities = future.result()
                df.at[drm_idx, 'annotation'] = entities

    else:
        # Apply the function to each row and extract only the entities part
        df['annotation'] = df.apply(
            lambda row: process_row_Gliner(args, tokenizerGliner, modelGlinerBio, modelGliner, glinerlabels, row)[1],
            axis=1
        )

    return df


def annotate(df, args, pipeInner, tokenizerGliner, modelGliner, modelGlinerBio, device="cpu"):

    if strtobool(args.debug):
        print("\nAnnotate using " + args.model_id)
        print("device=" + str(device))
        startAnnotate = time.time()

    if "gliner" in args.model_id:

        df['model'] = args.model_id
        df['annotation'] = None

        glinerlabels = ["location", "disease", "date", "numerical value", "number"]

        # Parallel CPU computation for Gliner:
        df = parallel_process_df_Gliner(args, df, tokenizerGliner, modelGlinerBio, modelGliner, glinerlabels)


        # for drm_idx, row in tqdm(df.iterrows()):
        #     context_to_annotate = row[args.source_column]
        #
        #     # Tokenize the text
        #     tokens = tokenizerGliner.tokenize(context_to_annotate)
        #
        #     # Process each chunk and predict entities
        #     entities = []
        #     offset = 0  # Initialize the offset
        #
        #     if "gliner_large_bio" in args.model_id:
        #         maxchunckslen=modelGlinerBio.config.max_len
        #     else:
        #         maxchunckslen = modelGliner.config.max_len
        #     for chunk in chunk_tokens(tokens, (maxchunckslen - 1)):
        #         # Convert tokens back to text for the chunk
        #         chunk_text = tokenizerGliner.convert_tokens_to_string(chunk)
        #         # Predict entities for the chunk
        #         if "gliner_large_bio" in args.model_id:
        #             chunk_entities = modelGlinerBio.predict_entities(chunk_text, glinerlabels, threshold=args.entities_filter_threshold)
        #         else:
        #             chunk_entities = modelGliner.predict_entities(chunk_text, glinerlabels, threshold=args.entities_filter_threshold)
        #
        #         # Adjust the start and end positions of entities to reflect their positions in the original text
        #         adjusted_entities = []
        #         for entity in chunk_entities:
        #             adjusted_entity = {
        #                 'text': entity['text'],
        #                 'score': entity['score'],
        #                 'start': entity['start'] + offset,
        #                 'end': entity['end'] + offset,
        #                 'label': entity['label']
        #             }
        #             adjusted_entities.append(adjusted_entity)
        #
        #         # Append adjusted entities to all_entities
        #         entities.extend(adjusted_entities)
        #
        #         # Update the offset for the next chunk by adding the length of the current chunk
        #         offset += len(chunk_text)
        #
        #     # Now `all entities` contains all entities with adjusted positions
        #     if entities and isinstance(entities, list):
        #
        #         # if strtobool(args.debug):
        #         #     for entity in entities:
        #         #         print(entity["text"], "=>", entity["label"])
        #
        #         for d in entities:
        #             d['entity_group'] = d.pop('label')  # Change 'label' to 'entity_group'
        #             d['word'] = d.pop('text')  # Change 'text' to 'word'
        #             d['entity_group'] = d['entity_group'].upper()
        #
        #         df.at[drm_idx, 'annotation'] = entities


        df_annot = df.explode('annotation').dropna(subset=['annotation']).reset_index(drop=True)

    elif "NCBO" in args.model_id:  #NCBO/BioPortal" annotator
        #https://data.bioontology.org/documentation#nav_annotator
        #https://bioportal.bioontology.org/annotatorplus

        key_bioportal = ""
        if args.bioportalkey_filename and os.path.exists(args.bioportalkey_filename):
            fkeyname = args.bioportalkey_filename
            with open(fkeyname) as f:
                key_bioportal = f.read()
        else:
            key_bioportal = os.environ['key_bioportal']

        df_annot = pd.DataFrame()
        for drm_idx, row in tqdm(df.iterrows()):
            df_BioPortalAnnotation=process_row_BioPortal_api(args, key_bioportal, row)

            if not df_BioPortalAnnotation.empty:

                df_BioPortalAnnotation = df_BioPortalAnnotation.sort_values(
                    by=['from', 'text', 'score', 'matchType'], ascending=[True, True, False, False])

                df_biop_minimised = df_BioPortalAnnotation.copy()

                # Group the dataframe by "from" and "to" columns
                grouped_biop = df_biop_minimised.groupby(['from', 'to'])
                # Get the index of the row with the maximum score in each group
                idx_biop = grouped_biop['score'].idxmax()
                # Filter the dataframe using these indices
                df_max_score_biop = df_biop_minimised.loc[idx_biop]
                # Optional: Reset index if you want a clean index
                df_max_score_biop = df_max_score_biop.reset_index(drop=True)

                # Create the "pippo" column by grouping and collecting "@id" values
                pippo_lists = grouped_biop['@id'].apply(list).reset_index()

                # Merge the lists back into the df_max_score_biop by 'from' and 'to'
                df_max_score_biop = df_max_score_biop.merge(pippo_lists, on=['from', 'to'])

                # Rename the merged column to "pippo"
                df_max_score_biop = df_max_score_biop.rename(columns={'@id_x': '@id'})
                df_max_score_biop = df_max_score_biop.rename(columns={'@id_y': 'ALLURIScontextFromNCBO'})
                # The df_max_score_biop now contains the "pippo" column with lists of "@id" values.

                # Filter the dataframe to keep only rows where the score is greater than 3.0
                df_max_score_biop = df_max_score_biop[df_max_score_biop['score'] > 3.0]

                if "semantic_groups" not in df_max_score_biop.columns:
                    # Drop the '@id' column
                    df_max_score_biop["semantic_groups"] = None

                # Specify the columns you want to keep
                columns_to_keep = ["score", "from", "to", "prefLabel", "text", "semantic_groups", "@id", "ALLURIScontextFromNCBO"]

                # Subset the dataframe to keep only the specified columns
                df_max_score_biop = df_max_score_biop[columns_to_keep]

                # Rename the specified columns
                df_max_score_biop = df_max_score_biop.rename(columns={"from": "start", "to": "end", "text": "word", "semantic_groups": "entity_group"})

                # Optional: Reset index if you want a clean index after filtering
                df_max_score_biop = df_max_score_biop.reset_index(drop=True)

                df_max_score_biop['score'] = df_max_score_biop['score'].round(2)

                # Each row of  "semantic_groups" --> 'entity_group' is a list of string ... keep only the first part if not NaN
                # df_max_score_biop['entity_group'] = df_max_score_biop['entity_group'].apply(lambda x: x[0] if pd.notna(x).all() else x)
                df_max_score_biop['entity_group'] = df_max_score_biop['entity_group'].apply(lambda x: x[0] if isinstance(x, list) and len(x) > 0 else (np.nan if x is None or (isinstance(x, float) and pd.isna(x)) else x))

                #df_max_score_biop = df_max_score_biop.dropna(subset=['entity_group'])

                all_empty_or_nan_or_empty_string = df_max_score_biop['entity_group'].replace('', pd.NA).isna().all()
                if not all_empty_or_nan_or_empty_string:
                    #print("Is the 'entity_group' column completely NaN, None, or empty strings?", all_empty_or_nan_or_empty_string)

                    # Identify the minimum score
                    min_score_biop = df_max_score_biop['score'].min()

                    # Apply filter to find rows where entity_group is None and score is the minimum one
                    conditionBiop = (df_max_score_biop['entity_group'].isna()) & (df_max_score_biop['score'] == min_score_biop)
                    df_max_score_biop = df_max_score_biop[~conditionBiop]

                # Replace all NaN values in 'entity_group' with 'BIOP'
                df_max_score_biop['entity_group'] = df_max_score_biop['entity_group'].fillna('BIOP')

                # To delete the columns "start" and "end"
                #df_max_score_biop = df_max_score_biop.drop(columns=['start', 'end', 'word', 'entity_group', 'score', 'prefLabel'])

                if not df_max_score_biop.empty:
                    row_df = pd.DataFrame([row] * len(df_max_score_biop), columns=row.index)
                    row_df['model'] = args.model_id  #'NCBO/BioPortal'
                    df_max_score_biop = pd.concat([row_df.reset_index(drop=True), df_max_score_biop.reset_index(drop=True)],
                                                  axis=1)
                    df_annot = pd.concat([df_annot, df_max_score_biop], ignore_index=True)


        #here I finish the for cycle


        if not df_annot.empty:

            mmax_score = df_annot['score'].max()
            mmin_score = df_annot['score'].min()

            if mmax_score == mmin_score:
                df_annot['score'] = 0.3
                df_annot.loc[df_annot['score'].notnull(), 'score'] = 0.7
            else:
                # Apply the transformation function
                #df_annot = rescale_exponential_to_linear(df_annot, 'score', new_min=0.5, new_max=1.0)
                df_annot = rescale_exponential_to_logarithmic(df_annot, 'score', new_min=0.7, new_max=1.0)

            columnsDict = ['start', 'end', 'word', 'entity_group', 'score', 'prefLabel']

            df_annot['annotation'] = df_annot.apply(row_to_dict_string, axis=1, columnsDict=columnsDict)

            # Convert JSON string to a dictionary, if needed
            df_annot['annotation'] = df_annot['annotation'].apply(
                lambda x: json.loads(x) if isinstance(x, str) else x
            )

            df_annot = df_annot.drop(columns=columnsDict)

    else:

        HF_dataset = datasets.Dataset.from_pandas(pd.DataFrame(data=df))

        # tsk = "token-classification"
        #
        # pipe = pipeline(
        #     tsk,
        #     model=args.model_id,
        #     aggregation_strategy="simple",
        #     device=device,
        # )

        if strtobool(args.debug):
            print('Annotating texts...')

        annotated_texts = [
            # out for out in tqdm(pipe(KeyDataset(HF_dataset, args.source_column), batch_size=args.batch_size))
            # out for out in tqdm(ner_pipeline(KeyDataset(HF_dataset, args.source_column), batch_size=args.batch_size))
            out for out in tqdm(pipeInner(KeyDataset(HF_dataset, args.source_column), batch_size=args.batch_size))
        ]

        if strtobool(args.debug):
            print('looping annotations...')


        df['model'] = args.model_id
        df['annotation'] = annotated_texts

        df_annot = df.explode('annotation').dropna(subset=['annotation']).reset_index(drop=True)

        # # Initialize an empty dataframe with the same columns as df
        # df_annot = pd.DataFrame(columns=df.columns.tolist()) #+ ['model', 'annotation'])
        #
        # for drm_idx, txt_ents in enumerate(tqdm(annotated_texts)):
        #
        #     filtered_entities = get_filtered_entities(txt_ents)
        #
        #     for x_ent in filtered_entities:
        #         # Create a new row with the annotation
        #         new_row = df.iloc[drm_idx].copy()
        #         new_row['model'] = args.model_id
        #         new_row['annotation'] = x_ent
        #         df_annot = pd.concat([df_annot, new_row.to_frame().T], ignore_index=True)  # Append the new row to the dataframe


    if strtobool(args.debug):
        endAnnotate = time.time()
        hours, rem = divmod(endAnnotate - startAnnotate, 3600)
        minutes, seconds = divmod(rem, 60)
        print("...end annotation - Time... {:0>2}:{:0>2}:{:05.2f}\n".format(int(hours), int(minutes), seconds))
        print('\n')

    return df_annot

###
def is_cross_inside(df_sorted, args, valuecutCross=0.75):

    df_sorted['IsCrossInside'] = 0

    df_sorted = df_sorted.reset_index(drop=True)  # I reset the index so that I can loop on inner loop inside

    # this works but it is absolutely too heavy!!! Got: mask1 = (SentenceRef_array[:, None] == SentenceRef_array) & \
    #              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    # numpy.core._exceptions._ArrayMemoryError: Unable to allocate 73.4 TiB for an array with shape (8984940, 8984940) and data type bool
    #
    # # Assuming df_sorted is your DataFrame
    # df_sorted['IsCrossInside'] = 0
    #
    # # Convert necessary columns to numpy arrays
    # SentenceRef_array = df_sorted['SentenceRef'].to_numpy()
    # start_array = df_sorted['start'].to_numpy()
    # end_array = df_sorted['end'].to_numpy()
    # score_array = df_sorted['score'].to_numpy()
    # ToLink_array = df_sorted['ToLink'].to_numpy()
    #
    # # Identify NaN or empty strings in ToLink array
    # toLink_nan_or_empty = pd.isna(ToLink_array) | (ToLink_array == '')
    #
    # # Create the mask for the first set of conditions
    # mask1 = (SentenceRef_array[:, None] == SentenceRef_array) & \
    #         (start_array[:, None] <= start_array) & \
    #         (end_array[:, None] > start_array) & \
    #         (end_array[:, None] <= end_array) & \
    #         (score_array[:, None] < score_array) & \
    #         toLink_nan_or_empty[:, None]
    #
    # # Create the mask for the second set of conditions
    # mask2 = (SentenceRef_array[:, None] == SentenceRef_array) & \
    #         (end_array[:, None] >= end_array) & \
    #         (start_array[:, None] < end_array) & \
    #         (start_array[:, None] >= start_array) & \
    #         (score_array[:, None] < score_array) & \
    #         toLink_nan_or_empty[:, None]
    #
    # # Combine the masks
    # mask = mask1 | mask2
    #
    # # Aggregate the masks and set 'IsCrossInside' where the condition is True
    # df_sorted.loc[mask.any(axis=1), 'IsCrossInside'] = 1


    # df_sorted['IsCrossInside'] = 0
    #
    # #questo for e' altamente inefficiente...guardare per alternative piu' ottimizzate!!!
    # for index, row in df_sorted.iterrows():
    #
    #     print(row)
    #
    #     mask = (df_sorted['SentenceRef'] == row['SentenceRef']) & (df_sorted['start'] <= row['start']) & (df_sorted['end'] > row['start']) & (
    #             df_sorted['end'] <= row['end']) & (df_sorted['score'] < row['score']) & (df_sorted['ToLink'].isnull() | (df_sorted['ToLink'] == ''))
    #     mask |= (df_sorted['SentenceRef'] == row['SentenceRef']) & (df_sorted['end'] >= row['end']) & (df_sorted['start'] < row['end']) & (
    #             df_sorted['start'] >= row['start']) & (df_sorted['score'] < row['score']) & (df_sorted['ToLink'].isnull() | (df_sorted['ToLink'] == ''))
    #
    #     df_sorted.loc[mask, 'IsCrossInside'] = 1

    # Iterate over the DataFrame with an outer loop - I know that df_sorted is ordered by 'SentenceRef' and 'start'
    indexes_list = []
    scores_list = []
    IsToLinkContained = []
    for i, row_outer in tqdm(df_sorted.iterrows()):
        #print(row_outer)

        # if (i>0) and (df_sorted.iloc[i-1]['SentenceRef'] == row_outer['SentenceRef']): # if the previous row is of the same SenteceRef, it has been alredy computed and I skip
        #     continue
        if len(indexes_list)==0:
            scores_list.append(row_outer['score'])
            indexes_list.append(i)
            if (pd.isnull(row_outer['ToLink']==False) and len(row_outer['ToLink'])>0):
                IsToLinkContained.append(True)
            else:
                IsToLinkContained.append(False)
            # if (pd.isnull(row_outer['ToLink']) or row_outer['ToLink'] == ''):
            #     IsToLinkContained.append(False)
            # elif len(row_outer['ToLink'])>0:
            #     IsToLinkContained.append(True)
        else:
            if i in indexes_list:
                if (i == indexes_list[-1]):
                    # indexes_list = []
                    # scores_list = []
                    # IsToLinkContained = []
                    indexes_list = [indexes_list[-1]]
                    scores_list = [scores_list[-1]]
                    IsToLinkContained = [IsToLinkContained[-1]]
                else:
                    continue

        # Inner loop only needs to consider rows starting from the current position
        for j in range(i + 1, len(df_sorted)):
        #for j in range(0, len(df_sorted)):
        #    if i==j: continue

            #print(j)
            row_inner = df_sorted.iloc[j]

            # Break the inner loop if SentenceRef changes (due to sorting)
            if row_inner['SentenceRef'] != row_outer['SentenceRef']:
                break
            elif row_inner['start'] >= row_outer['end']:
                break
            else:
                scores_list.append(row_inner['score'])
                indexes_list.append(j)
                if (pd.isnull(row_inner['ToLink'] == False) and len(row_inner['ToLink']) > 0):
                    IsToLinkContained.append(True)
                else:
                    IsToLinkContained.append(False)

        if len(indexes_list)>1:
            first_true_index = -1
            try:
                first_true_index = IsToLinkContained.index(True)
                #print(f"The first index with value True is: {first_true_index}")
            except ValueError:
                first_true_index = -1
                #print("No True value found in the list")

            topinlist=-1
            if first_true_index >=0:
                topinlist = first_true_index
            else:
                topinlist = scores_list.index(max(scores_list))
                #print(f"The position with the maximum score is: {topinlist}")

            if topinlist >= 0:
                for xx in range(0, len(indexes_list)):
                    if xx == topinlist:
                        continue
                    df_sorted.at[indexes_list[xx], 'IsCrossInside'] = 1

        else:
            indexes_list = []
            scores_list = []
            IsToLinkContained = []

    #Now I want to delete all the words that are contained within another . For example I want to drop "osis" from Legionellosis
    if not df_sorted.empty:

        # punctuation_and_space = set(string.punctuation + ' ')
        # condition = df_sorted.apply(lambda row:
        #                             (row['IsCrossInside'] == 0) or
        #                             ((row['IsCrossInside'] == 1) and
        #                             ((row[args.source_column][row['start'] - 1] in punctuation_and_space) if row[
        #                                                                                        'start'] - 1 >= 0 else True) and
        #                             ((row[args.source_column][row['end']] in punctuation_and_space) if row['end'] <= len(
        #                                 row[args.source_column]) else True)),
        #                             axis=1)

        condition = df_sorted.apply(lambda row:
                                    (row['IsCrossInside'] == 0) or
                                    ( (row['IsCrossInside'] == 1) and (row['score'] >=valuecutCross )),
                                    axis=1)

        # Filter the DataFrame to keep only rows where the condition is False
        df_sorted = df_sorted[condition]

    return df_sorted

def entitiesFusion(df_annotated, args):

    if strtobool(args.debug):
        print("\nStart entities fusion and filtering ...")

    areJson = df_annotated["annotation"].apply(is_json)
    if False in areJson.unique():
        for idxr, rr in df_annotated.iterrows():
            # for idxr in range(df_annotated["annotation"].shape[0]):
            if areJson[idxr] == False:
                print("PROBLEM WITH JSON AT INDEX " + str(idxr) + ":\n" + df_annotated["annotation"][idxr])
                replacement_empty_myjson = '{\"entity_group\": \"\", \"score\": \"\", "word": \"\", \"start\": \"\", \"end\": \"\"}'  # {'entity_group': 'ORG', 'score': 0.9999951, 'word': 'Health Ministry', 'start': 4, 'end': 19}
                df_annotated.at[idxr, "annotation"] = replacement_empty_myjson
                print(" ...... Then replacing it with empty JSON --> " + df_annotated["annotation"][idxr])

    try:

        # df_extract = df_annotated.apply(lambda x: pd.Series(
        #    json.loads(x['annotation'], default=default_serializer).values(),
        #    index=json.loads(x['annotation'], default=default_serializer).keys()), axis=1)
        df_extract = df_annotated.apply(lambda x: pd.Series(x['annotation'].values(),
                                                            index=x['annotation'].keys()), axis=1)

        # Check if '@id' column exists in df_Extract
        if '@id' in df_extract.columns:
            # Drop the '@id' column
            df_extract = df_extract.drop(columns='@id')

        df_annotated = pd.merge(df_annotated, df_extract, left_index=True, right_index=True)

    except Exception as err:
        logging.error(
            f'FAILED to extract json results\n\tError: {err}\nLeaving it as a single column then and not decompressing! Have a check...')

    #


    # Delete all the rows where EXACT MATCHING NOT MET:
    # Apply the conditions
    condition_to_delete = (
            df_annotated[args.source_column].str.startswith('"') &
            df_annotated[args.source_column].str.endswith('"') &
            (df_annotated[args.source_column].apply(strip_quotes).str.lower() != df_annotated['word'].str.lower())
    )

    # Now Filter out the rows where condition_to_delete is True
    df_annotated = df_annotated[~condition_to_delete].copy()
    #

    #delete all the rows with score smaller than entities_filter_threshold:
    if args.entities_filter_threshold > 0:
        #df_annotated = df_annotated[df_annotated['score'] >= args.entities_filter_threshold]
        df_annotated = df_annotated[df_annotated['score'] > args.entities_filter_threshold]
        if df_annotated.empty:
            return df_annotated

    # #delete all the rows where the concept that was asked to link, in the column "ToLink", is present and it is different from the column "word", that was recognised
    # df_annotated = df_annotated[(df_annotated['ToLink'] == df_annotated['word']) | df_annotated['ToLink'].isna()]

    # in all the rows having a value not null for the column "ToLink", compare this value to that of the column "word". If they are different, set the value in "ToLink" to None
    # df_annotated.loc[
    #     (~df_annotated['ToLink'].isnull()) & (df_annotated['ToLink'] != df_annotated['word']), 'ToLink'] = None
    df_annotated.loc[
        (~df_annotated['ToLink'].isnull()) & (
                    df_annotated['ToLink'].str.casefold() != df_annotated['word'].str.casefold()), 'ToLink'] = None

    # now fill all the values of the column "toLink" that are empty with the values of the row "word":
    # df_annotated['ToLink'] = df_annotated['ToLink'].fillna(df_annotated['word'])

    if "IsGeo" not in df_annotated.columns:
        #df_annotated["IsGeo"] = None  # 0
        df_annotated.loc[:, "IsGeo"] = None
    if "IsBio" not in df_annotated.columns:
        #df_annotated["IsBio"] = None  # 0
        df_annotated.loc[:, "IsBio"] = None

    df_annotated.loc[df_annotated['entity_group'] == 'LOCATION', 'entity_group'] = "LOC"
    df_annotated.loc[df_annotated['entity_group'] == 'LOC', 'IsGeo'] = 1

    #df_annotated.loc[df_annotated['entity_group'] == 'DISEASE', 'IsBio'] = 1
    df_annotated.loc[df_annotated['entity_group'].str.lower().str.contains('disease'), 'IsBio'] = 1
    df_annotated.loc[(df_annotated['model'].str.contains('Medical-NER')) & (
                df_annotated['entity_group'].isin(['LOC', 'DATE', 'PER', 'ORG', 'DOSAGE', 'LAB_VALUE', 'DURATION']) == False), 'IsBio'] = 1
    df_annotated.loc[(df_annotated['model'].str.contains('NCBO')) & (
            df_annotated['entity_group'].isin(['CONC']) == False), 'IsBio'] = 1

    # !!! THIS CHECK HAS BEEN PLACED HERE BECAUSE NCBO Bioportal put Start+1 !!! SO I NEED TO DROP A ONE TO DROP DUPLICATES AND MAX SCORES
    df_annotated.loc[df_annotated['model'].str.lower().str.contains('ncbo'), 'start'] -= 1

    # !!! THIS CHECK HAS BEEN PLACED HERE BECAUSE MEDICAL-NER PUT A VALUE OF START WHICH IS THE REAL ONE MINUS ONE, IN THE CASE THAT THE IDENTIFIED ENTITY STARTS WITH A SPACE!!! SO I NEED TO ADD A ONE TO DROP DUPLICATES AND MAX SCORES
    df_annotated.loc[(df_annotated['model'] == 'blaze999/Medical-NER') &
                     df_annotated.apply(lambda row: row[args.source_column][row['start']] == ' ',
                                        axis=1), 'start'] += 1

    # !!! THIS CHECK HAS BEEN PLACED HERE BECAUSE GLINER MODELS PUT A VALUE OF START and of END WHICH IS THE REAL ONE MINUS ONE, IN THE CASE THAT THE IDENTIFIED ENTITY STARTS WITH A SPACE!!! SO I NEED TO ADD A ONE to START and END TO DROP DUPLICATES AND MAX SCORES
    df_annotated.loc[df_annotated['model'].str.lower().str.contains('gliner') &
                     df_annotated.apply(lambda row: row[args.source_column][row['start']] == ' ',
                                        axis=1), 'end'] += 1
    df_annotated.loc[df_annotated['model'].str.lower().str.contains('gliner')  &
                      df_annotated.apply(lambda row: row[args.source_column][row['start']] == ' ',
                                         axis=1), 'start'] += 1


    #### Here I drop all the identified NER words which are strictly contained in other words
        # Apply first the function to the "args.source_column" to create lists of words
    df_annotated['extracted_words'] = df_annotated[args.source_column].apply(extract_words,putInLower=True)
        # Filter the DataFrame
    df_annotated = df_annotated[df_annotated.apply(lambda row: all_words_in_list(row['word'], row['extracted_words'], putInLower=True), axis=1)]
        # drop the 'extracted_words' column afterwards:
    df_annotated = df_annotated.drop(columns=['extracted_words'])
    #####


    # This operation sorts the DataFrame by the "ToLink" column in descending order, with null values at the end, and then uses the drop_duplicates method to drop all duplicate rows,
    # except the first one, based on all columns except "ToLink".
    # This way, it will keep the rows with the non-null value in "ToLink" if there are multiple rows with the same values in all columns except "ToLink".
    df_annotated = df_annotated.sort_values(by='ToLink', ascending=False, na_position='last')

    for col in df_annotated.columns:
        if df_annotated[col].apply(lambda x: isinstance(x, dict)).any():
            if strtobool(args.debug):
                print(
                    f"Column '{col}' contains dictionaries...converting it to strings otherwise it will not work the concat etc..")
            df_annotated[col] = df_annotated[col].apply(lambda x: str(x))

    df_annotated = df_annotated.drop_duplicates(subset=[col for col in df_annotated.columns if
                                                        col != 'ToLink' and col != 'ALLURIScontextFromNCBO' and not df_annotated[col].apply(
                                                            lambda x: isinstance(x, dict)).any()], keep='first')

    # df_annotated = df_annotated.loc[df_annotated.groupby(['SentenceRef', 'ToLink', args.source_column, 'end', 'start', df_annotated['word'].str.lower()])['score'].idxmax()]
    # df_annotated = df_annotated.loc[df_annotated.groupby(['SentenceRef', args.source_column, 'end', 'start', df_annotated['word'].str.lower()])['score'].idxmax()]

    df_annotated_Geo = df_annotated.loc[df_annotated.groupby(
        ['SentenceRef', args.source_column, 'end', 'start', df_annotated['word'].str.lower(), 'IsGeo'])[
        'score'].idxmax()]
    df_annotated_Bio = df_annotated.loc[df_annotated.groupby(
        ['SentenceRef', args.source_column, 'end', 'start', df_annotated['word'].str.lower(), 'IsBio'])[
        'score'].idxmax()]
    df_annotated_all = df_annotated.loc[
        df_annotated.groupby(['SentenceRef', args.source_column, 'end', 'start', df_annotated['word'].str.lower()])[
            'score'].idxmax()]

    # now you can concat
    df_annotated_combined = pd.concat([df_annotated_Geo, df_annotated_Bio, df_annotated_all])
    df_annotated_combined = df_annotated_combined.drop_duplicates(subset=[col for col in df_annotated_combined.columns if
                                                        col != 'ToLink' and col != 'ALLURIScontextFromNCBO' and not df_annotated_combined[col].apply(
                                                            lambda x: isinstance(x, dict)).any()], keep='first')

    # df_annotated_combined['IsBioGeo'] = df_annotated_combined['IsGeo'].fillna(0) + df_annotated_combined['IsBio'].fillna(0)
    # df_annotated_combined['IsBioGeo'] = pd.to_numeric(df_annotated_combined['IsGeo'], errors='coerce').fillna(0) + pd.to_numeric(df_annotated_combined['IsBio'], errors='coerce').fillna(0)
    # df_annotated_combined['IsBioGeo'] = df_annotated_combined['IsGeo'].infer_objects(copy=False).fillna(0) + df_annotated_combined['IsBio'].infer_objects(copy=False).fillna(0)
    df_annotated_combined.loc[:, "IsBioGeo"] = df_annotated_combined.loc[:, 'IsGeo'].infer_objects(copy=False).fillna(0) + df_annotated_combined.loc[:, 'IsBio'].infer_objects(copy=False).fillna(0)

    df_annotated_combined = df_annotated_combined.loc[df_annotated_combined.groupby(
        ['SentenceRef', args.source_column, 'end', 'start', df_annotated['word'].str.lower()])['IsBioGeo'].idxmax()]

    df_annotated_combined = df_annotated_combined.loc[
        df_annotated_combined.groupby(
            ['SentenceRef', args.source_column, 'end', 'start', df_annotated['word'].str.lower(), 'IsBioGeo'])[
            'score'].idxmax()]

    df_annotated_combined = df_annotated_combined.drop('IsBioGeo', axis=1)
    df_annotated_combined.loc[df_annotated_combined['IsBio'] == 0, 'IsBio'] = None
    df_annotated_combined.loc[df_annotated_combined['IsGeo'] == 0, 'IsGeo'] = None

    df_annotated_combined = df_annotated_combined.sort_values(by=['SentenceRef', 'start', 'ToLink', 'word', 'score'], ascending=[True, True, True, True, False])
    #df_annotated_combined = df_annotated_combined.reindex(range(len(df_annotated_combined)))

    return df_annotated_combined


def geonames_api_call(word,args, key_geonames,cache_map_geonames):

    context = ""
    singleContext = None
    globalContext = None
    singleTriples = None
    globalTriples = None

    if cache_map_geonames is not None:
        if word in cache_map_geonames:
            if context in cache_map_geonames[word]:
                url_text = cache_map_geonames[word][context]
                if strtobool(args.debug):
                    print("RETRIEVED CACHED RESULT FOR:\n", word, " => ", url_text, "\n")
                return url_text, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames

    #url = f"http://api.geonames.org/search?name_equals={word}&maxRows=1&featureClass=A&featureCode=ADM1&featureCode=ADM2&featureCode=ADM3&featureCode=ADM4&featureCode=ADM5&type=json&username={key_geonames}"
    url = f"http://api.geonames.org/search?name_equals={word}&maxRows=1&type=json&username={key_geonames}"
    response = requests.get(url)

    try:
        data = response.json()
        if data['geonames']:
            # geoname = data['geonames'][0]['name']
            geonameId = data['geonames'][0]['geonameId']
            geonameUrl = "https://sws.geonames.org/" + str(geonameId) + "/"

            if cache_map_geonames is not None:
                if not word in cache_map_geonames:
                    cache_map_geonames[word] = {}
                cache_map_geonames[word][context] = geonameUrl

            return geonameUrl, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames
        else:

            if cache_map_geonames is not None:
                if not word in cache_map_geonames:
                    cache_map_geonames[word] = {}
                cache_map_geonames[word][context] = None

            return None, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames
    
    except Exception as err:
        
        # if cache_map_geonames is not None:
        #     if not word in cache_map_geonames:
        #         cache_map_geonames[word] = {}
        #     cache_map_geonames[word][context] = None

        return None, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames




def getLinearTextualContextFromTriples(word,labelTriplesLIST, text_splitter, args, map_query_input_output, cleanInput=True, questionText=""):

    # trial
    #return None, map_query_input_output

    word = word.lower()
    word = word.capitalize()

    labelTriples=""

    if labelTriplesLIST and getattr(args, 'maxTriplesContextComputation', None):  # it means it exists
        if args.maxTriplesContextComputation > 0:
            if len(labelTriplesLIST) > args.maxTriplesContextComputation:
                labelTriplesLIST = labelTriplesLIST[:args.maxTriplesContextComputation]

    if (strtobool(args.UseRetrieverForContextCreation) == True):

        # if strtobool(args.debug):
        #     print("Start reranking - num passages : ", len(labelTriplesLIST), "\n")
        #     startRerank = time.time()
        #
        # labelTriples = ""
        # passages = []
        # nn = 200
        #
        # OverallListRAGtriples = []
        # labelTriplesLIST_RAGGED = []
        #
        # if len(labelTriplesLIST) <= nn:
        #     passages = []
        #     for i, triple in enumerate(labelTriplesLIST, start=1):
        #         # for triple in labelTriplesLIST:
        #         TriplesString = (" ".join(str(element).capitalize() for element in triple))
        #         passages.append(TriplesString)
        #
        #     df_retrieved = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20,
        #                                       min_threshold=0.7)
        #
        #     if not df_retrieved.empty:
        #         # labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist()
        #         labelTriplesLIST_RAGGED = df_retrieved['Passage'].apply(lambda x: (x,)).tolist()
        #         labelTriplesAPP = ". ".join(
        #             " ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED)
        #
        #         if not labelTriples:
        #             labelTriples = labelTriplesAPP
        #         else:
        #             labelTriples = labelTriples + ". " + labelTriplesAPP
        #
        # else:
        #
        #     OverallListRAGtriples = labelTriplesLIST.copy()
        #
        #     while len(OverallListRAGtriples) > nn:
        #         Oinnerlistiterative = []
        #         for i, triple in enumerate(OverallListRAGtriples, start=1):
        #             # for triple in labelTriplesLIST:
        #             TriplesString = (" ".join(str(element).capitalize() for element in triple))
        #             passages.append(TriplesString)
        #             # Check if the current index is a multiple of nn
        #             if i % nn == 0:
        #                 # print("elaborate RAG triples")
        #
        #                 # df_retrieved_Base = RAG_retrieval_Base(questionText, passages, min_threshold=0.7, max_num_passages=20)
        #                 # df_retrievedZscore = RAG_retrieval_Z_scores(questionText, passages, z_threshold=1.0,  max_num_passages=20, min_threshold=0.7)
        #                 # df_retrievedPercentile = RAG_retrieval_Percentile(questionText, passages, percentile=90,  max_num_passages=20, min_threshold=0.7)
        #                 df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1,
        #                                                       max_num_passages=20,
        #                                                       min_threshold=0.7)
        #
        #                 passages = []
        #
        #                 df_retrieved = df_retrievedtopk.copy()
        #                 if not df_retrieved.empty:
        #                     # labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist()
        #                     labelTriplesLIST_RAGGED = df_retrieved['Passage'].apply(lambda x: (x,)).tolist()
        #                     if not Oinnerlistiterative:
        #                         Oinnerlistiterative = labelTriplesLIST_RAGGED
        #                     else:
        #                         Oinnerlistiterative.extend(labelTriplesLIST_RAGGED)
        #
        #         if passages:
        #             df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20,
        #                                                   min_threshold=0.7)
        #
        #             df_retrieved = df_retrievedtopk.copy()
        #             if not df_retrieved.empty:
        #                 # labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist()
        #                 labelTriplesLIST_RAGGED = df_retrieved['Passage'].apply(lambda x: (x,)).tolist()
        #                 if not Oinnerlistiterative:
        #                     Oinnerlistiterative = labelTriplesLIST_RAGGED
        #                 else:
        #                     Oinnerlistiterative.extend(labelTriplesLIST_RAGGED)
        #
        #         OverallListRAGtriples = Oinnerlistiterative.copy()
        #
        #     if OverallListRAGtriples:
        #         labelTriplesAPP = ". ".join(
        #             " ".join(str(element).capitalize() for element in triple) for triple in OverallListRAGtriples)
        #
        #         if not labelTriples:
        #             labelTriples = labelTriplesAPP
        #         else:
        #             labelTriples = labelTriples + ". " + labelTriplesAPP
        #
        #         labelTriples = labelTriples.strip().replace("..", ".").strip()
        #
        # # labelTriples = ""
        # # passages = []
        # # nn=200
        # # for i, triple in enumerate(labelTriplesLIST, start=1):
        # # #for triple in labelTriplesLIST:
        # #     TriplesString = (" ".join(str(element).capitalize() for element in triple))
        # #     passages.append(TriplesString)
        # #     # Check if the current index is a multiple of nn
        # #     if i % nn == 0:
        # #         #print("elaborate RAG triples")
        # #
        # #         #df_retrieved_Base = RAG_retrieval_Base(questionText, passages, min_threshold=0.7, max_num_passages=20)
        # #         #df_retrievedZscore = RAG_retrieval_Z_scores(questionText, passages, z_threshold=1.0,  max_num_passages=20, min_threshold=0.7)
        # #         #df_retrievedPercentile = RAG_retrieval_Percentile(questionText, passages, percentile=90,  max_num_passages=20, min_threshold=0.7)
        # #         df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20, min_threshold=0.7)
        # #
        # #         passages = []
        # #
        # #         df_retrieved = df_retrievedtopk.copy()
        # #         if not df_retrieved.empty:
        # #             #labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist()
        # #             labelTriplesLIST_RAGGED = df_retrieved['Passage'].apply(lambda x: (x,)).tolist()
        # #             labelTriplesAPP = ". ".join(" ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED)
        # #
        # #             if not labelTriples:
        # #                 labelTriples =labelTriplesAPP
        # #             else:
        # #                 labelTriples = labelTriples + ". " + labelTriplesAPP
        # #
        # # if passages:
        # #     df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20, min_threshold=0.7)
        # #
        # #     df_retrieved = df_retrievedtopk.copy()
        # #     if not df_retrieved.empty:
        # #         #labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist()
        # #           labelTriplesLIST_RAGGED = df_retrieved['Passage'].apply(lambda x: (x,)).tolist()
        # #         labelTriplesAPP = ". ".join(" ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED)
        # #         if not labelTriples:
        # #             labelTriples = labelTriplesAPP
        # #         else:
        # #             labelTriples = labelTriples + ". " + labelTriplesAPP
        # #
        # # if labelTriples:
        # #     labelTriples.strip().replace("..",".").strip()
        #
        # if strtobool(args.debug):
        #     numfinal = 0
        #     if OverallListRAGtriples:
        #         numfinal = len(OverallListRAGtriples)
        #     elif labelTriplesLIST_RAGGED:
        #         numfinal = len(labelTriplesLIST_RAGGED)
        #     print("End reranking - found final passages : ", numfinal, "\n")
        #     #
        #     endRerank = time.time()
        #     hours, rem = divmod(endRerank - startRerank, 3600)
        #     minutes, seconds = divmod(rem, 60)
        #     print("Rerank Time... {:0>2}:{:0>2}:{:05.2f}\n".format(int(hours), int(minutes), seconds))
        #     #

        # if len(labelTriplesLIST) > 10000:
        #      print("debug")

        if strtobool(args.debug):
            print("Start reranking2 - num passages : ", len(labelTriplesLIST), "\n")
            startRerank2 = time.time()

        labelTriples = ""

        try:

            passages = []
            for i, triple in enumerate(labelTriplesLIST, start=1):
                # for triple in labelTriplesLIST:
                TriplesString = (" ".join(str(element).capitalize() for element in triple))
                passages.append(TriplesString)

            nback = 1
            if len(passages) <= 10:
                nback = len(passages)
            elif len(passages) <= 1000:
                nback = 10+int(0.1 * len(passages))  # 10% of the number of passages
            elif len(passages) <= 5000:
                nback = 200
            elif len(passages) <= 10000:
                nback = 300
            else:
                nback = 400

            df_retrieved = RAG_retrieval_Base(questionText, passages, min_threshold=0, max_num_passages=nback)

            if not df_retrieved.empty:

                countRetr = 0
                min_threshold = 0.80
                countRetr = (df_retrieved['score'] > min_threshold).sum()

                countRetrThreshold = int(nback / 2)
                if nback > 10:
                    countRetrThreshold = 10
                else:
                    countRetrThreshold = int(nback/2)
                if countRetrThreshold <=0:
                    countRetrThreshold = 1

                while countRetr <= countRetrThreshold:
                    min_threshold = min_threshold - 0.05
                    countRetr = (df_retrieved['score'] >= min_threshold).sum()
                    if min_threshold < 0.2:
                        break

                # countRetrThreshold = int(0.1 + nback)
                # if countRetrThreshold > 5:
                #     countRetrThreshold = 5
                #
                # countRetr=0
                # min_threshold = 0.90
                # countRetr = (df_retrieved['score'] > min_threshold).sum()
                # while countRetr<=countRetrThreshold:
                #     min_threshold = min_threshold - 0.05
                #     if min_threshold<0.7:
                #         countRetrThreshold=0
                #     if min_threshold == 0:
                #         min_threshold = 0.01
                #     countRetr = (df_retrieved['score'] > min_threshold).sum()
                #     if min_threshold <= 0.01:
                #         break

                if countRetr > 0:
                    df_retrieved = df_retrieved[df_retrieved['score'] > min_threshold]

                    # labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist()
                    labelTriplesLIST_RAGGED = df_retrieved['Passage'].apply(lambda x: (x,)).tolist()
                    labelTriplesAPP = ". ".join(
                        " ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED)

                    if not labelTriples:
                        labelTriples = labelTriplesAPP
                    else:
                        labelTriples = labelTriples + ". " + labelTriplesAPP

                else:
                    labelTriplesLIST_RAGGED = []
                    labelTriples = ""


            if strtobool(args.debug):
                numfinal = 0
                if labelTriplesLIST_RAGGED:
                    numfinal = len(labelTriplesLIST_RAGGED)
                print("End reranking2 - found final passages : ", numfinal, "\n")
                endRerank2 = time.time()
                hours, rem = divmod(endRerank2 - startRerank2, 3600)
                minutes, seconds = divmod(rem, 60)
                print("Rerank2 Time... {:0>2}:{:0>2}:{:05.2f}\n".format(int(hours), int(minutes), seconds))
                #

        except Exception as err:
            print("SOMETHING HAPPENED on PASSAGE RERANKING for Question :"+questionText+"\n")
            print(err)
            #status_code: 422, body: type='validation_error' url='https://www.mixedbread.ai/api-reference' message='Your request is invalid. Please check your input and try again.' details=[[{'type': 'too_long', 'loc': ['body', 'input', 'list[str]'], 'msg': 'List should have at most 1000 items after validation, not 4249',


    else:  # NO RAG on triples
        labelTriples = ". ".join(" ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST)


    if not(labelTriples) or labelTriples.strip=="":
        logging.warning("getLinearTextualContextFromTriples - No text or prompt supplied! No relevant contextual triples retrieved...Skypping it! Word: "+str(word))
        return "", map_query_input_output

    if token_counter(labelTriples, args.model_name) > args.tokens_max:  # THE CONTEXT IS TOO BIG, BIGGER THAN tokens_max, I need to split
        texts = text_splitter.create_documents([labelTriples])
        labelTriples = texts[0].page_content
        if not (labelTriples) or labelTriples.strip == "":
            logging.warning("after splitting ...No text or prompt supplied! Skypping it! Word: "+str(word))
            return "", map_query_input_output


    contextText = ""

    if (strtobool(args.UseRetrieverForContextCreation) == True):

        contextText = labelTriples

    else:  #USE the LLM for summarise the triples

        # Can you elaborate and express better the following notes, delimited by triple backticks, about "{word}"?
        # Don't add explanations for your answer. Do not invent. Don't use a structure or indenting. Be concise. Don't discard relevant information.
        # made of RDF-like statements,

        # myPromt = f"""
        # Can you elaborate and express better the given notes below, delimited by triple backticks, about "{word}"?
        # Don't add explanations for your answer.
        # Do not invent.
        # Don't use a structure or indenting.
        # Be concise but exhaustive. Don't discard information reported in the notes.
        # """
        myPromt = f"""
        Can you reformulate the following notes, provided between triple backticks, into clear and complete sentences about "{word}"? 
        Ensure the rewriting is human-readable and easily interpretable. Maintain conciseness and exhaustiveness, including all information from the notes. 
        Avoid using note formats or lists, and refrain from inventing additional information.
        """
        myDelimiter = "```"

        if cleanInput==True:
            labelTriples = cleanInputText(labelTriples)

        # try to read cache

        if map_query_input_output is not None:
            key = args.model_name + "__" + str(args.temperature) + "__" + myPromt

            if key in map_query_input_output:
                if labelTriples in map_query_input_output[key]:
                    output = map_query_input_output[key][labelTriples]
                    # if input_text.strip() == "":
                    #     print("here")

                    # if handler == api_call_dglc:
                    #     output = clean_gpt_out(output) #clean output

                    if strtobool(args.debug):
                        print("RETRIEVED CACHED RESULT FOR:\n", myPromt, "\n", myDelimiter, word, myDelimiter, "\n=>\n", output, "\n")

                    return output, map_query_input_output

        #  call

        try:

            contextText = ""
            if args.service_provider == "gptjrc":
                contextText = call_model(input_text=labelTriples, prompt=myPromt, model=args.model_name,
                                  temperature=args.temperature, delimiter=myDelimiter,
                                  InContextExamples=[],
                                  handler=api_call_gptjrc,
                                  verbose=True, args=args)
            # elif args.service_provider == "HFonPremises":
            #     contextText = call_model(input_text=labelTriples, prompt=myPromt, model=args.model_name,
            #                              temperature=args.temperature, delimiter=myDelimiter,
            #                              InContextExamples=[],
            #                              handler=api_call_HFonPremises,
            #                              verbose=True, args=args)



            if contextText:
                if not isinstance(contextText, str):
                    contextText = contextText['choices'][0]['message']['content']

            if map_query_input_output is not None:
                if not key in map_query_input_output:
                    map_query_input_output[key] = {}

                if contextText:
                    if contextText != "":
                        map_query_input_output[key][labelTriples] = contextText


        except Exception as err:
            return None, map_query_input_output



    return contextText, map_query_input_output



def getUrlBioAndAllOtherBioConcepts(word, args, key_virtuoso, cache_map_virtuoso, endpoint, VirtuosoUsername, contextWordVirtuoso, UseBioportalForLinking=False, questionText="" ):
    #UseBioportalForLinking = False #trial to del

    if strtobool(args.debug):
        print("--- start getUrlBioAndAllOtherBioConcepts for " + word.lower())

    entityBioeUrl = None
    ALLURIScontext = []

    key_bioportal = ""
    if args.bioportalkey_filename and os.path.exists(args.bioportalkey_filename):
        fkeyname = args.bioportalkey_filename
        with open(fkeyname) as f:
            key_bioportal = f.read()
    else:
        key_bioportal = os.environ['key_bioportal']

    # Check if args.KG_restriction exists and is not empty
    if getattr(args, 'KG_restriction', None):

        # api call
        if strtobool(args.debug):
            print("--- " + word.lower())

        # args.KG_restriction exists and is not empty
        if strtobool(args.debug):
            print("KG_restriction is provided and not empty:", args.KG_restriction)

        from_clauses = ' '.join([f"FROM <{choice}>" for choice in args.KG_restriction])

        # Construct the full SPARQL query
        query = f"""
                                prefix skosxl: <http://www.w3.org/2008/05/skos-xl#> 
                                SELECT ?concept ?label (COUNT(?edge) AS ?score)
                                {from_clauses}
                                WHERE {{
                                  ?concept skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?label .
                                  FILTER (LCASE(STR(?label)) = "{word.lower()}")
                                  ?concept ?edge ?o .
                                }}
                                GROUP BY ?concept ?label
                                ORDER BY DESC(?score)
                            """

        ### this is for Bioportal url api:
        onto_clauses = ""
        # for choice in args.KG_restriction:
        #     if choice == "SNOMEDCT":
        #         choice = "SNOMED"
        #     elif choice == "OBOREL":
        #         choice = "RO"
        #     elif choice == "PTRANS":
        #         choice = "TRANS"
        #     elif choice == "FOODON":
        #         choice = "FoodOn"
        #     elif choice == "GEOSPARQL":
        #         choice = "GeoSPARQL"
        #     elif choice == "NCBITAXON":
        #         choice = "NCBITAXON,NCBITaxon_"
        #     onto_clauses = onto_clauses + choice + ","
        for choice in args.KG_restriction:
            if choice == "SNOMED":
                choice="SNOMEDCT"
            elif choice == "RO":
                choice = "OBOREL"
            elif choice == "TRANS":
                choice = "PTRANS"
            elif choice == "FoodOn":
                choice = "FOODON"
            elif choice == "GeoSPARQL":
                choice = "GEOSPARQL"
            # elif choice == "NCBITAXON":
            #     choice = "NCBITAXON,NCBITaxon_"
            elif choice == "NCBITaxon_":
                choice = "NCBITAXON"
            if choice in ONLY_Ontologies_OnBIOPORTAL:
                onto_clauses=onto_clauses+choice+","

        if onto_clauses and onto_clauses[-1] == ",":
            onto_clauses = onto_clauses[:-1]

        url = f"https://services.data.bioontology.org/annotatorplus/?text={word.lower()}&ontologies={onto_clauses}&longest_only=true&exclude_numbers=true&whole_word_only=true&exclude_synonyms=false&negation=false&experiencer=false&temporality=false&score_threshold=0&confidence_threshold=0&display_links=false&display_context=false&score=cvalue&apikey={key_bioportal}"

    else:
        # args.KG_restriction does not exist or is empty

        kg_restriction = getattr(args, 'KG_restriction', None)
        if kg_restriction is not None and len(kg_restriction) == 0:
            print("KG_restriction is provided but empty")
            return None, None, cache_map_virtuoso

        if strtobool(args.debug):
            print("--- " + word.lower())
            print("KG_restriction is not provided or empty - Consider all the KGs in the virtuoso endpoint")

        query = f"""
                    prefix skosxl: <http://www.w3.org/2008/05/skos-xl#> 
                    SELECT ?concept ?label (COUNT(?edge) AS ?score)
                    WHERE {{
                      ?concept skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?label .
                      FILTER (LCASE(STR(?label)) = "{word.lower()}")
                      ?concept ?edge ?o .
                    }}
                    GROUP BY ?concept ?label
                    ORDER BY DESC(?score)
                """

        ### this is for Bioportal url api:
        url = f"https://services.data.bioontology.org/annotatorplus/?text={word.lower()}&ontologies=AEO,BFO,BIM,BCGO,CL,CHIRO,CHEBI,DCM,FMA,GO,GENO,GEOSPARQL,HL7,DOID,HP,HP_O,IDO,IAO,ICD10,LOINC,MESH,MONDO,NCIT,NCBITAXON,NIFCELL,NIFSTD,GML,OBCS,OCHV,OHPI,OPB,PTRANS,PLOSTHES,RADLEX,OBOREL,STY,SO,SNOMEDCT,STATO,SYMP,FOODON,UBERON,VO&longest_only=true&exclude_numbers=true&whole_word_only=true&exclude_synonyms=false&negation=false&experiencer=false&temporality=false&score_threshold=0&confidence_threshold=0&display_links=false&display_context=false&score=cvalue&apikey={key_bioportal}"

    try:

        if UseBioportalForLinking == False:

            if strtobool(args.debug):
                print("Use Virtuoso Sparql endpoint for linking ... " + word.lower())

            responseText = sparqlQuery(endpoint, questionText, VirtuosoUsername, key_virtuoso, strtobool(args.USE_CACHE))

            # Parse the response as JSON
            results = json.loads(responseText)

            if len(results) > 0 and results['results']['bindings']:

                entityBioeUrl = str(results['results']['bindings'][0]['concept']['value'])

                if cache_map_virtuoso is not None:
                    if not word in cache_map_virtuoso:
                        cache_map_virtuoso[word] = {}
                    cache_map_virtuoso[word][contextWordVirtuoso] = entityBioeUrl

                # # loop the results
                for result in results['results']['bindings']:
                    # print(result)

                    contextConcept = result['concept']['value']
                    if contextConcept not in ALLURIScontext:
                        ALLURIScontext.append(contextConcept)
                        if cache_map_virtuoso is not None:
                            if not word in cache_map_virtuoso:
                                cache_map_virtuoso[word] = {}
                            cache_map_virtuoso[word][contextConcept] = None

                if ALLURIScontext and isinstance(ALLURIScontext, list):
                    ALLURIScontext = list(set(ALLURIScontext))

                if cache_map_virtuoso is not None:
                    if not word in cache_map_virtuoso:
                        cache_map_virtuoso[word] = {}
                    cache_map_virtuoso[word]['ALLURIScontext'] = ALLURIScontext

            else:

                if cache_map_virtuoso is not None:
                    if not word in cache_map_virtuoso:
                        cache_map_virtuoso[word] = {}
                    cache_map_virtuoso[word][contextWordVirtuoso] = None
                    cache_map_virtuoso[word]['ALLURIScontext'] = []

        else: #this is instead using Bioportal API for linking

            if strtobool(args.debug):
                print("Use Bioportal for linking ... " + word.lower())

            response = requests.get(url)

            try:
                data = response.json()

                if not data:
                    # nothing found from Bioportal
                    #return None, None, None, None, None, cache_map_virtuoso
                    return None, None, cache_map_virtuoso

                dff = pd.DataFrame(data)
                dff = dff.drop(columns=['hierarchy', 'mappings'])

                # If the columns are dictionary-like, use pd.json_normalize:
                expanded_annotated_class = pd.json_normalize(dff['annotatedClass'])

                expanded_annotations = pd.DataFrame(dff['annotations'].tolist(), index=dff.index)
                expanded_annotations = pd.json_normalize(expanded_annotations[0])

                # Join the expanded columns back to the original DataFrame
                df_expanded = dff.drop(columns=['annotatedClass', 'annotations']).join(expanded_annotated_class).join(
                    expanded_annotations)

                # Snomed id replacement because in our internal knolwedgebase we have this base uri
                df_expanded['@id'] = df_expanded['@id'].str.replace(
                    "http://purl.bioontology.org/ontology/SNOMEDCT/",
                    "http://snomed.info/id/"
                )

                if not df_expanded.empty:

                    df_expanded = df_expanded.sort_values(
                        by=['from', 'text', 'score', 'matchType'], ascending=[True, True, False, False])

                    df_expanded = df_expanded.drop_duplicates(subset=['@id'])

                    # Filter rows where 'prefLabel' is exactly equal to 'word.lower()' or 'word.lower()' is in 'synonym'
                    # filtered_df = df_expanded[
                    #     df_expanded['prefLabel'].str.lower() == word.lower() |
                    #     df_expanded['synonym'].apply(
                    #         lambda x: True if isinstance(x, list) and len(x) > 0 and word.lower() in [item.lower() for
                    #                                                                                   item in x] else (
                    #             np.nan if x is None or (isinstance(x, float) and pd.isna(x)) else x))
                    #     ]

                    if 'prefLabel' in df_expanded.columns:
                        if 'synonym' in df_expanded.columns:
                            df_expanded = df_expanded[
                                df_expanded['prefLabel'].apply(
                                    lambda x: isinstance(x, str) and x.lower() == word.lower()
                                ) |
                                df_expanded['synonym'].apply(
                                    lambda x: isinstance(x, list) and any(item.lower() == word.lower() for item in x)
                                )
                                ]
                        else:
                            df_expanded = df_expanded[
                                df_expanded['prefLabel'].apply(
                                    lambda x: isinstance(x, str) and x.lower() == word.lower()
                                )
                            ]

                    if df_expanded.empty:
                        # nothing found from Bioportal
                        #return None, None, None, None, None, cache_map_virtuoso
                        return None, None, cache_map_virtuoso

                    # Specify the columns you want to keep
                    columns_to_keep = ["score", "from", "to", "prefLabel", "text", "@id"]

                    # Subset the dataframe to keep only the specified columns
                    df_expanded = df_expanded[columns_to_keep]

                    # Rename the specified columns
                    df_expanded = df_expanded.rename(
                        columns={"from": "start", "to": "end", "text": "word"})

                    # Optional: Reset index if you want a clean index after filtering
                    df_expanded = df_expanded.reset_index(drop=True)

                    df_expanded['score'] = df_expanded['score'].round(2)

                    # Find the index of the row with the maximum 'score'
                    max_score_index = df_expanded['score'].idxmax()

                    max_score_row = df_expanded.loc[df_expanded['score'].idxmax()]

                    entityBioeUrl = str(max_score_row['@id'])

                    if cache_map_virtuoso is not None:
                        if not word in cache_map_virtuoso:
                            cache_map_virtuoso[word] = {}
                        cache_map_virtuoso[word][contextWordVirtuoso] = entityBioeUrl

                    # Drop the row with the maximum 'score'
                    #df_expanded = df_expanded.drop(max_score_index)

                    # Reset the index if desired (optional)
                    df_expanded.reset_index(drop=True, inplace=True)

                    # Iterate over each row in the DataFrame
                    for index, row in df_expanded.iterrows():
                        # Append the '@id' value to the list
                        if row['@id'] is not None and pd.notna(row['@id']):
                            contextConcept=row['@id']
                            ALLURIScontext.append(contextConcept)
                            if cache_map_virtuoso is not None:
                                if not word in cache_map_virtuoso:
                                    cache_map_virtuoso[word] = {}
                                cache_map_virtuoso[word][contextConcept] = None

                    if ALLURIScontext and isinstance(ALLURIScontext, list):
                        ALLURIScontext = list(set(ALLURIScontext))

                    if cache_map_virtuoso is not None:
                        if not word in cache_map_virtuoso:
                            cache_map_virtuoso[word] = {}
                        cache_map_virtuoso[word]['ALLURIScontext'] = ALLURIScontext


                    return entityBioeUrl, ALLURIScontext, cache_map_virtuoso

                else:
                    #nothing found from Bioportal
                    #return None, None, None, None, None, cache_map_virtuoso
                    return None, None, cache_map_virtuoso


            except Exception as err:
                logging.error(
                    f'ERROR ON BioPortal Annotator API Call\n\tError: {err}\n TextToAnnotate: {word.lower()}\n Have a check...')
                #return None, None, None, None, None, cache_map_virtuoso
                return None, None, cache_map_virtuoso

    except Exception as err:

        # if cache_map_virtuoso is not None:
        #     if not word in cache_map_virtuoso:
        #         cache_map_virtuoso[word] = {}
        #     cache_map_virtuoso[word][contextWordVirtuoso] = None

        #return None, None, None, None, None, cache_map_virtuoso
        return None, None, cache_map_virtuoso


    return entityBioeUrl, ALLURIScontext, cache_map_virtuoso



#@mem.cache
def virtuoso_api_call(word, text_splitter, args, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, id=None, iALLURIScontextFromNCBO=None,UseBioportalForLinking=True,questionText=""):

    if strtobool(args.debug):
        print(f"\n----- Starting virtuoso_api_call for {word}")

    word = word.lower()
    word = strip_quotes(word)

    endpoint = 'https://api-vast.jrc.service.ec.europa.eu/sparql'
    VirtuosoUsername = 'dba'

    if getattr(args, 'KG_restriction', None):
        contextWordVirtuoso = ', '.join(sorted(args.KG_restriction))
    else:
        contextWordVirtuoso = ""

    singleContext = None
    globalContext = None
    sssingleTriples = None
    ggglobalTriples = None
    unique_listLabelTriples = []
    unique_listGlobalTriples  = []

    ALLURIScontext = []

    url_text = None
    if id:
        url_text = id

    if iALLURIScontextFromNCBO and isinstance(iALLURIScontextFromNCBO, list):
        ALLURIScontext=iALLURIScontextFromNCBO
        ALLURIScontext = list(set(ALLURIScontext))

    if (cache_map_virtuoso is not None) and (not url_text):
        if word in cache_map_virtuoso:
            if contextWordVirtuoso in cache_map_virtuoso[word]:
                url_text = cache_map_virtuoso[word][contextWordVirtuoso]
                if strtobool(args.debug):
                    print("RETRIEVED CACHED RESULT FOR:\n", word, " => ", url_text, "\n")
                if not url_text:
                    return None, None, None, None, None, None, cache_map_virtuoso, load_map_query_input_output

    if url_text and not ALLURIScontext:
        if cache_map_virtuoso is not None:
            if word in cache_map_virtuoso:
                if 'ALLURIScontext' in cache_map_virtuoso[word]:
                    ALLURIScontext = cache_map_virtuoso[word]['ALLURIScontext']

    entityBioeUrl = None
    if url_text and ALLURIScontext:
        entityBioeUrl = url_text

    else:

        try:
            entityBioeUrl, ALLURIScontext, cache_map_virtuoso =  getUrlBioAndAllOtherBioConcepts(word, args, key_virtuoso, cache_map_virtuoso, endpoint, VirtuosoUsername, contextWordVirtuoso, UseBioportalForLinking=UseBioportalForLinking, questionText=questionText )
            if ALLURIScontext and isinstance(ALLURIScontext, list):
                ALLURIScontext = list(set(ALLURIScontext))
        except Exception as err:

            # if cache_map_virtuoso is not None:
            #     if not word in cache_map_virtuoso:
            #         cache_map_virtuoso[word] = {}
            #     cache_map_virtuoso[word][contextWordVirtuoso] = None

            return None, None, None, None, None, None, cache_map_virtuoso, load_map_query_input_output



    if entityBioeUrl:

        if strtobool(args.computeEntityContext) and (strtobool(args.computeEntityGlobalContext)==False):

            if strtobool(args.debug):
                print("START computeEntityContext")

            unique_listLabelTriples = []
            singleContext = None

            if cache_map_virtuoso is not None:
                if entityBioeUrl in cache_map_virtuoso:
                    if "LabelTriples" in cache_map_virtuoso[entityBioeUrl]:
                        unique_listLabelTriples = cache_map_virtuoso[entityBioeUrl]["LabelTriples"]
                        if strtobool(args.debug):
                            print("RETRIEVED CACHED RESULT FOR:\n", entityBioeUrl, " => ", "LabelTriples", "\n")
                    if ("SingleContext" in cache_map_virtuoso[entityBioeUrl]) and (strtobool(args.UseRetrieverForContextCreation)==False):
                        singleContext = cache_map_virtuoso[entityBioeUrl]["SingleContext"]
                        if strtobool(args.debug):
                            print("RETRIEVED CACHED RESULT FOR:\n", entityBioeUrl, " => ", "SingleContext", "\n")


            if not singleContext:
                if unique_listLabelTriples:
                    singleContext, load_map_query_input_output = getLinearTextualContextFromTriples(word, unique_listLabelTriples,
                                                                                                    text_splitter, args,
                                                                                                    load_map_query_input_output,cleanInput=True,questionText=questionText)
                else:

                    query = f"""
                    prefix skosxl: <http://www.w3.org/2008/05/skos-xl#> 
                    SELECT DISTINCT ?labelS ?labelP ?labelO
                    WHERE {{
                      {{
                        <{entityBioeUrl}> ?p ?o.
                        <{entityBioeUrl}> skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelS .
                        ?p skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelP .
                        ?o skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelO .
                      }}
                      UNION
                      {{
                        SELECT ?labelS ?labelP ?labelO
                        WHERE {{
                          <{entityBioeUrl}> ?p ?labelO .
                          <{entityBioeUrl}> skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelS .
                            ?p skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelP .
                            FILTER (isLiteral(?labelO))
                        }}
                      }}
                      UNION
                      {{
                        SELECT DISTINCT ?labelS ?labelP  ?labelO
                        WHERE {{   
                            <{entityBioeUrl}> ?ppp ?ooo .
                            ?ooo rdf:type owl:Restriction .
                            ?ooo owl:onProperty ?p .
                            ?ooo owl:someValuesFrom ?o .
                        <{entityBioeUrl}> skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelS . 
                         ?p skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelP . 
                         ?o skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelO . 
                        }}
                      }}
                    }}
                    """

                    try:
                        responseText = sparqlQuery(endpoint, query, VirtuosoUsername, key_virtuoso, strtobool(args.USE_CACHE))

                        # Parse the response as JSON
                        results = json.loads(responseText)

                        if len(results) > 0 and results['results']['bindings']:

                            # word = "subClassOf"
                            # split_word = split_camel_case(word)

                            # # loop the results
                            labelTriples=""
                            listLabelTriples = []
                            pattern = r'\^\^<http:.*?>'
                            for result in results['results']['bindings']:
                                 #print(result)

                                 ss = str(result['labelS']['value']).strip().replace("..",".").replace("@en","")
                                 ss = re.sub(pattern, '', ss)
                                 pp = split_camel_case(str(result['labelP']['value'])).replace("_"," ").strip().replace("..",".").replace("@en","")
                                 pp = re.sub(pattern, '', pp)
                                 oo = str(result['labelO']['value']).replace("_"," ").strip().replace("..",".").replace("@en","")
                                 oo = re.sub(pattern, '', oo)
                                 listLabelTriples.append([ss, pp, oo])

                            # Remove duplicates while preserving order
                            unique_listLabelTriples = list(dict.fromkeys(tuple(triple) for triple in listLabelTriples))

                            # # If you need the result back in list of lists format
                            # unique_listLabelTriples = [list(triple) for triple in unique_listLabelTriples]

                            if unique_listLabelTriples:
                                if cache_map_virtuoso is not None:
                                    if not entityBioeUrl in cache_map_virtuoso:
                                        cache_map_virtuoso[entityBioeUrl] = {}
                                    cache_map_virtuoso[entityBioeUrl]["LabelTriples"] = unique_listLabelTriples

                                singleContext, load_map_query_input_output = getLinearTextualContextFromTriples(word, unique_listLabelTriples, text_splitter, args, load_map_query_input_output,cleanInput=True,questionText=questionText)


                    except Exception as err:
                        singleContext = None

                if singleContext and (strtobool(args.UseRetrieverForContextCreation)==False):
                    if cache_map_virtuoso is not None:
                        if not entityBioeUrl in cache_map_virtuoso:
                            cache_map_virtuoso[entityBioeUrl] = {}
                        cache_map_virtuoso[entityBioeUrl]["SingleContext"] = singleContext


        if strtobool(args.computeEntityGlobalContext):

            if strtobool(args.debug):
                print("START computeEntityGlobalContext")

            unique_listGlobalTriples = []
            globalContext = None

            if cache_map_virtuoso is not None:
                if word in cache_map_virtuoso:
                    if ("GlobalTriples"+" "+contextWordVirtuoso).strip() in cache_map_virtuoso[word]:
                        unique_listGlobalTriples = cache_map_virtuoso[word][("GlobalTriples"+" "+contextWordVirtuoso).strip()]
                        if strtobool(args.debug):
                            print("RETRIEVED CACHED RESULT FOR:\n", word, " => ", ("GlobalTriples"+" "+contextWordVirtuoso).strip(), "\n")
                    if (("GlobalContext"+" "+contextWordVirtuoso).strip() in cache_map_virtuoso[word]) and (strtobool(args.UseRetrieverForContextCreation)==False):
                        globalContext = cache_map_virtuoso[word][("GlobalContext"+" "+contextWordVirtuoso).strip()]
                        if strtobool(args.debug):
                            print("RETRIEVED CACHED RESULT FOR:\n", word, " => ", ("GlobalContext"+" "+contextWordVirtuoso).strip(), "\n")


            if not globalContext:

                BreakenBeforeAll = False
                if unique_listGlobalTriples:
                    globalContext, load_map_query_input_output = getLinearTextualContextFromTriples(word, unique_listGlobalTriples,
                                                                                                    text_splitter, args,
                                                                                                    load_map_query_input_output,cleanInput=True,questionText=questionText)
                else:

                    if not ALLURIScontext:
                        if cache_map_virtuoso is not None:
                            if word in cache_map_virtuoso:
                                ALLURIScontext = list(cache_map_virtuoso[word].keys())
                                ALLURIScontext = [element for element in ALLURIScontext if element and ("GlobalTriples" in element == False) and ("GlobalContext" in element == False) and "http" in element ]

                    if not ALLURIScontext:

                        # THIS CASE SHOULD BE VERY DIFFICULT TO HAPPEN...IT WILL HAPPEN IN CASE IT COMES FROM AN INITIAL BIOPORTAL ANNOTATION; WHICH ALREADY RECOGNISED THE FIRST URL

                        # # Print the error message to stderr
                        # print("THIS CASE SHOULD NEVER HAPPEN!!!! Check what's happening...exiting now...")
                        # # Exit the program with a non-zero status code (commonly used to indicate an error)
                        # sys.exit(1)

                        try:
                            entityBioeUrl, ALLURIScontext, cache_map_virtuoso = getUrlBioAndAllOtherBioConcepts(word,
                                                                                                                args,
                                                                                                                key_virtuoso,
                                                                                                                cache_map_virtuoso,
                                                                                                                endpoint,
                                                                                                                VirtuosoUsername,
                                                                                                                contextWordVirtuoso,
                                                                                                                UseBioportalForLinking=UseBioportalForLinking,
                                                                                                                questionText=questionText)
                            if ALLURIScontext and isinstance(ALLURIScontext, list):
                                ALLURIScontext = list(set(ALLURIScontext))

                        except Exception as err:

                            # if cache_map_virtuoso is not None:
                            #     if not word in cache_map_virtuoso:
                            #         cache_map_virtuoso[word] = {}
                            #     cache_map_virtuoso[word][contextWordVirtuoso] = None

                            return None, None, None, None, None, None, cache_map_virtuoso, load_map_query_input_output



                    if not ALLURIScontext:
                        # Print the error message to stderr
                        print("THIS CASE SHOULD NEVER HAPPEN NOW!!!! Check what's happening...exiting now...")
                        # Exit the program with a non-zero status code (commonly used to indicate an error)
                        sys.exit(1)

                    else:

                        for xxUrl in ALLURIScontext:

                            unique_listLabelTriples = []
                            # singleContext = None

                            if cache_map_virtuoso is not None:
                                if xxUrl in cache_map_virtuoso:
                                    if "LabelTriples" in cache_map_virtuoso[xxUrl]:
                                        unique_listLabelTriples = cache_map_virtuoso[xxUrl]["LabelTriples"]
                                        if strtobool(args.debug):
                                            print("RETRIEVED CACHED RESULT FOR:\n", xxUrl, " => ",
                                                  "LabelTriples", "\n")
                                    # if "SingleContext" in cache_map_virtuoso[xxUrl] and (strtobool(args.UseRetrieverForContextCreation)==False):
                                    #     singleContext = cache_map_virtuoso[xxUrl]["SingleContext"]
                                    #     if strtobool(args.debug):
                                    #         print("RETRIEVED CACHED RESULT FOR:\n", xxUrl, " => ",
                                    #               "SingleContext", "\n")

                            # if not singleContext:
                            #     if unique_listLabelTriples:
                            #         singleContext, load_map_query_input_output = getLinearTextualContextFromTriples(
                            #             word, unique_listLabelTriples,
                            #             text_splitter, args,
                            #             load_map_query_input_output, cleanInput=True, questionText=questionText)
                            #     else:

                            if not unique_listLabelTriples:

                                query = f"""
                                    prefix skosxl: <http://www.w3.org/2008/05/skos-xl#> 
                                    SELECT DISTINCT ?labelS ?labelP ?labelO
                                    WHERE {{
                                      {{
                                        <{xxUrl}> ?p ?o.
                                        <{xxUrl}> skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelS .
                                        ?p skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelP .
                                        ?o skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelO .
                                      }}
                                      UNION
                                      {{
                                        SELECT ?labelS ?labelP ?labelO
                                        WHERE {{
                                          <{xxUrl}> ?p ?labelO .
                                          <{xxUrl}> skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelS .
                                            ?p skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelP .
                                            FILTER (isLiteral(?labelO))
                                        }}
                                      }}
                                      UNION
                                      {{
                                        SELECT DISTINCT ?labelS ?labelP  ?labelO
                                        WHERE {{   
                                            <{xxUrl}> ?ppp ?ooo .
                                            ?ooo rdf:type owl:Restriction .
                                            ?ooo owl:onProperty ?p .
                                            ?ooo owl:someValuesFrom ?o .
                                        <{xxUrl}> skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelS . 
                                         ?p skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelP . 
                                         ?o skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelO . 
                                        }}
                                      }}
                                    }}
                                    """

                                try:
                                    responseText = sparqlQuery(endpoint, query, VirtuosoUsername, key_virtuoso, strtobool(args.USE_CACHE))

                                    # Parse the response as JSON
                                    results = json.loads(responseText)

                                    if len(results) > 0 and results['results']['bindings']:

                                        # word = "subClassOf"
                                        # split_word = split_camel_case(word)

                                        # # loop the results
                                        labelTriples = ""
                                        listLabelTriples = []
                                        pattern = r'\^\^<http:.*?>'
                                        for result in results['results']['bindings']:
                                            # print(result)

                                            ss = str(result['labelS']['value']).strip().replace("..", ".").replace("@en","")
                                            ss = re.sub(pattern, '', ss)
                                            pp = split_camel_case(str(result['labelP']['value'])).replace("_"," ").strip().replace("..", ".").replace("@en","")
                                            pp = re.sub(pattern, '', pp)
                                            oo = str(result['labelO']['value']).replace("_"," ").strip().replace("..", ".").replace("@en","")
                                            oo = re.sub(pattern, '', oo)
                                            listLabelTriples.append([ss, pp, oo])

                                        # Remove duplicates while preserving order
                                        unique_listLabelTriples = list(
                                            dict.fromkeys(tuple(triple) for triple in listLabelTriples))

                                        # # If you need the result back in list of lists format
                                        # unique_listLabelTriples = [list(triple) for triple in unique_listLabelTriples]

                                        if unique_listLabelTriples:
                                            if cache_map_virtuoso is not None:
                                                if not xxUrl in cache_map_virtuoso:
                                                    cache_map_virtuoso[xxUrl] = {}
                                                cache_map_virtuoso[xxUrl][
                                                    "LabelTriples"] = unique_listLabelTriples

                                            # singleContext, load_map_query_input_output = getLinearTextualContextFromTriples(
                                            #     word, unique_listLabelTriples, text_splitter, args, load_map_query_input_output, cleanInput=True, questionText=questionText)
                                            #
                                            # if singleContext and (strtobool(args.UseRetrieverForContextCreation)==False):
                                            #     if cache_map_virtuoso is not None:
                                            #         if not xxUrl in cache_map_virtuoso:
                                            #             cache_map_virtuoso[xxUrl] = {}
                                            #         cache_map_virtuoso[xxUrl][
                                            #             "SingleContext"] = singleContext

                                except Exception as err:
                                    unique_listLabelTriples = []


                            if unique_listLabelTriples:
                                unique_listGlobalTriples.extend(unique_listLabelTriples)


                                # This is to speed up, so I break here the global, but in this case I will not store the triples for the other uris in the cache, which maybe useful in the future
                                # #if token_counter(str(unique_listGlobalTriples),args.model_name) > args.tokens_max:

                                if getattr(args, 'maxTriplesContextComputation', None):  #it means it exists
                                    if args.maxTriplesContextComputation > 0:
                                        if len(unique_listGlobalTriples) > args.maxTriplesContextComputation:
                                            unique_listGlobalTriples = unique_listGlobalTriples[:args.maxTriplesContextComputation]
                                            BreakenBeforeAll = True
                                            break # BREAK THE FOR LOOP IF THE GLOBAL CONTEXT IS ALREADY TOO BIG, BIGGER THAN tokens_max


                    if unique_listGlobalTriples:
                        # Remove duplicates while preserving order
                        unique_listGlobalTriples = list(
                            dict.fromkeys(tuple(triple) for triple in unique_listGlobalTriples))

                        if cache_map_virtuoso is not None:
                            if not word in cache_map_virtuoso:
                                cache_map_virtuoso[word] = {}
                            if BreakenBeforeAll == False:
                                cache_map_virtuoso[word][("GlobalTriples"+" "+contextWordVirtuoso).strip()] = unique_listGlobalTriples

                        globalContext, load_map_query_input_output = getLinearTextualContextFromTriples(word,
                                                                                                        unique_listGlobalTriples,
                                                                                                        text_splitter, args,
                                                                                                        load_map_query_input_output, cleanInput=True, questionText=questionText)

                if globalContext and (strtobool(args.UseRetrieverForContextCreation)==False):
                    if cache_map_virtuoso is not None:
                        if not word in cache_map_virtuoso:
                            cache_map_virtuoso[word] = {}
                        if BreakenBeforeAll == False:
                            cache_map_virtuoso[word][("GlobalContext"+" "+contextWordVirtuoso).strip()] = globalContext

    if unique_listLabelTriples:
        sssingleTriples = " ,., ".join(
            " ,,, ".join(str(element).capitalize() for element in triple) for triple in unique_listLabelTriples)
        while "\\n" in sssingleTriples:
            sssingleTriples = sssingleTriples.replace("\\n", " ")
            sssingleTriples = sssingleTriples.strip()
        while "\t" in sssingleTriples:
            sssingleTriples = sssingleTriples.replace("\t", " ")
            sssingleTriples = sssingleTriples.strip()

    if unique_listGlobalTriples:
        ggglobalTriples = " ,., ".join(
            " ,,, ".join(str(element).capitalize() for element in triple) for triple in unique_listGlobalTriples)
        while "\\n" in ggglobalTriples:
            ggglobalTriples = ggglobalTriples.replace("\\n", " ")
            ggglobalTriples = ggglobalTriples.strip()
        while "\t" in ggglobalTriples:
            ggglobalTriples = ggglobalTriples.replace("\t", " ")
            ggglobalTriples = ggglobalTriples.strip()

    return entityBioeUrl, ALLURIScontext, singleContext, globalContext, sssingleTriples, ggglobalTriples, cache_map_virtuoso, load_map_query_input_output



# def process_geo_row(row, key_geonames, cache_map_geonames):
#     if row['IsGeo'] == 1:
#         result, cache_map_geonames = geonames_api_call(row['word'], key_geonames, cache_map_geonames)
#         return result, cache_map_geonames
#     else:
#         return None, cache_map_geonames


def process_row4Linking(row, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output):

    result = ""  #None
    singleContext = ""  #None
    globalContext = ""  #None
    singleTriples = ""  #None
    globalTriples = ""  #None
    ALLURIScontext = []

    try:

        if row.empty:
            return result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames, cache_map_virtuoso, load_map_query_input_output, row.name

        InRagMode=False
        if hasattr(args, 'useBioKgRAG') and (strtobool(args.useBioKgRAG)==True):
            InRagMode = True

        if (InRagMode==False):
            if row['IsGeo'] == 1:

                if strtobool(args.debug):
                    print(f"\n----- IsGeo ... COMPUTING {row['word']} IN THE TEXT:")
                    print(row[args.source_column])

                result, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames = geonames_api_call(row['word'], args, key_geonames, cache_map_geonames)

            elif row['IsBio'] == 1:

                # Check if '@id' column exists in df_Extract
                iiid = None
                # Check if the '@id' exists in the Series
                if '@id' in row:
                    # Check if the value is not None or NaN
                    if row['@id'] is not None and not pd.isna(row['@id']):
                        # Assign the value to the variable iiid
                        iiid = row['@id']
                iiiALLURIScontextFromNCBO = None
                if 'ALLURIScontextFromNCBO' in row:
                    if row['ALLURIScontextFromNCBO'] is not None and isinstance(row['ALLURIScontextFromNCBO'], list): #and not pd.isna(row['ALLURIScontextFromNCBO']):
                        iiiALLURIScontextFromNCBO=row['ALLURIScontextFromNCBO']
                        iiiALLURIScontextFromNCBO = list(set(iiiALLURIScontextFromNCBO))

                if strtobool(args.debug):
                    print(f"\n----- isBio COMPUTING ... {row['word']} IN THE TEXT:")
                    print(row[args.source_column])
                result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_virtuoso, load_map_query_input_output = virtuoso_api_call(row['word'], text_splitter, args, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, id=iiid, iALLURIScontextFromNCBO=iiiALLURIScontextFromNCBO, UseBioportalForLinking=True, questionText=row[args.source_column])

            else:
                if row['model'] == "Forced":
                    # Check if '@id' column exists in df_Extract
                    iiid = None
                    # Check if the '@id' exists in the Series
                    if '@id' in row:
                        # Check if the value is not None or NaN
                        if row['@id'] is not None and not pd.isna(row['@id']):
                            # Assign the value to the variable iiid
                            iiid = row['@id']
                    iiiALLURIScontextFromNCBO = None
                    if 'ALLURIScontextFromNCBO' in row:
                        if row['ALLURIScontextFromNCBO'] is not None and isinstance(row['ALLURIScontextFromNCBO'],
                                                                                    list):  # and not pd.isna(row['ALLURIScontextFromNCBO']):
                            iiiALLURIScontextFromNCBO = row['ALLURIScontextFromNCBO']
                            iiiALLURIScontextFromNCBO = list(set(iiiALLURIScontextFromNCBO))

                    if strtobool(args.debug):
                        print(f"\n----- isForced COMPUTING ... {row['word']} IN THE TEXT:")
                        print(row[args.source_column])

                    result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_virtuoso, load_map_query_input_output = virtuoso_api_call(
                        row['word'], text_splitter, args, key_virtuoso, cache_map_virtuoso, load_map_query_input_output,
                        id=iiid, iALLURIScontextFromNCBO=iiiALLURIScontextFromNCBO,UseBioportalForLinking=True,questionText=row[args.source_column])

                    if not result:  #try annotation without bioportal
                        result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_virtuoso, load_map_query_input_output = virtuoso_api_call(
                            row['word'], text_splitter, args, key_virtuoso, cache_map_virtuoso, load_map_query_input_output,
                            id=iiid, iALLURIScontextFromNCBO=iiiALLURIScontextFromNCBO, UseBioportalForLinking=False,questionText=row[args.source_column])

                else:
                    # this case is only considered if it is not IsBio or IsGeo or RagMode, etcc..but there is already an annoptation for the NEL coming from NCBO
                    if '@id' in row:
                        # Check if the value is not None or NaN
                        if row['@id'] is not None and not pd.isna(row['@id']):
                            # Assign the value to the variable iiid
                            iiid = row['@id']

                            iiiALLURIScontextFromNCBO = None
                            if 'ALLURIScontextFromNCBO' in row:
                                if row['ALLURIScontextFromNCBO'] is not None and isinstance(
                                        row['ALLURIScontextFromNCBO'],
                                        list):  # and not pd.isna(row['ALLURIScontextFromNCBO']):
                                    iiiALLURIScontextFromNCBO = row['ALLURIScontextFromNCBO']
                                    iiiALLURIScontextFromNCBO = list(set(iiiALLURIScontextFromNCBO))

                            if strtobool(args.debug):
                                print(f"\n----- It is not IsBio or IsGeo, but it has id from NCBO ...forcing COMPUTING ... {row['word']} IN THE TEXT:")
                                print(row[args.source_column])

                            result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_virtuoso, load_map_query_input_output = virtuoso_api_call(
                                row['word'], text_splitter, args, key_virtuoso, cache_map_virtuoso,
                                load_map_query_input_output,
                                id=iiid, iALLURIScontextFromNCBO=iiiALLURIScontextFromNCBO, UseBioportalForLinking=True,
                                questionText=row[args.source_column])


        else:
            if (row['IsBio'] == 1) or ( (pd.isnull(row["IsBio"]) or row["IsBio"] == '' or row['IsBio'] == 0 or row["IsBio"] is None) and (row['entity_group'] == "MISC") ):

                if strtobool(args.debug):
                    print(f"\n----- InRagMode ...COMPUTING ... {row['word']} IN THE TEXT:")
                    print(row[args.source_column])

                    # Check if '@id' column exists in df_Extract
                    iiid = None
                    # Check if the '@id' exists in the Series
                    if '@id' in row:
                        # Check if the value is not None or NaN
                        if row['@id'] is not None and not pd.isna(row['@id']):
                            # Assign the value to the variable iiid
                            iiid = row['@id']
                    iiiALLURIScontextFromNCBO = None
                    if 'ALLURIScontextFromNCBO' in row:
                        if row['ALLURIScontextFromNCBO'] is not None and isinstance(row['ALLURIScontextFromNCBO'], list):
                            iiiALLURIScontextFromNCBO = row['ALLURIScontextFromNCBO']
                            iiiALLURIScontextFromNCBO = list(set(iiiALLURIScontextFromNCBO))

                result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_virtuoso, load_map_query_input_output = virtuoso_api_call(
                    row['word'], text_splitter, args, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, id=iiid, iALLURIScontextFromNCBO=iiiALLURIScontextFromNCBO,UseBioportalForLinking=True,questionText=row[args.source_column])

        return result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames, cache_map_virtuoso, load_map_query_input_output, row.name

    except Exception as e:
        #print(f"Error occurred: {e}")
        return result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames, cache_map_virtuoso, load_map_query_input_output, row.name


def parallel_process_Row4Linking(df, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output):
    results = []

    with ThreadPoolExecutor(max_workers=args.num_cores_eLinking) as executor:
        # Submit tasks to ThreadPoolExecutor
        futures = [executor.submit(process_row4Linking, row, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output)
                   for _, row in df.iterrows()]

        # Collect results
        for future in as_completed(futures):
            try:

                result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames_Inner, cache_map_virtuoso_Inner, load_map_query_input_output_Inner, drm_idx = future.result()

                df.at[drm_idx,'namedEntity'] = result
                df.at[drm_idx, 'ALLURIScontext'] = ALLURIScontext
                df.at[drm_idx,'Context'] = singleContext
                df.at[drm_idx,'ContextGlobal'] = globalContext
                df.at[drm_idx, 'Triples'] = singleTriples
                df.at[drm_idx, 'TriplesGlobal'] = globalTriples
                # Recursively update cache_map files with values from _Inner --> actually, this will never happen, because when you pass a mutable object like a dictionary to a function,
                # changes made to that object within the function are reflected outside the function as well. This is because the function receives a reference to the original object,
                # rather than a copy of it.
                # cache_map_geonames, cache_map_virtuoso, load_map_query_input_output are like global variables, which is very good for the parallelization!
                if (cache_map_geonames == cache_map_geonames_Inner)==False:
                    update_nested_dict(cache_map_geonames, cache_map_geonames_Inner)
                if (cache_map_virtuoso == cache_map_virtuoso_Inner) == False:
                    update_nested_dict(cache_map_virtuoso, cache_map_virtuoso_Inner)
                if (load_map_query_input_output == load_map_query_input_output_Inner) == False:
                    update_nested_dict(load_map_query_input_output, load_map_query_input_output_Inner)

            except Exception as e:
                print(f"Error occurred: {e}")

    return df, cache_map_geonames, cache_map_virtuoso, load_map_query_input_output


def elinking(df_annotated_combined, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, device):

    if "ALLURIScontext" not in df_annotated_combined.columns:
        df_annotated_combined["ALLURIScontext"] = None

    if args.num_cores_eLinking>1:
        # parallel elinking process
        #result
        df_annotated_combined, cache_map_geonames_AFTER, cache_map_virtuoso_AFTER, load_map_query_input_output_AFTER = parallel_process_Row4Linking(df_annotated_combined, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output)

        #if isinstance(result, list):
        #    result=pd.Series(result)  # I need this after the parallel processing

    else:
        # single processing
        result = df_annotated_combined.apply(lambda row: process_row4Linking(row, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output), axis=1)

        #
        try:
            df_annotated_combined['namedEntity'] = result.str[0]
            df_annotated_combined['ALLURIScontext'] = result.str[1]
            df_annotated_combined['Context'] = result.str[2]
            df_annotated_combined['ContextGlobal'] = result.str[3]
            df_annotated_combined['Triples'] = result.str[4]
            df_annotated_combined['TriplesGlobal'] = result.str[5]
            cache_map_geonames_AFTER = result.str[6].iloc[-1]
            cache_map_virtuoso_AFTER = result.str[7].iloc[-1]
            load_map_query_input_output_AFTER = result.str[8].iloc[-1]  #
        except Exception as e:
            # print(f"Error occurred: {e}")
            df_annotated_combined['namedEntity'] = ""
            df_annotated_combined['ALLURIScontext'] = ""
            df_annotated_combined['Context'] = ""
            df_annotated_combined['ContextGlobal'] = ""
            df_annotated_combined['Triples'] = ""
            df_annotated_combined['TriplesGlobal'] = ""
            cache_map_geonames_AFTER = cache_map_geonames
            cache_map_virtuoso_AFTER = cache_map_virtuoso
            load_map_query_input_output_AFTER = load_map_query_input_output


    def fill_alluriscontext(row):
        if not row['ALLURIScontext'] and pd.notnull(row['namedEntity']):
            return [row['namedEntity']]
        return row['ALLURIScontext']

    # Apply the function to the DataFrame
    df_annotated_combined['ALLURIScontext'] = df_annotated_combined.apply(fill_alluriscontext, axis=1)


    if args.num_cores_eLinking>1:
        # if parallel elinking process I need to sort again, because rows in late will arrive later
        df_annotated_combined = df_annotated_combined.sort_values(by=['SentenceRef', 'start', 'ToLink', 'word', 'score'],
                                                              ascending=[True, True, True, True, False])

    return df_annotated_combined, cache_map_geonames_AFTER, cache_map_virtuoso_AFTER, load_map_query_input_output_AFTER





if __name__ == '__main__':

    # df_rrr = pd.read_csv("ToAnnotateTrial2_OutputAnnotated-withTriples.csv", sep=',', header=0, dtype=str, encoding='utf-8')
    # print("try")

    # accelerator = Accelerator()
    # # accelerator = Accelerator(mixed_precision='fp16', device_placement=False)  #fp16 bf16
    # device = accelerator.device
    # print("Using accelerator device = " + str(device))

    start = time.time()

    encod = encoding_getter('microsoft/deberta-v3-large')
    text_splitter = TokenTextSplitter(
        # separators=separators,
        encoding_name=encod.name,
        chunk_size=80000,
        chunk_overlap=50,
        length_function=len,
        add_start_index=True,
    )

    #models_List = ["FacebookAI/xlm-roberta-large-finetuned-conll03-english", "Babelscape/wikineural-multilingual-ner",  "blaze999/Medical-NER", "urchade/gliner_large-v2.1", "urchade/gliner_large_bio-v0.1", "NCBO/BioPortal"  ]  #"urchade/gliner_large-v2.1",  "knowledgator/gliner-multitask-large-v0.5"
    #models_List = ["urchade/gliner_large_bio-v0.1"]
    #models_List = [ "Babelscape/wikineural-multilingual-ner", "urchade/gliner_large-v2.1", "NCBO/BioPortal"]
    models_List = ["NCBO/BioPortal"]

    # Babelscape/wikineural-multilingual-ner
    # https://huggingface.co/Babelscape/wikineural-multilingual-ner?
    # https://aclanthology.org/2021.findings-emnlp.215.pdf
    # PERSON → PER, ORG → ORG, GPE → LOC,
    # LOC → LOC, FAC → LOC, PRODUCT → MISC,
    # EVENT → MISC, WORK_OF_ART → MISC,
    # LAW → O, LANGUAGE → MISC, NORP → MISC,
    # DATA → O, TIME → O, PERCENT → O,
    # MONEY → O, QUANTITY → O, ORDINAL → O,
    # CARDINAL → O
    # --> "LOC" "MISC"

    # blaze999/Medical-NER
    # https://huggingface.co/blaze999/Medical-NER
    # 41 recognised entities: https://huggingface.co/blaze999/Medical-NER/blob/main/config.json
    #

    # FacebookAI/xlm-roberta-large-finetuned-conll03-english
    # https://huggingface.co/FacebookAI/xlm-roberta-large-finetuned-conll03-english

    # urchade/gliner_large-v2.1
    # https://huggingface.co/urchade/gliner_large-v2.1
    #
    # urchade/gliner_large_bio-v0.1
    # https://huggingface.co/urchade/gliner_large_bio-v0.1

    # model = GLiNER.from_pretrained("urchade/gliner_large-v2.1")
    # model = GLiNER.from_pretrained("urchade/gliner_large_bio-v0.1")
    #
    # text = """
    # The doctor diagnosed the patient with basal cell carcinoma, a common type of skin cancer. The Health Ministry has detected about 4000 suspected chikungunya cases nationwide this year [2008], Minister Datuk Liow Tiong Lai said Wednesday [17 Dec 2008].
    # He said the disease which was 1st detected in Johor had spread to Negeri Sembilan, Melaka, Perak, Selangor and the latest Kedah.  He said the disease was 1st detected to have spread to Malaysia from Africa in 1997.
    # """
    #
    # glinerlabels = ["location", "disease"]
    #
    # entities = model.predict_entities(text, glinerlabels)
    #
    # for entity in entities:
    #    print(entity["text"], "=>", entity["label"])

    df_annotated = pd.DataFrame()
    for model_id in models_List:

        parser = argparse.ArgumentParser()

        parser.add_argument("--model_id", type=str, default=model_id, help="model to use")
        parser.add_argument("--debug", type=str, default="True", help="set debug mode")

        # parser.add_argument("--input_dir", type=str, default="/eos/jeodpp/data/products/JRC-OpenData/DIGLIFE/ner2/")   #/eos/jeodpp/home/users/consose/PycharmProjects/ner-virtuoso/
        # parser.add_argument("--filename", type=str, default="OutputAnnotatedTexts-LLMs-ENSEMBLE.csv")  #ToAnnotateTrial.csv
        # parser.add_argument("--source_column", type=str, default="texts")
        # #
        parser.add_argument("--input_dir", type=str,
                            default="/eos/jeodpp/home/users/consose/PycharmProjects/ner-virtuoso/")  #
        parser.add_argument("--filename", type=str,
                            default="ToAnnotateTrial2.csv")  #ToAnnotateTrial.csv   #ToAnnotateTrial2.csv  #ToAnnotateTrialGliner.csv  #ToAnnotateTrial3.csv
        parser.add_argument("--source_column", type=str, default="ContextToAnnotate")  # ""

        parser.add_argument("--entities_filter_threshold", type=int, default=0.7)

        parser.add_argument("--SEED", type=int, default=41)
        parser.add_argument("--batch_size", type=int, default=4)  # 2 - 4 - 8 - 16
        parser.add_argument("--num_cores_Gliner", type=int, default=0, help="parallel processing for Gliner annotation")  # 0 means use the GPU for Gliner !

        parser.add_argument("--entity_linking", type=str, default="True", help="whether to make entities linking or not")
        parser.add_argument("--geonameskey_filename", type=str, default="GEONAMES-API.key", help="file location where it is stored the geonames api key")
        parser.add_argument("--virtuosokey_filename", type=str, default="VIRTUOSO-dba.key", help="file location where it is stored the virtuoso endpoint dba pwd")
        parser.add_argument("--bioportalkey_filename", type=str, default="NCBO-BioPortal.key", help="file location where it is stored the NCBO BioPortal api key")

        # Add the --KG_restriction argument
        # Since nargs='+' is used, this will accept one or more values from the command line
        KGchoices = POSSIBLE_KGchoices_List
        # KGchoices = ['SNOMED', 'LOINC', 'ICD10', 'NCIT']
        # KGchoices = ['SNOMED', 'LOINC', 'ICD10', 'MESH', 'NCIT']  # restricts the input to these values only
        if KGchoices:
            KGchoices.sort()
        parser.add_argument("--KG_restriction", nargs='+', choices=KGchoices, default=KGchoices,
                            help="List of ontologies to which restrict the entity linking task.")

        # consose 20250502:
        if Counter(KGchoices) == Counter(POSSIBLE_KGchoices_List):
            parser.add_argument("--USE_CACHE", type=str, default="True",  #True
                                help="whether to use cache for the NER and NEL tasks or not")
        else:
            # print("Lists do not have the same elements")
            parser.add_argument("--USE_CACHE", type=str, default="False",
                                help="whether to use cache for the NER and NEL tasks or not")

        parser.add_argument("--num_cores_eLinking", type=int, default=10, help="parallel processing for the entity linking process")

        parser.add_argument("--computeEntityContext", type=str, default="False", help="whether to extract a readable context from the extracted triples for the concept")
        parser.add_argument("--computeEntityGlobalContext", type=str, default="False", help="whether to extract a readable context from the extracted triples of all the entities extracted from the endpoint for the concept")
        parser.add_argument("--maxTriplesContextComputation", type=int, default=20000,
                            help="maximum number of triples to consider for global context computation")  # if 0 or None it is not considered
        parser.add_argument("--UseRetrieverForContextCreation", type=str, default="True",
                            help="whether to use a retriever for the creation of the context of the entities from the triples coming from the KGs")

        parser.add_argument("--service_provider", type=str, default="gptjrc", help="llm service provider")   # "gptjrc", "HFonPremises"
        parser.add_argument("--model_name", type=str, default="llama-3.1-70b-instruct-fp8", help="llm to use")     # llama-3.1-70b-instruct "llama-3.1-70b-instruct"  "llama-3.1-70b-instruct-fp8"
        parser.add_argument("--tokens_max", type=int, default=80000, help="max number of tokens to supply to the llm")

        parser.add_argument("--max_new_tokens", type=int, default=4096, help="max number of tokens for the output of the LLM in premises")

        parser.add_argument("--temperature", type=int, default=0.01)

        args = parser.parse_args()

        print("ARGS:")
        print(args)

        # %% n machine learning tasks, particularly when dealing with models that have stochasticity involved (like text generation), it's important to set seeds for random number generators to ensure reproducibility of results. In the case of using models from the transformers library, you need to set seeds for both Python's random module, NumPy, and PyTorch to ensure that the results are the same every time you run the code.
        # Before you create the pipeline and run the text generation, set the seeds like this:
        random.seed(args.SEED)
        np.random.seed(args.SEED)
        torch.manual_seed(args.SEED)
        torch.cuda.manual_seed_all(args.SEED)
        ###

        # cache_map_bioportal = None
        # if strtobool(args.USE_CACHE):
        #     cachebioportal_filename = "CACHE_bioportal.json"
        #     if os.path.exists(cachebioportal_filename):
        #         with open(cachebioportal_filename) as f:
        #             cache_map_bioportal = json.load(f)
        #     else:
        #         cache_map_bioportal = {}
        #
        # key_bioportal = ""
        # if args.bioportalkey_filename and os.path.exists(args.bioportalkey_filename):
        #     fkeyname = args.bioportalkey_filename
        #     with open(fkeyname) as f:
        #         key_bioportal = f.read()

        in_filename = args.input_dir + args.filename
        df_ToAnnotate = pd.read_csv(in_filename, sep=',', header=0, dtype=str, encoding='utf-8')

        #df_ToAnnotate = df_ToAnnotate.head()  #for debugging
        #df_ToAnnotate = df_ToAnnotate.head(1)  # for debugging

        if "ToLink" not in df_ToAnnotate.columns:
            df_ToAnnotate["ToLink"] = None

        if "SentenceRef" not in df_ToAnnotate.columns:
            df_ToAnnotate["SentenceRef"] = None
            df_ToAnnotate = df_ToAnnotate[['SentenceRef'] + [col for col in df_ToAnnotate.columns if
                                                             col != 'SentenceRef']]  # this moves it to the first position

        #df_ToAnnotate['SentenceRef'] = df_ToAnnotate[args.source_column].rank(method='dense', ascending=True).astype(int)

        # df_ToAnnotate['rank'] = df_ToAnnotate.index + 1
        # df_ToAnnotate['rank2'] = df_ToAnnotate['rank'].argsort().groupby(df_ToAnnotate[args.source_column]).transform('min').astype(int)
        # df_ToAnnotate['rank2'] = df_ToAnnotate['rank2'].rank(method='dense').astype(int)

        df_ToAnnotate['SentenceRef'] = df_ToAnnotate.index + 1
        df_ToAnnotate['SentenceRef'] = df_ToAnnotate['SentenceRef'].argsort().groupby(df_ToAnnotate[args.source_column]).transform('min').astype(int)
        df_ToAnnotate['SentenceRef'] = df_ToAnnotate['SentenceRef'].rank(method='dense').astype(int)

        # device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        # if strtobool(args.debug):
        #     print(f"Device: {device}...")
        #     if device.type == "cuda":
        #         print("GPU number:", torch.cuda.current_device())

        tsk = "token-classification"

        pipe = None
        tokenizerGliner = None
        modelGliner = None
        modelGlinerBio = None
        if (("/gliner" in args.model_id) == False) and (("NCBO" in args.model_id) == False):
            pipe = pipeline(
                tsk,
                model=args.model_id,
                aggregation_strategy="simple",
                device=device,
            )
        elif ("/gliner" in args.model_id):
            if not tokenizerGliner:
                tokenizerGliner = AutoTokenizer.from_pretrained('microsoft/deberta-v3-large')
            if "_bio-" in args.model_id:
                if args.num_cores_Gliner > 0:
                    modelGlinerBio = GLiNER.from_pretrained(args.model_id)  #"urchade/gliner_large_bio-v0.1")
                else:
                    modelGlinerBio = GLiNER.from_pretrained(args.model_id, map_location=device)
            else:
                if args.num_cores_Gliner > 0:
                    modelGliner = GLiNER.from_pretrained(args.model_id) #"knowledgator/gliner-multitask-large-v0.5" - "urchade/gliner_large-v2.1"
                else:
                    modelGliner = GLiNER.from_pretrained(args.model_id, map_location=device)

        new_annotations = annotate(df_ToAnnotate, args, pipe, tokenizerGliner, modelGliner, modelGlinerBio, device)

        if not new_annotations.empty:
            if df_annotated.empty:
                # If df_annotated is empty, just assign new_annotations to it
                df_annotated = new_annotations
            else:
                # If df_annotated is not empty, concatenate new_annotations to it
                df_annotated = pd.concat([df_annotated, new_annotations], ignore_index=True)

    if not df_annotated.empty:
        df_annotated_combined = entitiesFusion(df_annotated,args)
        if strtobool(args.debug):
            print("\nStart is_cross_inside function ...")
        df_annotated_combined = is_cross_inside(df_annotated_combined, args, 0.75)
    else:
        df_annotated_combined = df_annotated


    #### GPT@JRC API
    if args.service_provider == "gptjrc":
        key_gptjrc = ""
        fkeyname = "GPTJRC-APItoken.key"
        if os.path.exists(fkeyname):
            with open(fkeyname) as f:
                key_gptjrc = f.read()
        else:
            key_gptjrc = os.environ['key_gptjrc']
        if key_gptjrc and key_gptjrc != "":
            setup_gptjrc(key_gptjrc)
    #####

    cache_prefix_fp = "LLMQUERYNER"
    cache_nameLLMs = cache_prefix_fp + "___" + "__".join(
        [args.service_provider, args.model_name, str(args.temperature)]).replace(
        " ", "_") + ".json"

    load_map_query_input_output = None
    if strtobool(args.USE_CACHE):
        if os.path.exists(cache_nameLLMs):
            with open(cache_nameLLMs) as f:
                load_map_query_input_output = json.load(f)
        else:
            load_map_query_input_output = {}

    ### entity linking part:
    if strtobool(args.entity_linking):

        cache_map_geonames = None
        if strtobool(args.USE_CACHE):
            cacheGeonames_filename = "CACHE_geonames.json"
            if os.path.exists(cacheGeonames_filename):
                with open(cacheGeonames_filename) as f:
                    cache_map_geonames = json.load(f)
            else:
                cache_map_geonames = {}

        key_geonames = ""
        if args.geonameskey_filename and os.path.exists(args.geonameskey_filename):
            fkeyname = args.geonameskey_filename
            with open(fkeyname) as f:
                key_geonames = f.read()
        else:
            key_geonames = os.environ['key_geonames']

        cache_map_virtuoso = None
        if strtobool(args.USE_CACHE):
            cacheVirtuoso_filename = "CACHE_virtuoso.json"
            if os.path.exists(cacheVirtuoso_filename):
                with open(cacheVirtuoso_filename) as f:
                    cache_map_virtuoso = json.load(f)
            else:
                cache_map_virtuoso = {}

        key_virtuoso = ""
        if args.virtuosokey_filename and os.path.exists(args.virtuosokey_filename):
            fkeyname = args.virtuosokey_filename
            with open(fkeyname) as f:
                key_virtuoso = f.read()
        else:
            key_virtuoso = os.environ['key_virtuoso']


        # Here for the EXACT MATCHING "" - if the desired term has not been identified in the NER, add to the dataframe:

        # Define the condition to find missing SentenceRefs
        missing_sentence_refs = ~df_ToAnnotate['SentenceRef'].isin(df_annotated_combined['SentenceRef'])

        # Define the condition to check if ContextToAnnotate starts and ends with quotes
        quoted_context = df_ToAnnotate[args.source_column].str.startswith('"') & df_ToAnnotate[
            args.source_column].str.endswith('"')

        # Combine both conditions
        condition = missing_sentence_refs & quoted_context

        # Select rows from df_ToAnnotate that meet the condition
        rows_to_add = df_ToAnnotate[condition].copy()

        rows_to_add['model'] = "Forced"
        rows_to_add['entity_group'] = "MISC"
        rows_to_add['word'] = rows_to_add[args.source_column]
        rows_to_add['word'] = rows_to_add[args.source_column].apply(strip_quotes)
        rows_to_add['score'] = 1.0
        rows_to_add['start'] = int(1)
        rows_to_add['end'] = rows_to_add['word'].apply(len)+int(1)
        rows_to_add['IsGeo'] = None
        rows_to_add['IsBio'] = None
        rows_to_add['IsCrossInside'] = 0.0

        # Append these rows to df_annotated_combined
        df_annotated_combined = pd.concat([df_annotated_combined, rows_to_add], ignore_index=True)

        df_annotated_combined = df_annotated_combined.sort_values(
            by=['SentenceRef', 'start', 'ToLink', 'word', 'score'], ascending=[True, True, True, True, False])

        # Now df_annotated_combined contains the additional rows



        df_annotated_combined, cache_map_geonames_AFTER, cache_map_virtuoso_AFTER,  load_map_query_input_output_AFTER = elinking(df_annotated_combined, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, device)

        if strtobool(args.USE_CACHE):
            if cache_map_geonames_AFTER is not None:
                with open(cacheGeonames_filename, "w") as f:
                    json.dump(cache_map_geonames_AFTER, f)
                    
            if cache_map_virtuoso_AFTER is not None:
                with open(cacheVirtuoso_filename, "w") as f:
                    json.dump(cache_map_virtuoso_AFTER, f)

            if load_map_query_input_output_AFTER is not None:
                with open(cache_nameLLMs, "w") as f:
                    json.dump(load_map_query_input_output_AFTER, f)

        ### end entity linking part



    # # delete Triples and TriplesGlobal if not needed, too space
    all_emptyTriples = None
    all_emptyGlobalTriples = None
    if 'Triples' in df_annotated_combined.columns:
        all_emptyTriples = df_annotated_combined['Triples'].apply(lambda x: pd.isnull(x) or x == '' or x is None).all()
        #print("All values in 'Triples' are empty (NaN, None, or ''):", all_emptyTriples)
    if 'TriplesGlobal' in df_annotated_combined.columns:
        all_emptyGlobalTriples = df_annotated_combined['TriplesGlobal'].apply(lambda x: pd.isnull(x) or x == '' or x is None).all()
        #print("All values in 'TriplesGlobal' are empty (NaN, None, or ''):", all_emptyGlobalTriples)

    if '@id' in df_annotated_combined.columns:
        df_annotated_combined = df_annotated_combined.rename(columns={'@id': '@idfromNCBO'})
    #     df_annotated_combined.drop('@id', axis=1, inplace=True)
    #
    if 'ALLURIScontextFromNCBO' in df_annotated_combined.columns:
         df_annotated_combined.drop('ALLURIScontextFromNCBO', axis=1, inplace=True)

    if (all_emptyTriples == False) or (all_emptyGlobalTriples==False):
        output_texts_filename = args.input_dir + args.filename.replace(".csv", "_OutputAnnotated-withTriples.csv")
        df_annotated_combined.to_csv(output_texts_filename, sep=',', header=True, index=False, encoding='utf-8')
        df_annotated_combined.drop('Triples', axis=1, inplace=True)
        df_annotated_combined.drop('TriplesGlobal', axis=1, inplace=True)

    output_texts_filename = args.input_dir + args.filename.replace(".csv", "_OutputAnnotated.csv")
    df_annotated_combined.to_csv(output_texts_filename, sep=',', header=True, index=False, encoding='utf-8')

    print("\nEnd script")

    #
    end = time.time()
    hours, rem = divmod(end - start, 3600)
    minutes, seconds = divmod(rem, 60)
    print("Overall Computational Time... {:0>2}:{:0>2}:{:05.2f}\n".format(int(hours), int(minutes), seconds))
    #