appStore/target.py

# set path
import glob, os, sys; 
sys.path.append('../utils')

#import needed libraries
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import streamlit as st
from utils.target_classifier import load_targetClassifier, target_classification
import logging
logger = logging.getLogger(__name__)
from utils.config import get_classifier_params
from utils.preprocessing import paraLengthCheck
from io import BytesIO
import xlsxwriter
import plotly.express as px
from utils.target_classifier import label_dict
from appStore.rag import run_query
from math import exp
import re
import json

import nltk
from nltk.corpus import stopwords
nltk.download('stopwords')

import openai
openai_api_key = os.environ["OPEN_AI_KEY"]

# Declare all the necessary variables
classifier_identifier = 'target'
params  = get_classifier_params(classifier_identifier)

@st.cache_data
def to_excel(df,sectorlist):
    len_df = len(df)
    output = BytesIO()
    writer = pd.ExcelWriter(output, engine='xlsxwriter')
    df.to_excel(writer, index=False, sheet_name='Sheet1')
    workbook = writer.book
    worksheet = writer.sheets['Sheet1']
    worksheet.data_validation('S2:S{}'.format(len_df), 
                              {'validate': 'list', 
                               'source': ['No', 'Yes', 'Discard']})
    worksheet.data_validation('X2:X{}'.format(len_df), 
                              {'validate': 'list', 
                               'source': sectorlist + ['Blank']})
    worksheet.data_validation('T2:T{}'.format(len_df), 
                              {'validate': 'list', 
                               'source': sectorlist + ['Blank']})
    worksheet.data_validation('U2:U{}'.format(len_df), 
                              {'validate': 'list', 
                               'source': sectorlist + ['Blank']})                               
    worksheet.data_validation('V2:V{}'.format(len_df), 
                              {'validate': 'list', 
                               'source': sectorlist + ['Blank']})
    worksheet.data_validation('W2:U{}'.format(len_df), 
                              {'validate': 'list', 
                               'source': sectorlist + ['Blank']})                            
    writer.save()
    processed_data = output.getvalue()
    return processed_data

def app():
    
    ### Main app code ###
    with st.container():
        
        if 'key1' in st.session_state:
           
            # Load the existing dataset
            df = st.session_state.key1

            # Filter out all paragraphs that do not have a reference to groups 
            df = df[df['Vulnerability Label'].apply(lambda x: len(x) > 0 and 'Other' not in x)]

            # Load the classifier model
            classifier = load_targetClassifier(classifier_name=params['model_name'])
         
            st.session_state['{}_classifier'.format(classifier_identifier)] = classifier
                
            df = target_classification(haystack_doc=df,
                                        threshold= params['threshold'])

            # Rename column 
            df.rename(columns={'Target Label': 'Specific action/target/measure mentioned'}, inplace=True)


            st.session_state.key2 = df



vc_prompt="""
    You are assessing the accuracy of a multi-label classifier. The classifier seeks to assess the relevance of a given passage of context to any of 18 classes:

    'Agricultural communities',
    'Children',
    'Coastal communities',
    'Ethnic, racial or other minorities',
    'Fishery communities',
    'Informal sector workers',
    'Members of indigenous and local communities',
    'Migrants and displaced persons',
    'Older persons',
    'Other',
    'Persons living in poverty',
    'Persons with disabilities',
    'Persons with pre-existing health conditions',
    'Residents of drought-prone regions',
    'Rural populations',
    'Sexual minorities (LGBTQI+)',
    'Urban populations',
    'Women and other genders'

    If there is a semantic relevance or keyword(s) match between labels and context, then assess accuracy as a boolean True.
    Assessing class relevance may be tricky in some cases as the context can use technical language which is sometimes ambiguous. Please take your time to ensure a robust assessment.
    If you can't decide, err on the side of the classifier, and assume it is correct. 

    Use the examples below for reference:

    EXAMPLE 1

    LABEL: ['Agricultural communities', 'Residents of drought-prone regions']

    CONTEXT: "Future climatic predictions for Kenya indicate possible temperature increase of 1C by 2020 and 2.3C by 2050. These changes unless effectively mitigated, will likely result to erosion of the productive assets and the weakening of coping strategies and resilience of rain-fed farming systems, especially in the arid and semi-arid lands."
    
    RESPONSE: True

    EXAMPLE 2

    LABEL: ['Fishery communities']

    CONTEXT: "The reduced water availability resulting from frequent droughts also limits aquaculture development. Forests and agroforestry The farmed fisheries resources include the trout fish in cold water high altitude areas and tilapia, catfish, common carp for warmer water low altitude areas. Figure 5 shows the quantities and monetary value of fish produced in Kenya between 2005 and 2016."

    RESPONSE: True

    EXAMPLE 3

    LABEL: ['Persons with disabilities']

    CONTEXT: "In addressing climate change issues, public entities are required to undertake public awareness and consultations, and ensure gender mainstreaming, in line with the Constitution and the Climate Change Bill (2014).  5. Means of implementation Kenya's contribution will be implemented with both domestic and international support."

    RESPONSE: False

    EXAMPLE 4

    LABEL: ['Children', 'Women and other genders']

    CONTEXT: "Enhance quality control and food safety by relevant institutions along crop, livestock and fisheries value chains. Enhance use of low greenhouse gas emitting fish production technologies and practices. Promote integrated farming systems comprising crops, livestock, aquaculture and farm forestry. Create awareness and capacity build women, youth and venerable groups (WY&VG) on CSA."

    RESPONSE: True

    EXAMPLE 5

    LABEL: ['Ethnic, racial or other minorities']

    CONTEXT: "Harmonize livestock vaccinations across the bordering counties and across the international borders. Facilitate management of veterinary drug residues, carcasses and agrochemicals. Promote efficient use of farm mechanization. Promote mechanized and animal powered conservation tillage practices as compared to conventional tillage. Promote value addition of farm produce through cottage industries."

    RESPONSE: False

    EXAMPLE 6

    LABEL: ['Agricultural communities']

    CONTEXT: "There is also no traceability mechanism for produce and products from farm to folk. Value addition will ensure longer shelf life, reduced transaction costs and higher incomes. Summary of Actions: Identify and promote existing value addition technologies. Incentivize the private sector to invest in agricultural value addition."

    RESPONSE: False

    EXAMPLE 7

    LABEL: ['Agricultural communities', 'Rural populations']

    CONTEXT: "Kenya's total greenhouse gas (GHG) emissions are relatively low, standing at 73 MtCO2eq in 2010, out of which 75%/ are from the land use, land-use change and forestry (LULUCF) and agriculture sectors. This may be explained by the reliance on wood fuel by a large proportion of the population coupled with the increasing demand for agricultural land and urban development."

    RESPONSE: True

    Return the assessment as a boolean True or False.
    Return only the boolean, and nothing else.

    Now assess the following sample:


    """

tma_prompt="""
    You are assessing the accuracy of a binary ('YES'/'NO') classifier. The classifier classifies a given passage of text as to whether it contains reference to a target, measure, action, and plans
    in the context of the United Nations Framework Convention on Climate Change (UNFCCC) and the Paris Agreement.
    The text is extracted from Nationally Determined Contributions (NDCs) documents.
    The concepts of targets, measures, actions, and plans are defined below:

    1. Targets

        •   Definition: Targets in the NDCs refer to the specific, quantified objectives that each country sets for itself to reduce greenhouse gas (GHG) emissions and mitigate climate change. These targets reflect the level of ambition a country is willing to commit to in its climate action.
        •   Example in NDCs: A common form of a target is a percentage reduction in GHG emissions by a certain year, such as “reducing emissions by 50% by 2030 compared to 1990 levels.” Targets can also be sector-specific, such as setting renewable energy capacity goals.

    2. Measures

        •   Definition: Measures are the policies, regulations, and actions that are implemented to achieve the targets set in the NDCs. These are the instruments through which a country can ensure that it is on the right path to meet its climate goals.
        •   Example in NDCs: Measures can include implementing a carbon tax, introducing renewable energy incentives, or regulations to improve energy efficiency in buildings or transportation sectors. These could also involve reforestation or land-use changes to enhance carbon sinks.

    3. Actions

        •   Definition: Actions refer to the specific activities, projects, or steps that are undertaken to implement the measures and meet the set targets. Actions are the tangible efforts that contribute to reducing emissions or adapting to climate change impacts.
        •   Example in NDCs: Actions might include building solar or wind power plants, electrifying transportation systems, or retrofitting existing infrastructure to make it more energy efficient. Actions are often the ground-level, operational steps that translate plans into reality.

    
    If you agree with the classifier, then assess accuracy as a boolean True.
    Note - assessing targets, measures and actions may be tricky in some cases as the text can use technical language which is sometimes ambiguous. 
    Please take your time to ensure a robust assessment.
    If you can't decide, err on the side of the classifier, and assume it is correct. 

    EXAMPLE 1:

    LABEL: 'YES'

    CONTEXT: "This will lead to more climate related vulnerabilities thereby predisposing farming communities to food insecurity and more poverty. In response to this scenario, the Government has been exploring innovative and transformative measures to assist stakeholders across the agricultural value chains to manage the effects of current and projected change of climate patterns."

    RESPONSE: True

    EXAMPLE 2:

    LABEL: 'NO'

    CONTEXT: "Kenya's total greenhouse gas (GHG) emissions are relatively low, standing at 73 MtCO2eq in 2010, out of which 75%/ are from the land use, land-use change and forestry (LULUCF) and agriculture sectors. This may be explained by the reliance on wood fuel by a large proportion of the population coupled with the increasing demand for agricultural land and urban development."

    RESPONSE: True

    EXAMPLE 3:

    LABEL: 'YES'

    CONTEXT: "1.1 National Circumstances Kenya is located in the Greater Horn of Africa region, which is highly vulnerable to the impacts of climate change. More than 80% of the country’s landmass is arid and semi-arid land (ASAL) with poor infrastructure, and other developmental challenges."

    RESPONSE: True

    Return the assessment as a boolean True or False.
    Return only the boolean, and nothing else.

    Now assess the following sample:

    """



def send_to_chatgpt_api(context, label, prompt, openai_api_key, logprobs_flag=None, logprobs_n=None):
    
    # Combine the result object and context with the new prompt
    combined_message = f"""
    {prompt}
    LABEL: {label}
    CONTEXT: {context}
    RESPONSE:
    """

    # Set up the OpenAI API
    openai.api_key = openai_api_key


    # Send the combined message to the ChatGPT API
    response = openai.ChatCompletion.create(
        # model="gpt-4o-mini",
        model="gpt-4o-mini-2024-07-18",
        # model="gpt-4o-2024-08-06",
        messages=[
            {"role": "system", "content": "You are ChatGPT."},
            {"role": "user", "content": combined_message}
        ],
        logprobs=logprobs_flag,  # whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message..
        top_logprobs=logprobs_n,
    )
    
    # Get the response from the API
    if logprobs_flag:
        gpt_response = response.choices[0].logprobs.content[0].top_logprobs[0]

    else:
        gpt_response = response.choices[0].message['content']
    
    return gpt_response


# Fuzzy matching
def levenshtein_distance(a, b):
    # Initialize the matrix
    dp = [[0 for _ in range(len(b) + 1)] for _ in range(len(a) + 1)]

    # Base cases
    for i in range(len(a) + 1):
        dp[i][0] = i
    for j in range(len(b) + 1):
        dp[0][j] = j

    # Fill the matrix
    for i in range(1, len(a) + 1):
        for j in range(1, len(b) + 1):
            if a[i - 1] == b[j - 1]:
                cost = 0
            else:
                cost = 1
            dp[i][j] = min(dp[i - 1][j] + 1,      # Deletion
                           dp[i][j - 1] + 1,      # Insertion
                           dp[i - 1][j - 1] + cost)  # Substitution

    # Return the Levenshtein distance
    return dp[-1][-1]

def similarity_score(a, b):
    max_len = max(len(a), len(b))
    if max_len == 0:
        return 1.0
    return (max_len - levenshtein_distance(a, b)) / max_len

def remove_stopwords(text):
    stop_words = set(stopwords.words('english'))
    # Tokenize the string and filter out stopwords
    return ' '.join([word for word in text.split() if word.lower() not in stop_words])

def fuzzy_match_sequence(sequence, long_string, threshold=0.4):
    # If the sequence is a single string, split it into phrases (based on commas or similar punctuation)
    if isinstance(sequence, str):
        sequence = re.split(r',\s*|\s+', sequence)

    # Remove stopwords from both the sequence and the long string
    sequence = [remove_stopwords(phrase) for phrase in sequence]
    long_string = remove_stopwords(long_string)

    # Ensure that the input is now a list or tuple
    if not isinstance(sequence, (list, tuple)):
        sequence = list(sequence)

    # Split the long string into words
    long_string_words = long_string.split()

    # Perform Levenshtein-based fuzzy matching and calculate overall similarity score
    total_score = 0
    matches = []
    count_high_prob_matches = 0

    for word in sequence:
        # Find the best match for the current word in the long string
        best_match_score, best_match_word = max((similarity_score(word, ls_word), ls_word) for ls_word in long_string_words)
        
        # Only count matches with a similarity score above the threshold
        if best_match_score >= threshold:
            count_high_prob_matches += 1
            # Cap the total score at 1 and calculate contribution
            total_score += min(best_match_score, 1 - total_score)  # Ensure the score doesn't go above 1

        matches.append(f"Keyword '{word}' matched '{best_match_word}' with a similarity score of {best_match_score:.2f}")

    # The total score should never exceed 1, ensure it is capped at 1
    total_score = round(min(total_score, 1),2)

    return total_score


def target_display(model_sel_name, doc_name):
    
    ### TABLE Output ###

    # Assign dataframe a name
    df = st.session_state['key2']
    st.write(df)

    ### RAG Output by group ##

    # Expand the DataFrame
    df_expand = (
        df.query("`Specific action/target/measure mentioned` == 'YES'")
        .explode('Vulnerability Label')
        )
    # Group by 'Vulnerability Label' and concatenate 'text'
    df_agg = df_expand.groupby('Vulnerability Label')['text'].agg('; '.join).reset_index()

    # st.write(df_agg)

    st.markdown("----")
    st.markdown('**SUMMARY OF GOALS BY VULNERABILITY LABEL:**')

    # Check if the results are already in session state
    if 'results_df' not in st.session_state:
        # Initialize an empty list to store the results
        summary_list = []
        results_list = []

        # Process the data in the loop
        for i in range(0, len(df_agg)):
            st.write(df_agg['Vulnerability Label'].iloc[i])
            
            # Run query to get the result
            result = run_query(
                context=df_agg['text'].iloc[i], 
                label=df_agg['Vulnerability Label'].iloc[i], 
                model_sel_name=model_sel_name
            )

            # Store the Vulnerability Label and the response in a list of dictionaries
            summary_list.append({
                'document': doc_name,
                'text': df_agg['text'].iloc[i],
                'label': df_agg['Vulnerability Label'].iloc[i],
                'summary': result.get_full_content()
            })


        # Process the data in the loop
        for i in range(0, len(df)):

            # Send the result to the ChatGPT API and get the labeled response
            vc_response = send_to_chatgpt_api(
                context = df['text'].iloc[i],
                label = df['Vulnerability Label'].iloc[i],
                prompt = vc_prompt,
                openai_api_key=openai_api_key,
                logprobs_flag=True,
                logprobs_n=1)

            tma_response = send_to_chatgpt_api(
                context = df['text'].iloc[i],
                label = df['Specific action/target/measure mentioned'].iloc[i],
                prompt = tma_prompt,
                openai_api_key=openai_api_key,
                logprobs_flag=True,
                logprobs_n=1)


            # Convert logprobs to % scale
            vc_prob = np.round(np.exp(vc_response.logprob),2)
            vc_token = vc_response.token

            # Convert contrary predictions to probability of positive prediction (inverse)
            if vc_token == 'False':
                vc_prob_cnv = round(1 - vc_prob,2)
            else:
                vc_prob_cnv = vc_prob

            # Do some fuzzy matching to check for class-related keywords in the text
            vc_keywords = fuzzy_match_sequence(str(df['Vulnerability Label'].iloc[i]), str(df['text'].iloc[i]))

            # Compute vulnerability classifciation eval
            vc_eval = False
            if vc_prob_cnv > 0.5 or vc_keywords > 0:
                vc_eval = True

            # Convert logprobs to % scale
            tma_prob = np.round(np.exp(tma_response.logprob),2)
            tma_token = tma_response.token

            # Convert contrary predictions to probability of positive prediction (inverse)
            if tma_token == 'False':
                tma_prob_cnv = round(1 - tma_prob,2)
            else:
                tma_prob_cnv = tma_prob

            # Compute TMA classification eval
            tma_eval = False
            if tma_prob_cnv > 0.5:
                tma_eval = True

            # Store the Vulnerability Label and the response in a list of dictionaries
            results_list.append({
                'document': doc_name,
                'text': df['text'].iloc[i],
                'page': df['page'].iloc[i],
                'label': df['Vulnerability Label'].iloc[i],
                'target': df['Specific action/target/measure mentioned'].iloc[i],
                'VC_prob': vc_prob_cnv,
                'VC_keywords': vc_keywords,
                'VC_eval': vc_eval,
                'TMA_prob': tma_prob_cnv,
                'TMA_eval': tma_eval,
                'VC_check': None,
                'TMA_check': None,
            })

        # Once the loop is done, convert results to a DataFrame and store in session state
        st.session_state['results_df'] = pd.DataFrame(results_list)
        st.session_state['summary_df'] = pd.DataFrame(summary_list)

    df_full = st.session_state['key1']
    num_paragraphs = len(df_full['Vulnerability Label'])
    num_references = len(df['Vulnerability Label'])

    meta_list = []
    # Store the Vulnerability Label and the response in a list of dictionaries
    meta_list.append({
        'document': doc_name,
        'paragraphs': num_paragraphs,
        'references': num_references,
    })

    st.session_state['meta_df'] = pd.DataFrame(meta_list)

    # Retrieve the results from session state
    meta_df = st.session_state['meta_df']
    summary_df = st.session_state['summary_df']
    results_df = st.session_state['results_df']

    # Use an in-memory buffer to hold the Excel file
    excel_buffer = BytesIO()

    # Create an Excel writer and write each DataFrame to a separate sheet
    with pd.ExcelWriter(excel_buffer, engine='xlsxwriter') as writer:
        meta_df.to_excel(writer, sheet_name='Meta', index=False)
        summary_df.to_excel(writer, sheet_name='Summary', index=False)
        results_df.to_excel(writer, sheet_name='Results', index=False)

    # Ensure the buffer is ready for downloading
    excel_buffer.seek(0)

    # Create a download button for the Excel file
    st.download_button(
        label="Download LLM Evaluation",
        data=excel_buffer,
        file_name='eval_' + str.split(doc_name,".")[0] + '.xlsx',
        mime='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
    )