File size: 1,254 Bytes
d7b7dc6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
import pandas as pd

def load_dataframe(data_path):
    df = pd.read_csv(data_path)
    return df

def generate_prompt(source_passage):
    return f"""You are a chat bot answering questions using data. You must stick to the answers provided solely by the text in the passage provided. 
    You are asked the question 'Provide a concise summary of the following passage, covering the core pieces of information described.'
    Passage:
    {source_passage}
    """
    
def format_results(hallucination_scores, model_name, revision, precision, accuracy, hallucination_rate, answer_rate, avg_summary_len):
    # Define the structure of the results (JSON)
    results = {
        "config": {
            "model_dtype": precision, # Precision with which you ran the evaluation
            "model_name": model_name, # Name of the model
            "model_sha": revision # Hash of the model 
        },
        "results": {
            "hallucination_eval": {
                "HEM Scores": hallucination_scores,
                "Accuracy": accuracy,
                "Hallucination Rate": hallucination_rate,
                "Answer Rate": answer_rate, 
                "Average Summary Length": avg_summary_len,
            }
        }
    }
    return results