import streamlit as st import pandas as pd import torch from transformers import pipeline import datetime from rapidfuzz import process, fuzz # Load the CSV file df = pd.read_csv("anomalies.csv", quotechar='"') # Convert 'real' column to standard float format and then to strings df['real'] = df['real'].apply(lambda x: f"{x:.2f}") # Fill NaN values and convert all columns to strings df = df.fillna('').astype(str) # Function to filter the DataFrame using RapidFuzz for dates def filter_dataframe_by_date(df, date_str, threshold=80): # Apply fuzzy matching on the 'ds' (date) column matches = process.extract(date_str, df['ds'], scorer=fuzz.token_sort_ratio, limit=None) filtered_rows = [match[2] for match in matches if match[1] >= threshold] return df.iloc[filtered_rows] # Function to filter the DataFrame using RapidFuzz for groups def filter_dataframe_by_group(df, group_keyword, threshold=80): # Apply fuzzy matching on the 'Group' column matches = process.extract(group_keyword, df['Group'], scorer=fuzz.token_sort_ratio, limit=None) filtered_rows = [match[2] for match in matches if match[1] >= threshold] return df.iloc[filtered_rows] # Function to generate a response using the TAPAS model def response(user_question, df): a = datetime.datetime.now() # Extract date and group keywords from the user question date_str = "December 2022" # Example; you'd extract this from the user question group_keyword = "IPVA" # Filter the DataFrame by date and group subset_df = filter_dataframe_by_date(df, date_str) subset_df = filter_dataframe_by_group(subset_df, group_keyword) # Initialize the TAPAS model tqa = pipeline(task="table-question-answering", model="google/tapas-large-finetuned-wtq", tokenizer_kwargs={"clean_up_tokenization_spaces": False}) # Debugging information print("Filtered DataFrame shape:", subset_df.shape) print("Filtered DataFrame head:\n", subset_df.head()) print("User question:", user_question) # Query the TAPAS model try: answer = tqa(table=subset_df, query=user_question)['answer'] except IndexError as e: print(f"Error: {e}") answer = "Error occurred: " + str(e) query_result = { "Resposta": answer } b = datetime.datetime.now() print("Time taken:", b - a) return query_result # Streamlit interface st.markdown("""