Spaces:
Sleeping
Sleeping
File size: 4,012 Bytes
ae6eb20 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 |
from sentence_transformers import SentenceTransformer
from scipy.spatial.distance import cosine
import numpy as np
from data_ret import search_relevant_data # Assuming this function fetches the data from some source
import streamlit as st
# Load the Sentence Transformer model for similarity search
def load_similarity_model():
st.write("Loading similarity model...") # Show status on Streamlit
retriever_model = SentenceTransformer("all-mpnet-base-v2")
st.write("Similarity model loaded.")
return retriever_model
# Create embeddings for the retrieved documents
def create_embeddings(documents, model):
if not documents:
st.write("No documents provided for embedding.")
return np.array([]) # Return empty array if no documents
st.write(f"Creating embeddings for {len(documents)} documents...") # Show progress
embeddings = []
# Track progress of the embedding creation using Streamlit's progress bar
progress_bar = st.progress(0)
step = 1 / len(documents) # This ensures the progress bar value stays within [0.0, 1.0]
# Include 'text' in the document text along with 'question' and 'answer'
document_texts = [doc['question'] + " " + doc['answer'] + " " + doc.get('text', '') for doc in documents]
for i, doc_text in enumerate(document_texts):
embedding = model.encode(doc_text)
embeddings.append(embedding)
progress_bar.progress(i * step) # Update the progress bar within valid range
embeddings = np.array(embeddings)
st.write(f"Embeddings created with shape: {embeddings.shape}")
return embeddings
# Retrieve documents based on the question embedding
def retrieve_documents(question_embedding, document_embeddings, top_k=5):
if document_embeddings.size == 0:
st.write("No document embeddings available for retrieval.")
return []
st.write("Calculating similarities between question and documents...")
similarities = np.array([1 - cosine(question_embedding, doc_embedding) for doc_embedding in document_embeddings])
# Get indices of top K similarities (highest similarity first)
top_indices = similarities.argsort()[-top_k:][::-1] # Sort in descending order
return top_indices
# Main function to get the context from the most relevant documents based on topic and question
def get_relevant_context(question, topic):
try:
st.write("Searching for relevant documents based on the topic...")
relevant_documents = search_relevant_data(topic) # Use dynamic topic for search query
st.write(f"Found {len(relevant_documents)} relevant documents.")
if not relevant_documents:
return "No relevant documents found."
retriever_model = load_similarity_model() # Load the similarity model
# Create document embeddings and show progress
document_embeddings = create_embeddings(relevant_documents, retriever_model)
if document_embeddings.size == 0:
return "No embeddings created for relevant documents."
st.write("Generating question embedding and retrieving relevant documents...")
question_embedding = retriever_model.encode(question)
relevant_doc_indices = retrieve_documents(question_embedding, document_embeddings)
if len(relevant_doc_indices) == 0:
return "No relevant documents found after embedding."
# Extract context from the top relevant documents
contexts = []
for idx in relevant_doc_indices:
doc = relevant_documents[idx]
context = doc.get('answer', '') + " " + doc.get('text', '')
if context.strip():
contexts.append(context)
if not contexts:
return "No valid contexts available for answering."
# Return the combined context for question answering
return " ".join(contexts)
except Exception as e:
st.write(f"Error processing question: {str(e)}")
return f"Error: {str(e)}"
|