Spaces:
Sleeping
Sleeping
File size: 2,860 Bytes
ae6eb20 5dc73ed ae6eb20 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
from datasets import load_dataset
# Load the dataset (specify split as 'train' to load the training data)
dataset = load_dataset('tom-010/google_natural_questions_answerability', split='train')
# Function to filter based on a query/topic and return relevant data
def search_relevant_data(topic="Artificial Intelligence", max_words=250, top_n=250):
# Filter the dataset based on the presence of the topic in 'question', 'answer', or 'text' fields
filtered_data = dataset.filter(
lambda x: (
(x['question'] is not None and topic.lower() in x['question'].lower()) or
(x['answer'] is not None and topic.lower() in x['answer'].lower()) or
(x['text'] is not None and topic.lower() in x['text'].lower())
)
)
# Ensure we only select up to the available number of rows
#num_to_select = min(top_n, len(filtered_data)) # Choose the minimum of top_n and available data
#filtered_data = filtered_data.select(range(num_to_select)) # Select up to 'num_to_select' rows
filtered_data = filtered_data.select(range(min(top_n, len(filtered_data))))
# Create a list to store the relevant data
relevant_documents = []
# Display and store an excerpt of the answer for each relevant entry
for entry in filtered_data:
# Check the type of 'entry' first to ensure it's a dictionary
if isinstance(entry, dict):
question = entry.get('question', '') # Accessing the 'question' field safely
answer = entry.get('answer', '') # Accessing the 'answer' field safely
text = entry.get('text', '') # Accessing the 'text' field safely
# Only store the first 'max_words' words of the answer or text
answer_excerpt = ' '.join(answer.split()[:max_words]) if answer else ""
text_excerpt = ' '.join(text.split()[:max_words]) if text else ""
# Append relevant document information to the list
relevant_documents.append({
"question": question,
"answer": answer_excerpt,
"text": text_excerpt
})
# Debugging: Print a preview of the data (optional)
#print(f"Question: {question[:20]}...") # Print first 20 chars of the question
#print(f"Answer (first {max_words} words): {answer_excerpt[:20]}...") # Print first 20 words of the answer
#print(f"Text (first {max_words} words): {text_excerpt[:20]}...") # Print first 20 words of the text
#print("-" * 50)
else:
print("Unexpected entry format:", entry)
return relevant_documents # Return the list of relevant documents
# Sample search query
#relevant_data = search_relevant_data("vatican city") # Change to the desired query/topic
#print(f"Found {len(relevant_data)} relevant documents.")
|