|
|
|
import os |
|
import spacy |
|
import gradio as gr |
|
from sentence_transformers import SentenceTransformer |
|
from sklearn.metrics.pairwise import cosine_similarity |
|
import numpy as np |
|
import zipfile |
|
|
|
|
|
zip_path = '/mnt/data/en_core_web_sm.zip' |
|
|
|
|
|
extraction_dir = '/mnt/data/en_core_web_sm' |
|
|
|
|
|
with zipfile.ZipFile(zip_path, 'r') as zip_ref: |
|
zip_ref.extractall(extraction_dir) |
|
|
|
|
|
nlp = spacy.load(extraction_dir) |
|
|
|
|
|
|
|
model = SentenceTransformer('sentence-transformers/all-mpnet-base-v2', device='cpu') |
|
|
|
|
|
with open('testo.txt', 'r', encoding='utf-8') as file: |
|
text = file.read() |
|
|
|
|
|
doc = nlp(text) |
|
sentences = [sent.text for sent in doc.sents] |
|
|
|
|
|
embeddings = model.encode(sentences, batch_size=8, show_progress_bar=True) |
|
|
|
|
|
|
|
def find_relevant_sentences(query): |
|
query_embedding = model.encode([query]) |
|
similarities = cosine_similarity(query_embedding, embeddings).flatten() |
|
|
|
|
|
threshold = 0.2 |
|
filtered_results = [(idx, sim) for idx, sim in enumerate(similarities) if sim >= threshold] |
|
|
|
if not filtered_results: |
|
return ["No relevant sentences found."] |
|
|
|
|
|
filtered_results.sort(key=lambda x: x[1], reverse=True) |
|
|
|
|
|
top_n = 4 |
|
relevant_sentences = [sentences[idx] for idx, _ in filtered_results[:top_n]] |
|
|
|
|
|
unique_sentences = list(dict.fromkeys(relevant_sentences)) |
|
doc = nlp(" ".join(unique_sentences)) |
|
grouped_results = [sent.text.strip() for sent in doc.sents] |
|
|
|
return grouped_results |
|
|
|
|
|
examples = [ |
|
["irresponsible use of the machine?"], |
|
["If I have a problem how can I get help? "], |
|
["precautions when using the cutting machine"], |
|
["How do I change the knife of the cutting machine?"] |
|
["Uso irresponsable de la máquina cortadora ?"] |
|
] |
|
|
|
|
|
iface = gr.Interface( |
|
fn=find_relevant_sentences, |
|
inputs=gr.Textbox(label="Insert your query"), |
|
outputs=gr.Textbox(label="Relevant sentences"), |
|
examples=examples, |
|
title="Manual Querying System", |
|
description="Enter a question about the machine, and this tool will find the most relevant sentences from the manual." |
|
) |
|
|
|
|
|
iface.launch() |
|
|