Spaces:
Sleeping
Sleeping
File size: 3,457 Bytes
8ca60f2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 |
import streamlit as st
from transformers import AutoTokenizer, AutoModelForCausalLM
from sentence_transformers import SentenceTransformer, util
import PyPDF2
from docx import Document
# Load the tokenizer and model for sentence embeddings
@st.cache_resource
def load_model():
tokenizer = AutoTokenizer.from_pretrained("rakeshkiriyath/gpt2Medium_text_to_sql")
model = AutoModelForCausalLM.from_pretrained("rakeshkiriyath/gpt2Medium_text_to_sql")
sentence_model = SentenceTransformer('all-MiniLM-L6-v2') # Smaller, faster sentence embeddings model
return tokenizer, model, sentence_model
# Extract text from a PDF file
def extract_text_from_pdf(pdf_file):
pdf_reader = PyPDF2.PdfReader(pdf_file)
text = ""
for page in pdf_reader.pages:
text += page.extract_text()
return text
# Extract text from a Word document
def extract_text_from_word(docx_file):
doc = Document(docx_file)
text = ""
for paragraph in doc.paragraphs:
text += paragraph.text + "\n"
return text
# Compare sentences for similarity
def compare_sentences(doc1_sentences, doc2_sentences, sentence_model):
similar_sentences = []
for i, sent1 in enumerate(doc1_sentences):
best_match = None
best_score = 0
for j, sent2 in enumerate(doc2_sentences):
score = util.pytorch_cos_sim(sentence_model.encode(sent1), sentence_model.encode(sent2)).item()
if score > best_score: # Higher similarity score
best_score = score
best_match = (i, j, score, sent1, sent2)
if best_match and best_score > 0.6: # Threshold for similarity
similar_sentences.append(best_match)
return similar_sentences
# Streamlit UI
def main():
st.title("Comparative Analysis of Two Documents")
st.sidebar.header("Upload Files")
# Upload files
uploaded_file1 = st.sidebar.file_uploader("Upload the First Document (PDF/Word)", type=["pdf", "docx"])
uploaded_file2 = st.sidebar.file_uploader("Upload the Second Document (PDF/Word)", type=["pdf", "docx"])
if uploaded_file1 and uploaded_file2:
# Extract text from the uploaded documents
text1 = extract_text_from_pdf(uploaded_file1) if uploaded_file1.name.endswith(".pdf") else extract_text_from_word(uploaded_file1)
text2 = extract_text_from_pdf(uploaded_file2) if uploaded_file2.name.endswith(".pdf") else extract_text_from_word(uploaded_file2)
# Split text into sentences
doc1_sentences = text1.split('. ')
doc2_sentences = text2.split('. ')
# Load model
tokenizer, model, sentence_model = load_model()
# Perform sentence comparison
similar_sentences = compare_sentences(doc1_sentences, doc2_sentences, sentence_model)
# Display results
st.header("Comparative Analysis Results")
if similar_sentences:
for match in similar_sentences:
doc1_index, doc2_index, score, sent1, sent2 = match
st.markdown(f"**Document 1 Sentence {doc1_index + 1}:** {sent1}")
st.markdown(f"**Document 2 Sentence {doc2_index + 1}:** {sent2}")
st.markdown(f"**Similarity Score:** {score:.2f}")
st.markdown("---")
else:
st.info("No significantly similar sentences found.")
else:
st.warning("Please upload two documents to compare.")
if __name__ == "__main__":
main()
|