import streamlit as st from transformers import AutoTokenizer, AutoModelForCausalLM from sentence_transformers import SentenceTransformer, util import PyPDF2 from docx import Document # Load the tokenizer and model for sentence embeddings @st.cache_resource def load_model(): tokenizer = AutoTokenizer.from_pretrained("rakeshkiriyath/gpt2Medium_text_to_sql") model = AutoModelForCausalLM.from_pretrained("rakeshkiriyath/gpt2Medium_text_to_sql") sentence_model = SentenceTransformer('all-MiniLM-L6-v2') # Smaller, faster sentence embeddings model return tokenizer, model, sentence_model # Extract text from a PDF file def extract_text_from_pdf(pdf_file): pdf_reader = PyPDF2.PdfReader(pdf_file) text = "" for page in pdf_reader.pages: text += page.extract_text() return text # Extract text from a Word document def extract_text_from_word(docx_file): doc = Document(docx_file) text = "" for paragraph in doc.paragraphs: text += paragraph.text + "\n" return text # Compare sentences for similarity def compare_sentences(doc1_sentences, doc2_sentences, sentence_model): similar_sentences = [] for i, sent1 in enumerate(doc1_sentences): best_match = None best_score = 0 for j, sent2 in enumerate(doc2_sentences): score = util.pytorch_cos_sim(sentence_model.encode(sent1), sentence_model.encode(sent2)).item() if score > best_score: # Higher similarity score best_score = score best_match = (i, j, score, sent1, sent2) if best_match and best_score > 0.6: # Threshold for similarity similar_sentences.append(best_match) return similar_sentences # Streamlit UI def main(): st.title("Comparative Analysis of Two Documents") st.sidebar.header("Upload Files") # Upload files uploaded_file1 = st.sidebar.file_uploader("Upload the First Document (PDF/Word)", type=["pdf", "docx"]) uploaded_file2 = st.sidebar.file_uploader("Upload the Second Document (PDF/Word)", type=["pdf", "docx"]) if uploaded_file1 and uploaded_file2: # Extract text from the uploaded documents text1 = extract_text_from_pdf(uploaded_file1) if uploaded_file1.name.endswith(".pdf") else extract_text_from_word(uploaded_file1) text2 = extract_text_from_pdf(uploaded_file2) if uploaded_file2.name.endswith(".pdf") else extract_text_from_word(uploaded_file2) # Split text into sentences doc1_sentences = text1.split('. ') doc2_sentences = text2.split('. ') # Load model tokenizer, model, sentence_model = load_model() # Perform sentence comparison similar_sentences = compare_sentences(doc1_sentences, doc2_sentences, sentence_model) # Display results st.header("Comparative Analysis Results") if similar_sentences: for match in similar_sentences: doc1_index, doc2_index, score, sent1, sent2 = match st.markdown(f"**Document 1 Sentence {doc1_index + 1}:** {sent1}") st.markdown(f"**Document 2 Sentence {doc2_index + 1}:** {sent2}") st.markdown(f"**Similarity Score:** {score:.2f}") st.markdown("---") else: st.info("No significantly similar sentences found.") else: st.warning("Please upload two documents to compare.") if __name__ == "__main__": main()