Upload 13 files

.env

@@ -0,0 +1,2 @@
+[email protected]
+EMAIL_PASSWORD=mwvm tluh heuy fowf

Update_tracking.py

@@ -0,0 +1,169 @@
+import requests
+from bs4 import BeautifulSoup
+import time
+import streamlit as st
+import threading
+from datetime import datetime, timedelta
+import smtplib
+from email.message import EmailMessage
+import os
+from dotenv import load_dotenv
+from google.oauth2.service_account import Credentials
+from googleapiclient.discovery import build
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Google Sheets Configuration
+SHEET_ID = '1bZjlA-UJrBhWS2jHlEQ-7nbmDvxpEoKylgxHW51Hhzc'  # Google Sheets ID
+RANGE = 'Sheet1!A:D'  # The range where you want to append the data
+
+# Predefined list of URLs to track
+TRACKING_URLS = [
+    "https://gdpr-info.eu/recitals/no-1/"]
+
+# Event to signal thread termination
+stop_event = threading.Event()
+
+# Authenticate Google Sheets API
+def authenticate_google_sheets():
+    creds = Credentials.from_service_account_file(
+        'Credentials.json',
+        scopes=['https://www.googleapis.com/auth/spreadsheets']
+    )
+    service = build('sheets', 'v4', credentials=creds)
+    return service
+
+# Append data to Google Sheets
+def append_to_google_sheets(service, url, title, content, timestamp):
+    values = [
+        [url, title, content[:200], timestamp]  # Prepare row to append
+    ]
+    body = {'values': values}
+    try:
+        service.spreadsheets().values().append(
+            spreadsheetId=SHEET_ID,
+            range=RANGE,
+            valueInputOption="RAW",
+            body=body
+        ).execute()
+        st.write(f"Data appended to Google Sheets at {timestamp}.")
+    except Exception as e:
+        st.error(f"Error appending to Google Sheets: {e}")
+
+# Send email notification
+def send_email_notification(to_email, url, title, content, timestamp):
+    sender_email = os.getenv("EMAIL_ADDRESS")
+    sender_password = os.getenv("EMAIL_PASSWORD")
+    smtp_server = "smtp.gmail.com"
+    smtp_port = 587
+
+    if not sender_email or not sender_password:
+        st.error("Environment variables not loaded. Check your .env file.")
+        return
+
+    msg = EmailMessage()
+    msg["Subject"] = f"Website Update Notification for {url}"
+    msg["From"] = sender_email
+    msg["To"] = to_email
+    msg.set_content(f"""
+    Website: {url}
+    Title: {title}
+    Content (preview): {content[:200]}...
+    Tracked at: {timestamp}
+    """)
+
+    try:
+        with smtplib.SMTP(smtp_server, smtp_port) as server:
+            server.starttls()
+            server.login(sender_email, sender_password)
+            server.send_message(msg)
+            st.success(f"Notification email sent to {to_email}")
+    except smtplib.SMTPException as e:
+        st.error(f"SMTP Error: {e}")
+
+# Fetch website data
+def fetch_website_data(url):
+    try:
+        response = requests.get(url, timeout=10)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.text, 'html.parser')
+        title = soup.title.string.strip() if soup.title else 'No title available'
+        paragraphs = soup.find_all('p')
+        content = ' '.join([p.text.strip() for p in paragraphs]) if paragraphs else 'New Notification available'
+        return title, content
+    except requests.exceptions.RequestException as e:
+        st.error(f"Error fetching website data: {e}")
+        return "Error occurred", "New notification detected. No content available due to an error."
+
+# Track websites and store updates in Google Sheets
+def track_websites(urls, recipient_email, interval=60, max_duration=20*60):
+    st.write(f"Started tracking for {recipient_email}")
+    service = authenticate_google_sheets()
+    last_updates = {}  # To track changes in website content
+
+    start_time = datetime.now()  # Record the start time
+    end_time = start_time + timedelta(seconds=max_duration)  # Set end time (20 minutes later)
+
+    while not stop_event.is_set() and datetime.now() < end_time:
+        for url in urls:
+            title, content = fetch_website_data(url)
+            if title and content:
+                timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+                # Check for updates
+                if url not in last_updates or last_updates[url] != (title, content):
+                    last_updates[url] = (title, content)
+
+                    # Append to Google Sheets
+                    append_to_google_sheets(service, url, title, content, timestamp)
+
+                    # Send notification email
+                    try:
+                        send_email_notification(recipient_email, url, title, content, timestamp)
+                    except Exception as e:
+                        st.error(f"Error sending email notification: {e}")
+
+        # Wait for the next interval or until stop_event is set
+        stop_event.wait(interval)
+
+    st.write("Stopped tracking after 20 minutes.")
+
+# Display tracking status
+def display_tracking_status():
+    st.title("Update Tracking System with Notifications")
+
+    email_input = st.text_input("Enter your email for notifications:")
+
+    # Maintain thread state
+    if "tracking_thread" not in st.session_state:
+        st.session_state["tracking_thread"] = None
+
+    if email_input:
+        # Start tracking
+        if st.button("Tracking"):
+            if st.session_state["tracking_thread"] is None or not st.session_state["tracking_thread"].is_alive():
+                stop_event.clear()  # Clear the stop flag to allow tracking
+                thread = threading.Thread(target=track_websites, args=(TRACKING_URLS, email_input), daemon=True)
+                thread.start()
+                st.session_state["tracking_thread"] = thread
+                st.success(f"Notifications will be sent to {email_input}.")
+            else:
+                st.warning("Tracking Updates is already running.")
+
+        # Stop tracking
+        if st.button("Stop Tracking"):
+            if st.session_state["tracking_thread"] is not None and st.session_state["tracking_thread"].is_alive():
+                stop_event.set()  # Signal the thread to stop
+                st.session_state["tracking_thread"].join()  # Wait for the thread to finish
+                st.session_state["tracking_thread"] = None
+                st.success("Tracking stopped.")
+            else:
+                st.warning("No active tracking to stop.")
+
+# Main function
+def main():
+    display_tracking_status()
+
+if __name__ == "__main__":
+    main()

app.py

@@ -0,0 +1,17 @@
+import streamlit as st
+import Update_tracking
+import legal_document_analysis
+from rag_pipeline import extract_text_from_pdf, create_vector_store, create_qa_pipeline
+
+# Streamlit App Navigation
+def main():
+    st.sidebar.title("Navigation")
+    page = st.sidebar.radio("Choose a page", ["Update Tracking", "Legal Document Analysis"])
+
+    if page == "Update Tracking":
+        Update_tracking.display_tracking_status()  # Ensure the correct function name
+    elif page == "Legal Document Analysis":
+        legal_document_analysis.display_legal_analysis_page()
+
+if __name__ == "__main__":
+    main()

credentials.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "graphite-setup-448304-u1",
+  "private_key_id": "c745314b1cd8b8a135a10c4819dabdb0d3d9a552",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDKKXcJFGxBxdan\nBPI+M1js/YcBI3646efu1U88oWG7CrZDYEs6ZMTx46mdsgaiJ3LCmt6NRI691rdt\nOl2rh3KZ/rMLQXyyQvMfePwk4t1f/MFbHK75pD+nUWsUlbXawnGfZXRJxV9gC0F8\nindrqRFXqfrJaCjsexW5WdKEt5FAb4OSRSEAEzO6AC7VOAnME+ctOvzI8pg8Cuaw\nkJf8/0fTN/1Fgmygmj3om16J3gYVmxvaTYjZnrM6hQhvHFvTCkoLYk4DSXhIBxce\nzD/qLgoqwFGjlhrveb8FXErzTXlulAed/R49HJnbSKsL+Nq2guthoYV/j3Bg+TZ4\ngwPdlWQxAgMBAAECggEAC/jb6eud1CDUWxCDzItzqhhDCx4YInkGsVoMTtvDSwWB\nwPh2SAtg7u7DtBkvg6GYrh3oDV9ooSUyuGsfEjoLn0NKROHAUdm9ZmkerIwLXmR8\namoG0jWoB99+WwoZKo7+ldkXI4nG4cwU2RxloVhyNJn2RkBvAP8yjIcdXJr25QUC\nqA7v+grCR2CcxBkbRXoh/BQ+4wJQjT12eW7ybYXgxJb8HK8fWcGWXv8ir+KPNOrL\nONMhUS8rHyJ3i/9i/uwMP61pqmKf5x1gBTM1h7Wr+8tTIbCsrv1D8DSwOqvHrgTb\nDzUDKmpZFBUD0xyKETR5r7xTpje1M/xcTQlOskllyQKBgQDmhxz3kcnEHN88J1Jq\nvCXD4+0pBF+tzNc9ZmSnu2W+EBq5mAMu7sDTGRZtj/GisyYgm8YMybGnh38IPh5f\nOM+dqcXOM68ARi1srkGXPbLOMksAALhx9NVkbAZvm8Y6wIw6S5V/SsRiW8dq0VTM\nW2ncwUfn9gV3jstdAokjsZTM2QKBgQDgf/l8AZO6MHl/lujz480HP1ye/bkxhPnZ\ndsyHJG7zKRjQsqa1Kj006mGVLtcaC+zhNxGpeLrBbK/+OzyiBuM5OX0UXeS8zfIp\nPkXliSNarWIncgTCEwrcZOe1HFhIYYfd8JKebpaMtPOGYs2ZaKALMNXLDAJv5kSe\nrB0y0nabGQKBgEBKLAzrDo7v0l3vRW/Y2ttfpFNNEfXha6GG/ck9XOl9ikZ6G2Eq\nWoJEu9mATE4okhiD+N62HSJwij/dOxEpQdVhdHXGOibka2Sk0EwCe+w/YGU2zjmR\nozVnXwAfPFBERJc5Xw7p5gzcNagXiwOW9tYG3SvWk729B///ZgmbS7k5AoGBAIWX\nwgeoDJe0duiVDgTsHzrb2qsGAylTnzZoOmeFJ7ztTMYMOaX+/7M4cl9LI1dtl0Wh\n9pGptUCbA9O+vmMQxDBuZF81TIOJ7jvawc9In1A7cvhUIj63BDSIm1WsC2DvIOHS\nYf2Dg3UxzOTAcOaEWxCtu9t7Rwv9tAHUG//9O/UpAoGBALNfN69s+vhpPX4Ioq3B\nyv0bAk3QU4NZHnX7d48eFbWSS5Hhbboo/T/0KPcOf7ynOptsLeV+HqLS/WLXsJ5J\npKw7mW7pb0VoLv+JokWeAtIc4c0Ft2EZtvb+6t3GcrrDJsFBshUdwblrP9sl6i2X\nYUd3Ck2TaXpXirfFdUgByhLl\n-----END PRIVATE KEY-----\n",
+  "client_email": "[email protected]",
+  "client_id": "105183693282963617063",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/sheet-865%40graphite-setup-448304-u1.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}

legal_document_analysis.py

@@ -0,0 +1,328 @@
+import streamlit as st
+from groq import Groq
+from PyPDF2 import PdfReader
+from docx import Document
+from tiktoken import get_encoding, Encoding
+import concurrent.futures
+import matplotlib.pyplot as plt
+import io
+import base64
+import os
+
+# Groq API client initialization
+client = Groq(api_key="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")  # Replace with your actual API key.
+
+def extract_text_from_pdf(file):
+    reader = PdfReader(file)
+    text = ""
+    for page in reader.pages:
+        text += page.extract_text()
+    return text
+
+def extract_text_from_docx(file):
+    doc = Document(file)
+    text = "\n".join([paragraph.text for paragraph in doc.paragraphs])
+    return text
+
+def preprocess_text(text):
+    return " ".join(text.replace("\n", " ").replace("\r", " ").split())
+
+def get_default_encoding():
+    return get_encoding("cl100k_base")
+
+def split_into_chunks(text, token_limit=5500):
+    encoding = get_default_encoding()
+    words = text.split()
+    chunks = []
+    current_chunk = []
+    current_tokens = 0
+
+    for word in words:
+        word_tokens = len(encoding.encode(word + " "))
+        if current_tokens + word_tokens > token_limit:
+            chunks.append(" ".join(current_chunk))
+            current_chunk = [word]
+            current_tokens = word_tokens
+        else:
+            current_chunk.append(word)
+            current_tokens += word_tokens
+
+    if current_chunk:
+        chunks.append(" ".join(current_chunk))
+    return chunks
+
+def summarize_text(text):
+    try:
+        response = client.chat.completions.create(
+            messages=[{
+                "role": "user",
+                "content": f"Summarize the following legal document in a concise manner: {text}"
+            }],
+            model="llama-3.1-8b-instant",
+            stream=False
+        )
+        if response and response.choices:
+            return response.choices[0].message.content
+        else:
+            return "Error: Received an empty or invalid response from Groq API."
+    except Exception as e:
+        return f"Error generating summary: {e}"
+
+def summarize_large_text(text, chunk_limit=5000):
+    chunks = split_into_chunks(text, token_limit=chunk_limit)
+    summaries = []
+    for chunk in chunks:
+        summaries.append(summarize_text(chunk))
+    return " ".join(summaries)
+
+def detect_key_clauses(text):
+    key_clauses = [
+        {"clause": "confidentiality", "summary": "Confidentiality clauses ensure that sensitive information remains protected."},
+        {"clause": "liability", "summary": "Liability clauses outline the responsibility for damages or losses incurred."},
+        {"clause": "termination", "summary": "Termination clauses specify the conditions under which a contract may be ended."},
+        {"clause": "force majeure", "summary": "Force majeure clauses excuse parties from performance obligations due to unforeseen events."},
+        {"clause": "governing law", "summary": "Governing law clauses specify which jurisdiction's laws will govern the contract."},
+        {"clause": "dispute resolution", "summary": "Dispute resolution clauses specify how conflicts between parties will be resolved."},
+        {"clause": "amendment", "summary": "Amendment clauses outline the process for changing the terms of the contract."},
+        {"clause": "warranty", "summary": "Warranty clauses provide assurances regarding the quality or condition of goods or services."},
+    ]
+    
+    detected_clauses = []
+    for clause in key_clauses:
+        if clause["clause"].lower() in text.lower():
+            clause_start = text.lower().find(clause["clause"].lower())
+            context = text[clause_start - 50: clause_start + 200]
+            explanation = f"The document mentions '{clause['clause']}' clause. Context: {context.strip()}..."
+            detected_clauses.append({
+                "clause": clause["clause"].capitalize(),
+                "summary": clause["summary"],
+                "explanation": explanation
+            })
+    
+    return detected_clauses
+
+def detect_hidden_obligations_or_dependencies(text, summary):
+    hidden_obligations = [
+        {"phrase": "dependent upon", "summary": "This suggests that some action is conditional upon another."},
+        {"phrase": "if", "summary": "This indicates that certain conditions must be met to fulfill the obligation."},
+        {"phrase": "may be required", "summary": "Implies that the party could be obligated to perform an action under specific conditions."},
+        {"phrase": "should", "summary": "Implies a recommendation or requirement, though not explicitly mandatory."},
+        {"phrase": "obligated to", "summary": "Indicates a clear, binding duty to perform an action."},
+    ]
+    
+    hidden_dependencies = []
+    
+    for item in hidden_obligations:
+        if item["phrase"].lower() in text.lower() or item["phrase"].lower() in summary.lower():
+            phrase_start = text.lower().find(item["phrase"].lower())
+            context = text[phrase_start - 50: phrase_start + 200]
+            hidden_dependencies.append({
+                "phrase": item["phrase"],
+                "summary": item["summary"],
+                "context": context.strip()
+            })
+    
+    return hidden_dependencies
+
+def detect_risks(text, summary):
+    risk_phrases = [
+        {"phrase": "penalty", "summary": "Penalty clauses may impose financial or legal consequences on the parties involved."},
+        {"phrase": "liability", "summary": "Liability clauses may indicate potential financial responsibility or legal risks."},
+        {"phrase": "default", "summary": "Default clauses can expose parties to consequences for failure to perform obligations."},
+        {"phrase": "breach", "summary": "Breach of contract can lead to serious legal consequences including financial penalties."},
+        {"phrase": "suspension", "summary": "Suspension clauses may indicate risks of halting services or operations in case of non-compliance."},
+    ]
+    
+    detected_risks = []
+    
+    for item in risk_phrases:
+        if item["phrase"].lower() in text.lower() or item["phrase"].lower() in summary.lower():
+            phrase_start = text.lower().find(item["phrase"].lower())
+            context = text[phrase_start - 50: phrase_start + 200]
+            detected_risks.append({
+                "phrase": item["phrase"],
+                "summary": item["summary"],
+                "context": context.strip()
+            })
+    
+    return detected_risks
+
+def plot_risk_pie_chart(detected_clauses, hidden_obligations, detected_risks):
+    # Calculate counts for each category
+    num_clauses = len(detected_clauses)
+    num_obligations = len(hidden_obligations)
+    num_risks = len(detected_risks)
+
+    # Create a pie chart
+    labels = ['Detected Key Clauses', 'Hidden Obligations or Dependencies', 'Detected Risks']
+    sizes = [num_clauses, num_obligations, num_risks]
+    colors = ['#ff9999','#66b3ff','#99ff99']
+
+    fig, ax = plt.subplots()
+    ax.pie(sizes, labels=labels, autopct='%1.1f%%', startangle=90, colors=colors, wedgeprops={'edgecolor': 'black'})
+    ax.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.
+
+    # Create a buffer to save the plot as an image in memory
+    buf = io.BytesIO()
+    plt.savefig(buf, format="png")
+    buf.seek(0)
+    
+    # Encode the image to base64
+    img_str = base64.b64encode(buf.read()).decode('utf-8')
+    buf.close()
+    
+    return img_str
+
+def generate_analysis_document(document_text, summary, detected_clauses, hidden_obligations, detected_risks):
+    doc = Document()
+    doc.add_heading('Legal Document Analysis', level=1)
+
+    doc.add_heading('Extracted Document Text', level=2)
+    doc.add_paragraph(document_text)
+
+    doc.add_heading('Summary', level=2)
+    doc.add_paragraph(summary)
+
+    doc.add_heading('Key Clauses', level=2)
+    if detected_clauses:
+        for clause in detected_clauses:
+            doc.add_paragraph(f"Clause: {clause['clause']}")
+            doc.add_paragraph(f"Summary: {clause['summary']}")
+            doc.add_paragraph(f"Explanation: {clause['explanation']}")
+    else:
+        doc.add_paragraph("No key clauses detected.")
+
+    doc.add_heading('Hidden Obligations or Dependencies', level=2)
+    if hidden_obligations:
+        for obligation in hidden_obligations:
+            doc.add_paragraph(f"Phrase: {obligation['phrase']}")
+            doc.add_paragraph(f"Summary: {obligation['summary']}")
+            doc.add_paragraph(f"Context: {obligation['context']}")
+    else:
+        doc.add_paragraph("No hidden obligations detected.")
+
+    doc.add_heading('Risks', level=2)
+    if detected_risks:
+        for risk in detected_risks:
+            doc.add_paragraph(f"Risk Phrase: {risk['phrase']}")
+            doc.add_paragraph(f"Summary: {risk['summary']}")
+            doc.add_paragraph(f"Context: {risk['context']}")
+    else:
+        doc.add_paragraph("No risks detected.")
+
+    return doc
+
+def display_legal_analysis_page():
+    st.title("Legal Document Analysis with Groq API")
+
+    uploaded_file = st.file_uploader("Upload your legal document (PDF or DOCX)", type=["pdf", "docx"])
+    if uploaded_file:
+        if uploaded_file.name.endswith(".pdf"):
+            document_text = preprocess_text(extract_text_from_pdf(uploaded_file))
+        elif uploaded_file.name.endswith(".docx"):
+            document_text = preprocess_text(extract_text_from_docx(uploaded_file))
+        else:
+            st.error("Unsupported file type!")
+            return
+
+        tabs = st.tabs(["Document Text", "Summary", "Key Clauses", "Hidden Obligations or Dependencies", "Risk Analysis"])
+
+
+        with tabs[0]:
+            st.subheader("Extracted Legal Document Text")
+            st.text_area("Document Text", document_text, height=300)
+
+        with tabs[1]:
+            st.subheader("Quick Summary")
+            summary = summarize_large_text(document_text)
+            if "Error" in summary:
+                st.warning("Summary generation failed.")
+                summary = "Summary not available."
+            st.write(summary)
+
+        with tabs[2]:
+            st.subheader("Detected Key Clauses")
+            
+            detected_clauses = detect_key_clauses(document_text)
+            if not detected_clauses:
+                st.write("No key clauses detected.")
+            else:
+                # Count occurrences of each detected clause
+                clause_counts = {}
+                for clause in detected_clauses:
+                    clause_counts[clause['clause']] = clause_counts.get(clause['clause'], 0) + 1
+
+                # Create a bar chart for detected clauses
+                if clause_counts:
+                    labels = list(clause_counts.keys())
+                    values = list(clause_counts.values())
+
+                    fig, ax = plt.subplots()
+                    ax.bar(labels, values, color='skyblue')
+
+                    # Rotate x-axis labels for better visibility
+                    plt.xticks(rotation=45, ha='right')
+
+                    # Add titles and labels
+                    ax.set_title("Detected Key Clauses Visualization")
+                    ax.set_xlabel("Clause")
+                    ax.set_ylabel("Count")
+
+                    # Display the plot
+                    st.pyplot(fig)
+
+                # Display details of each clause
+                for clause in detected_clauses:
+                    if st.button(f"Show Explanation for {clause['clause']} Clause"):
+                        st.write(f"**Clause: {clause['clause']}**")
+                        st.write(f"Summary: {clause['summary']}\nExplanation: {clause['explanation']}")
+
+        with tabs[3]:
+            st.subheader("Detected Hidden Obligations or Dependencies")
+            hidden_obligations = detect_hidden_obligations_or_dependencies(document_text, summary)
+            if not hidden_obligations:
+                st.write("No hidden obligations or dependencies detected.")
+            else:
+                for item in hidden_obligations:
+                    st.write(f"**Phrase: {item['phrase']}**")
+                    st.write(f"Summary: {item['summary']}\nContext: {item['context']}")
+
+        with tabs[4]:
+            st.subheader("Risk Analysis & Visualization")
+            
+            detected_clauses = detect_key_clauses(document_text)
+            hidden_obligations = detect_hidden_obligations_or_dependencies(document_text, summary)
+            detected_risks = detect_risks(document_text, summary)
+
+            # Generate and display the pie chart
+            img_str = plot_risk_pie_chart(detected_clauses, hidden_obligations, detected_risks)
+            st.image(f"data:image/png;base64,{img_str}", use_column_width=True)
+
+            # Display the detected risks after the visualization
+            st.write("### Detected Risks:")
+            if detected_risks:
+                for risk in detected_risks:
+                    st.write(f"**{risk['phrase']}**: {risk['summary']}")
+
+            # Optionally, show other categories (Key Clauses, Hidden Obligations) after risks
+            st.write("### Detected Key Clauses:")
+            for clause in detected_clauses:
+                st.write(f"**{clause['clause']}**: {clause['explanation']}")
+
+            st.write("### Hidden Obligations or Dependencies:")
+            for obligation in hidden_obligations:
+                st.write(f"**{obligation['phrase']}**: {obligation['summary']}")
+
+            # Generate the full analysis document for download
+            analysis_doc = generate_analysis_document(document_text, summary, detected_clauses, hidden_obligations, detected_risks)
+
+            with st.expander("Download Analysis"):
+                output_path = "analysis_report.docx"
+                analysis_doc.save(output_path)
+
+                with open(output_path, "rb") as f:
+                    st.download_button("Download Analysis", data=f, file_name="analysis_report.docx", mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document")
+
+
+if __name__ == "__main__":
+    display_legal_analysis_page()

rag_pipeline.py

@@ -0,0 +1,76 @@
+import os
+from dotenv import load_dotenv
+from transformers import pipeline
+from PyPDF2 import PdfReader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores import FAISS
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.chains import RetrievalQA
+from langchain.llms import HuggingFaceHub
+
+# Load environment variables from .env file
+load_dotenv()
+
+def extract_text_from_pdf(pdf_file):
+    """Extracts text from a PDF file."""
+    reader = PdfReader(pdf_file)
+    text = ""
+    for page in reader.pages:
+        text += page.extract_text() or ""
+    return text
+
+def create_vector_store(text, embeddings_model="sentence-transformers/all-MiniLM-L6-v2"):
+    """Creates a FAISS vector store from the input text."""
+    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+    texts = text_splitter.split_text(text)
+    embeddings = HuggingFaceEmbeddings(model_name=embeddings_model)
+    return FAISS.from_texts(texts, embeddings)
+
+def create_qa_pipeline(vector_store, llm_model="EleutherAI/gpt-neo-2.7B"):
+    """Creates a Retrieval-based Question-Answering pipeline."""
+    
+    # Get the Hugging Face API token from the environment variable
+    huggingfacehub_api_token = os.getenv("HUGGINGFACEHUB_API_TOKEN")
+    
+    if huggingfacehub_api_token is None:
+        raise ValueError("HuggingFace Hub API token is missing! Please set the 'HUGGINGFACEHUB_API_TOKEN' in your .env file.")
+    
+    retriever = vector_store.as_retriever()
+
+    # Initialize Hugging Face LLM with the API token
+    llm = HuggingFaceHub(
+        repo_id=llm_model,  # specify the repo_id (e.g., gpt-neo-2.7B)
+        huggingfacehub_api_token=huggingfacehub_api_token, 
+        task="text-generation"  # specify the task (e.g., text-generation for language models)
+    )
+    
+    return RetrievalQA.from_chain_type(llm, retriever=retriever)
+
+def process_pdf_and_answer(pdf_path):
+    """Processes the PDF and returns answers to the text inside."""
+    
+    # Extract text from the PDF
+    text = extract_text_from_pdf(pdf_path)
+
+    # Create a FAISS vector store
+    vector_store = create_vector_store(text)
+
+    # Create a QA pipeline
+    qa_pipeline = create_qa_pipeline(vector_store)
+
+    # Answer the question
+    # Since you no longer need to ask a question manually, just extract some context
+    answer = qa_pipeline.run("Extract key information from the PDF.")  # Modify to get a summary or key data
+    return answer
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description="RAG Pipeline for PDF analysis")
+    parser.add_argument("--pdf", type=str, required=True, help="Path to the PDF file")
+    args = parser.parse_args()
+
+    pdf_path = args.pdf
+
+    # Process the PDF and get results
+    answer = process_pdf_and_answer(pdf_path)
+    print(f"Answer: {answer}")

requirements.txt

@@ -0,0 +1,17 @@
+streamlit==1.20.0
+groq==0.15.0
+PyPDF2==2.10.0
+python-docx==0.8.11
+tiktoken==0.3.0
+matplotlib==3.6.3
+concurrent.futures; python_version<'3.9'
+requests==2.28.1
+beautifulsoup4==4.11.1
+python-dotenv==0.21.0
+google-auth==2.16.0
+google-auth-oauthlib==0.5.3
+google-auth-httplib2==0.1.0
+google-api-python-client==2.80.0
+transformers==4.25.0
+langchain==0.0.154
+huggingface-hub==0.13.1