Upload 9 files

README.md

@@ -1,12 +1,94 @@
 ---
-title: Voice Activated RAG System
-emoji: 🐢
-colorFrom: blue
-colorTo: purple
+title: Voice-Activated RAG System
+emoji: 🗣️
+colorFrom: indigo
+colorTo: blue
 sdk: gradio
-sdk_version: 5.36.2
-app_file: app.py
+sdk_version: 5.34.2
+app_file: main.py
 pinned: false
 ---
+# Voice to LLM & Sentiment Analyzer with RAG
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## Overview
+This project is a voice-driven application that integrates various machine learning models and APIs to provide sentiment analysis and context retrieval from uploaded PDF documents. It utilizes voice input to query the system, processes the audio using Whisper for transcription, retrieves relevant context from the uploaded PDFs using a Retrieval-Augmented Generation (RAG) approach, and provides a spoken audio output of the LLM response using Gemini TTS (Google GenAI API).
+
+## Features
+- Voice input for user queries
+- PDF document processing and context retrieval
+- Sentiment analysis using a pre-trained emotion classification model
+- Text-to-Speech output for the model's responses (using Gemini TTS)
+- Integration with the Groq API for advanced language model capabilities
+
+## Modular Structure
+
+This project is organized into modular components for maintainability and clarity:
+
+- `main.py`: Entry point to run the app.
+- `ui.py`: Gradio UI layout and event wiring.
+- `tts_gemini.py`: Gemini TTS logic (text-to-speech).
+- `emotion.py`: Emotion detection and tone mapping.
+- `rag.py`: PDF processing, chunking, embedding, FAISS, and context retrieval.
+- `llm.py`: LLM prompt construction and response logic.
+
+## Setup Instructions
+1. Clone the repository:
+   ```
+   git clone <repository-url>
+   cd GoComet-C4
+   ```
+
+2. Run the setup script to create a virtual environment and install dependencies (Linux/macOS):
+   ```
+   bash setup.sh
+   ```
+   On Windows, run each command in `setup.sh` manually in your terminal:
+   ```
+   python -m venv venv
+   venv\Scripts\activate
+   pip install -r requirements.txt
+   set GROQ_API_KEY="<your-groq-api-key>"
+   set GEMINI_API_KEY="<your-gemini-api-key>"
+   ```
+
+3. Activate the virtual environment:
+   - Linux/macOS:
+     ```
+     source venv/bin/activate
+     ```
+   - Windows:
+     ```
+     venv\Scripts\activate
+     ```
+
+## Usage
+To run the application, execute the following command:
+```
+python main.py
+```
+
+Once the application is running, you can upload PDF files and use the microphone to speak your queries. The application will process the audio, retrieve context from the PDFs, analyze sentiment, and provide the LLM output, sentiment, transcript, context, and a spoken audio response (.wav) in the interface. TTS audio files will be saved in a `tts_outputs` directory in your project root.
+
+## Dependencies
+This project requires the following Python libraries:
+- gradio
+- whisper
+- groq
+- transformers
+- PyPDF2
+- sentence-transformers
+- faiss-cpu
+- soundfile
+- numpy
+- google-genai
+
+Install these dependencies using the `requirements.txt` file provided in the project.
+
+## Latency Logging
+
+After each run, the latency (processing time in seconds) for each pipeline component is logged in `logs/latency_log.csv`:
+
+| Whisper STT (s) | Document Retrieval (s) | Sentiment Analysis (s) | Response Gen (LLM) (s) | TTS Synthesis (s) | Total (s) |
+|-----------------|-----------------------|-----------------------|------------------------|-------------------|-----------|
+
+This file accumulates results from all runs, allowing you to analyze and monitor performance over time.

emotion.py

@@ -0,0 +1,27 @@
+from transformers import pipeline
+
+# Emotion to tone instruction mapping
+emotion_tone_map = {
+    "Sadness": "Be comforting, empathetic, and gentle.",
+    "Anger": "Stay calm, respectful, and de-escalate.",
+    "Love": "Be warm, appreciative, and encouraging.",
+    "Surprise": "Be affirming and help clarify what's surprising.",
+    "Fear": "Be reassuring and emphasize safety/facts.",
+    "Happiness": "Be enthusiastic and congratulatory.",
+    "Neutral": "Be informative and straightforward.",
+    "Disgust": "Be clinical, non-judgmental, and clarify facts.",
+    "Shame": "Be kind, avoid blame, and uplift the user.",
+    "Guilt": "Be compassionate and reduce self-blame.",
+    "Confusion": "Be extra clear and explain step-by-step.",
+    "Desire": "Be supportive and help guide constructively.",
+    "Sarcasm": "Stay serious, clarify misunderstandings politely.",
+}
+
+emotion_classifier = pipeline("text-classification", model="boltuix/bert-emotion")
+
+def get_emotion_and_tone(text):
+    emotions = emotion_classifier(text)
+    detected_emotion = emotions[0]["label"].capitalize() if emotions else "Neutral"
+    emotion = detected_emotion if detected_emotion in emotion_tone_map else "Neutral"
+    tone_instruction = emotion_tone_map.get(emotion, "Be informative and polite.")
+    return emotion, tone_instruction 

gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

llm.py

@@ -0,0 +1,32 @@
+import os
+from groq import Groq
+
+GROQ_API_KEY = os.getenv("GROQ_API_KEY")
+if not GROQ_API_KEY:
+    raise RuntimeError("GROQ_API_KEY environment variable not set. Please set it to your Groq API key.")
+
+groq_client = Groq(api_key=GROQ_API_KEY)
+
+def get_llm_response(user_text, context, emotion, tone_instruction):
+    prompt = f"""
+You are a helpful and emotionally aware assistant.
+The user's emotional state is: {emotion}.
+
+{tone_instruction}
+
+Using the following context, answer the user's question:
+---
+{context}
+---
+Question: {user_text}
+"""
+    completion = groq_client.chat.completions.create(
+        model="meta-llama/llama-4-scout-17b-16e-instruct",
+        messages=[{"role": "user", "content": prompt}],
+        temperature=1,
+        max_completion_tokens=1024,
+        top_p=1,
+        stream=False,
+        stop=None,
+    )
+    return completion.choices[0].message.content 

main.py

@@ -0,0 +1,7 @@
+from ui import demo
+
+def main():
+    demo.launch()
+
+if __name__ == "__main__":
+    main() 

rag.py

@@ -0,0 +1,55 @@
+import os
+import PyPDF2
+from sentence_transformers import SentenceTransformer
+import faiss
+import numpy as np
+from rapidfuzz import fuzz
+
+embedder = SentenceTransformer("all-MiniLM-L6-v2")
+faiss_index = None
+pdf_chunks = []
+chunk_texts = []
+
+def process_pdfs(pdf_files):
+    global faiss_index, pdf_chunks, chunk_texts
+    all_text = ""
+    chunk_texts = []
+    for pdf_file in pdf_files:
+        reader = PyPDF2.PdfReader(pdf_file.name)
+        for page in reader.pages:
+            all_text += page.extract_text() + "\n"
+    chunk_size = 500
+    pdf_chunks = [all_text[i:i+chunk_size] for i in range(0, len(all_text), chunk_size)]
+    chunk_texts = pdf_chunks
+    embeddings = embedder.encode(pdf_chunks, convert_to_numpy=True)
+    dim = embeddings.shape[1]
+    faiss_index = faiss.IndexFlatL2(dim)
+    faiss_index.add(embeddings)
+    return f"Processed {len(pdf_chunks)} chunks from {len(pdf_files)} PDF(s)."
+
+def semantic_search(query, top_k=3):
+    global faiss_index, chunk_texts
+    if faiss_index is None or not chunk_texts:
+        return []
+    query_emb = embedder.encode([query], convert_to_numpy=True)
+    D, I = faiss_index.search(query_emb, top_k)
+    return [chunk_texts[i] for i in I[0] if i < len(chunk_texts)]
+
+def keyword_search(query, top_k=3):
+    global chunk_texts
+    if not chunk_texts:
+        return []
+    scored = [(chunk, fuzz.partial_ratio(query.lower(), chunk.lower())) for chunk in chunk_texts]
+    scored = sorted(scored, key=lambda x: x[1], reverse=True)
+    return [chunk for chunk, score in scored[:top_k]]
+
+def retrieve_context(query, top_k=3):
+    semantic_results = semantic_search(query, top_k)
+    keyword_results = keyword_search(query, top_k)
+    combined = []
+    seen = set()
+    for chunk in semantic_results + keyword_results:
+        if chunk not in seen:
+            combined.append(chunk)
+            seen.add(chunk)
+    return "\n".join(combined) 

requirements.txt

@@ -0,0 +1,11 @@
+gradio
+groq
+transformers
+PyPDF2
+sentence-transformers
+faiss-cpu
+soundfile
+numpy
+google-genai
+rapidfuzz
+openai-whisper

tts_gemini.py

@@ -0,0 +1,41 @@
+import os
+import wave
+import uuid
+from google import genai
+from google.genai import types
+
+GEMINI_API_KEY = os.getenv("GEMINI_API_KEY")
+
+def wave_file(filename, pcm, channels=1, rate=24000, sample_width=2):
+    with wave.open(filename, "wb") as wf:
+        wf.setnchannels(channels)
+        wf.setsampwidth(sample_width)
+        wf.setframerate(rate)
+        wf.writeframes(pcm)
+
+def tts_gemini(text, api_key=GEMINI_API_KEY):
+    output_dir = os.path.join(os.getcwd(), "tts_outputs")
+    os.makedirs(output_dir, exist_ok=True)
+    file_name = os.path.join(output_dir, f"tts_{uuid.uuid4().hex}.wav")
+    try:
+        client = genai.Client(api_key=api_key)
+        response = client.models.generate_content(
+            model="gemini-2.5-flash-preview-tts",
+            contents=text,
+            config=types.GenerateContentConfig(
+                response_modalities=["AUDIO"],
+                speech_config=types.SpeechConfig(
+                    voice_config=types.VoiceConfig(
+                        prebuilt_voice_config=types.PrebuiltVoiceConfig(
+                            voice_name='Kore',
+                        )
+                    )
+                ),
+            )
+        )
+        data = response.candidates[0].content.parts[0].inline_data.data
+        wave_file(file_name, data)
+        return file_name
+    except Exception as e:
+        print(f"[Gemini TTS ERROR] {e}")
+        return None 

ui.py

@@ -0,0 +1,83 @@
+import gradio as gr
+from rag import process_pdfs, retrieve_context
+from emotion import get_emotion_and_tone
+from llm import get_llm_response
+from tts_gemini import tts_gemini
+import whisper
+import time
+import csv
+import os
+
+# Load Whisper model once
+whisper_model = whisper.load_model("base")
+
+LOG_DIR = "logs"
+LOG_FILE = os.path.join(LOG_DIR, "latency_log.csv")
+
+# Ensure log directory and CSV header
+os.makedirs(LOG_DIR, exist_ok=True)
+if not os.path.exists(LOG_FILE):
+    with open(LOG_FILE, mode="w", newline="") as f:
+        writer = csv.writer(f)
+        writer.writerow([
+            "Whisper STT (s)", "Document Retrieval (s)", "Sentiment Analysis (s)", "Response Gen (LLM) (s)", "TTS Synthesis (s)", "Total (s)"
+        ])
+
+def process_audio_with_rag(audio):
+    t0 = time.time()
+    stt_start = time.time()
+    result = whisper_model.transcribe(audio)
+    text = result["text"]
+    stt_end = time.time()
+    retrieval_start = time.time()
+    context = retrieve_context(text)
+    retrieval_end = time.time()
+    sentiment_start = time.time()
+    emotion, tone_instruction = get_emotion_and_tone(text)
+    sentiment_end = time.time()
+    llm_start = time.time()
+    llm_output = get_llm_response(text, context, emotion, tone_instruction)
+    llm_end = time.time()
+    tts_start = time.time()
+    tts_path = tts_gemini(llm_output)
+    tts_end = time.time()
+    t1 = time.time()
+    stt_latency = stt_end - stt_start
+    retrieval_latency = retrieval_end - retrieval_start
+    sentiment_latency = sentiment_end - sentiment_start
+    llm_latency = llm_end - llm_start
+    tts_latency = tts_end - tts_start
+    total_latency = t1 - t0
+    # Log to CSV (latency only)
+    with open(LOG_FILE, mode="a", newline="") as f:
+        writer = csv.writer(f)
+        writer.writerow([
+            f"{stt_latency:.3f}",
+            f"{retrieval_latency:.3f}",
+            f"{sentiment_latency:.3f}",
+            f"{llm_latency:.3f}",
+            f"{tts_latency:.3f}",
+            f"{total_latency:.3f}"
+        ])
+    return llm_output, emotion, text, context, tts_path
+
+demo = gr.Blocks()
+with demo:
+    gr.Markdown("""
+    # Voice to LLM & Sentiment Analyzer with RAG (PDF Upload & Gemini TTS)
+    """)
+    with gr.Row():
+        pdf_input = gr.Files(label="Upload PDF(s)", type="filepath")
+        pdf_status = gr.Textbox(label="PDF Processing Status")
+    pdf_input.upload(process_pdfs, inputs=pdf_input, outputs=pdf_status)
+    audio_input = gr.Audio(sources=["microphone"], type="filepath", label="Speak your query")
+    llm_output = gr.Textbox(label="LLM Output")
+    sentiment_output = gr.Textbox(label="Sentiment")
+    transcript_output = gr.Textbox(label="Transcribed Text")
+    context_output = gr.Textbox(label="Retrieved Context from PDFs")
+    tts_output = gr.Audio(label="LLM Output (Gemini TTS)")
+    audio_input.change(
+        process_audio_with_rag,
+        inputs=audio_input,
+        outputs=[llm_output, sentiment_output, transcript_output, context_output, tts_output]
+    )