Trying to make it multimodal

app.py

@@ -6,6 +6,7 @@ import uuid
 from gtts import gTTS
 import google.generativeai as genai
 from io import BytesIO
+import PyPDF2
 
 # Set your API key
 api_key = "AIzaSyAHD0FwX-Ds6Y3eI-i5Oz7IdbJqR6rN7pg"  # Replace with your actual API key
@@ -14,7 +15,7 @@ genai.configure(api_key=api_key)
 # Configure the generative AI model
 generation_config = genai.GenerationConfig(
     temperature=0.9,
-    max_output_tokens=3000
+    max_output_tokens=4000
 )
 
 # Safety settings configuration
@@ -48,6 +49,9 @@ st.title("Gemini Chatbot")
 # Model Selection Dropdown
 selected_model = st.selectbox("Select a Gemini 1.5 model:", ["gemini-1.5-flash-latest", "gemini-1.5-pro-latest"])
 
+# TTS Option Checkbox
+enable_tts = st.checkbox("Enable Text-to-Speech")
+
 # Helper functions for image processing and chat history management
 def get_image_base64(image):
     image = image.convert("RGB")
@@ -67,12 +71,22 @@ def display_chat_history():
         if 'text' in parts:
             st.markdown(f"{role.title()}: {parts['text']}")
         elif 'data' in parts:
-            st.image(Image.open(io.BytesIO(base64.b64decode(parts['data']))), caption='Uploaded Image')
+            mime_type = parts.get('mime_type', '')
+            if mime_type.startswith('image'):
+                st.image(Image.open(io.BytesIO(base64.b64decode(parts['data']))), caption='Uploaded Image')
+            elif mime_type == 'application/pdf':
+                st.write("PDF Content:") 
+                pdf_reader = PyPDF2.PdfReader(io.BytesIO(base64.b64decode(parts['data'])))
+                for page_num in range(len(pdf_reader.pages)):
+                    page = pdf_reader.pages[page_num]
+                    st.write(page.extract_text())
+            elif mime_type.startswith('video'):
+                st.video(io.BytesIO(base64.b64decode(parts['data'])))
 
 def get_chat_history_str():
     chat_history_str = "\n".join(
         f"{entry['role'].title()}: {part['text']}" if 'text' in part
-        else f"{entry['role'].title()}: (Image)"
+        else f"{entry['role'].title()}: (File: {part.get('mime_type', '')})"
         for entry in st.session_state['chat_history']
         for part in entry['parts']
     )
@@ -91,43 +105,44 @@ def send_message():
             if 'text' in part:
                 prompts.append(part['text'])
             elif 'data' in part:
-                # Add the image in base64 format to prompt_parts for vision model
-                prompt_parts.append({"data": part['data'], "mime_type": "image/jpeg"})
-                prompts.append("[Image]")
+                prompts.append(f"(File: {part.get('mime_type', '')})")
+                prompt_parts.append(part)  # Add the entire part
 
     # Append the user input to the prompts list
     if user_input:
         prompts.append(user_input)
         st.session_state['chat_history'].append({"role": "user", "parts": [{"text": user_input}]})
-        # Also add the user text input to prompt_parts
         prompt_parts.append({"text": user_input})
 
     # Handle uploaded files
     if uploaded_files:
         for uploaded_file in uploaded_files:
-            base64_image = get_image_base64(Image.open(uploaded_file))
-            prompts.append("[Image]")
-            prompt_parts.append({"data": base64_image, "mime_type": "image/jpeg"})
+            file_content = uploaded_file.read()
+            base64_data = base64.b64encode(file_content).decode()
+            prompts.append(f"(File: {uploaded_file.type})")
+            prompt_parts.append({
+                "mime_type": uploaded_file.type,
+                "data": base64_data
+            })
             st.session_state['chat_history'].append({
                 "role": "user",
-                "parts": [{"mime_type": uploaded_file.type, "data": base64_image}]
+                "parts": [{"mime_type": uploaded_file.type, "data": base64_data}]
             })
 
-    # Determine if vision model should be used 
+    # Determine if vision model should be used
     use_vision_model = any(part.get('mime_type') == 'image/jpeg' for part in prompt_parts)
 
-    # Use the selected model 
+    # Use the selected model
     model_name = selected_model
     if use_vision_model and "pro" not in model_name:
         st.warning(f"The selected model ({model_name}) does not support image inputs. Choose a 'pro' model for image capabilities.")
         return
-    
+
     model = genai.GenerativeModel(
         model_name=model_name,
         generation_config=generation_config,
         safety_settings=safety_settings
     )
-
     chat_history_str = "\n".join(prompts)
 
     if use_vision_model:
@@ -142,14 +157,15 @@ def send_message():
 
     # After generating the response from the model, append it to the chat history
     if response_text:
-        st.session_state['chat_history'].append({"role": "model", "parts":[{"text": response_text}]})
+        st.session_state['chat_history'].append({"role": "model", "parts": [{"text": response_text}]})
 
-        # Convert the response text to speech
-        tts = gTTS(text=response_text, lang='en')
-        tts_file = BytesIO()
-        tts.write_to_fp(tts_file)
-        tts_file.seek(0)
-        st.audio(tts_file, format='audio/mp3')
+        # Convert the response text to speech if enabled
+        if enable_tts:
+            tts = gTTS(text=response_text, lang='en')
+            tts_file = BytesIO()
+            tts.write_to_fp(tts_file)
+            tts_file.seek(0)
+            st.audio(tts_file, format='audio/mp3')
 
     # Clear the input fields after sending the message
     st.session_state.user_input = ''
@@ -168,8 +184,8 @@ user_input = st.text_area(
 
 # File uploader for images
 uploaded_files = st.file_uploader(
-    "Upload images:",
-    type=["png", "jpg", "jpeg"],
+    "Upload files:",
+    type=["png", "jpg", "jpeg", "mp4", "pdf"],  # Added mp4 and pdf
     accept_multiple_files=True,
     key=st.session_state.file_uploader_key
 )