Update examples

app.py

@@ -62,33 +62,114 @@ def call_aml_endpoint(payload, url, api_key):
         logger.error(f"Error message: {error_message}")
         return {"error": error_message}
 
-def load_audio_from_url(url):
-    """Load audio from a URL using soundfile
+def improved_fetch_audio_from_url(url):
+    """Improved function to fetch audio data from URL and convert to base64
     Args:
         url (str): URL of the audio file
     Returns:
-        tuple: (sample_rate, audio_data) if successful, None otherwise
-        str: file path to the temporary saved audio file
+        tuple: (mime_type, base64_encoded_data) if successful, (None, None) otherwise
     """
     try:
         # Get the audio file from the URL
-        response = requests.get(url)
-        response.raise_for_status()  # Raise exception for bad status codes
+        logger.info(f"Fetching audio from URL: {url}")
         
-        # For other formats that soundfile supports directly (WAV, FLAC, etc.)
-        audio_data, sample_rate = sf.read(BytesIO(response.content))
+        # Use a session with increased timeout
+        session = requests.Session()
+        response = session.get(url, timeout=30)
+        response.raise_for_status()
         
-        # Save to a temporary file to be used by the chatbot
+        # Determine MIME type based on URL
         file_extension = os.path.splitext(url)[1].lower()
-        if not file_extension:
-            file_extension = '.wav'  # Default to .wav if no extension
-            
+        mime_type = None
+        
+        if file_extension == '.wav':
+            mime_type = "audio/wav"
+        elif file_extension == '.mp3':
+            mime_type = "audio/mpeg"
+        elif file_extension == '.flac':
+            mime_type = "audio/flac"
+        elif file_extension in ['.m4a', '.aac']:
+            mime_type = "audio/aac"
+        elif file_extension == '.ogg':
+            mime_type = "audio/ogg"
+        else:
+            # Try to detect the MIME type from headers
+            content_type = response.headers.get('Content-Type', '')
+            if content_type.startswith('audio/'):
+                mime_type = content_type
+            else:
+                mime_type = "audio/wav"  # Default to WAV
+        
+        logger.info(f"Detected MIME type: {mime_type}")
+        
+        # Save content to a temporary file for debugging
         temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=file_extension)
-        sf.write(temp_file.name, audio_data, sample_rate)
+        temp_file.write(response.content)
+        temp_file.close()
+        
+        logger.info(f"Saved audio to temporary file: {temp_file.name}")
+        
+        # Read the file to verify it's valid
+        try:
+            # For WAV files, try to read with soundfile to verify
+            if mime_type == "audio/wav":
+                data, samplerate = sf.read(temp_file.name)
+                logger.info(f"Successfully read audio file: {len(data)} samples, {samplerate}Hz")
+        except Exception as e:
+            logger.warning(f"Could not verify audio with soundfile: {e}")
+            # Continue anyway, the file might still be valid
+        
+        # Convert to base64
+        with open(temp_file.name, "rb") as f:
+            audio_content = f.read()
+            
+        base64_audio = base64.b64encode(audio_content).decode('utf-8')
+        logger.info(f"Successfully encoded audio to base64, length: {len(base64_audio)}")
+        
+        # Clean up temporary file
+        try:
+            os.unlink(temp_file.name)
+        except:
+            pass
+            
+        return mime_type, base64_audio
+    except Exception as e:
+        logger.error(f"Error fetching audio from URL: {e}", exc_info=True)
+        return None, None
+
+def fetch_image_from_url(url):
+    """Fetch image data from URL and convert to base64
+    Args:
+        url (str): URL of the image file
+    Returns:
+        tuple: (mime_type, base64_encoded_data) if successful, (None, None) otherwise
+    """
+    try:
+        # Get the image file from the URL
+        logger.info(f"Fetching image from URL: {url}")
+        response = requests.get(url)
+        response.raise_for_status()
         
-        return (sample_rate, audio_data), temp_file.name
+        # Determine MIME type based on URL
+        file_extension = os.path.splitext(url)[1].lower()
+        if file_extension in ['.jpg', '.jpeg']:
+            mime_type = "image/jpeg"
+        elif file_extension == '.png':
+            mime_type = "image/png"
+        elif file_extension == '.gif':
+            mime_type = "image/gif"
+        elif file_extension in ['.bmp', '.tiff', '.webp']:
+            mime_type = f"image/{file_extension[1:]}"
+        else:
+            mime_type = "image/jpeg"  # Default to JPEG
+        
+        # Convert to base64
+        base64_image = base64.b64encode(response.content).decode('utf-8')
+        
+        logger.info(f"Successfully fetched and encoded image, mime type: {mime_type}")
+        return mime_type, base64_image
     except Exception as e:
-        logger.error(f"Error loading audio from URL: {e}")
+        logger.error(f"Error fetching image from URL: {e}")
         return None, None
 
 def encode_base64_from_file(file_path):
@@ -185,6 +266,100 @@ def process_message(history, message, conversation_state):
     
     return history, gr.MultimodalTextbox(value=None, interactive=False), conversation_state
 
+def process_audio_example_direct(example_text, example_audio_url, history, conversation_state):
+    """Process an audio example directly from a URL."""
+    try:
+        logger.info(f"Processing audio example with text: {example_text}, URL: {example_audio_url}")
+        
+        # Initialize history and conversation_state if they're None
+        if history is None:
+            history = []
+        
+        if conversation_state is None:
+            conversation_state = []
+        
+        # Fetch audio and convert to base64 directly using improved function
+        mime_type, base64_audio = improved_fetch_audio_from_url(example_audio_url)
+        
+        if not mime_type or not base64_audio:
+            error_msg = f"Failed to load audio from {example_audio_url}"
+            logger.error(error_msg)
+            history.append({"role": "user", "content": f"{example_text} (Audio URL: {example_audio_url})"})
+            history.append({"role": "assistant", "content": f"Error: {error_msg}"})
+            return history, conversation_state
+        
+        logger.info(f"Successfully loaded audio, mime type: {mime_type}, base64 length: {len(base64_audio)}")
+        
+        # Add text message to history for display
+        history.append({"role": "user", "content": example_text})
+        
+        # Add to conversation state
+        content_items = [
+            {"type": "text", "text": example_text},
+            {"type": "audio_url", "audio_url": {"url": f"data:{mime_type};base64,{base64_audio}"}}
+        ]
+        
+        conversation_state.append({
+            "role": "user",
+            "content": content_items
+        })
+        
+        logger.info("Successfully prepared conversation state, now generating response")
+        
+        # Generate bot response
+        return bot_response(history, conversation_state)
+    except Exception as e:
+        logger.error(f"Error processing audio example: {e}", exc_info=True)
+        if history is None:
+            history = []
+        history.append({"role": "user", "content": f"{example_text} (Audio URL: {example_audio_url})"})
+        history.append({"role": "assistant", "content": f"Error: {str(e)}"})
+        return history, conversation_state
+
+def process_image_example_direct(example_text, example_image_url, history, conversation_state):
+    """Process an image example directly from a URL."""
+    try:
+        # Initialize history and conversation_state if they're None
+        if history is None:
+            history = []
+        
+        if conversation_state is None:
+            conversation_state = []
+            
+        # Fetch image and convert to base64 directly
+        mime_type, base64_image = fetch_image_from_url(example_image_url)
+        
+        if not mime_type or not base64_image:
+            error_msg = f"Failed to load image from {example_image_url}"
+            logger.error(error_msg)
+            history.append({"role": "user", "content": f"{example_text} (Image URL: {example_image_url})"})
+            history.append({"role": "assistant", "content": f"Error: {error_msg}"})
+            return history, conversation_state
+        
+        # Add text message to history for display
+        history.append({"role": "user", "content": example_text})
+        
+        # Add to conversation state
+        content_items = [
+            {"type": "text", "text": example_text},
+            {"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{base64_image}"}}
+        ]
+        
+        conversation_state.append({
+            "role": "user",
+            "content": content_items
+        })
+        
+        # Generate bot response
+        return bot_response(history, conversation_state)
+    except Exception as e:
+        logger.error(f"Error processing image example: {e}", exc_info=True)
+        if history is None:
+            history = []
+        history.append({"role": "user", "content": f"{example_text} (Image URL: {example_image_url})"})
+        history.append({"role": "assistant", "content": f"Error: {str(e)}"})
+        return history, conversation_state
+
 def bot_response(history, conversation_state):
     """Generate bot response based on conversation state."""
     if not conversation_state:
@@ -252,11 +427,57 @@ def bot_response(history, conversation_state):
     
     return history, conversation_state
 
+def enable_input():
+    """Re-enable the input box after bot responds."""
+    return gr.MultimodalTextbox(interactive=True)
+
+def update_debug(conversation_state):
+    """Update debug output with the last payload that would be sent."""
+    if not conversation_state:
+        return {}
+    
+    # Create a payload from the conversation
+    payload = {
+        "input_data": {
+            "input_string": conversation_state
+        }
+    }
+    
+    # Remove base64 data to avoid cluttering the UI
+    sanitized_payload = json.loads(json.dumps(payload))
+    for item in sanitized_payload["input_data"]["input_string"]:
+        if "content" in item and isinstance(item["content"], list):
+            for content_item in item["content"]:
+                if "image_url" in content_item:
+                    parts = content_item["image_url"]["url"].split(",")
+                    if len(parts) > 1:
+                        content_item["image_url"]["url"] = parts[0] + ",[BASE64_DATA_REMOVED]"
+                if "audio_url" in content_item:
+                    parts = content_item["audio_url"]["url"].split(",")
+                    if len(parts) > 1:
+                        content_item["audio_url"]["url"] = parts[0] + ",[BASE64_DATA_REMOVED]"
+    
+    return sanitized_payload
+
+# Add this near the beginning of your Blocks definition, before you define your components
+css = """
+#small-audio audio {
+    height: 20px !important;
+    width: 100px !important;
+}
+#small-audio .wrap {
+    max-width: 220px !important;
+}
+#small-audio .audio-container {
+    min-height: 0px !important;
+}
+"""
+
 # Create Gradio demo
-with gr.Blocks(theme=gr.themes.Soft()) as demo:
-    title = gr.Markdown("# Azure ML Multimodal Chatbot Demo")
+with gr.Blocks(theme=gr.themes.Soft(), css=css) as demo:
+    title = gr.Markdown("# Phi-4-Multimodal Playground")
     description = gr.Markdown("""
-    This demo allows you to interact with a multimodal AI model through Azure ML.
+    This demo allows you to interact with the [Phi-4-Multimodal AI model](https://aka.ms/phi-4-multimodal/techreport).
     You can type messages, upload images, or record audio to communicate with the AI.
     """)
     
@@ -264,7 +485,7 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
     conversation_state = gr.State([])
     
     with gr.Row():
-        with gr.Column(scale=4):
+        with gr.Column(scale=2):
             chatbot = gr.Chatbot(
                 type="messages",
                 avatar_images=(None, "https://upload.wikimedia.org/wikipedia/commons/d/d3/Phi-integrated-information-symbol.png",),
@@ -283,131 +504,107 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
                 clear_btn = gr.ClearButton([chatbot, chat_input], value="Clear conversation")
                 clear_btn.click(lambda: [], None, conversation_state)  # Also clear the conversation state
                 gr.HTML("<div style='text-align: right; margin-top: 5px;'><small>Powered by Azure ML</small></div>")
-                
-            # Define function to handle example submission directly
-            def handle_example_submission(text, files, history, conv_state):
-                """
-                Process an example submission directly including bot response
-                This bypasses the regular chat_input.submit flow
-                """
-                # Create a message object similar to what would be submitted by the user
-                message = {"text": text, "files": files if files else []}
-                
-                # Use the same processing function as normal submissions
-                new_history, _, new_conv_state = process_message(history, message, conv_state)
-                
-                # Then immediately trigger the bot response
-                final_history, final_conv_state = bot_response(new_history, new_conv_state)
-                
-                # Re-enable the input box
-                chat_input.update(interactive=True)
-                
-                # Return everything needed
-                return final_history, final_conv_state
         
         with gr.Column(scale=1):
             gr.Markdown("### Examples")
             
             with gr.Tab("Text Only"):
-                # For text examples, just submit them directly
-                def run_text_example(example_text, history, conv_state):
-                    # Process the example directly
-                    return handle_example_submission(example_text, [], history, conv_state)
-                
                 text_examples = gr.Examples(
                     examples=[
                         ["Tell me about Microsoft Azure cloud services."],
                         ["What can you help me with today?"],
                         ["Explain the difference between AI and machine learning."],
                     ],
-                    inputs=[gr.Textbox(visible=False)],
-                    outputs=[chatbot, conversation_state],
-                    fn=lambda text, h=chatbot, c=conversation_state: run_text_example(text, h, c),
-                    label="Text Examples (Click to run the example)"
+                    inputs=chat_input,
+                    label="Text Examples"
                 )
             
             with gr.Tab("Text & Audio"):
-                # Function to handle loading both text and audio from URL and sending directly
-                def run_audio_example(example_text, example_audio_url, history, conv_state):
-                    try:
-                        # Download and process the audio from URL
-                        print(f"Downloading audio from: {example_audio_url}")
-                        response = requests.get(example_audio_url)
-                        response.raise_for_status()
-                        
-                        # Save to a temporary file
-                        file_extension = os.path.splitext(example_audio_url)[1].lower()
-                        if not file_extension:
-                            file_extension = '.wav'  # Default to .wav if no extension
-                            
-                        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=file_extension)
-                        temp_file.write(response.content)
-                        temp_file.close()
-                        
-                        print(f"Saved audio to temporary file: {temp_file.name}")
-                        
-                        # Process the example directly
-                        return handle_example_submission(example_text, [temp_file.name], history, conv_state)
-                    except Exception as e:
-                        print(f"Error processing audio example: {e}")
-                        # If an error occurs, just add the text to history
-                        history.append({"role": "user", "content": f"{example_text} (Error loading audio: {e})"})
-                        return history, conv_state
+                gr.Markdown("### Audio Examples")
                 
-                audio_examples = gr.Examples(
-                    examples=[
-                        ["Transcribe this audio clip", "https://diamondfan.github.io/audio_files/english.weekend.plan.wav"],
-                        ["What language is being spoken in this recording?", "https://www2.cs.uic.edu/~i101/SoundFiles/BabyElephantWalk60.wav"],
-                    ],
-                    inputs=[
-                        gr.Textbox(visible=False),
-                        gr.Textbox(visible=False)
-                    ],
-                    outputs=[chatbot, conversation_state],
-                    fn=lambda text, url, h=chatbot, c=conversation_state: run_audio_example(text, url, h, c),
-                    label="Audio Examples (Click to run the example)"
+                # Example 1
+                gr.Markdown("**Example 1: Transcribe this audio clip**")
+                gr.Audio("https://diamondfan.github.io/audio_files/english.weekend.plan.wav", 
+                         label="Preview", elem_id="small-audio")
+
+                example1_btn = gr.Button("Run it", size="sm")
+                    
+                gr.Markdown("-----")
+                
+                # Example 2
+                gr.Markdown("**Example 2: Translate audio transcription to English**")
+                gr.Audio("https://diamondfan.github.io/audio_files/japanese.seattle.trip.report.wav", 
+                         label="Preview", elem_id="small-audio")
+                example2_btn = gr.Button("Run it", size="sm")
+                
+                # Define handlers for audio examples
+                def run_audio_example1():
+                    return process_audio_example_direct(
+                        "Transcribe this audio clip", 
+                        "https://diamondfan.github.io/audio_files/english.weekend.plan.wav",
+                        [], []
+                    )
+                
+                def run_audio_example2():
+                    return process_audio_example_direct(
+                        "Translate audio transcription to English", 
+                        "https://diamondfan.github.io/audio_files/japanese.seattle.trip.report.wav",
+                        [], []
+                    )
+                
+                # Connect buttons to handlers
+                example1_btn.click(
+                    run_audio_example1,
+                    inputs=[],
+                    outputs=[chatbot, conversation_state]
+                )
+                
+                example2_btn.click(
+                    run_audio_example2,
+                    inputs=[],
+                    outputs=[chatbot, conversation_state]
                 )
             
             with gr.Tab("Text & Image"):
-                # Function to handle loading both text and image from URL and sending directly
-                def run_image_example(example_text, example_image_url, history, conv_state):
-                    try:
-                        # Download the image from URL
-                        print(f"Downloading image from: {example_image_url}")
-                        response = requests.get(example_image_url)
-                        response.raise_for_status()
-                        
-                        # Save to a temporary file
-                        file_extension = os.path.splitext(example_image_url)[1].lower()
-                        if not file_extension:
-                            file_extension = '.jpg'  # Default to .jpg if no extension
-                            
-                        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=file_extension)
-                        temp_file.write(response.content)
-                        temp_file.close()
-                        
-                        print(f"Saved image to temporary file: {temp_file.name}")
-                        
-                        # Process the example directly
-                        return handle_example_submission(example_text, [temp_file.name], history, conv_state)
-                    except Exception as e:
-                        print(f"Error processing image example: {e}")
-                        # If an error occurs, just add the text to history
-                        history.append({"role": "user", "content": f"{example_text} (Error loading image: {e})"})
-                        return history, conv_state
+                gr.Markdown("### Image Examples")
                 
-                image_examples = gr.Examples(
-                    examples=[
-                        ["What's in this image?", "https://storage.googleapis.com/demo-image/dog.jpg"],
-                        ["Describe this chart", "https://matplotlib.org/stable/_images/sphx_glr_bar_stacked_001.png"],
-                    ],
-                    inputs=[
-                        gr.Textbox(visible=False),
-                        gr.Textbox(visible=False)
-                    ],
-                    outputs=[chatbot, conversation_state],
-                    fn=lambda text, url, h=chatbot, c=conversation_state: run_image_example(text, url, h, c),
-                    label="Image Examples (Click to run the example)"
+                # Example 1
+                gr.Markdown("**Example 1: What's in this image?**")
+                gr.Image("https://upload.wikimedia.org/wikipedia/commons/thumb/3/31/Hanoi_Temple_of_Literature.jpg/640px-Hanoi_Temple_of_Literature.jpg", label="Preview")
+                img_example1_btn = gr.Button("Run it", size="sm")
+                
+                # Example 2
+                gr.Markdown("**Example 2: Describe this chart**")
+                gr.Image("https://upload.wikimedia.org/wikipedia/commons/thumb/0/0a/Places_to_visit_in_Vietnam_-_SOTC.jpg/640px-Places_to_visit_in_Vietnam_-_SOTC.jpg", label="Preview")
+                img_example2_btn = gr.Button("Run it", size="sm")
+                
+                # Define handlers for image examples
+                def run_image_example1():
+                    return process_image_example_direct(
+                        "What's in this image?", 
+                        "https://upload.wikimedia.org/wikipedia/commons/thumb/3/31/Hanoi_Temple_of_Literature.jpg/640px-Hanoi_Temple_of_Literature.jpg",
+                        [], #chatbot.value, 
+                        [], #conversation_state.value
+                    )
+                
+                def run_image_example2():
+                    return process_image_example_direct(
+                        "Describe this chart", 
+                        "https://upload.wikimedia.org/wikipedia/commons/thumb/0/0a/Places_to_visit_in_Vietnam_-_SOTC.jpg/640px-Places_to_visit_in_Vietnam_-_SOTC.jpg",
+                        [], []
+                    )
+                
+                # Connect buttons to handlers
+                img_example1_btn.click(
+                    run_image_example1,
+                    inputs=[],
+                    outputs=[chatbot, conversation_state]
+                )
+                
+                img_example2_btn.click(
+                    run_image_example2,
+                    inputs=[],
+                    outputs=[chatbot, conversation_state]
                 )
             
             gr.Markdown("### Instructions")
@@ -415,6 +612,7 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
             - Type a question or statement
             - Upload images or audio files
             - You can combine text with media files
+            - Support 2 modalities at the same time
             - The model can analyze images and transcribe audio
             - For best results with images, use JPG or PNG files
             - For audio, use WAV, MP3, or FLAC files
@@ -425,7 +623,7 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
             This chatbot can:
             - Answer questions and provide explanations
             - Describe and analyze images
-            - Transcribe and analyze audio content 
+            - Transcribe, translate, summarize, and analyze audio content 
             - Process multiple inputs in the same message
             - Maintain context throughout the conversation
             """)
@@ -436,38 +634,6 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
                     value={}
                 )
     
-    def update_debug(conversation_state):
-        """Update debug output with the last payload that would be sent."""
-        if not conversation_state:
-            return {}
-        
-        # Create a payload from the conversation
-        payload = {
-            "input_data": {
-                "input_string": conversation_state
-            }
-        }
-        
-        # Remove base64 data to avoid cluttering the UI
-        sanitized_payload = json.loads(json.dumps(payload))
-        for item in sanitized_payload["input_data"]["input_string"]:
-            if "content" in item and isinstance(item["content"], list):
-                for content_item in item["content"]:
-                    if "image_url" in content_item:
-                        parts = content_item["image_url"]["url"].split(",")
-                        if len(parts) > 1:
-                            content_item["image_url"]["url"] = parts[0] + ",[BASE64_DATA_REMOVED]"
-                    if "audio_url" in content_item:
-                        parts = content_item["audio_url"]["url"].split(",")
-                        if len(parts) > 1:
-                            content_item["audio_url"]["url"] = parts[0] + ",[BASE64_DATA_REMOVED]"
-        
-        return sanitized_payload
-    
-    def enable_input():
-        """Re-enable the input box after bot responds."""
-        return gr.MultimodalTextbox(interactive=True)
-    
     # Set up event handlers
     msg_submit = chat_input.submit(
         process_message, [chatbot, chat_input, conversation_state], [chatbot, chat_input, conversation_state], queue=False
@@ -478,10 +644,8 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
     )
     
     msg_response.then(enable_input, None, chat_input)
-    # btn_response.then(enable_input, None, chat_input)
     
     # Update debug info
-    # msg_response.then(update_debug, conversation_state, debug_output)
-    # btn_response.then(update_debug, conversation_state, debug_output)
+    msg_response.then(update_debug, conversation_state, debug_output)
 
 demo.launch(share=True, debug=True)