Merge pull request #6 from jdalfons/develop

.github/workflows/check_file_size.yml

@@ -0,0 +1,16 @@
+name: Check file size
+on:               # or directly `on: [push]` to run the action on every push on any branch
+  pull_request:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check large files
+        uses: ActionsDesk/[email protected]
+        with:
+          filesizelimit: 10485760 # this is 10MB so we can sync to HF Spaces

.github/workflows/sync_hf.yml

@@ -0,0 +1,20 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push https://HF_USERNAME:[email protected]/spaces/jdalfonso/SISE-ULTIMATE-CHALLENGE main

.gitignore

@@ -2,6 +2,7 @@
 __pycache__/
 *.py[cod]
 *$py.class
+.idea/
 
 # C extensions
 *.so
@@ -178,6 +179,10 @@ dataset/
 old/
 *.wav
 data/*
-
+*.pth
+old/
 # Mac
 .DS_Store
+.idea
+wav2vec2_emotion/
+dataset/

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.streamlit/config.toml

@@ -1,4 +1,4 @@
 [theme]
-base="dark"
+base="light"
 primaryColor="#7c99b4"
 

README.md

@@ -1,5 +1,5 @@
 # SISE  Ultimate Challenge
-![Logo du Ultimate Challenge SISE](img/logo.png)
+![Logo du Ultimate Challenge SISE](img/logo_01.png)
 
 Ceci est le Ultimate Challenge pour le Master SISE.
 

app.py

@@ -1,34 +1,248 @@
 import streamlit as st
-from streamlit_option_menu import option_menu
-from views.application import application
-from views.about import about
-
-# Set the logo
-st.sidebar.image("img/logo.png", use_container_width=True)
-
-# Create a sidebar with navigation options
-# Sidebar navigation with streamlit-option-menu
-with st.sidebar:
-    # st.image("img/logo.png", use_container_width=True)
-    # st.markdown("<h1 style='text-align: center;'>SecureIA Dashboard</h1>", unsafe_allow_html=True)
-    # Navigation menu with icons
-    selected_tab = option_menu(
-        menu_title=None,  # Added menu_title parameter
-        options=["Application", "About"],
-        icons=["robot", "bar-chart", "robot"],
-        menu_icon="cast",
-        default_index=0,
-        # styles={
-        # "container": {"padding": "5px", "background-color": "#f0f2f6"},
-        # "icon": {"color": "orange", "font-size": "18px"},
-        # "nav-link": {"font-size": "16px", "text-align": "left", "margin": "0px", "color": "black"},
-        # "nav-link-selected": {"background-color": "#4CAF50", "color": "white"},
-        # }
+import pandas as pd
+import numpy as np
+import os
+import time
+import matplotlib.pyplot as plt
+from datetime import datetime
+import tempfile
+import io
+import json
+from model.transcriber import transcribe_audio
+from predict import predict_emotion
+
+# You'll need to install this package:
+# pip install streamlit-audiorec
+from st_audiorec import st_audiorec
+
+# Page configuration
+st.set_page_config(
+    page_title="Emotion Analyser",
+    page_icon="🎤",
+    layout="wide"
+)
+
+# Initialize session state variables if they don't exist
+if 'audio_data' not in st.session_state:
+    st.session_state.audio_data = []
+if 'current_audio_index' not in st.session_state:
+    st.session_state.current_audio_index = -1
+if 'audio_history_csv' not in st.session_state:
+    # Define columns for our CSV storage
+    st.session_state.audio_history_csv = pd.DataFrame(
+        columns=['timestamp', 'file_path', 'transcription', 'emotion', 'probabilities']
+    )
+if 'needs_rerun' not in st.session_state:
+    st.session_state.needs_rerun = False
+
+# Function to ensure we keep only the last 10 entries
+def update_audio_history(new_entry):
+    # Add the new entry
+    st.session_state.audio_history_csv = pd.concat([st.session_state.audio_history_csv, pd.DataFrame([new_entry])], ignore_index=True)
+    
+    # Keep only the last 10 entries
+    if len(st.session_state.audio_history_csv) > 10:
+        st.session_state.audio_history_csv = st.session_state.audio_history_csv.iloc[-10:]
+    
+    # Save to CSV
+    st.session_state.audio_history_csv.to_csv('audio_history.csv', index=False)
+
+# Function to process audio and get results
+def process_audio(audio_path):
+    try:
+        # Get transcription
+        transcription = transcribe_audio(audio_path)
+        
+        # Get emotion prediction
+        predicted_emotion, probabilities = predict_emotion(audio_path)
+        
+        # Update audio history
+        new_entry = {
+            'timestamp': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+            'file_path': audio_path,
+            'transcription': transcription,
+            'emotion': predicted_emotion,
+            'probabilities': str(probabilities)  # Convert dict to string for storage
+        }
+        update_audio_history(new_entry)
+        
+        # Update current index
+        st.session_state.current_audio_index = len(st.session_state.audio_history_csv) - 1
+        
+        return transcription, predicted_emotion, probabilities
+    except Exception as e:
+        st.error(f"Error processing audio: {str(e)}")
+        return None, None, None
+
+# Function to split audio into 10-second segments
+def split_audio(audio_file, segment_length=10):
+    # This is a placeholder - in a real implementation, you'd use a library like pydub
+    # to split the audio file into segments
+    st.warning("Audio splitting functionality is a placeholder. Implement with pydub or similar library.")
+    # For now, we'll just return the whole file as a single segment
+    return [audio_file]
+
+# Function to display emotion visualization
+def display_emotion_chart(probabilities):
+    emotions = list(probabilities.keys())
+    values = list(probabilities.values())
+    
+    fig, ax = plt.subplots(figsize=(10, 5))
+    bars = ax.bar(emotions, values, color=['red', 'gray', 'green'])
+    
+    # Add data labels on top of bars
+    for bar in bars:
+        height = bar.get_height()
+        ax.text(bar.get_x() + bar.get_width()/2., height + 0.02,
+                f'{height:.2f}', ha='center', va='bottom')
+    
+    ax.set_ylim(0, 1.1)
+    ax.set_ylabel('Probability')
+    ax.set_title('Emotion Prediction Results')
+    
+    st.pyplot(fig)
+
+# Trigger rerun if needed (replaces experimental_rerun)
+if st.session_state.needs_rerun:
+    st.session_state.needs_rerun = False
+    st.rerun()  # Using st.rerun() instead of experimental_rerun
+
+# Main App Layout
+st.image("./img/logo_01.png", width=400)
+
+# Create two columns for the main layout
+col1, col2 = st.columns([1, 1])
+
+with col1:
+    st.header("Audio Input")
+    
+    # Method selection
+
+    tab1, tab2 = st.tabs(["Record Audio", "Upload Audio"])
+    
+    with tab1:
+        st.write("Record your audio (max 10 seconds):")
+        
+        # Using streamlit-audiorec for better recording functionality
+        wav_audio_data = st_audiorec()
+        
+        if wav_audio_data is not None:
+            # Save the recorded audio to a temporary file
+            with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as tmp_file:
+                tmp_file.write(wav_audio_data)
+                tmp_file_path = tmp_file.name
+            
+            st.success("Audio recorded successfully!")
+            
+            # Process button
+            if st.button("Process Recorded Audio"):
+                # Process the audio
+                with st.spinner("Processing audio..."):
+                    transcription, emotion, probs = process_audio(tmp_file_path)
+                    # Set flag for rerun instead of calling experimental_rerun
+                    if transcription is not None:
+                        st.success("Audio processed successfully!")
+                        st.session_state.needs_rerun = True
+    
+    with tab2:
+        uploaded_file = st.file_uploader("Upload an audio file (WAV format)", type=['wav'])
+        
+        if uploaded_file is not None:
+            # Save the uploaded file to a temporary location
+            with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as tmp_file:
+                tmp_file.write(uploaded_file.getbuffer())
+                tmp_file_path = tmp_file.name
+            
+            st.audio(uploaded_file, format="audio/wav")
+            
+            # Process button
+            if st.button("Process Uploaded Audio"):
+                # Split audio into 10-second segments
+                with st.spinner("Processing audio..."):
+                    segments = split_audio(tmp_file_path)
+                    
+                    # Process each segment
+                    for i, segment_path in enumerate(segments):
+                        st.write(f"Processing segment {i+1}...")
+                        transcription, emotion, probs = process_audio(segment_path)
+                        
+                    # Set flag for rerun instead of calling experimental_rerun
+                    st.success("Audio processed successfully!")
+                    st.session_state.needs_rerun = True
+
+with col2:
+    st.header("Results")
+    
+    # Display results if available
+    if st.session_state.current_audio_index >= 0 and len(st.session_state.audio_history_csv) > 0:
+        current_data = st.session_state.audio_history_csv.iloc[st.session_state.current_audio_index]
+        
+        # Transcription
+        st.subheader("Transcription")
+        st.text_area("", value=current_data['transcription'], height=100, key="transcription_area")
+        
+        # Emotion
+        st.subheader("Detected Emotion")
+        st.info(f"🎭 Predicted emotion: **{current_data['emotion']}**")
+        
+        # Convert string representation of dict back to actual dict
+        try:
+            import ast
+            probs = ast.literal_eval(current_data['probabilities'])
+            display_emotion_chart(probs)
+        except Exception as e:
+            st.error(f"Error parsing probabilities: {str(e)}")
+            st.write(f"Raw probabilities: {current_data['probabilities']}")
+    else:
+        st.info("Record or upload audio to see results")
+
+# Audio History and Analytics Section
+st.header("Audio History and Analytics")
+
+if len(st.session_state.audio_history_csv) > 0:
+    # Display a select box to choose from audio history
+    timestamps = st.session_state.audio_history_csv['timestamp'].tolist()
+    selected_timestamp = st.selectbox(
+        "Select audio from history:",
+        options=timestamps,
+        index=len(timestamps) - 1  # Default to most recent
     )
     
+    # Update current index when selection changes
+    selected_index = st.session_state.audio_history_csv[
+        st.session_state.audio_history_csv['timestamp'] == selected_timestamp
+    ].index[0]
+    
+    # Only update if different
+    if st.session_state.current_audio_index != selected_index:
+        st.session_state.current_audio_index = selected_index
+        st.session_state.needs_rerun = True
     
-if selected_tab == "Application":
-    application()
-elif selected_tab == "About":
-    about()
-        
+    # Analytics button
+    if st.button("Run Analytics on Selected Audio"):
+        st.subheader("Analytics Results")
+        
+        # Get the selected audio data
+        selected_data = st.session_state.audio_history_csv.iloc[selected_index]
+        
+        # Display analytics (this is where you would add more sophisticated analytics)
+        st.write(f"Selected Audio: {selected_data['timestamp']}")
+        st.write(f"Emotion: {selected_data['emotion']}")
+        st.write(f"File Path: {selected_data['file_path']}")
+        
+        # Add any additional analytics you want here
+        
+        # Try to play the selected audio
+        try:
+            if os.path.exists(selected_data['file_path']):
+                st.audio(selected_data['file_path'], format="audio/wav")
+            else:
+                st.warning("Audio file not found - it may have been deleted or moved.")
+        except Exception as e:
+            st.error(f"Error playing audio: {str(e)}")
+else:
+    st.info("No audio history available. Record or upload audio to create history.")
+
+# Footer
+st.markdown("---")
+st.caption("Audio Emotion Analyzer - Processes audio in 10-second segments and predicts emotions")

config.py

@@ -0,0 +1,25 @@
+import os
+import torch
+from dotenv import load_dotenv
+
+# Charger les variables d'environnement
+load_dotenv()
+HF_API_KEY = os.getenv("HF_API_KEY")
+
+if not HF_API_KEY:
+    raise ValueError("Le token Hugging Face n'a pas été trouvé dans .env")
+
+# Labels d'émotions
+LABELS = {"colere": 0, "neutre": 1, "joie": 2}
+#LABELS = ["colere", "neutre", "joie"]
+NUM_LABELS = len(LABELS)
+
+# Choisir le device
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+# Modèle Wav2Vec2
+MODEL_NAME = "facebook/wav2vec2-large-xlsr-53-french"
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__)) 
+BEST_MODEL_NAME = os.path.join(BASE_DIR, "model","fr-speech-emotion-model.pth")  # Monte d'un niveau pour aller à la racine
+

model/emotion_classifier.py

@@ -0,0 +1,54 @@
+
+
+# Prédit 33% environ partout (dans le cas 3 classes)
+
+# class EmotionClassifier(nn.Module):
+#     def __init__(self, feature_dim, num_labels):
+#         super(EmotionClassifier, self).__init__()
+#         self.fc1 = nn.Linear(feature_dim, 256)
+#         self.relu = nn.ReLU()
+#         self.dropout = nn.Dropout(0.3)
+#         self.fc2 = nn.Linear(256, num_labels)
+
+#     def forward(self, x):
+#         x = self.fc1(x)
+#         x = self.relu(x)
+#         x = self.dropout(x)
+#         return self.fc2(x)
+    
+
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class Attention(nn.Module):
+    """Mécanisme d’attention permettant de pondérer l’importance des caractéristiques audio"""
+    def __init__(self, hidden_dim):
+        super(Attention, self).__init__()
+        self.attention_weights = nn.Linear(hidden_dim, 1)
+
+    def forward(self, lstm_output):
+        # lstm_output: (batch_size, sequence_length, hidden_dim)
+        attention_scores = self.attention_weights(lstm_output)  # (batch_size, sequence_length, 1)
+        attention_weights = torch.softmax(attention_scores, dim=1)  # Normalisation softmax
+        weighted_output = lstm_output * attention_weights  # Pondération des features
+        return weighted_output.sum(dim=1)  # Somme pondérée sur la séquence
+
+class EmotionClassifier(nn.Module):
+    """Modèle de classification des émotions basé sur BiLSTM et attention"""
+    def __init__(self, feature_dim, num_labels, hidden_dim=128):
+        super(EmotionClassifier, self).__init__()
+        self.lstm = nn.LSTM(feature_dim, hidden_dim, batch_first=True, bidirectional=True)
+        self.attention = Attention(hidden_dim * 2)  # Bidirectionnel → hidden_dim * 2
+        self.fc = nn.Linear(hidden_dim * 2, num_labels)  # Couche de classification finale
+
+    def forward(self, x):
+        lstm_out, _ = self.lstm(x)  # (batch_size, sequence_length, hidden_dim*2)
+        attention_out = self.attention(lstm_out)  # (batch_size, hidden_dim*2)
+        logits = self.fc(attention_out)  # (batch_size, num_labels)
+        return logits
+
+
+
+

model/feature_extractor.py

@@ -0,0 +1,6 @@
+import torch
+from transformers import Wav2Vec2Model, Wav2Vec2Processor
+from config import MODEL_NAME, DEVICE
+
+processor = Wav2Vec2Processor.from_pretrained(MODEL_NAME)
+feature_extractor = Wav2Vec2Model.from_pretrained(MODEL_NAME).to(DEVICE)

model/transcriber.py

@@ -0,0 +1,35 @@
+import os
+import torch
+import librosa
+from transformers import Wav2Vec2Processor, Wav2Vec2ForCTC
+
+# Charger le modèle et le processeur
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+MODEL_NAME = "facebook/wav2vec2-large-xlsr-53-french"
+
+processor = Wav2Vec2Processor.from_pretrained(MODEL_NAME)
+model = Wav2Vec2ForCTC.from_pretrained(MODEL_NAME).to(device)
+model.eval()
+
+def transcribe_audio(audio_path, sampling_rate=16000):
+    # Charger l'audio
+    audio, sr = librosa.load(audio_path, sr=sampling_rate)
+
+    # Transformer l'audio en entrée pour le modèle
+    input_values = processor(audio, sampling_rate=sampling_rate, return_tensors="pt").input_values.to(device)
+
+    # Obtenir les prédictions
+    with torch.no_grad():
+        logits = model(input_values).logits
+
+    # Décoder les prédictions en texte
+    predicted_ids = torch.argmax(logits, dim=-1)
+    transcription = processor.batch_decode(predicted_ids)[0]
+    return transcription
+
+# Exemple d'utilisation
+if __name__ == "__main__":
+    base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "data"))
+    audio_path = os.path.join(base_path, "colere", "c1af.wav")
+    texte = transcribe_audio(audio_path)
+    print(f"Transcription : {texte}")

predict.py

@@ -0,0 +1,57 @@
+import sys
+import os
+import torch
+import librosa
+import numpy as np
+from model.emotion_classifier import EmotionClassifier
+from utils.preprocessing import collate_fn
+from config import DEVICE, NUM_LABELS, BEST_MODEL_NAME
+
+# Charger le modèle entraîné
+feature_dim = 40  # Nombre de MFCCs utilisés
+model = EmotionClassifier(feature_dim, NUM_LABELS).to(DEVICE)
+model.load_state_dict(torch.load(BEST_MODEL_NAME, map_location=DEVICE))
+model.eval()  # Mode évaluation
+
+# Labels des émotions
+LABELS = {0: "colère", 1: "neutre", 2: "joie"}
+
+# Fonction pour prédire l’émotion d’un fichier audio avec probabilités
+def predict_emotion(audio_path, max_length=128):
+    # Charger l’audio
+    y, sr = librosa.load(audio_path, sr=16000)
+
+    # Extraire les MFCCs
+    mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=40)
+
+    # Ajuster la taille des MFCCs avec padding/troncature
+    if mfcc.shape[1] > max_length:  
+        mfcc = mfcc[:, :max_length]  # Tronquer si trop long
+    else:
+        pad_width = max_length - mfcc.shape[1]
+        mfcc = np.pad(mfcc, pad_width=((0, 0), (0, pad_width)), mode='constant')
+
+    # Convertir en tenseur PyTorch
+    input_tensor = torch.tensor(mfcc.T, dtype=torch.float32).unsqueeze(0).to(DEVICE)  # (1, max_length, 40)
+
+    # Prédiction avec le modèle
+    with torch.no_grad():
+        logits = model(input_tensor)
+        probabilities = torch.nn.functional.softmax(logits, dim=-1).cpu().numpy().flatten()  # Convertir en probabilités
+        predicted_class = torch.argmax(logits, dim=-1).item()
+
+    # Associer les probabilités aux labels
+    probabilities_dict = {LABELS[i]: float(probabilities[i]) for i in range(NUM_LABELS)}
+
+    return LABELS[predicted_class], probabilities_dict
+
+
+# Exemple d'utilisation
+if __name__ == "__main__":
+    base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "data"))
+    audio_file = os.path.join(base_path, "colere", "c1ac.wav")
+
+    predicted_emotion, probabilities = predict_emotion(audio_file)
+
+    print(f"🎤 L'émotion prédite est : {predicted_emotion}")
+    print(f"📊 Probabilités par classe : {probabilities}")

requirements.txt

@@ -15,3 +15,5 @@ scikit-learn
 huggingface
 huggingface_hub
 pyaudio
+streamlit_audiorec
+dotenv