Duplicate from jmparejaz/Audio_to_text_classification

Co-authored-by: Jose Mario Pareja <[email protected]>

.gitattributes

@@ -0,0 +1,33 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: Sound Emotion Recognition
+emoji: 💻
+colorFrom: pink
+colorTo: blue
+sdk: gradio
+sdk_version: 3.6
+app_file: app.py
+pinned: false
+license: mit
+duplicated_from: jmparejaz/Audio_to_text_classification
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,107 @@
+import os
+os.system("pip install git+https://github.com/openai/whisper.git")
+import gradio as gr
+import whisper
+from huggingface_hub import from_pretrained_keras
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from transformers import pipeline
+from sklearn.preprocessing import StandardScaler
+import logging
+import librosa
+import numpy as np
+import pickle
+
+
+
+#call tokenizer and NLP model for text classification
+tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment-latest")
+model_nlp = AutoModelForSequenceClassification.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment-latest")
+
+
+# call whisper model for audio/speech processing
+model = whisper.load_model("small")
+
+# call model for audio emotions
+reloaded_model = from_pretrained_keras('jmparejaz/RAVDESS-CREMAD_AudioEmotionClassifier')
+
+# call scaler and decoder
+with open("scaler.pkl", "rb") as f:
+    scaler = pickle.load(f)
+
+with open("encoder.pkl", "rb") as f:
+    encoder = pickle.load(f)
+
+
+
+def inference_audio(audio):
+    audio = whisper.load_audio(audio)
+    audio = whisper.pad_or_trim(audio)
+    
+    mel = whisper.log_mel_spectrogram(audio).to(model.device)
+    
+    _, probs = model.detect_language(mel)
+    
+    options = whisper.DecodingOptions(fp16 = False)
+    result = whisper.decode(model, mel, options)
+    
+    return result.text
+
+def inference_text(audio):
+    text =inference_audio(audio)
+
+    sentiment_task = pipeline("sentiment-analysis", model=model_nlp, tokenizer=tokenizer)
+    res=sentiment_task(text)[0]
+
+    return text,res['label'],res['score']
+
+    
+def extract_features(data):
+    # ZCR
+    result = np.array([])
+    zcr = np.mean(librosa.feature.zero_crossing_rate(y=data).T, axis=0)
+    result=np.hstack((result, zcr)) # stacking horizontally
+
+    # Chroma_stft
+    stft = np.abs(librosa.stft(data))
+    chroma_stft = np.mean(librosa.feature.chroma_stft(S=stft, sr=sample_rate).T, axis=0)
+    result = np.hstack((result, chroma_stft)) # stacking horizontally
+
+    # MFCC
+    mfcc = np.mean(librosa.feature.mfcc(y=data, sr=sample_rate).T, axis=0)
+    result = np.hstack((result, mfcc)) # stacking horizontally
+
+    # Root Mean Square Value
+    rms = np.mean(librosa.feature.rms(y=data).T, axis=0)
+    result = np.hstack((result, rms)) # stacking horizontally
+
+    # MelSpectogram
+    mel = np.mean(librosa.feature.melspectrogram(y=data, sr=sample_rate).T, axis=0)
+    result = np.hstack((result, mel)) # stacking horizontally
+    
+    return result
+"""
+def audio_emotions(audio):
+    sr,data = audio
+    features_audio = extract_features(data)
+    features_audio = np.array(features_audio)
+    scaled_features=scaler.transform(features_audio)
+    scaled_features = np.expand_dims(scaled_features, axis=2)
+    prediction=reloaded_model.predict(scaled_features)
+    y_pred = encoder.inverse_transform(prediction)
+    return y_pred
+"""
+def main(audio):
+    r1,r2,r3=inference_text(audio)
+    #r3=audio_emotions(audio)
+    return r1,r2,r3
+    
+
+audio = gr.Audio(
+                    label="Input Audio",
+                    show_label=False,
+                    source="microphone",
+                    type="filepath"
+                )
+
+
+app=gr.Interface(title="Sentiment Audio Analysis",fn=main,inputs=audio, outputs=["text","text","text"]).launch(debug = True)

encoder.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c6a3ef0f2c45329f271e0c533784148f2e9fbe6ed814f22933699660d8e5a14
+size 430

requirements.txt

@@ -0,0 +1,3 @@
+librosa
+tensorflow
+scikit-learn==1.0.2

scaler.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aac2a609b867b59002822ad08d86679c11338c24776e83ad2d2dd51eaba9cf53
+size 4346