mlp

app/main.py

@@ -1,4 +1,4 @@
-from fastapi import FastAPI, HTTPException, Header, Depends
+from fastapi import FastAPI, File, UploadFile, HTTPException, Depends
 from pydantic import BaseModel
 import os
 from pymongo import MongoClient
@@ -7,6 +7,25 @@ from langchain_community.vectorstores import MongoDBAtlasVectorSearch
 import uvicorn
 from dotenv import load_dotenv
 from fastapi.middleware.cors import CORSMiddleware
+from uuid import uuid4
+
+from typing import Optional
+import joblib
+import librosa
+import numpy as np
+
+import pandas as pd 
+import numpy as np
+import matplotlib.pylab as plt
+import seaborn as sns
+
+from glob import glob
+import librosa.display
+import IPython.display as ipd
+import soundfile as sf
+
+from itertools import cycle
+
 
 load_dotenv()
 
@@ -127,5 +146,127 @@ async def get_answer(item: Item, token: str = Depends(verify_token)):
         # If there's an error, return a 500 error with the error's details
         raise HTTPException(status_code=500, detail=str(e))
 
+# mlp
+mlp_model = joblib.load('app/mlp_model.pkl')
+mlp_pca = joblib.load('app/pca.pkl')
+mlp_scaler = joblib.load('app/scaler.pkl')
+mlp_label_encoder = joblib.load('app/label_encoder.pkl')
+
+def preprocess_audio(path, save_dir):
+    y, sr = librosa.load(path)
+
+    # remove silence
+    intervals = librosa.effects.split(y, top_db=20)
+    # Concatenate non-silent intervals
+    y_no_gaps = np.concatenate([y[start:end] for start, end in intervals])
+
+    file_name_without_extension = os.path.basename(path).split('.')[0]
+    y_trimmed, _ = librosa.effects.trim(y_no_gaps, top_db = 20) 
+    D = librosa.stft(y)
+    S_db = librosa.amplitude_to_db(np.abs(D), ref=np.max)
+    S = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=128*2,)
+    S_db_mel = librosa.amplitude_to_db(np.abs(S), ref=np.max)
+    
+    # Apply noise reduction (example using spectral subtraction)
+    y_denoised = librosa.effects.preemphasis(y_trimmed)
+
+    # Apply dynamic range compression
+    y_compressed = librosa.effects.preemphasis(y_denoised)
+
+    # Augmentation (example of time stretching)
+#     y_stretched = librosa.effects.time_stretch(y_compressed, rate=1.2)
+
+    # Silence Removal
+    y_silence_removed, _ = librosa.effects.trim(y_compressed)
+
+    # Equalization (example: apply high-pass filter)
+    y_equalized = librosa.effects.preemphasis(y_silence_removed)
+    
+    # Define target sample rate
+    target_sr = sr
+
+#     # Data Augmentation (example: pitch shifting)
+#     y_pitch_shifted = librosa.effects.pitch_shift(y_normalized, sr=target_sr, n_steps=2)
+
+
+    # Split audio into non-silent intervals
+
+    
+    # Normalize the audio signal
+    y_normalized = librosa.util.normalize(y_equalized)
+
+    # Feature Extraction (example: MFCCs)
+#     mfccs = librosa.feature.mfcc(y=y_normalized, sr=target_sr, n_mfcc=20)
+
+    
+    output_file_path = os.path.join(save_dir, f"{file_name_without_extension}.wav")
+
+    # Write the audio data to the output file in .wav format
+    sf.write(output_file_path, y_normalized, target_sr)
+    
+    return 'success'
+
+
+def extract_features(file_path):
+    # Load the audio file
+    y, sr = librosa.load(file_path, sr=None, dtype=np.float32)
+    
+    # Extract MFCCs
+    mfccs = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=20)
+    mfccs_mean = pd.Series(mfccs.mean(axis=1), index=[f'mfcc_{i}' for i in range(mfccs.shape[0])])
+    
+    # Extract Spectral Features
+    spectral_centroids = pd.Series(np.mean(librosa.feature.spectral_centroid(y=y, sr=sr)), index=['spectral_centroid'])
+    spectral_rolloff = pd.Series(np.mean(librosa.feature.spectral_rolloff(y=y, sr=sr)), index=['spectral_rolloff'])
+    spectral_flux = pd.Series(np.mean(librosa.onset.onset_strength(y=y, sr=sr)), index=['spectral_flux'])
+    spectral_contrast = pd.Series(np.mean(librosa.feature.spectral_contrast(S=np.abs(librosa.stft(y)), sr=sr), axis=1), index=[f'spectral_contrast_{i}' for i in range(librosa.feature.spectral_contrast(S=np.abs(librosa.stft(y)), sr=sr).shape[0])])
+    
+    # Extract Pitch
+    pitches, magnitudes = librosa.piptrack(y=y, sr=sr)
+    pitch_mean = pd.Series(np.mean(pitches[pitches != 0]), index=['pitch_mean'])  # Average only non-zero values
+    
+    # Extract Zero Crossings
+    zero_crossings = pd.Series(np.mean(librosa.feature.zero_crossing_rate(y)), index=['zero_crossings'])
+    
+    # Combine all features into a single Series
+    features = pd.concat([mfccs_mean, spectral_centroids, spectral_rolloff, spectral_flux, spectral_contrast, pitch_mean, zero_crossings])
+    return features
+
+
+@app.post("/mlp")
+async def handle_audio(file: UploadFile = File(...)):
+    try:
+        # Ensure that we are handling an MP3 file
+        if file.content_type not in ["audio/mpeg", "audio/wav", "audio/mp3"]:
+            raise HTTPException(status_code=400, detail="Invalid file type. Please upload an MP3 file.")
+
+        # Read the file's content
+        contents = await file.read()
+        temp_filename = f"app/{uuid4().hex}.mp3"
+
+        # Save file to a temporary file if needed or process directly from memory
+        with open(temp_filename, "wb") as f:
+            f.write(contents)
+
+        preprocess_audio(temp_filename, 'app')
+        
+        # Here you would add the feature extraction logic
+        features = extract_features(temp_filename)
+        print("Extracted Features:", features)
+
+        features = mlp_scaler.fit_transform(features)
+        features = mlp_pca.fit_transform(features)
+
+        # Dummy example to proceed with an inference
+        results = mlp_model.predict(features)
+
+        # Clean up (optional, especially if dealing with large files or sensitive data)
+        os.remove(temp_filename)
+
+        return {"message": "File processed successfully", "prediction": results.tolist()}
+    except Exception as e:
+        # Handle possible exceptions
+        raise HTTPException(status_code=500, detail=str(e))
+
 # if __name__ == "__main__":
 #     uvicorn.run("main:app", host="0.0.0.0", port=8080, reload=False)

requirements.txt

@@ -10,4 +10,8 @@ pymongo>=3.11
 tiktoken==0.6.0
 langchain-openai==0.0.8
 python-dotenv
-upstash-redis
+upstash-redis
+librosa
+soundfile
+opensmile
+eyeD3