eval/scripts/run_commercial_comparison.sh

#!/bin/bash
# Run Whisper Model WER Evaluation - Fine-tunes vs Commercial APIs

set -e

echo "Whisper Model WER Evaluation - Fine-tunes vs Commercial APIs"
echo "============================================================="
echo ""
echo "Comparing fine-tuned models vs commercial providers:"
echo "  - Deepgram Nova 3"
echo "  - OpenAI Whisper API"
echo "  - AssemblyAI"
echo "  - Gladia"
echo ""

# Check if venv exists
if [ ! -d ".venv" ]; then
    echo "Creating virtual environment..."
    uv venv
fi

# Activate venv
echo "Activating virtual environment..."
source .venv/bin/activate

# Install/update dependencies
echo "Installing dependencies..."
uv pip install -r requirements.txt

echo ""
echo "Starting evaluation..."
echo "============================================================="
echo ""

# Run evaluation
python scripts/evaluate_vs_commercial.py

echo ""
echo "Evaluation complete!"
echo ""
echo "Results are available in:"
echo "  - results/latest/commercial_comparison_report.txt"
echo "  - results/latest/commercial_comparison_results.json"
echo "  - results/latest/commercial_comparison_chart.txt"
echo ""