Update benchmark.sh
Browse files- benchmark.sh +51 -0
benchmark.sh
CHANGED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# clone whisper.cpp
|
| 2 |
+
git clone https://github.com/ggerganov/whisper.cpp.git
|
| 3 |
+
cd whisper.cpp
|
| 4 |
+
# clone dataset
|
| 5 |
+
git clone https://huggingface.co/datasets/kotoba-tech/kotoba-whisper-eval
|
| 6 |
+
# convert to 16khz
|
| 7 |
+
ffmpeg -i kotoba-whisper-eval/audio/long_interview_1.mp3 -ar 16000 -ac 1 -c:a pcm_s16le kotoba-whisper-eval/audio/long_interview_1.wav
|
| 8 |
+
ffmpeg -i kotoba-whisper-eval/audio/manzai1.mp3 -ar 16000 -ac 1 -c:a pcm_s16le kotoba-whisper-eval/audio/manzai1.wav
|
| 9 |
+
ffmpeg -i kotoba-whisper-eval/audio/manzai2.mp3 -ar 16000 -ac 1 -c:a pcm_s16le kotoba-whisper-eval/audio/manzai2.wav
|
| 10 |
+
ffmpeg -i kotoba-whisper-eval/audio/manzai3.mp3 -ar 16000 -ac 1 -c:a pcm_s16le kotoba-whisper-eval/audio/manzai3.wav
|
| 11 |
+
# clone weight
|
| 12 |
+
wget https://huggingface.co/kotoba-tech/kotoba-whisper-v1.0-ggml/resolve/main/ggml-kotoba-whisper-v1.0.bin -P ./models
|
| 13 |
+
# benchmark main model
|
| 14 |
+
SECONDS=0
|
| 15 |
+
make -j && ./main -m models/ggml-kotoba-whisper-v1.0.bin -f kotoba-whisper-eval/audio/long_interview_1.wav
|
| 16 |
+
TIME_INTERVIEW=$SECONDS
|
| 17 |
+
SECONDS=0
|
| 18 |
+
make -j && ./main -m models/ggml-kotoba-whisper-v1.0.bin -f kotoba-whisper-eval/audio/manzai1.wav
|
| 19 |
+
TIME_MANZAI1=$SECONDS
|
| 20 |
+
SECONDS=0
|
| 21 |
+
make -j && ./main -m models/ggml-kotoba-whisper-v1.0.bin -f kotoba-whisper-eval/audio/manzai2.wav
|
| 22 |
+
TIME_MANZAI2=$SECONDS
|
| 23 |
+
SECONDS=0
|
| 24 |
+
make -j && ./main -m models/ggml-kotoba-whisper-v1.0.bin -f kotoba-whisper-eval/audio/manzai3.wav
|
| 25 |
+
TIME_MANZAI3=$SECONDS
|
| 26 |
+
# clone weight (quantized)
|
| 27 |
+
wget https://huggingface.co/kotoba-tech/kotoba-whisper-v1.0-ggml/resolve/main/ggml-kotoba-whisper-v1.0-q5_0.bin -P ./models
|
| 28 |
+
# benchmark quantized model
|
| 29 |
+
SECONDS=0
|
| 30 |
+
make -j && ./main -m models/ggml-kotoba-whisper-v1.0-q5_0.bin -f kotoba-whisper-eval/audio/long_interview_1.wav
|
| 31 |
+
TIME_INTERVIEW_Q=$SECONDS
|
| 32 |
+
SECONDS=0
|
| 33 |
+
make -j && ./main -m models/ggml-kotoba-whisper-v1.0-q5_0.bin -f kotoba-whisper-eval/audio/manzai1.wav
|
| 34 |
+
TIME_MANZAI1_Q=$SECONDS
|
| 35 |
+
SECONDS=0
|
| 36 |
+
make -j && ./main -m models/ggml-kotoba-whisper-v1.0-q5_0.bin -f kotoba-whisper-eval/audio/manzai2.wav
|
| 37 |
+
TIME_MANZAI2_Q=$SECONDS
|
| 38 |
+
SECONDS=0
|
| 39 |
+
make -j && ./main -m models/ggml-kotoba-whisper-v1.0-q5_0.bin -f kotoba-whisper-eval/audio/manzai3.wav
|
| 40 |
+
TIME_MANZAI3_Q=$SECONDS
|
| 41 |
+
# summary
|
| 42 |
+
echo "MAIN MODEL"
|
| 43 |
+
echo "interview: $TIME_INTERVIEW"
|
| 44 |
+
echo "manzai1 : $TIME_MANZAI1"
|
| 45 |
+
echo "manzai2 : $TIME_MANZAI2"
|
| 46 |
+
echo "manzai3 : $TIME_MANZAI3"
|
| 47 |
+
echo "QUANTIZED MODEL"
|
| 48 |
+
echo "interview: $TIME_INTERVIEW_Q"
|
| 49 |
+
echo "manzai1 : $TIME_MANZAI1_Q"
|
| 50 |
+
echo "manzai2 : $TIME_MANZAI2_Q"
|
| 51 |
+
echo "manzai3 : $TIME_MANZAI3_Q"
|