Merge remote-tracking branch 'origin/master'

Cargo.lock

@@ -2456,18 +2456,18 @@ dependencies = [
 
 [[package]]
 name = "whisper-rs"
-version = "0.9.0-rc.2"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa2ccc5c685f7f5050478d253e6c67f0b79baeb0cd0a069aa286275069b958d0"
+checksum = "2c950fb18ad556b053ba615b88fd4d01ed6020be740c3371eb0fc4aec64a0639"
 dependencies = [
  "whisper-rs-sys",
 ]
 
 [[package]]
 name = "whisper-rs-sys"
-version = "0.7.2"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c69a0d37a415512fb94332cff2be611d26f65473959b5debbd959997149640e"
+checksum = "094a5bd86f6f52562bbc74c28f27cd80197e54656cfb7213cf4ba37b5246cc9e"
 dependencies = [
  "bindgen",
  "cfg-if",

Cargo.toml

@@ -20,13 +20,24 @@ tokio = { version = "1.33", features = ["macros", "rt-multi-thread", "sync"] }
 tokio-stream = "0.1"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-whisper-rs = "0.9.0-rc.2"
-whisper-rs-sys = "0.7"
+whisper-rs = "0.8"
+whisper-rs-sys = "0.6"
 
 [dependencies.poem]
 version = "1.3"
 features = ["websocket", "static-files"]
 
 [target.aarch64-apple-darwin.dependencies.whisper-rs]
-version = "0.9.0-rc.2"
-features = ["coreml"]
+version = "0.8"
+features = ["coreml"]
+
+[target.aarch64-apple-darwin.dependencies.whisper-rs-sys]
+version = "0.6"
+
+[target.x86_64-unknown-linux-gnu.dependencies.whisper-rs]
+version = "0.8"
+features = ["cuda"]
+
+[target.aarch64-unknown-linux-gnu.dependencies.whisper-rs]
+version = "0.8"
+features = ["cuda"]

Dockerfile

@@ -0,0 +1,46 @@
+# Build stage
+FROM nvidia/cuda:12.2.2-devel-ubuntu22.04 as builder
+
+# Install python3.11 and build dependencies
+RUN apt-get update
+RUN apt-get install -y software-properties-common
+#RUN add-apt-repository ppa:deadsnakes/ppa
+
+RUN apt-get update
+RUN apt-get install -y libssl-dev cmake python3-dev curl pkg-config clang
+
+# install rust toolchain
+RUN curl https://sh.rustup.rs -sSf | sh -s  -- --default-toolchain stable -y
+ENV PATH=/root/.cargo/bin:$PATH
+
+# Just copy the manifest files to cache dependencies
+COPY Cargo.toml Cargo.lock ./
+
+# Download dependencies
+RUN mkdir -p src/bin && echo "fn main() {println!(\"if you see this, the build broke\")}" > src/bin/bigbot.rs
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    cargo build --release
+
+COPY ./src ./src
+
+# Build the project with release profile
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    cargo build --release
+
+# Runtime stage
+FROM nvidia/cuda:12.2.2-runtime-ubuntu22.04 as runtime
+
+# Install runtime dependencies
+RUN apt-get update && apt-get install -y ca-certificates python3-dev && rm -rf /var/lib/apt/lists/*
+
+# Just copy the binary from the build stage
+COPY --from=builder /target/release/polyhedron /usr/local/bin/polyhedron
+COPY ./models/ggml-large-encoder.mlmodelc ./models/ggml-large-encoder.mlmodelc
+COPY ./models/ggml-large.bin ./models/ggml-large.bin
+COPY config.yaml config.yaml
+COPY ./static ./static
+
+# Run the binary
+CMD ["polyhedron"]

config/dev.yaml

@@ -5,7 +5,7 @@ whisper:
     length_ms: 10000
     keep_ms: 200
     step_ms: 5000
-    model: "models/ggml-medium.en.bin" #"models/ggml-base.bin"
+    model: "models/ggml-large-q5_0.bin" #"models/ggml-base.bin"
     max_prompt_tokens: 128
     params:
         #n_threads: 4

src/config.rs

@@ -38,10 +38,8 @@ pub(crate) struct WhisperParams {
     pub(crate) language: Option<String>,
 }
 
-const _NONE: [c_int; 0] = [];
-
 impl WhisperParams {
-    pub(crate) fn to_full_params<'a, 'b>(&'a self, _tokens: &'b [c_int]) -> FullParams<'a, 'b> {
+    pub(crate) fn to_full_params<'a, 'b>(&'a self, tokens: &'b [c_int]) -> FullParams<'a, 'b> {
         let mut param = FullParams::new(Default::default());
         param.set_print_progress(self.print_progress);
         param.set_print_special(self.print_special);
@@ -60,6 +58,7 @@ impl WhisperParams {
         param.set_speed_up(self.speed_up);
         // param.set_tdrz_enable(self.tinydiarize);
         param.set_temperature_inc(self.temperature_inc);
+        param.set_tokens(tokens);
 
         param
     }

static/index.html

@@ -5,7 +5,7 @@
     <link rel="icon" type="image/svg+xml" href="/vite.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>Vite + React</title>
-    <script type="module" crossorigin src="/assets/index-835823f8.js"></script>
+    <script type="module" crossorigin src="/assets/index-34871d29.js"></script>
     <link rel="stylesheet" href="/assets/index-983f9492.css">
   </head>
   <body>