first

Dockerfile

@@ -1,30 +1,30 @@
-FROM nvidia/cuda:12.1.1-cudnn8-devel-ubuntu22.04
+ARG CUDA_IMAGE="12.1.1-devel-ubuntu22.04"
+FROM nvidia/cuda:${CUDA_IMAGE}
 
-ARG DEBIAN_FRONTEND=noninteractive
+# We need to set the host to 0.0.0.0 to allow outside access
+ENV HOST 0.0.0.0
 
-ENV PYTHONUNBUFFERED=1
+RUN apt-get update && apt-get upgrade -y \
+    && apt-get install -y git build-essential \
+    python3 python3-pip gcc wget \
+    ocl-icd-opencl-dev opencl-headers clinfo \
+    libclblast-dev libopenblas-dev \
+    && mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
 
-RUN curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
-  && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
-    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
-    sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
+COPY . .
 
-RUN sed -i -e '/experimental/ s/^#//g' /etc/apt/sources.list.d/nvidia-container-toolkit.list    
+# setting build related env vars
+ENV CUDA_DOCKER_ARCH=all
+ENV LLAMA_CUBLAS=1
 
-RUN apt-get update && apt-get install --no-install-recommends -y \
-  build-essential \
-  python3.9 \
-  python3-pip \
-  git \
-  ffmpeg \
-  nvidia-container-toolkit \
-  && apt-get clean && rm -rf /var/lib/apt/lists/*
+# Install depencencies
+RUN python3 -m pip install --upgrade pip pytest cmake \
+    scikit-build setuptools fastapi uvicorn sse-starlette \
+    pydantic-settings starlette-context gradio huggingface_hub hf_transfer
 
-WORKDIR /code
+# Install llama-cpp-python (build with cuda)
+RUN CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python
 
-COPY ./requirements.txt /code/requirements.txt
-
-# Set up a new user named "user" with user ID 1000
 RUN useradd -m -u 1000 user
 # Switch to the "user" user
 USER user
@@ -33,11 +33,12 @@ ENV HOME=/home/user \
 	PATH=/home/user/.local/bin:$PATH \
     PYTHONPATH=$HOME/app \
 	PYTHONUNBUFFERED=1 \
+	GRADIO_ALLOW_FLAGGING=never \
+	GRADIO_NUM_PORTS=1 \
+	GRADIO_SERVER_NAME=0.0.0.0 \
+	GRADIO_THEME=huggingface \
 	SYSTEM=spaces
 
-RUN pip3 install --no-cache-dir --upgrade -r /code/requirements.txt
-
-# Set the working directory to the user's home directory
 WORKDIR $HOME/app
 
 # Copy the current directory contents into the container at $HOME/app setting the owner to the user