Upload 4 files

Dockerfile

@@ -0,0 +1,42 @@
+FROM ubuntu:22.04
+
+# Install system dependencies
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential \
+    libssl-dev \
+    zlib1g-dev \
+    libboost-system-dev \
+    libboost-filesystem-dev \
+    cmake \
+    git \
+    python3-pip \
+    curl \
+    wget && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies
+RUN pip3 install huggingface-hub openai gradio
+
+# Build llama.cpp
+RUN git clone https://github.com/ggerganov/llama.cpp && \
+    cd llama.cpp && \
+    mkdir build && \
+    cd build && \
+    cmake .. -DLLAMA_BUILD_SERVER=ON -DLLAMA_BUILD_EXAMPLES=ON -DCMAKE_BUILD_TYPE=Release && \
+    cmake --build . --config Release --target llama-server -j $(nproc)
+
+# Download model
+RUN mkdir -p /models && \
+    wget -O /models/model.q2_k_l.gguf https://huggingface.co/unsloth/DeepSeek-R1-Distill-Qwen-32B-GGUF/resolve/main/DeepSeek-R1-Distill-Qwen-32B-Q2_K_L.gguf
+
+# Copy app and startup script
+COPY app.py /app.py
+COPY start.sh /start.sh
+RUN chmod +x /start.sh
+
+# Expose ports
+EXPOSE 7860 8080
+
+# Start services
+CMD ["/start.sh"]

README.md

@@ -1,12 +1,15 @@
----
-title: DeepSeek R1 Distill Qwen 32B Demo GGUF
-emoji: 🌍
-colorFrom: gray
-colorTo: blue
-sdk: gradio
-sdk_version: 5.13.1
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: DeepSeek R1 Distill Qwen 32B Q2_K_L Demo
+emoji: 📈
+colorFrom: purple
+colorTo: red
+sdk: docker
+sdk_version: 5.13.0
+app_file: app.py
+app_port: 7860
+pinned: false
+license: mit
+thumbnail: >-
+  https://cdn-uploads.huggingface.co/production/uploads/62f847d692950415b63c6011/KdOkloioovSNbYhV2BO5v.png
+short_description: DeepSeek R1 Distill Qwen 32B Demo GGUF(Q2_K_L) Fully in CPU
+---

app.py

@@ -0,0 +1,201 @@
+import time
+import gradio as gr
+from openai import OpenAI
+
+DESCRIPTION = '''
+# DeepSeek-R1 Distill Qwen-32B Demo
+A reasoning model trained using RL (Reinforcement Learning) that demonstrates structured reasoning capabilities.
+'''
+
+CSS = """
+.spinner {
+    animation: spin 1s linear infinite;
+    display: inline-block;
+    margin-right: 8px;
+}
+@keyframes spin {
+    from { transform: rotate(0deg); }
+    to { transform: rotate(360deg); }
+}
+.thinking-summary {
+    cursor: pointer;
+    padding: 8px;
+    background: #f5f5f5;
+    border-radius: 4px;
+    margin: 4px 0;
+}
+.thought-content {
+    padding: 10px;
+    background: #f8f9fa;
+    border-radius: 4px;
+    margin: 5px 0;
+}
+.thinking-container {
+    border-left: 3px solid #e0e0e0;
+    padding-left: 10px;
+    margin: 8px 0;
+}
+details:not([open]) .thinking-container {
+    border-left-color: #4CAF50;
+}
+"""
+
+client = OpenAI(base_url="http://localhost:8080/v1", api_key="no-key-required")
+
+def user(message, history):
+    return "", history + [[message, None]]
+
+class ParserState:
+    __slots__ = ['answer', 'thought', 'in_think', 'start_time', 'last_pos']
+    def __init__(self):
+        self.answer = ""
+        self.thought = ""
+        self.in_think = False
+        self.start_time = 0
+        self.last_pos = 0
+
+def parse_response(text, state):
+    buffer = text[state.last_pos:]
+    state.last_pos = len(text)
+    
+    while buffer:
+        if not state.in_think:
+            think_start = buffer.find('<think>')
+            if think_start != -1:
+                state.answer += buffer[:think_start]
+                state.in_think = True
+                state.start_time = time.perf_counter()
+                buffer = buffer[think_start + 7:]
+            else:
+                state.answer += buffer
+                break
+        else:
+            think_end = buffer.find('</think>')
+            if think_end != -1:
+                state.thought += buffer[:think_end]
+                state.in_think = False
+                buffer = buffer[think_end + 8:]
+            else:
+                state.thought += buffer
+                break
+    
+    elapsed = time.perf_counter() - state.start_time if state.in_think else 0
+    return state, elapsed
+
+def format_response(state, elapsed):
+    answer_part = state.answer.replace('<think>', '').replace('</think>', '')
+    collapsible = []
+
+    if state.thought or state.in_think:
+        status = (f"🌀 Thinking for {elapsed:.0f} seconds" 
+                  if state.in_think else f"✅ Thought for {elapsed:.0f} seconds")
+        collapsible.append(
+            f"<details open><summary>{status}</summary>\n\n<div class='thinking-container'>\n{state.thought}\n</div>\n</details>"
+        )
+
+    return collapsible, answer_part
+
+def generate_response(history, temperature, top_p, max_tokens, active_gen):
+    messages = [{"role": "user", "content": history[-1][0]}]
+    full_response = ""
+    state = ParserState()
+    last_update = 0
+    
+    try:
+        stream = client.chat.completions.create(
+            model="",
+            messages=messages,
+            temperature=temperature,
+            top_p=top_p,
+            max_tokens=max_tokens,
+            stream=True
+        )
+        
+        for chunk in stream:
+            if not active_gen[0]:
+                break
+            
+            if chunk.choices[0].delta.content:
+                full_response += chunk.choices[0].delta.content
+                state, elapsed = parse_response(full_response, state)
+                
+                collapsible, answer_part = format_response(state, elapsed)
+                history[-1][1] = "\n\n".join(collapsible + [answer_part])  # Markdown-safe
+                yield history
+        
+        # Final update
+        state, elapsed = parse_response(full_response, state)
+        collapsible, answer_part = format_response(state, elapsed)
+        history[-1][1] = "\n\n".join(collapsible + [answer_part])  # Markdown-safe
+        yield history
+        
+    except Exception as e:
+        history[-1][1] = f"Error: {str(e)}"
+        yield history
+    finally:
+        active_gen[0] = False
+
+with gr.Blocks(css=CSS) as demo:
+    gr.Markdown(DESCRIPTION)
+    active_gen = gr.State([False])
+    
+    chatbot = gr.Chatbot(
+        elem_id="chatbot",
+        height=500,
+        show_label=False,
+        render_markdown=True
+    )
+
+    with gr.Row():
+        msg = gr.Textbox(
+            label="Message",
+            placeholder="Type your message...",
+            container=False,
+            scale=4
+        )
+        submit_btn = gr.Button("Send", variant='primary', scale=1)
+    
+    with gr.Column(scale=2):
+        with gr.Row():
+            clear_btn = gr.Button("Clear", variant='secondary')
+            stop_btn = gr.Button("Stop", variant='stop')
+        
+        with gr.Accordion("Parameters", open=False):
+            temperature = gr.Slider(minimum=0.1, maximum=1.5, value=0.6, label="Temperature")
+            top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, label="Top-p")
+            max_tokens = gr.Slider(minimum=2048, maximum=16384, value=4096, step=64, label="Max Tokens")
+
+    gr.Examples(
+        examples=[
+            ["How many r's are in the word strawberry?"],
+            ["Write 10 funny sentences that end in a fruit!"],
+            ["Explain how RL for LLMs can be done!"]
+        ],
+        inputs=msg,
+        label="Example Prompts"
+    )
+    
+    submit_event = submit_btn.click(
+        user, [msg, chatbot], [msg, chatbot], queue=False
+    ).then(
+        lambda: [True], outputs=active_gen
+    ).then(
+        generate_response, [chatbot, temperature, top_p, max_tokens, active_gen], chatbot
+    )
+    
+    msg.submit(
+        user, [msg, chatbot], [msg, chatbot], queue=False
+    ).then(
+        lambda: [True], outputs=active_gen
+    ).then(
+        generate_response, [chatbot, temperature, top_p, max_tokens, active_gen], chatbot
+    )
+    
+    stop_btn.click(
+        lambda: [False], None, active_gen, cancels=[submit_event]
+    )
+    
+    clear_btn.click(lambda: None, None, chatbot, queue=False)
+
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7860)

start.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Start llama-server in background
+cd /llama.cpp/build
+./bin/llama-server --host 0.0.0.0 --port 8080 --model /models/model.q2_k_l.gguf --ctx-size 16384 &
+
+# Wait for server to initialize
+echo "Waiting for server to start..."
+until curl -s "http://localhost:8080/v1/models" >/dev/null; do
+    sleep 1
+done
+
+echo "Server is ready. Starting Gradio app."
+
+# Start Gradio UI
+cd /
+python3 app.py