init app

README.md

@@ -1,14 +1,14 @@
 ---
-title: VPTQ Demo
-emoji: 💬
-colorFrom: yellow
-colorTo: purple
+title: VPTQ demo
+emoji: 🚀
+colorFrom: blue
+colorTo: green
 sdk: gradio
 sdk_version: 4.36.1
 app_file: app.py
-pinned: false
+pinned: true
 license: mit
-short_description: VPTQ online demo
+short_description: Vector Post-Training Quantization (VPTQ) Demo
 ---
 
-An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
+An example chatbot using [VPTQ](https://github.com/microsoft/VPTQ), [huggingface community](https://huggingface.co/spaces/VPTQ-community/).

app.py

@@ -1,12 +1,41 @@
+import spaces
 import gradio as gr
 from huggingface_hub import InferenceClient
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+from vptq.app_utils import get_chat_loop_generator
+
+# Update model list with annotations
+model_list_with_annotations = {
+    # "VPTQ-community/Meta-Llama-3.1-70B-Instruct-v8-k65536-65536-woft": "Llama 3.1 70B @ 4bit",
+    # "VPTQ-community/Meta-Llama-3.1-70B-Instruct-v8-k65536-256-woft": "Llama 3.1 70B @ 3bit",
+    # "VPTQ-community/Meta-Llama-3.1-70B-Instruct-v16-k65536-65536-woft": "Llama 3.1 70B @ 2bit",
+    # "VPTQ-community/Qwen2.5-72B-Instruct-v8-k65536-65536-woft": "Qwen2.5 72B @ 4 bits",
+    # "VPTQ-community/Qwen2.5-72B-Instruct-v8-k65536-256-woft": "Qwen2.5 72B @ 3 bits",
+    # "VPTQ-community/Qwen2.5-72B-Instruct-v16-k65536-65536-woft": "Qwen2.5 72B @ 3 bits",
+    # "VPTQ-community/Qwen2.5-32B-Instruct-v8-k65536-65536-woft": "Qwen2.5 32B @ 4 bits",
+    "VPTQ-community/Qwen2.5-32B-Instruct-v8-k65536-256-woft": "Qwen2.5 32B @ 3 bits",
+    "VPTQ-community/Qwen2.5-32B-Instruct-v16-k65536-0-woft": "Qwen2.5 32B @ 2 bits"
+}
+
+# Create a list of choices with annotations for the dropdown
+model_list_with_annotations_display = [f"{key} ({value})" for key, value in model_list_with_annotations.items()]
 
+model_keys = list(model_list_with_annotations.keys())
+current_model_g = model_keys[0]
+chat_completion = get_chat_loop_generator(current_model_g)
 
+@spaces.GPU
+def update_title_and_chatmodel(model):
+    model = str(model)
+    global chat_completion
+    global current_model_g
+    if model != current_model_g:
+        current_model_g = model
+        chat_completion = get_chat_loop_generator(current_model_g)
+    return model
+
+
+@spaces.GPU
 def respond(
     message,
     history: list[tuple[str, str]],
@@ -27,37 +56,56 @@ def respond(
 
     response = ""
 
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
+    for message in chat_completion(
+            messages,
+            max_tokens=max_tokens,
+            stream=True,
+            temperature=temperature,
+            top_p=top_p,
     ):
-        token = message.choices[0].delta.content
+        token = message
 
         response += token
         yield response
 
+
+css = """
+h1 {
+  text-align: center;
+  display: block;
+}
 """
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
 
+chatbot = gr.Chatbot(label="Gradio ChatInterface")
+with gr.Blocks() as demo:
+    with gr.Column(scale=1):
+        title_output = gr.Markdown("Please select a model to run")
+        chat_demo = gr.ChatInterface(
+            respond,
+            additional_inputs_accordion=gr.Accordion(
+                label="⚙️ Parameters", open=False, render=False
+            ),
+            fill_height=False,
+            additional_inputs=[
+                gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
+                gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
+                gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+                gr.Slider(
+                    minimum=0.1,
+                    maximum=1.0,
+                    value=0.95,
+                    step=0.05,
+                    label="Top-p (nucleus sampling)"
+                ),
+            ],
+        )
+        model_select = gr.Dropdown(
+            choices=model_list_with_annotations_display,
+            label="Models",
+            value=model_list_with_annotations_display[0],
+            info="Model & Estimated Quantized Bitwidth"
+        )
+        model_select.change(update_title_and_chatmodel, inputs=[model_select], outputs=title_output)
 
 if __name__ == "__main__":
     demo.launch()

requirements.txt

@@ -1 +1,2 @@
-huggingface_hub==0.22.2
+huggingface_hub>=0.22.2
+https://github.com/microsoft/VPTQ/releases/download/v0.0.1/vptq-0.0.1-cp310-cp310-manylinux1_x86_64.whl