Update app.py

app.py

@@ -33,6 +33,10 @@ import requests
 
 from huggingface_hub import snapshot_download
 
+default_temperature = 0.15
+default_max_context = 16384
+default_max_output = 512
+
 default_bpw = "4.0bpw"
 available_models = [
     "2.5bpw",
@@ -49,14 +53,18 @@ for model in available_models:
     dirs.update({model: snapshot_download(repo_id="turboderp/pixtral-12b-exl2", revision=model)})
 
 @spaces.GPU(duration=45)
-def run_inference(message, history, model_picked):
-    if model_picked == None:
+def run_inference(message, history, model_picked, temperature, context_size, max_output):
+    if not model_picked:
         model_picked = default_bpw
+    if not temperature:
+        temperature = default_temperature
+    if not context_size:
+        context_size = default_max_context
+    if not max_output:
+        max_output = default_max_output
         
     local_dir = dirs[model_picked]
     
-    print(message)
-    print(history)
     # Loading only once GPU available
     config = ExLlamaV2Config(local_dir)
     config.max_seq_len = 16384
@@ -141,9 +149,10 @@ The current version of ExllamaV2 running is the dev branch, not the master branc
 
 The model at **4bpw and 16k context size fits in less than 12GB of VRAM**!
 
-The current settings are:  
+The current default settings are:  
+- Model Quant: 4.0bpw
 - Context Size: 16k tokens
-- Max Output: 1024 tokens
+- Max Output: 512 tokens
 - Temperature: 0.15
 
 You can select other quants and experiment!
@@ -157,5 +166,8 @@ examples = [
 ]
 
 drop = gr.Dropdown(available_models, label="EXL2 Quant", value=default_bpw)
-demo = gr.ChatInterface(fn=run_inference, examples = examples, title="Pixtral 12B EXL2", multimodal=True, description=description, additional_inputs = drop)
+temperature_gradio = gradio.Slider(minimum = 0, maximum = 1, label="Temperature", value=default_temperature, step = 0.05)
+context_size_gradio = gradio.Slider(minimum = 256, maximum = 32768, label="Context Size", value=default_max_context, step = 1)
+output_length_gradio = gradio.Slider(minimum = 1, maximum = 4096, label="Max Ouput Length", value=default_max_output, step = 1)
+demo = gr.ChatInterface(fn=run_inference, examples = examples, title="Pixtral 12B EXL2", multimodal=True, description=description, additional_inputs = [drop,  temperature_gradio, context_size_gradio, output_length_gradio])
 demo.queue().launch()