Update app.py

app.py

@@ -6,9 +6,9 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStream
 import gradio as gr
 from threading import Thread
 
-MODEL_LIST = ["CohereForAI/aya-expanse-8b"]
+MODEL_LIST = ["CohereForAI/aya-expanse-32b"]
 HF_TOKEN = os.environ.get("HF_TOKEN", None)
-MODEL = "CohereForAI/aya-expanse-8b"
+MODEL = "CohereForAI/aya-expanse-32b"
 
 TITLE = "<h1><center>Mawred T2 Wip </center></h1>"
 
@@ -34,20 +34,20 @@ h3 {
 device = "cuda" # for GPU usage or "cpu" for CPU usage
 
 quantization_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_compute_dtype=torch.bfloat16,
-    bnb_4bit_use_double_quant=True,
-    bnb_4bit_quant_type= "nf4")
+    load_in_8bit=True,
+    bnb_8bit_compute_dtype=torch.bfloat16,
+    bnb_8bit_use_double_quant=True,
+    bnb_8bit_quant_type= "nf4")
 
 tokenizer = AutoTokenizer.from_pretrained(MODEL)
 model = AutoModelForCausalLM.from_pretrained(
     MODEL,
     torch_dtype=torch.bfloat16,
     device_map="auto",
-    # quantization_config=quantization_config
+    quantization_config=quantization_config
 )
 
-@spaces.GPU()
+@spaces.GPU(660)
 def stream_chat(
     message: str, 
     history: list,