Testing T4

app.py

@@ -6,6 +6,7 @@ from transformers import (
     StoppingCriteria,
     StoppingCriteriaList,
     TextIteratorStreamer,
+    BitsAndBytesConfig,
 )
 import gradio as gr
 import torch
@@ -18,14 +19,18 @@ max_new_tokens = 1024
 
 print(f"Starting to load the model {model_name}")
 
+bnb_config = BitsAndBytesConfig(
+    llm_int8_enable_fp32_cpu_offload=True,
+    load_in_8bit=True,
+)
+
 m = AutoModelForCausalLM.from_pretrained(
     model_name,
-    llm_int8_enable_fp32_cpu_offload=True,
     device_map = {
         "transformer.h": 0,
         "": "cpu",
     },
-    #load_in_8bit=True,
+    quantization_config=bnb_config,
 )
 
 m.config.pad_token_id = m.config.eos_token_id