Upload handler.py

handler.py

@@ -8,6 +8,7 @@ class EndpointHandler():
         self.model = AutoModelForCausalLM.from_pretrained(path,
                                                           torch_dtype=torch.bfloat16,
                                                           device_map="auto",
+                                                          load_in_8bit=True,
                                                           trust_remote_code=True)
         self.tokenizer = AutoTokenizer.from_pretrained(path)
         self.device = "cuda" if torch.cuda.is_available() else "cpu"