Update handler.py

handler.py

@@ -8,9 +8,8 @@ device = "cuda" if torch.cuda.is_available() else "cpu"
 class EndpointHandler():
     def __init__(self, path=""):
         config = PeftConfig.from_pretrained(path)
-        model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, load_in_8bit=True, device_map='auto')
+        model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, device_map="auto")#, load_in_8bit=True, device_map='auto')
         self.tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
-        # Load the Lora model
         self.model = PeftModel.from_pretrained(model, path)
 
     def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]: