Update app.py

app.py

@@ -31,13 +31,8 @@ streamer = TextStreamer(tokenizer)
 #model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, token = access_token, device_map="cuda", load_in_8bit=True, offload_folder="offload/")
 #model = PeftModel.from_pretrained(model, peft_model, offload_folder="offload/")
 
-#model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, device_map="cuda:0")
-model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True)
-model = PeftModel.from_pretrained(model, peft_model)
-
-#device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
-device = 'cuda:0'
-model.to(device)
+model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, token = access_token, device_map="cuda", load_in_8bit=True, offload_folder="offload/")
+model = PeftModel.from_pretrained(model, peft_model, offload_folder="offload/")
 
 model = model.eval()