Update app.py

app.py

@@ -29,7 +29,8 @@ streamer = TextStreamer(tokenizer)
 
 #model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, token = access_token, device_map="cuda", load_in_8bit=True, offload_folder="offload/")
 #model = PeftModel.from_pretrained(model, peft_model, offload_folder="offload/")
-model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, token = access_token, device_map="cuda", load_in_8bit=True)
+
+model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, device_map="cuda")
 model = PeftModel.from_pretrained(model, peft_model)
 
 model = model.eval()
@@ -398,9 +399,9 @@ def ask(symbol, weeks_before, withbasic):
   #model=model.to(device)
     
   print(model.device)
+    
   inputs = tokenizer(pt, return_tensors='pt')
-
-  inputs = {key: value.to(model.device('cuda:0')) for key, value in inputs.items()}
+  inputs = {key: value.to(model.device) for key, value in inputs.items()}
   #inputs = {key: value.to(model.device) for key, value in inputs.items()}
   #inputs = {key: value.to('cuda:0') for key, value in inputs.items()}
   print("Inputs loaded onto devices.")