Update app.py

app.py

@@ -21,17 +21,17 @@ access_token = os.environ["TOKEN"]
 model = "NousResearch/Llama-2-7b-hf"
 peft_model = "FinGPT/fingpt-forecaster_sz50_llama2-7B_lora"
 
-tokenizer = AutoTokenizer.from_pretrained(model, token = access_token, device_map="cuda:0", trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained(model, token = access_token, device_map="auto", trust_remote_code=True)
 tokenizer.pad_token = tokenizer.eos_token
 tokenizer.padding_side = "right"
 
 streamer = TextStreamer(tokenizer)
 
-model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, token = access_token, device_map="cuda:0", load_in_8bit=True, offload_folder="offload/")
-model = PeftModel.from_pretrained(model, peft_model,  device_map="cuda:0", offload_folder="offload/")
+model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, token = access_token, device_map="auto", offload_folder="offload/")
+model = PeftModel.from_pretrained(model, peft_model,  device_map="auto", offload_folder="offload/")
 
-#device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
-device = 'cuda:0'
+device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
+#device = 'cuda:0'
 model = model.to(device)
 model = model.eval()
 
@@ -401,9 +401,9 @@ def ask(symbol, weeks_before, withbasic):
   print(model.device)
     
   inputs = tokenizer(pt, return_tensors='pt')
+  inputs = {key: value.to(model.device) for key, value in inputs.items()}
   #inputs = {key: value.to(model.device) for key, value in inputs.items()}
-  #inputs = {key: value.to(model.device) for key, value in inputs.items()}
-  inputs = {key: value.to('cuda:0') for key, value in inputs.items()}
+  #inputs = {key: value.to('cuda:0') for key, value in inputs.items()}
   print("Inputs loaded onto devices.")
 
   res = model.generate(