Update app.py

app.py

@@ -17,25 +17,26 @@ from datetime import date, datetime, timedelta
 
 access_token = os.environ["TOKEN"]
 
-# load model
-#model = "NousResearch/Llama-2-7b-chat-hf"
-model = "NousResearch/Llama-2-7b-chat-hf"
-peft_model = "FinGPT/fingpt-forecaster_sz50_llama2-7B_lora"
-
-tokenizer = AutoTokenizer.from_pretrained(model, token = access_token, trust_remote_code=True)
-tokenizer.pad_token = tokenizer.eos_token
-tokenizer.padding_side = "right"
-
-streamer = TextStreamer(tokenizer)
-
-#model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, token = access_token, device_map="cuda", load_in_8bit=True, offload_folder="offload/")
-#model = PeftModel.from_pretrained(model, peft_model, offload_folder="offload/")
-
-model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, token = access_token, load_in_8bit=True, offload_folder="offload/")
-model = PeftModel.from_pretrained(model, peft_model, offload_folder="offload/")
-
+base_model = AutoModelForCausalLM.from_pretrained(
+    'NousResearch/Llama-2-7b-chat-hf',
+    token=access_token,
+    trust_remote_code=True, 
+    device_map="auto",
+    load_in_8bit=True,
+    offload_folder="offload/"
+)
+model = PeftModel.from_pretrained(
+    base_model,
+    'FinGPT/fingpt-forecaster_dow30_llama2-7b_lora',
+    offload_folder="offload/"
+)
 model = model.eval()
 
+tokenizer = AutoTokenizer.from_pretrained(
+    'NousResearch/Llama-2-7b-chat-hf',
+    token=access_token
+)
+
 # Inference Data
 # get company news online