Update app.py

app.py

@@ -17,12 +17,19 @@ from datetime import date, datetime, timedelta
 
 access_token = os.environ["TOKEN"]
 
-base_model = "NousResearch/Llama-2-13b-hf" 
-peft_model = "FinGPT/fingpt-sentiment_llama2-13b_lora"
-tokenizer =  AutoTokenizer.from_pretrained(base_model, trust_remote_code=True)
+# load model
+model = "NousResearch/Llama-2-7b-hf"
+peft_model = "FinGPT/fingpt-forecaster_sz50_llama2-7B_lora"
+
+tokenizer = AutoTokenizer.from_pretrained(model, token = access_token, trust_remote_code=True)
 tokenizer.pad_token = tokenizer.eos_token
-model = AutoModelForCausalLM.from_pretrained(base_model, trust_remote_code=True, device_map = "cuda:0", load_in_8bit = True,)
-model = PeftModel.from_pretrained(model, peft_model)
+tokenizer.padding_side = "right"
+
+streamer = TextStreamer(tokenizer)
+
+model = AutoModelForCausalLM.from_pretrained(model, trust_remote_code=True, token = access_token, device_map="cuda", load_in_8bit=True, offload_folder="offload/")
+model = PeftModel.from_pretrained(model, peft_model, offload_folder="offload/")
+
 model = model.eval()
 
 # Inference Data