Change of settings again for TPU

app.py

@@ -44,7 +44,7 @@ REPO_ID = "sugiv/leetmonkey-peft-gguf"
 
 # Load the model
 model_path = hf_hub_download(repo_id=REPO_ID, filename=MODEL_NAME, cache_dir="./models")
-llm = Llama(model_path=model_path, n_ctx=1500, n_threads=16, n_gpu_layers=-1, verbose=False, mlock=True) ## TPU
+llm = Llama(model_path=model_path, n_ctx=1024, n_threads=8, n_gpu_layers=-1, verbose=False, mlock=True) ## TPU
 #llm = Llama(model_path=model_path, n_ctx=1024, n_threads=16, n_gpu_layers=0, verbose=False, mlock=False) ## CPU only
 #llm = Llama(model_path=model_path, n_ctx=1024, n_threads=16, n_gpu_layers=10, verbose=False, mlock=False) ## Nvidia
 logger.info("8-bit model loaded successfully")
@@ -54,7 +54,7 @@ token_to_problem_solution = {}
 
 # Generation parameters
 generation_kwargs = {
-    "max_tokens": 1024,
+    "max_tokens": 512,
     "stop": ["```", "### Instruction:", "### Response:"],
     "echo": False,
     "temperature": 0.05,