Update app.py

app.py

@@ -3,15 +3,20 @@ import pandas as pd
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM
 
-# Load the tokenizer and quantized model
-model_name = "meta-llama/Meta-Llama-3.1-8B"
+# Load the tokenizer
+model_name = "your-llama-model"  # Replace with the LLaMA model name
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 
-# Use bitsandbytes to load the model in 8-bit precision
-model = AutoModelForCausalLM.from_pretrained(model_name, load_in_8bit=True, device_map='auto')
+# Load the model
+model = AutoModelForCausalLM.from_pretrained(model_name)
 
-# Move model to the appropriate device (GPU/CPU)
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+# Apply dynamic quantization for CPU
+model = torch.quantization.quantize_dynamic(
+    model, {torch.nn.Linear}, dtype=torch.qint8
+)
+
+# Move model to CPU
+device = torch.device("cpu")
 model = model.to(device)
 
 # Set the padding token to the end-of-sequence token