Upload app.py

app.py

@@ -11,6 +11,7 @@ tokenizer = AutoTokenizer.from_pretrained(model_path)
 model = AutoModelForCausalLM.from_pretrained(
     model_path,
     device_map="auto",
+    load_in_8bit=True,  # Use 8-bit quantization instead of 4-bit
     torch_dtype=torch.float16,
     trust_remote_code=True
 )
@@ -58,16 +59,17 @@ def generate_response(prompt, max_length=512, temperature=0.7, top_p=0.9, top_k=
             
         inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
         
-        outputs = model.generate(
-            **inputs,
-            max_length=max_length,
-            temperature=temperature,
-            num_return_sequences=1,
-            pad_token_id=tokenizer.eos_token_id,
-            do_sample=True,
-            top_p=top_p,
-            top_k=top_k,
-        )
+        with torch.no_grad():  # Disable gradient computation for inference
+            outputs = model.generate(
+                **inputs,
+                max_length=max_length,
+                temperature=temperature,
+                num_return_sequences=1,
+                pad_token_id=tokenizer.eos_token_id,
+                do_sample=True,
+                top_p=top_p,
+                top_k=top_k,
+            )
         
         response = tokenizer.decode(outputs[0], skip_special_tokens=True)
         return response