neuralmagic
/

Meta-Llama-3.1-8B-Instruct-FP8

Text Generation

text-generation-inference

compressed-tensors

Model card Files Files and versions Community

Lin-K76 commited on Jul 23, 2024

Commit

8db78f8

·

verified ·

1 Parent(s): c285c71

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -109,7 +109,7 @@ model_stub = "meta-llama/Meta-Llama-3.1-8B-Instruct"
 model_name = model_stub.split("/")[-1]
 device_map = calculate_offload_device_map(
-    model_stub, reserve_for_hessians=False, num_gpus=2, torch_dtype=torch.float16
 )
 model = SparseAutoModelForCausalLM.from_pretrained(

 model_name = model_stub.split("/")[-1]
 device_map = calculate_offload_device_map(
+    model_stub, reserve_for_hessians=False, num_gpus=1, torch_dtype=torch.float16
 )
 model = SparseAutoModelForCausalLM.from_pretrained(