Update README.md

README.md

@@ -114,17 +114,124 @@ print("==========================================")
 
 ## Evaluation
 
+The model was evaluated using [mistral-evals](https://github.com/neuralmagic/mistral-evals) for vision-related tasks and using [lm_evaluation_harness](https://github.com/neuralmagic/lm-evaluation-harness) for select text-based benchmarks. The evaluations were conducted using the following commands:
 
 <details>
 <summary>Evaluation Commands</summary>
+  
+### Vision Tasks
+- vqav2
+- docvqa
+- mathvista
+- mmmu
+- chartqa
 
 ```
+vllm serve neuralmagic/pixtral-12b-quantized.w8a8 --tensor_parallel_size 1 --max_model_len 25000 --trust_remote_code --max_num_seqs 8 --gpu_memory_utilization 0.9 --dtype float16 --limit_mm_per_prompt image=7
+
+python -m eval.run eval_vllm \
+        --model_name neuralmagic/pixtral-12b-quantized.w8a8 \
+        --url http://0.0.0.0:8000 \
+        --output_dir ~/tmp \
+        --eval_name <vision_task_name>
+```
+
+### Text-based Tasks
+#### MMLU
+  
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="<model_name>",dtype=auto,add_bos_token=True,max_model_len=4096,tensor_parallel_size=<n>,gpu_memory_utilization=0.8,enable_chunked_prefill=True,trust_remote_code=True \
+  --tasks mmlu \
+  --num_fewshot 5 \
+  --batch_size auto \
+  --output_path output_dir
+
 ```
 
+#### MGSM
+
+```
+lm_eval \
+  --model vllm \
+  --model_args pretrained="<model_name>",dtype=auto,max_model_len=4096,max_gen_toks=2048,max_num_seqs=128,tensor_parallel_size=<n>,gpu_memory_utilization=0.9 \
+  --tasks mgsm_cot_native \
+  --num_fewshot 0 \
+  --batch_size auto \
+  --output_path output_dir
+
+```
 </details>
 
+
 ### Accuracy
 
+<table>
+  <thead>
+    <tr>
+      <th>Category</th>
+      <th>Metric</th>
+      <th>Qwen/Qwen2-VL-72B-Instruct</th>
+      <th>neuralmagic/Qwen2-VL-72B-Instruct-FP8-Dynamic</th>
+      <th>Recovery (%)</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td rowspan="6"><b>Vision</b></td>
+      <td>MMMU (val, CoT)<br><i>explicit_prompt_relaxed_correctness</i></td>
+      <td>62.11</td>
+      <td>60.67</td>
+      <td>97.68%</td>
+    </tr>
+    <tr>
+      <td>VQAv2 (val)<br><i>vqa_match</i></td>
+      <td>82.51</td>
+      <td>82.44</td>
+      <td>99.91%</td>
+    </tr>
+    <tr>
+      <td>DocVQA (val)<br><i>anls</i></td>
+      <td>95.01</td>
+      <td>95.10</td>
+      <td>100.09%</td>
+    </tr>
+    <tr>
+      <td>ChartQA (test, CoT)<br><i>anywhere_in_answer_relaxed_correctness</i></td>
+      <td>83.40</td>
+      <td>83.68</td>
+      <td>100.34%</td>
+    </tr>
+    <tr>
+      <td>Mathvista (testmini, CoT)<br><i>explicit_prompt_relaxed_correctness</i></td>
+      <td>66.57</td>
+      <td>67.07</td>
+      <td>100.75%</td>
+    </tr>
+    <tr>
+      <td><b>Average Score</b></td>
+      <td><b>77.12</b></td>
+      <td><b>77.39</b></td>
+      <td><b>100.35%</b></td>
+    </tr>
+    <tr>
+      <td rowspan="2"><b>Text</b></td>
+      <td>MGSM (CoT)</td>
+      <td>68.60</td>
+      <td>67.78</td>
+      <td>98.80%</td>
+    </tr>
+    <tr>
+      <td>MMLU (5-shot)</td>
+      <td>82.70</td>
+      <td>82.60</td>
+      <td>99.88%</td>
+    </tr>
+  </tbody>
+</table>
+
+
 ## Inference Performance