Update README.md
Browse files
README.md
CHANGED
@@ -56,20 +56,19 @@ Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.gi
|
|
56 |
lm_eval --model hf --model_args pretrained="Intel/gemma-7b-int4-inc",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 32
|
57 |
```
|
58 |
|
59 |
-
| Metric |
|
60 |
| -------------- | ------ | ------ |
|
61 |
-
| Avg. | 0.
|
62 |
-
| mmlu | 0.
|
63 |
-
| lambada_openai | 0.
|
64 |
-
| hellaswag | 0.
|
65 |
-
| winogrande | 0.
|
66 |
-
| piqa | 0.
|
67 |
-
| truthfulqa_mc1 | 0.
|
68 |
-
| openbookqa | 0.
|
69 |
-
| boolq | 0.
|
70 |
-
|
|
71 |
-
|
|
72 |
-
| arc_challenge | 0.4991 | 0.4940 |
|
73 |
|
74 |
|
75 |
|
|
|
56 |
lm_eval --model hf --model_args pretrained="Intel/gemma-7b-int4-inc",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 32
|
57 |
```
|
58 |
|
59 |
+
| Metric | BF16 | int4 |
|
60 |
| -------------- | ------ | ------ |
|
61 |
+
| Avg. | 0.6302 | 0.6262 |
|
62 |
+
| mmlu | 0.6199 | 0.6111 |
|
63 |
+
| lambada_openai | 0.7316 | 0.7252 |
|
64 |
+
| hellaswag | 0.6059 | 0.6033 |
|
65 |
+
| winogrande | 0.7506 | 0.7451 |
|
66 |
+
| piqa | 0.8014 | 0.8058 |
|
67 |
+
| truthfulqa_mc1 | 0.3121 | 0.2889 |
|
68 |
+
| openbookqa | 0.3220 | 0.3380 |
|
69 |
+
| boolq | 0.8339 | 0.8281 |
|
70 |
+
| arc_easy | 0.8253 | 0.8152 |
|
71 |
+
| arc_challenge | 0.4991 | 0.5017 |
|
|
|
72 |
|
73 |
|
74 |
|