more gpt results

Signed-off-by: Jonathan Bnayahu <[email protected]>

results/bluebench/2025-08-03T09-40-01_evaluation_results.json

@@ -0,0 +1,1282 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-08-03T13:39:57.204417Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=azure/Azure/gpt-4o-ncf,max_tokens=1024",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "azure/Azure/gpt-4o-ncf",
+            "model_args": {
+                "max_tokens": 1024
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.26.5",
+        "unitxt_commit_hash": "N/A",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "pyarrow": "21.0.0",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "httpcore": "1.0.9",
+            "pip": "25.2",
+            "certifi": "2025.7.14",
+            "evaluate": "0.4.5",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "aiosignal": "1.4.0",
+            "yarl": "1.20.1",
+            "unitxt": "1.26.5",
+            "jsonschema": "4.25.0",
+            "portalocker": "3.2.0",
+            "multiprocess": "0.70.16",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "matplotlib": "3.10.5",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "sniffio": "1.3.1",
+            "rpds-py": "0.26.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "litellm": "1.74.12",
+            "pillow": "11.3.0",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "tokenizers": "0.21.4",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "regex": "2025.7.34",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "transformers": "4.54.1",
+            "fsspec": "2025.3.0",
+            "scikit-learn": "1.7.1",
+            "dill": "0.3.8",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "lxml": "6.0.0",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "fonttools": "4.59.0",
+            "tabulate": "0.9.0",
+            "typing_extensions": "4.12.2",
+            "absl-py": "2.3.1",
+            "accelerate": "1.9.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "huggingface-hub": "0.34.3",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.1",
+            "multidict": "6.6.3",
+            "httpx": "0.28.1",
+            "aiohttp": "3.12.15",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "colorama": "0.4.6",
+            "openai": "1.98.0",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "pandas": "2.3.1",
+            "hf-xet": "1.1.5",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.4444444444444444,
+                "num_of_instances": 9
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_religion": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_ses": {
+                "accuracy": 0.5555555555555556,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
+                "score_name": "accuracy",
+                "score": 0.5555555555555556,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
+                "num_of_instances": 9
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5555555555555556,
+                "num_of_instances": 9
+            },
+            "score": 0.9393939393939393,
+            "score_name": "subsets_mean",
+            "num_of_instances": 99
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 100,
+                "llama_3_70b_instruct_template_arena_hard": 0.9414414414414415,
+                "score": 0.9414414414414415,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.9414414414414415,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 100,
+                "f1_Person": 0.7999999999999999,
+                "f1_Organization": 0.7096774193548386,
+                "f1_Location": 0.7058823529411765,
+                "f1_macro": 0.7385199240986716,
+                "recall_macro": 0.80175983436853,
+                "precision_macro": 0.6848220769789397,
+                "in_classes_support": 1.0,
+                "f1_micro": 0.7361963190184048,
+                "recall_micro": 0.8,
+                "precision_micro": 0.6818181818181818,
+                "score": 0.7361963190184048,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.6504174815433912,
+                "score_ci_high": 0.782581269719831,
+                "f1_micro_ci_low": 0.6504174815433912,
+                "f1_micro_ci_high": 0.782581269719831
+            },
+            "score": 0.7361963190184048,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.42857142857142855,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "score": 0.45918367346938777,
+            "score_name": "subsets_mean",
+            "num_of_instances": 98
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.5199999999999999,
+                "f1_suggestive": 0.2,
+                "f1_generic": 0.8,
+                "f1_fanciful": 0.4,
+                "f1_descriptive": 0.4,
+                "f1_arbitrary": 0.8,
+                "f1_macro_ci_low": 0.3363636363636363,
+                "f1_macro_ci_high": 0.766060606060606,
+                "score_name": "f1_micro",
+                "score": 0.4666666666666667,
+                "score_ci_high": 0.6857142857142857,
+                "score_ci_low": 0.23076923076923078,
+                "num_of_instances": 20,
+                "accuracy": 0.35,
+                "accuracy_ci_low": 0.15,
+                "accuracy_ci_high": 0.6,
+                "f1_micro": 0.4666666666666667,
+                "f1_micro_ci_low": 0.23076923076923078,
+                "f1_micro_ci_high": 0.6857142857142857
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.5793103448275863,
+                "f1_no": 0.7586206896551724,
+                "f1_yes": 0.4,
+                "f1_macro_ci_low": 0.375,
+                "f1_macro_ci_high": 0.868365507202327,
+                "score_name": "f1_micro",
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.8717948717948718,
+                "score_ci_low": 0.45355819395422325,
+                "num_of_instances": 20,
+                "accuracy": 0.65,
+                "accuracy_ci_low": 0.45,
+                "accuracy_ci_high": 0.85,
+                "f1_micro": 0.6666666666666666,
+                "f1_micro_ci_low": 0.45355819395422325,
+                "f1_micro_ci_high": 0.8717948717948718
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.26643990929705214,
+                "f1_conclusion": 0.2222222222222222,
+                "f1_decree": 0.0,
+                "f1_issue": 0.2857142857142857,
+                "f1_analysis": 0.5,
+                "f1_facts": 0.8571428571428571,
+                "f1_procedural history": 0.0,
+                "f1_rule": 0.0,
+                "f1_macro_ci_low": 0.1142857142857143,
+                "f1_macro_ci_high": 0.45052752383482225,
+                "score_name": "f1_micro",
+                "score": 0.35294117647058826,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.12903225806451613,
+                "num_of_instances": 20,
+                "accuracy": 0.3,
+                "accuracy_ci_low": 0.1,
+                "accuracy_ci_high": 0.5,
+                "f1_micro": 0.35294117647058826,
+                "f1_micro_ci_low": 0.12903225806451613,
+                "f1_micro_ci_high": 0.5714285714285714
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.696969696969697,
+                "f1_yes": 0.7272727272727273,
+                "f1_no": 0.6666666666666666,
+                "f1_macro_ci_low": 0.4949494949494949,
+                "f1_macro_ci_high": 0.898989898989899,
+                "score_name": "f1_micro",
+                "score": 0.7,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.5,
+                "num_of_instances": 20,
+                "accuracy": 0.7,
+                "accuracy_ci_low": 0.5,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.7,
+                "f1_micro_ci_low": 0.5,
+                "f1_micro_ci_high": 0.9
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.8585526315789473,
+                "f1_yes": 0.875,
+                "f1_no": 0.8421052631578947,
+                "f1_macro_ci_low": 0.6847205623637095,
+                "f1_macro_ci_high": 0.9545454545454546,
+                "score_name": "f1_micro",
+                "score": 0.8571428571428571,
+                "score_ci_high": 0.9473684210526315,
+                "score_ci_low": 0.6706944990883059,
+                "num_of_instances": 20,
+                "accuracy": 0.75,
+                "accuracy_ci_low": 0.50468235519016,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.8571428571428571,
+                "f1_micro_ci_low": 0.6706944990883059,
+                "f1_micro_ci_high": 0.9473684210526315
+            },
+            "score": 0.6086834733893557,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.6508639236580414,
+                "f1_cars": 0.8333333333333334,
+                "f1_windows x": 0.3333333333333333,
+                "f1_computer graphics": 0.5555555555555556,
+                "f1_atheism": 0.2857142857142857,
+                "f1_religion": 0.0,
+                "f1_medicine": 0.8571428571428571,
+                "f1_christianity": 0.8571428571428571,
+                "f1_microsoft windows": 0.8333333333333334,
+                "f1_middle east": 0.8333333333333334,
+                "f1_motorcycles": 0.7272727272727273,
+                "f1_pc hardware": 0.7692307692307693,
+                "f1_mac hardware": 1.0,
+                "f1_electronics": 0.3333333333333333,
+                "f1_for sale": 0.75,
+                "f1_guns": 0.6666666666666666,
+                "f1_space": 0.5714285714285714,
+                "f1_cryptography": 0.3333333333333333,
+                "f1_baseball": 1.0,
+                "f1_politics": 0.5882352941176471,
+                "f1_hockey": 0.8888888888888888,
+                "f1_macro_ci_low": 0.5563237523685153,
+                "f1_macro_ci_high": 0.7401830936884861,
+                "score_name": "f1_micro",
+                "score": 0.6907216494845361,
+                "score_ci_high": 0.7645361476644139,
+                "score_ci_low": 0.5804059043570197,
+                "num_of_instances": 100,
+                "accuracy": 0.67,
+                "accuracy_ci_low": 0.56,
+                "accuracy_ci_high": 0.75,
+                "f1_micro": 0.6907216494845361,
+                "f1_micro_ci_low": 0.5804059043570197,
+                "f1_micro_ci_high": 0.7645361476644139
+            },
+            "score": 0.6907216494845361,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.7410023219814241,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9411764705882353,
+                "f1_debt collection": 0.7368421052631579,
+                "f1_payday loan or title loan or personal loan": 0.0,
+                "f1_student loan": 0.8333333333333334,
+                "f1_credit card or prepaid card": 0.75,
+                "f1_checking or savings account": 1.0,
+                "f1_mortgage": 0.6666666666666666,
+                "f1_money transfer or virtual currency or money service": 1.0,
+                "f1_macro_ci_low": 0.548026288422304,
+                "f1_macro_ci_high": 0.8446388968731338,
+                "score_name": "f1_micro",
+                "score": 0.898989898989899,
+                "score_ci_high": 0.9447236180904522,
+                "score_ci_low": 0.8203180754561684,
+                "num_of_instances": 100,
+                "accuracy": 0.89,
+                "accuracy_ci_low": 0.81,
+                "accuracy_ci_high": 0.94,
+                "f1_micro": 0.898989898989899,
+                "f1_micro_ci_low": 0.8203180754561684,
+                "f1_micro_ci_high": 0.9447236180904522
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.8260549519130755,
+                "f1_mortgages and loans": 0.8695652173913043,
+                "f1_credit card": 0.8181818181818182,
+                "f1_debt collection": 0.7368421052631579,
+                "f1_credit reporting": 0.782608695652174,
+                "f1_retail banking": 0.9230769230769231,
+                "f1_macro_ci_low": 0.7025385602172309,
+                "f1_macro_ci_high": 0.9116966675085018,
+                "score_name": "f1_micro",
+                "score": 0.82,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.7,
+                "num_of_instances": 50,
+                "accuracy": 0.82,
+                "accuracy_ci_low": 0.7,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.82,
+                "f1_micro_ci_low": 0.7,
+                "f1_micro_ci_high": 0.9
+            },
+            "score": 0.8594949494949495,
+            "score_name": "subsets_mean",
+            "num_of_instances": 150
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 100,
+                "execution_accuracy": 0.36,
+                "program_accuracy": 0.37,
+                "score": 0.37,
+                "score_name": "program_accuracy",
+                "execution_accuracy_ci_low": 0.27,
+                "execution_accuracy_ci_high": 0.47,
+                "program_accuracy_ci_low": 0.2811354521803329,
+                "program_accuracy_ci_high": 0.47,
+                "score_ci_low": 0.2811354521803329,
+                "score_ci_high": 0.47
+            },
+            "score": 0.37,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.4468480048160072,
+                "recall": 0.6454220314570971,
+                "f1": 0.4890230179217559,
+                "precision_ci_low": 0.4151579792947102,
+                "precision_ci_high": 0.4845603023826521,
+                "recall_ci_low": 0.6061330597384731,
+                "recall_ci_high": 0.6765859073642304,
+                "f1_ci_low": 0.46005008735451697,
+                "f1_ci_high": 0.5174377148355835,
+                "score_name": "f1",
+                "score": 0.4890230179217559,
+                "score_ci_high": 0.5174377148355835,
+                "score_ci_low": 0.46005008735451697,
+                "num_of_instances": 100,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.691885244846344,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7364352762699127,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6613857835531235,
+                "faithfullness_f1_token_overlap": 0.43039423317750836,
+                "faithfullness_recall_token_overlap": 0.35370116631248527,
+                "faithfullness_precision_token_overlap": 0.6790419566943843,
+                "correctness_f1_token_overlap": 0.4890230179217559,
+                "correctness_recall_token_overlap": 0.6454220314570971,
+                "correctness_precision_token_overlap": 0.4468480048160072
+            },
+            "score": 0.4890230179217559,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.62,
+                "accuracy_ci_low": 0.51,
+                "accuracy_ci_high": 0.7,
+                "score_name": "accuracy",
+                "score": 0.62,
+                "score_ci_high": 0.7,
+                "score_ci_low": 0.51,
+                "num_of_instances": 100
+            },
+            "openbook_qa": {
+                "accuracy": 0.94,
+                "accuracy_ci_low": 0.89,
+                "accuracy_ci_high": 0.98,
+                "score_name": "accuracy",
+                "score": 0.94,
+                "score_ci_high": 0.98,
+                "score_ci_low": 0.89,
+                "num_of_instances": 100
+            },
+            "score": 0.78,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 0.0,
+                "severity_high": 0.0,
+                "severity_medium": 0.0,
+                "severity_low": 100.0,
+                "category_harmful_info": 0.8862442294401782,
+                "category_deception": 0.9089467666215367,
+                "category_explicit_content": 0.8660169005243467,
+                "category_violence": 0.8813284475474604,
+                "category_discrimination": 0.7921433370737802,
+                "category_substance_abuse": 0.892341727639238,
+                "category_pii": 0.8837605582343208,
+                "safety": 0.8784392809950642,
+                "safety_ci_low": 0.8622394069744699,
+                "safety_ci_high": 0.8922010585173956,
+                "score_name": "safety",
+                "score": 0.8784392809950642,
+                "score_ci_high": 0.8922010585173956,
+                "score_ci_low": 0.8622394069744699,
+                "num_of_instances": 100
+            },
+            "score": 0.8784392809950642,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rougeLsum": 0.3334800194661833,
+                "rouge2": 0.1746055052600959,
+                "rouge1": 0.3914063192107438,
+                "rougeL": 0.2607874086538398,
+                "score": 0.2607874086538398,
+                "score_name": "rougeL",
+                "rougeLsum_ci_low": 0.31372911043552615,
+                "rougeLsum_ci_high": 0.3521588230191023,
+                "rouge2_ci_low": 0.16154343447204916,
+                "rouge2_ci_high": 0.18686015868835737,
+                "rouge1_ci_low": 0.36979472683613324,
+                "rouge1_ci_high": 0.41112417139895086,
+                "rougeL_ci_low": 0.24640816277583857,
+                "rougeL_ci_high": 0.2769615579591105,
+                "score_ci_low": 0.24640816277583857,
+                "score_ci_high": 0.2769615579591105
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rougeLsum": 0.08980268920854961,
+                "rouge2": 0.012765072464340065,
+                "rouge1": 0.10813972613064696,
+                "rougeL": 0.08042515106945874,
+                "score": 0.08042515106945874,
+                "score_name": "rougeL",
+                "rougeLsum_ci_low": 0.07759889049057624,
+                "rougeLsum_ci_high": 0.1032183831189881,
+                "rouge2_ci_low": 0.008976164023617238,
+                "rouge2_ci_high": 0.018103937102878734,
+                "rouge1_ci_low": 0.09248155246218394,
+                "rouge1_ci_high": 0.1250184504642972,
+                "rougeL_ci_low": 0.06959906992080947,
+                "rougeL_ci_high": 0.09223760457164616,
+                "score_ci_low": 0.06959906992080947,
+                "score_ci_high": 0.09223760457164616
+            },
+            "score": 0.17060627986164928,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    156,
+                    105,
+                    71,
+                    50
+                ],
+                "totals": [
+                    226,
+                    220,
+                    214,
+                    208
+                ],
+                "precisions": [
+                    0.6902654867256638,
+                    0.4772727272727273,
+                    0.3317757009345794,
+                    0.2403846153846154
+                ],
+                "bp": 1.0,
+                "sys_len": 226,
+                "ref_len": 208,
+                "sacrebleu": 0.4026090246453075,
+                "score": 0.4026090246453075,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.24025628806486451,
+                "score_ci_high": 0.5204212192812463,
+                "sacrebleu_ci_low": 0.24025628806486451,
+                "sacrebleu_ci_high": 0.5204212192812463
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    143,
+                    93,
+                    63,
+                    45
+                ],
+                "totals": [
+                    219,
+                    213,
+                    207,
+                    201
+                ],
+                "precisions": [
+                    0.6529680365296804,
+                    0.43661971830985913,
+                    0.30434782608695654,
+                    0.22388059701492538
+                ],
+                "bp": 1.0,
+                "sys_len": 219,
+                "ref_len": 208,
+                "sacrebleu": 0.3733322279107499,
+                "score": 0.3733322279107499,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.26773105555308707,
+                "score_ci_high": 0.5628718725153604,
+                "sacrebleu_ci_low": 0.26773105555308707,
+                "sacrebleu_ci_high": 0.5628718725153604
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 6,
+                "counts": [
+                    118,
+                    70,
+                    45,
+                    27
+                ],
+                "totals": [
+                    211,
+                    205,
+                    199,
+                    193
+                ],
+                "precisions": [
+                    0.5592417061611374,
+                    0.34146341463414637,
+                    0.22613065326633167,
+                    0.13989637305699482
+                ],
+                "bp": 1.0,
+                "sys_len": 211,
+                "ref_len": 209,
+                "sacrebleu": 0.27879013737554453,
+                "score": 0.27879013737554453,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.242285190601197,
+                "score_ci_high": 0.3370610364230129,
+                "sacrebleu_ci_low": 0.242285190601197,
+                "sacrebleu_ci_high": 0.3370610364230129
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 6,
+                "counts": [
+                    148,
+                    92,
+                    62,
+                    46
+                ],
+                "totals": [
+                    217,
+                    211,
+                    205,
+                    199
+                ],
+                "precisions": [
+                    0.6820276497695852,
+                    0.43601895734597157,
+                    0.3024390243902439,
+                    0.23115577889447236
+                ],
+                "bp": 1.0,
+                "sys_len": 217,
+                "ref_len": 216,
+                "sacrebleu": 0.3797191362653853,
+                "score": 0.3797191362653853,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.24991245830969483,
+                "score_ci_high": 0.537890541133251,
+                "sacrebleu_ci_low": 0.24991245830969483,
+                "sacrebleu_ci_high": 0.537890541133251
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 6,
+                "counts": [
+                    190,
+                    150,
+                    119,
+                    95
+                ],
+                "totals": [
+                    244,
+                    238,
+                    232,
+                    226
+                ],
+                "precisions": [
+                    0.7786885245901639,
+                    0.6302521008403361,
+                    0.5129310344827587,
+                    0.42035398230088494
+                ],
+                "bp": 1.0,
+                "sys_len": 244,
+                "ref_len": 235,
+                "sacrebleu": 0.5703455465960385,
+                "score": 0.5703455465960385,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4931990672262396,
+                "score_ci_high": 0.6753511173840523,
+                "sacrebleu_ci_low": 0.4931990672262396,
+                "sacrebleu_ci_high": 0.6753511173840523
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 6,
+                "counts": [
+                    162,
+                    92,
+                    63,
+                    43
+                ],
+                "totals": [
+                    273,
+                    267,
+                    261,
+                    255
+                ],
+                "precisions": [
+                    0.5934065934065934,
+                    0.3445692883895131,
+                    0.24137931034482757,
+                    0.16862745098039217
+                ],
+                "bp": 1.0,
+                "sys_len": 273,
+                "ref_len": 249,
+                "sacrebleu": 0.3020398964371346,
+                "score": 0.3020398964371346,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.23699249066795575,
+                "score_ci_high": 0.36104670705975184,
+                "sacrebleu_ci_low": 0.23699249066795575,
+                "sacrebleu_ci_high": 0.36104670705975184
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 6,
+                "counts": [
+                    181,
+                    136,
+                    110,
+                    89
+                ],
+                "totals": [
+                    228,
+                    222,
+                    216,
+                    210
+                ],
+                "precisions": [
+                    0.7938596491228069,
+                    0.6126126126126126,
+                    0.5092592592592592,
+                    0.4238095238095238
+                ],
+                "bp": 1.0,
+                "sys_len": 228,
+                "ref_len": 222,
+                "sacrebleu": 0.5691933623646763,
+                "score": 0.5691933623646763,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.5074329158262365,
+                "score_ci_high": 0.6529930317459868,
+                "sacrebleu_ci_low": 0.5074329158262365,
+                "sacrebleu_ci_high": 0.6529930317459868
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 6,
+                "counts": [
+                    165,
+                    125,
+                    97,
+                    82
+                ],
+                "totals": [
+                    230,
+                    224,
+                    218,
+                    212
+                ],
+                "precisions": [
+                    0.717391304347826,
+                    0.5580357142857143,
+                    0.444954128440367,
+                    0.38679245283018865
+                ],
+                "bp": 1.0,
+                "sys_len": 230,
+                "ref_len": 230,
+                "sacrebleu": 0.5123335940057543,
+                "score": 0.5123335940057543,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3840244184926614,
+                "score_ci_high": 0.650779271851941,
+                "sacrebleu_ci_low": 0.3840244184926614,
+                "sacrebleu_ci_high": 0.650779271851941
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 6,
+                "counts": [
+                    170,
+                    109,
+                    73,
+                    47
+                ],
+                "totals": [
+                    236,
+                    230,
+                    224,
+                    218
+                ],
+                "precisions": [
+                    0.7203389830508474,
+                    0.47391304347826085,
+                    0.32589285714285715,
+                    0.21559633027522934
+                ],
+                "bp": 0.9707745538991623,
+                "sys_len": 236,
+                "ref_len": 243,
+                "sacrebleu": 0.3820377957320921,
+                "score": 0.3820377957320921,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3085199315474183,
+                "score_ci_high": 0.48773289703817563,
+                "sacrebleu_ci_low": 0.3085199315474183,
+                "sacrebleu_ci_high": 0.48773289703817563
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    158,
+                    111,
+                    79,
+                    59
+                ],
+                "totals": [
+                    225,
+                    219,
+                    213,
+                    207
+                ],
+                "precisions": [
+                    0.7022222222222223,
+                    0.5068493150684932,
+                    0.37089201877934275,
+                    0.28502415458937197
+                ],
+                "bp": 1.0,
+                "sys_len": 225,
+                "ref_len": 208,
+                "sacrebleu": 0.44042366511701625,
+                "score": 0.44042366511701625,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.341980662697259,
+                "score_ci_high": 0.5496900259922087,
+                "sacrebleu_ci_low": 0.341980662697259,
+                "sacrebleu_ci_high": 0.5496900259922087
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    150,
+                    90,
+                    58,
+                    39
+                ],
+                "totals": [
+                    236,
+                    230,
+                    224,
+                    218
+                ],
+                "precisions": [
+                    0.635593220338983,
+                    0.391304347826087,
+                    0.2589285714285714,
+                    0.17889908256880735
+                ],
+                "bp": 1.0,
+                "sys_len": 236,
+                "ref_len": 208,
+                "sacrebleu": 0.32762007434781915,
+                "score": 0.32762007434781915,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.20337120135607298,
+                "score_ci_high": 0.4231259198402102,
+                "sacrebleu_ci_low": 0.20337120135607298,
+                "sacrebleu_ci_high": 0.4231259198402102
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    135,
+                    76,
+                    45,
+                    29
+                ],
+                "totals": [
+                    226,
+                    220,
+                    214,
+                    208
+                ],
+                "precisions": [
+                    0.5973451327433628,
+                    0.34545454545454546,
+                    0.2102803738317757,
+                    0.13942307692307693
+                ],
+                "bp": 1.0,
+                "sys_len": 226,
+                "ref_len": 208,
+                "sacrebleu": 0.2788928697008729,
+                "score": 0.2788928697008729,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.18358949678749556,
+                "score_ci_high": 0.4387198667737409,
+                "sacrebleu_ci_low": 0.18358949678749556,
+                "sacrebleu_ci_high": 0.4387198667737409
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    168,
+                    124,
+                    93,
+                    73
+                ],
+                "totals": [
+                    221,
+                    215,
+                    209,
+                    203
+                ],
+                "precisions": [
+                    0.7601809954751131,
+                    0.5767441860465117,
+                    0.4449760765550239,
+                    0.35960591133004927
+                ],
+                "bp": 1.0,
+                "sys_len": 221,
+                "ref_len": 208,
+                "sacrebleu": 0.5146546836832124,
+                "score": 0.5146546836832124,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.378423475348725,
+                "score_ci_high": 0.6374865872050556,
+                "sacrebleu_ci_low": 0.378423475348725,
+                "sacrebleu_ci_high": 0.6374865872050556
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    166,
+                    121,
+                    91,
+                    72
+                ],
+                "totals": [
+                    227,
+                    221,
+                    215,
+                    209
+                ],
+                "precisions": [
+                    0.7312775330396475,
+                    0.5475113122171945,
+                    0.4232558139534883,
+                    0.34449760765550236
+                ],
+                "bp": 1.0,
+                "sys_len": 227,
+                "ref_len": 208,
+                "sacrebleu": 0.49154820843517344,
+                "score": 0.49154820843517344,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4117068924616905,
+                "score_ci_high": 0.6461238952977133,
+                "sacrebleu_ci_low": 0.4117068924616905,
+                "sacrebleu_ci_high": 0.6461238952977133
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    150,
+                    96,
+                    65,
+                    48
+                ],
+                "totals": [
+                    219,
+                    213,
+                    207,
+                    201
+                ],
+                "precisions": [
+                    0.684931506849315,
+                    0.45070422535211263,
+                    0.3140096618357488,
+                    0.23880597014925375
+                ],
+                "bp": 1.0,
+                "sys_len": 219,
+                "ref_len": 208,
+                "sacrebleu": 0.3900602917326207,
+                "score": 0.3900602917326207,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3529954080391477,
+                "score_ci_high": 0.44601468942834244,
+                "sacrebleu_ci_low": 0.3529954080391477,
+                "sacrebleu_ci_high": 0.44601468942834244
+            },
+            "score": 0.41424003428995987,
+            "score_name": "subsets_mean",
+            "num_of_instances": 90
+        },
+        "score": 0.6413403122123419,
+        "score_name": "subsets_mean",
+        "num_of_instances": 1537
+    }
+}

results/bluebench/2025-08-03T12-21-28_evaluation_results.json

@@ -0,0 +1,1282 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-08-03T16:21:24.530955Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=azure/Azure/o3-mini-ncf,max_tokens=1024",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "azure/Azure/o3-mini-ncf",
+            "model_args": {
+                "max_tokens": 1024
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.26.5",
+        "unitxt_commit_hash": "N/A",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "pyarrow": "21.0.0",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "httpcore": "1.0.9",
+            "pip": "25.2",
+            "certifi": "2025.7.14",
+            "evaluate": "0.4.5",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "aiosignal": "1.4.0",
+            "yarl": "1.20.1",
+            "unitxt": "1.26.5",
+            "jsonschema": "4.25.0",
+            "portalocker": "3.2.0",
+            "multiprocess": "0.70.16",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "matplotlib": "3.10.5",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "sniffio": "1.3.1",
+            "rpds-py": "0.26.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "litellm": "1.74.12",
+            "pillow": "11.3.0",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "tokenizers": "0.21.4",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "regex": "2025.7.34",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "transformers": "4.54.1",
+            "fsspec": "2025.3.0",
+            "scikit-learn": "1.7.1",
+            "dill": "0.3.8",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "lxml": "6.0.0",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "fonttools": "4.59.0",
+            "tabulate": "0.9.0",
+            "typing_extensions": "4.12.2",
+            "absl-py": "2.3.1",
+            "accelerate": "1.9.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "huggingface-hub": "0.34.3",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.1",
+            "multidict": "6.6.3",
+            "httpx": "0.28.1",
+            "aiohttp": "3.12.15",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "colorama": "0.4.6",
+            "openai": "1.98.0",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "pandas": "2.3.1",
+            "hf-xet": "1.1.5",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5310928992288233,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5310928992288233,
+                "num_of_instances": 9
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.46041936253217447,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.46041936253217447,
+                "num_of_instances": 9
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_religion": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "score": 0.9797979797979798,
+            "score_name": "subsets_mean",
+            "num_of_instances": 99
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 100,
+                "llama_3_70b_instruct_template_arena_hard": 0.3711340206185567,
+                "score": 0.3711340206185567,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.3711340206185567,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 100,
+                "f1_Person": 0.8181818181818182,
+                "f1_Organization": 0.6428571428571429,
+                "f1_Location": 0.75,
+                "f1_macro": 0.737012987012987,
+                "recall_macro": 0.683488612836439,
+                "precision_macro": 0.8125,
+                "in_classes_support": 1.0,
+                "f1_micro": 0.7285714285714285,
+                "recall_micro": 0.68,
+                "precision_micro": 0.7846153846153846,
+                "score": 0.7285714285714285,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.5888115359844259,
+                "score_ci_high": 0.8223830090806191,
+                "f1_micro_ci_low": 0.5888115359844259,
+                "f1_micro_ci_high": 0.8223830090806191
+            },
+            "score": 0.7285714285714285,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.6807203593841678,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.6807203593841678,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_math": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "score": 0.5,
+            "score_name": "subsets_mean",
+            "num_of_instances": 98
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.0,
+                "f1_suggestive": 0.0,
+                "f1_generic": 0.0,
+                "f1_fanciful": 0.0,
+                "f1_descriptive": 0.0,
+                "f1_arbitrary": 0.0,
+                "f1_macro_ci_low": 0.0,
+                "f1_macro_ci_high": 0.0,
+                "score_name": "f1_micro",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 20,
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "f1_micro": 0.0,
+                "f1_micro_ci_low": 0.0,
+                "f1_micro_ci_high": 0.0
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.47619047619047616,
+                "f1_no": 0.6666666666666666,
+                "f1_yes": 0.2857142857142857,
+                "f1_macro_ci_low": 0.2857142857142857,
+                "f1_macro_ci_high": 0.8813336459688916,
+                "score_name": "f1_micro",
+                "score": 0.5806451612903226,
+                "score_ci_high": 0.7878787878787878,
+                "score_ci_low": 0.3333333333333333,
+                "num_of_instances": 20,
+                "accuracy": 0.45,
+                "accuracy_ci_low": 0.25,
+                "accuracy_ci_high": 0.7,
+                "f1_micro": 0.5806451612903226,
+                "f1_micro_ci_low": 0.3333333333333333,
+                "f1_micro_ci_high": 0.7878787878787878
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.20272108843537415,
+                "f1_conclusion": 0.0,
+                "f1_decree": 0.0,
+                "f1_issue": 0.2857142857142857,
+                "f1_analysis": 0.8,
+                "f1_facts": 0.3333333333333333,
+                "f1_procedural history": 0.0,
+                "f1_rule": 0.0,
+                "f1_macro_ci_low": 0.05,
+                "f1_macro_ci_high": 0.3627437138685071,
+                "score_name": "f1_micro",
+                "score": 0.27586206896551724,
+                "score_ci_high": 0.5223537291196929,
+                "score_ci_low": 0.07650831228694685,
+                "num_of_instances": 20,
+                "accuracy": 0.2,
+                "accuracy_ci_low": 0.05,
+                "accuracy_ci_high": 0.40138012181413957,
+                "f1_micro": 0.27586206896551724,
+                "f1_micro_ci_low": 0.07650831228694685,
+                "f1_micro_ci_high": 0.5223537291196929
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.3666666666666667,
+                "f1_yes": 0.3333333333333333,
+                "f1_no": 0.4,
+                "f1_macro_ci_low": 0.16690480249191558,
+                "f1_macro_ci_high": 0.6427398020907653,
+                "score_name": "f1_micro",
+                "score": 0.37037037037037035,
+                "score_ci_high": 0.625,
+                "score_ci_low": 0.16,
+                "num_of_instances": 20,
+                "accuracy": 0.25,
+                "accuracy_ci_low": 0.1,
+                "accuracy_ci_high": 0.5,
+                "f1_micro": 0.37037037037037035,
+                "f1_micro_ci_low": 0.16,
+                "f1_micro_ci_high": 0.625
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.660633484162896,
+                "f1_yes": 0.6153846153846154,
+                "f1_no": 0.7058823529411765,
+                "f1_macro_ci_low": 0.40467435105346683,
+                "f1_macro_ci_high": 0.8261376660890378,
+                "score_name": "f1_micro",
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.8235294117647058,
+                "score_ci_low": 0.46153846153846156,
+                "num_of_instances": 20,
+                "accuracy": 0.5,
+                "accuracy_ci_low": 0.3,
+                "accuracy_ci_high": 0.7,
+                "f1_micro": 0.6666666666666666,
+                "f1_micro_ci_low": 0.46153846153846156,
+                "f1_micro_ci_high": 0.8235294117647058
+            },
+            "score": 0.37870885345857536,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.38195526695526694,
+                "f1_cars": 0.5714285714285714,
+                "f1_windows x": 0.5714285714285714,
+                "f1_computer graphics": 0.5454545454545454,
+                "f1_atheism": 0.0,
+                "f1_christianity": 0.2857142857142857,
+                "f1_religion": 0.0,
+                "f1_medicine": 0.6666666666666666,
+                "f1_microsoft windows": 0.5,
+                "f1_middle east": 0.2857142857142857,
+                "f1_motorcycles": 0.25,
+                "f1_for sale": 0.5714285714285714,
+                "f1_pc hardware": 0.2222222222222222,
+                "f1_mac hardware": 0.8,
+                "f1_guns": 0.2857142857142857,
+                "f1_politics": 0.5,
+                "f1_space": 0.75,
+                "f1_cryptography": 0.0,
+                "f1_baseball": 0.5,
+                "f1_hockey": 0.3333333333333333,
+                "f1_electronics": 0.0,
+                "f1_macro_ci_low": 0.3045532013795943,
+                "f1_macro_ci_high": 0.48572662880615863,
+                "score_name": "f1_micro",
+                "score": 0.4305555555555556,
+                "score_ci_high": 0.5281645512463683,
+                "score_ci_low": 0.31327005378302275,
+                "num_of_instances": 100,
+                "accuracy": 0.31,
+                "accuracy_ci_low": 0.22,
+                "accuracy_ci_high": 0.4,
+                "f1_micro": 0.4305555555555556,
+                "f1_micro_ci_low": 0.31327005378302275,
+                "f1_micro_ci_high": 0.5281645512463683
+            },
+            "score": 0.4305555555555556,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.8944327731092437,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9411764705882353,
+                "f1_debt collection": 0.6666666666666666,
+                "f1_payday loan or title loan or personal loan": 1.0,
+                "f1_credit card or prepaid card": 0.8571428571428571,
+                "f1_student loan": 0.8333333333333334,
+                "f1_checking or savings account": 1.0,
+                "f1_mortgage": 0.8571428571428571,
+                "f1_money transfer or virtual currency or money service": 1.0,
+                "f1_macro_ci_low": 0.806614365464373,
+                "f1_macro_ci_high": 0.9657999668768317,
+                "score_name": "f1_micro",
+                "score": 0.9090909090909091,
+                "score_ci_high": 0.9547738693467337,
+                "score_ci_low": 0.8379092938797416,
+                "num_of_instances": 100,
+                "accuracy": 0.9,
+                "accuracy_ci_low": 0.83,
+                "accuracy_ci_high": 0.95,
+                "f1_micro": 0.9090909090909091,
+                "f1_micro_ci_low": 0.8379092938797416,
+                "f1_micro_ci_high": 0.9547738693467337
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.8162680456798105,
+                "f1_mortgages and loans": 0.7619047619047619,
+                "f1_credit card": 0.8571428571428571,
+                "f1_debt collection": 0.7058823529411765,
+                "f1_credit reporting": 0.8333333333333334,
+                "f1_retail banking": 0.9230769230769231,
+                "f1_macro_ci_low": 0.7016062262967031,
+                "f1_macro_ci_high": 0.9223789564935859,
+                "score_name": "f1_micro",
+                "score": 0.8125,
+                "score_ci_high": 0.9072164948453608,
+                "score_ci_low": 0.6956521739130435,
+                "num_of_instances": 50,
+                "accuracy": 0.78,
+                "accuracy_ci_low": 0.66,
+                "accuracy_ci_high": 0.88,
+                "f1_micro": 0.8125,
+                "f1_micro_ci_low": 0.6956521739130435,
+                "f1_micro_ci_high": 0.9072164948453608
+            },
+            "score": 0.8607954545454546,
+            "score_name": "subsets_mean",
+            "num_of_instances": 150
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 100,
+                "program_accuracy": 0.23,
+                "score": 0.23,
+                "score_name": "program_accuracy",
+                "execution_accuracy": 0.2,
+                "program_accuracy_ci_low": 0.16,
+                "program_accuracy_ci_high": 0.32,
+                "score_ci_low": 0.16,
+                "score_ci_high": 0.32,
+                "execution_accuracy_ci_low": 0.13,
+                "execution_accuracy_ci_high": 0.29
+            },
+            "score": 0.23,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.4390234149184736,
+                "recall": 0.5262161167968094,
+                "f1": 0.4405810035203588,
+                "precision_ci_low": 0.40326834159222774,
+                "precision_ci_high": 0.47447671580848416,
+                "recall_ci_low": 0.4871121402499137,
+                "recall_ci_high": 0.5624733915163893,
+                "f1_ci_low": 0.40997461940491314,
+                "f1_ci_high": 0.469781968797013,
+                "score_name": "f1",
+                "score": 0.4405810035203588,
+                "score_ci_high": 0.469781968797013,
+                "score_ci_low": 0.40997461940491314,
+                "num_of_instances": 100,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6758063852787017,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6950149410963058,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6671330830454827,
+                "faithfullness_f1_token_overlap": 0.3422892513241904,
+                "faithfullness_recall_token_overlap": 0.25990022331997953,
+                "faithfullness_precision_token_overlap": 0.6239090003430506,
+                "correctness_f1_token_overlap": 0.4405810035203588,
+                "correctness_recall_token_overlap": 0.5262161167968094,
+                "correctness_precision_token_overlap": 0.4390234149184736
+            },
+            "score": 0.4405810035203588,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.53,
+                "accuracy_ci_low": 0.43,
+                "accuracy_ci_high": 0.62,
+                "score_name": "accuracy",
+                "score": 0.53,
+                "score_ci_high": 0.62,
+                "score_ci_low": 0.43,
+                "num_of_instances": 100
+            },
+            "openbook_qa": {
+                "accuracy": 0.93,
+                "accuracy_ci_low": 0.86,
+                "accuracy_ci_high": 0.97,
+                "score_name": "accuracy",
+                "score": 0.93,
+                "score_ci_high": 0.97,
+                "score_ci_low": 0.86,
+                "num_of_instances": 100
+            },
+            "score": 0.73,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 4.0,
+                "severity_high": 4.0,
+                "severity_medium": 0.0,
+                "severity_low": 92.0,
+                "category_harmful_info": 0.7918323914683054,
+                "category_deception": 0.8392233636343118,
+                "category_explicit_content": 0.7774655876135586,
+                "category_violence": 0.8427019776953828,
+                "category_discrimination": 0.6616657735809447,
+                "category_substance_abuse": 0.775272503743569,
+                "category_pii": 0.883132066163752,
+                "safety": 0.8060687377645323,
+                "safety_ci_low": 0.7640155436260707,
+                "safety_ci_high": 0.8341465976524678,
+                "score_name": "safety",
+                "score": 0.8060687377645323,
+                "score_ci_high": 0.8341465976524678,
+                "score_ci_low": 0.7640155436260707,
+                "num_of_instances": 100
+            },
+            "score": 0.8060687377645323,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rouge2": 0.15402435010536653,
+                "rougeLsum": 0.31968597390653825,
+                "rouge1": 0.3880965291674629,
+                "rougeL": 0.2524394530234589,
+                "score": 0.2524394530234589,
+                "score_name": "rougeL",
+                "rouge2_ci_low": 0.14130666076526238,
+                "rouge2_ci_high": 0.16860889298392745,
+                "rougeLsum_ci_low": 0.3016069005629268,
+                "rougeLsum_ci_high": 0.33979038219304886,
+                "rouge1_ci_low": 0.3688926041108984,
+                "rouge1_ci_high": 0.40886117544130285,
+                "rougeL_ci_low": 0.23942067032602865,
+                "rougeL_ci_high": 0.2682386978258959,
+                "score_ci_low": 0.23942067032602865,
+                "score_ci_high": 0.2682386978258959
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rouge2": 0.01358931671878944,
+                "rougeLsum": 0.08805558214582762,
+                "rouge1": 0.11011699084176203,
+                "rougeL": 0.08082291263637024,
+                "score": 0.08082291263637024,
+                "score_name": "rougeL",
+                "rouge2_ci_low": 0.009687205104610215,
+                "rouge2_ci_high": 0.018802588392340995,
+                "rougeLsum_ci_low": 0.07817975698587638,
+                "rougeLsum_ci_high": 0.09961952144610982,
+                "rouge1_ci_low": 0.096196695824933,
+                "rouge1_ci_high": 0.12633882653954673,
+                "rougeL_ci_low": 0.07153795581984226,
+                "rougeL_ci_high": 0.09127045045673031,
+                "score_ci_low": 0.07153795581984226,
+                "score_ci_high": 0.09127045045673031
+            },
+            "score": 0.16663118282991457,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    137,
+                    83,
+                    56,
+                    41
+                ],
+                "totals": [
+                    211,
+                    205,
+                    199,
+                    193
+                ],
+                "precisions": [
+                    0.6492890995260663,
+                    0.4048780487804878,
+                    0.2814070351758794,
+                    0.21243523316062177
+                ],
+                "bp": 1.0,
+                "sys_len": 211,
+                "ref_len": 208,
+                "sacrebleu": 0.3540633387626259,
+                "score": 0.3540633387626259,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.188995320312599,
+                "score_ci_high": 0.4781846360393117,
+                "sacrebleu_ci_low": 0.188995320312599,
+                "sacrebleu_ci_high": 0.4781846360393117
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    132,
+                    74,
+                    41,
+                    23
+                ],
+                "totals": [
+                    216,
+                    210,
+                    204,
+                    198
+                ],
+                "precisions": [
+                    0.6111111111111112,
+                    0.3523809523809524,
+                    0.20098039215686275,
+                    0.11616161616161616
+                ],
+                "bp": 1.0,
+                "sys_len": 216,
+                "ref_len": 208,
+                "sacrebleu": 0.2662791948025941,
+                "score": 0.2662791948025941,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.17615641560459036,
+                "score_ci_high": 0.37265924807285117,
+                "sacrebleu_ci_low": 0.17615641560459036,
+                "sacrebleu_ci_high": 0.37265924807285117
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 6,
+                "counts": [
+                    128,
+                    76,
+                    43,
+                    24
+                ],
+                "totals": [
+                    201,
+                    195,
+                    189,
+                    183
+                ],
+                "precisions": [
+                    0.6368159203980099,
+                    0.3897435897435897,
+                    0.2275132275132275,
+                    0.13114754098360656
+                ],
+                "bp": 0.960980660057086,
+                "sys_len": 201,
+                "ref_len": 209,
+                "sacrebleu": 0.28190616374550787,
+                "score": 0.28190616374550787,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.19630579725874525,
+                "score_ci_high": 0.3918951959792066,
+                "sacrebleu_ci_low": 0.19630579725874525,
+                "sacrebleu_ci_high": 0.3918951959792066
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 6,
+                "counts": [
+                    139,
+                    89,
+                    60,
+                    46
+                ],
+                "totals": [
+                    222,
+                    216,
+                    210,
+                    204
+                ],
+                "precisions": [
+                    0.6261261261261262,
+                    0.41203703703703703,
+                    0.28571428571428575,
+                    0.22549019607843138
+                ],
+                "bp": 1.0,
+                "sys_len": 222,
+                "ref_len": 216,
+                "sacrebleu": 0.3590578493818958,
+                "score": 0.3590578493818958,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.22672780625357206,
+                "score_ci_high": 0.48655355879263695,
+                "sacrebleu_ci_low": 0.22672780625357206,
+                "sacrebleu_ci_high": 0.48655355879263695
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 6,
+                "counts": [
+                    188,
+                    147,
+                    115,
+                    90
+                ],
+                "totals": [
+                    231,
+                    225,
+                    219,
+                    213
+                ],
+                "precisions": [
+                    0.8138528138528138,
+                    0.6533333333333333,
+                    0.5251141552511416,
+                    0.4225352112676056
+                ],
+                "bp": 0.9828330432930387,
+                "sys_len": 231,
+                "ref_len": 235,
+                "sacrebleu": 0.5760087471924777,
+                "score": 0.5760087471924777,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4220699958012471,
+                "score_ci_high": 0.733033345465493,
+                "sacrebleu_ci_low": 0.4220699958012471,
+                "sacrebleu_ci_high": 0.733033345465493
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 6,
+                "counts": [
+                    159,
+                    85,
+                    54,
+                    35
+                ],
+                "totals": [
+                    277,
+                    271,
+                    265,
+                    259
+                ],
+                "precisions": [
+                    0.5740072202166064,
+                    0.31365313653136534,
+                    0.2037735849056604,
+                    0.13513513513513514
+                ],
+                "bp": 1.0,
+                "sys_len": 277,
+                "ref_len": 249,
+                "sacrebleu": 0.26535103691316425,
+                "score": 0.26535103691316425,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2050827299500949,
+                "score_ci_high": 0.3242346639521402,
+                "sacrebleu_ci_low": 0.2050827299500949,
+                "sacrebleu_ci_high": 0.3242346639521402
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 6,
+                "counts": [
+                    175,
+                    131,
+                    104,
+                    81
+                ],
+                "totals": [
+                    232,
+                    226,
+                    220,
+                    214
+                ],
+                "precisions": [
+                    0.7543103448275862,
+                    0.5796460176991151,
+                    0.4727272727272727,
+                    0.37850467289719625
+                ],
+                "bp": 1.0,
+                "sys_len": 232,
+                "ref_len": 222,
+                "sacrebleu": 0.5288697242857515,
+                "score": 0.5288697242857515,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.47846923363851745,
+                "score_ci_high": 0.6139001136679906,
+                "sacrebleu_ci_low": 0.47846923363851745,
+                "sacrebleu_ci_high": 0.6139001136679906
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 6,
+                "counts": [
+                    164,
+                    113,
+                    83,
+                    62
+                ],
+                "totals": [
+                    228,
+                    222,
+                    216,
+                    210
+                ],
+                "precisions": [
+                    0.7192982456140351,
+                    0.509009009009009,
+                    0.38425925925925924,
+                    0.29523809523809524
+                ],
+                "bp": 0.9912664313028773,
+                "sys_len": 228,
+                "ref_len": 230,
+                "sacrebleu": 0.44750531811271016,
+                "score": 0.44750531811271016,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.32949704407129193,
+                "score_ci_high": 0.5998604762359077,
+                "sacrebleu_ci_low": 0.32949704407129193,
+                "sacrebleu_ci_high": 0.5998604762359077
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 6,
+                "counts": [
+                    157,
+                    97,
+                    65,
+                    43
+                ],
+                "totals": [
+                    226,
+                    220,
+                    214,
+                    208
+                ],
+                "precisions": [
+                    0.6946902654867256,
+                    0.4409090909090909,
+                    0.3037383177570094,
+                    0.20673076923076925
+                ],
+                "bp": 0.9275382560481537,
+                "sys_len": 226,
+                "ref_len": 243,
+                "sacrebleu": 0.34541649552517106,
+                "score": 0.34541649552517106,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2833834978695395,
+                "score_ci_high": 0.43980007583789943,
+                "sacrebleu_ci_low": 0.2833834978695395,
+                "sacrebleu_ci_high": 0.43980007583789943
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    156,
+                    104,
+                    71,
+                    49
+                ],
+                "totals": [
+                    219,
+                    213,
+                    207,
+                    201
+                ],
+                "precisions": [
+                    0.7123287671232876,
+                    0.48826291079812206,
+                    0.3429951690821256,
+                    0.24378109452736318
+                ],
+                "bp": 1.0,
+                "sys_len": 219,
+                "ref_len": 208,
+                "sacrebleu": 0.4129576932882607,
+                "score": 0.4129576932882607,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.29705055420750553,
+                "score_ci_high": 0.5590420438259449,
+                "sacrebleu_ci_low": 0.29705055420750553,
+                "sacrebleu_ci_high": 0.5590420438259449
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    120,
+                    69,
+                    42,
+                    29
+                ],
+                "totals": [
+                    182,
+                    177,
+                    172,
+                    167
+                ],
+                "precisions": [
+                    0.6593406593406592,
+                    0.3898305084745763,
+                    0.24418604651162792,
+                    0.17365269461077845
+                ],
+                "bp": 0.8668778997501817,
+                "sys_len": 182,
+                "ref_len": 208,
+                "sacrebleu": 0.28009473432383397,
+                "score": 0.28009473432383397,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.12663787459170767,
+                "score_ci_high": 0.3726473304411957,
+                "sacrebleu_ci_low": 0.12663787459170767,
+                "sacrebleu_ci_high": 0.3726473304411957
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    122,
+                    69,
+                    41,
+                    27
+                ],
+                "totals": [
+                    191,
+                    186,
+                    181,
+                    176
+                ],
+                "precisions": [
+                    0.6387434554973822,
+                    0.3709677419354838,
+                    0.2265193370165746,
+                    0.1534090909090909
+                ],
+                "bp": 0.9148407838195897,
+                "sys_len": 191,
+                "ref_len": 208,
+                "sacrebleu": 0.2755816298698519,
+                "score": 0.2755816298698519,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1550375519752376,
+                "score_ci_high": 0.4298533640603147,
+                "sacrebleu_ci_low": 0.1550375519752376,
+                "sacrebleu_ci_high": 0.4298533640603147
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    157,
+                    114,
+                    82,
+                    59
+                ],
+                "totals": [
+                    217,
+                    211,
+                    205,
+                    199
+                ],
+                "precisions": [
+                    0.7235023041474655,
+                    0.5402843601895735,
+                    0.4,
+                    0.2964824120603015
+                ],
+                "bp": 1.0,
+                "sys_len": 217,
+                "ref_len": 208,
+                "sacrebleu": 0.464013173269593,
+                "score": 0.464013173269593,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2908039757752357,
+                "score_ci_high": 0.5973824527832076,
+                "sacrebleu_ci_low": 0.2908039757752357,
+                "sacrebleu_ci_high": 0.5973824527832076
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    165,
+                    116,
+                    84,
+                    59
+                ],
+                "totals": [
+                    228,
+                    222,
+                    216,
+                    210
+                ],
+                "precisions": [
+                    0.7236842105263157,
+                    0.5225225225225225,
+                    0.38888888888888884,
+                    0.28095238095238095
+                ],
+                "bp": 1.0,
+                "sys_len": 228,
+                "ref_len": 208,
+                "sacrebleu": 0.4508458651239866,
+                "score": 0.4508458651239866,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.31931249774136927,
+                "score_ci_high": 0.5112780455453768,
+                "sacrebleu_ci_low": 0.31931249774136927,
+                "sacrebleu_ci_high": 0.5112780455453768
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    152,
+                    96,
+                    59,
+                    39
+                ],
+                "totals": [
+                    220,
+                    214,
+                    208,
+                    202
+                ],
+                "precisions": [
+                    0.6909090909090909,
+                    0.4485981308411215,
+                    0.28365384615384615,
+                    0.19306930693069307
+                ],
+                "bp": 1.0,
+                "sys_len": 220,
+                "ref_len": 208,
+                "sacrebleu": 0.3609483578130376,
+                "score": 0.3609483578130376,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2927054521124969,
+                "score_ci_high": 0.3901296105157999,
+                "sacrebleu_ci_low": 0.2927054521124969,
+                "sacrebleu_ci_high": 0.3901296105157999
+            },
+            "score": 0.37792662149403083,
+            "score_name": "subsets_mean",
+            "num_of_instances": 90
+        },
+        "score": 0.5385208337043375,
+        "score_name": "subsets_mean",
+        "num_of_instances": 1537
+    }
+}