Spaces:
Runtime error
Runtime error
File size: 4,080 Bytes
382191a 8ceca60 382191a 8ceca60 382191a 8ceca60 382191a 8ceca60 382191a 8ceca60 382191a 8ceca60 382191a 8ceca60 382191a 8ceca60 382191a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 |
"""Config for analyzing GPT-MT."""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from zeno_build.evaluation.text_features.capitalization import input_capital_char_ratio
from zeno_build.evaluation.text_features.exact_match import avg_exact_match, exact_match
from zeno_build.evaluation.text_features.frequency import output_max_word_freq
from zeno_build.evaluation.text_features.length import (
doc_context_length,
input_length,
label_length,
output_length,
)
from zeno_build.evaluation.text_metrics.critique import (
avg_bert_score,
avg_chrf,
avg_comet,
avg_length_ratio,
bert_score,
chrf,
comet,
length_ratio,
)
from zeno_build.experiments import search_space
from modeling import remove_leading_language
lang_pairs: dict[str, list[str]] = {
# All language pairs used in any experiment
"all_lang_pairs": [
"csen",
"deen",
"defr",
"encs",
"ende",
"enha",
"enis",
"enja",
"enru",
"enuk",
"enzh",
"frde",
"haen",
"isen",
"jaen",
"ruen",
"uken",
"zhen",
],
# Language pairs used in the experiments on a limited number of language pairs
"limited_lang_pairs": [
"deen",
"defr",
"ende",
"enru",
"enzh",
"frde",
"ruen",
"zhen",
],
}
# The search space for the main experiments
main_space = search_space.CombinatorialSearchSpace(
{
"lang_pairs": search_space.Constant("all_lang_pairs"),
"model_preset": search_space.Categorical(
[
"text-davinci-003-zeroshot",
"text-davinci-003-RR-1-shot",
"text-davinci-003-RR-5-shot",
"text-davinci-003-QR-1-shot",
"text-davinci-003-QR-5-shot",
"gpt-3.5-turbo-0301-zeroshot",
"gpt-4-0314-zeroshot",
"gpt-4-0314-zeroshot-postprocess",
"MS-Translator",
"google-cloud",
"wmt-best",
]
),
}
)
@dataclass(frozen=True)
class GptMtConfig:
"""Config for gpt-MT models."""
path: str
base_model: str
prompt_strategy: str | None = None
prompt_shots: int | None = None
post_processors: list[Callable[[str], str]] | None = None
# The details of each model
model_configs = {
"text-davinci-003-RR-1-shot": GptMtConfig(
"text-davinci-003/RR/1-shot", "text-davinci-003", "RR", 1
),
"text-davinci-003-RR-5-shot": GptMtConfig(
"text-davinci-003/RR/5-shot", "text-davinci-003", "RR", 5
),
"text-davinci-003-QR-1-shot": GptMtConfig(
"text-davinci-003/QR/1-shot", "text-davinci-003", "QR", 1
),
"text-davinci-003-QR-5-shot": GptMtConfig(
"text-davinci-003/QR/5-shot", "text-davinci-003", "QR", 5
),
"text-davinci-003-zeroshot": GptMtConfig(
"text-davinci-003/zeroshot", "text-davinci-003", None, 0
),
"gpt-3.5-turbo-0301-zeroshot": GptMtConfig(
"gpt-3.5-turbo-0301/zeroshot", "gpt-3.5-turbo-0301", None, 0
),
"gpt-4-0314-zeroshot": GptMtConfig("gpt-4-0314/zeroshot", "gpt-4-0314", None, 0),
"gpt-4-0314-zeroshot-postprocess": GptMtConfig(
"gpt-4-0314/zeroshot", "gpt-4-0314", None, 0, [remove_leading_language]
),
"MS-Translator": GptMtConfig("MS-Translator", "MS-Translator"),
"google-cloud": GptMtConfig("google-cloud", "google-cloud"),
"wmt-best": GptMtConfig("wmt-best", "wmt-best"),
}
sweep_distill_functions = [chrf]
sweep_metric_function = avg_chrf
# The functions used for Zeno visualization
zeno_distill_and_metric_functions = [
output_length,
input_length,
label_length,
doc_context_length,
input_capital_char_ratio,
output_max_word_freq,
chrf,
comet,
length_ratio,
bert_score,
exact_match,
avg_chrf,
avg_comet,
avg_length_ratio,
avg_bert_score,
avg_exact_match,
]
|