import { Wav2Vec2CTCTokenizer } from "../../../src/tokenizers.js";
import { BASE_TEST_STRINGS, BERT_TEST_STRINGS } from "../test_strings.js";

export const TOKENIZER_CLASS = Wav2Vec2CTCTokenizer;
export const TEST_CONFIG = {
  "Xenova/wav2vec2-base-960h": {
    SIMPLE: {
      text: BASE_TEST_STRINGS.SIMPLE,
      tokens: ["H", "o", "w", "|", "a", "r", "e", "|", "y", "o", "u", "|", "d", "o", "i", "n", "g", "?"],
      ids: [11, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3],
      decoded: "H<unk> <unk> <unk> <unk>",
    },
    SIMPLE_WITH_PUNCTUATION: {
      text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION,
      tokens: ["Y", "o", "u", "|", "s", "h", "o", "u", "l", "d", "'", "v", "e", "|", "d", "o", "n", "e", "|", "t", "h", "i", "s"],
      ids: [22, 3, 3, 4, 3, 3, 3, 3, 3, 3, 27, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3],
      decoded: "Y<unk> <unk>'<unk> <unk> <unk>",
    },
    NUMBERS: {
      text: BASE_TEST_STRINGS.NUMBERS,
      tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "|", "0", "|", "1", "|", "2", "|", "3", "|", "4", "|", "5", "|", "6", "|", "7", "|", "8", "|", "9", "|", "1", "0", "|", "1", "0", "0", "|", "1", "0", "0", "0"],
      ids: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3],
      decoded: "<unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk>",
    },
    TEXT_WITH_NUMBERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS,
      tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."],
      ids: [6, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 3],
      decoded: "T<unk> <unk> <unk> <unk> <unk> <unk>",
    },
    PUNCTUATION: {
      text: BASE_TEST_STRINGS.PUNCTUATION,
      tokens: ["A", "\n", "'", "l", "l", "|", "!", "!", "t", "o", "?", "'", "d", "'", "'", "d", "|", "o", "f", ",", "|", "c", "a", "n", "'", "t", "."],
      ids: [7, 3, 27, 3, 3, 4, 3, 3, 3, 3, 3, 27, 3, 27, 27, 3, 4, 3, 3, 3, 4, 3, 3, 3, 27, 3, 3],
      decoded: "A<unk>'<unk> <unk>'<unk>'<unk> <unk> <unk>'<unk>",
    },
    PYTHON_CODE: {
      text: BASE_TEST_STRINGS.PYTHON_CODE,
      tokens: ["d", "e", "f", "|", "m", "a", "i", "n", "(", ")", ":", "\n", "\t", "p", "a", "s", "s"],
      ids: [3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3],
      decoded: "<unk> <unk>",
    },
    JAVASCRIPT_CODE: {
      text: BASE_TEST_STRINGS.JAVASCRIPT_CODE,
      tokens: ["l", "e", "t", "|", "a", "|", "=", "|", "o", "b", "j", ".", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";", "\n", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";"],
      ids: [3, 3, 3, 4, 3, 4, 3, 4, 3, 3, 3, 3, 3, 3, 12, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 12, 3, 3, 3, 3, 3, 3, 3, 3],
      decoded: "<unk> <unk> <unk> <unk>S<unk>S<unk>",
    },
    NEWLINES: {
      text: BASE_TEST_STRINGS.NEWLINES,
      tokens: ["T", "h", "i", "s", "\n", "\n", "i", "s", "\n", "a", "\n", "t", "e", "s", "t", "."],
      ids: [6, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3],
      decoded: "T<unk>",
    },
    BASIC: {
      text: BASE_TEST_STRINGS.BASIC,
      tokens: ["U", "N", "w", "a", "n", "t", "\u00e9", "d", ",", "r", "u", "n", "n", "i", "n", "g"],
      ids: [16, 9, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3],
      decoded: "UN<unk>",
    },
    CONTROL_TOKENS: {
      text: BASE_TEST_STRINGS.CONTROL_TOKENS,
      tokens: ["1", "\u0000", "2", "\ufffd", "3"],
      ids: [3, 3, 3, 3, 3],
      decoded: "<unk>",
    },
    HELLO_WORLD_TITLECASE: {
      text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE,
      tokens: ["H", "e", "l", "l", "o", "|", "W", "o", "r", "l", "d"],
      ids: [11, 3, 3, 3, 3, 4, 18, 3, 3, 3, 3],
      decoded: "H<unk> W<unk>",
    },
    HELLO_WORLD_LOWERCASE: {
      text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE,
      tokens: ["h", "e", "l", "l", "o", "|", "w", "o", "r", "l", "d"],
      ids: [3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3],
      decoded: "<unk> <unk>",
    },
    CHINESE_ONLY: {
      text: BASE_TEST_STRINGS.CHINESE_ONLY,
      tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"],
      ids: [3, 3, 3, 3, 3, 3],
      decoded: "<unk>",
    },
    LEADING_SPACE: {
      text: BASE_TEST_STRINGS.LEADING_SPACE,
      tokens: ["|", "|", "|", "l", "e", "a", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e"],
      ids: [4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3],
      decoded: "<unk> <unk>",
    },
    TRAILING_SPACE: {
      text: BASE_TEST_STRINGS.TRAILING_SPACE,
      tokens: ["t", "r", "a", "i", "l", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"],
      ids: [3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 4, 4],
      decoded: "<unk> <unk>",
    },
    SURROUNDING_SPACE: {
      text: BASE_TEST_STRINGS.SURROUNDING_SPACE,
      tokens: ["|", "|", "|", "s", "u", "r", "r", "o", "u", "n", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"],
      ids: [4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 4, 4],
      decoded: "<unk> <unk>",
    },
    DOUBLE_SPACE: {
      text: BASE_TEST_STRINGS.DOUBLE_SPACE,
      tokens: ["H", "i", "|", "|", "H", "e", "l", "l", "o"],
      ids: [11, 3, 4, 4, 11, 3, 3, 3, 3],
      decoded: "H<unk> H<unk>",
    },
    CURRENCY: {
      text: BASE_TEST_STRINGS.CURRENCY,
      tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"],
      ids: [3, 3, 3, 3, 4, 3, 3, 4, 13, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3],
      decoded: "<unk> <unk> R<unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk>",
    },
    CURRENCY_WITH_DECIMALS: {
      text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS,
      tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."],
      ids: [10, 4, 3, 3, 3, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3],
      decoded: "I <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk>",
    },
    ELLIPSIS: {
      text: BASE_TEST_STRINGS.ELLIPSIS,
      tokens: ["y", "o", "u", "\u2026", "|", "|"],
      ids: [3, 3, 3, 3, 4, 4],
      decoded: "<unk>",
    },
    TEXT_WITH_ESCAPE_CHARACTERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS,
      tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0"],
      ids: [3, 3, 3, 3, 3, 3],
      decoded: "<unk>",
    },
    TEXT_WITH_ESCAPE_CHARACTERS_2: {
      text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2,
      tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0", "y", "o", "u", "\u2026", "\u00a0", "\u00a0"],
      ids: [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3],
      decoded: "<unk>",
    },
    TILDE_NORMALIZATION: {
      text: BASE_TEST_STRINGS.TILDE_NORMALIZATION,
      tokens: ["w", "e", "i", "r", "d", "|", "\uff5e", "|", "e", "d", "g", "e", "|", "\uff5e", "|", "c", "a", "s", "e"],
      ids: [3, 3, 3, 3, 3, 4, 3, 4, 3, 3, 3, 3, 4, 3, 4, 3, 3, 3, 3],
      decoded: "<unk> <unk> <unk> <unk> <unk>",
    },
    SPIECE_UNDERSCORE: {
      text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE,
      tokens: ["\u2581", "T", "h", "i", "s", "|", "\u2581", "i", "s", "|", "\u2581", "a", "|", "\u2581", "t", "e", "s", "t", "|", "\u2581", "."],
      ids: [3, 6, 3, 3, 3, 4, 3, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3],
      decoded: "<unk>T<unk> <unk> <unk> <unk> <unk>",
    },
    POPULAR_EMOJIS: {
      text: BASE_TEST_STRINGS.POPULAR_EMOJIS,
      tokens: ["\ud83d\ude02", "|", "\ud83d\udc4d", "|", "\ud83e\udd23", "|", "\ud83d\ude0d", "|", "\ud83d\ude2d", "|", "\ud83c\udf89", "|", "\ud83d\ude4f", "|", "\ud83d\ude0a", "|", "\ud83d\udd25", "|", "\ud83d\ude01", "|", "\ud83d\ude05", "|", "\ud83e\udd17", "|", "\ud83d\ude06", "|", "\ud83d\udc4f", "|", "\u2764", "\ufe0f", "|", "\ud83d\udc9c", "|", "\ud83d\udc9a", "|", "\ud83d\udc97", "|", "\ud83d\udc99", "|", "\ud83d\udda4", "|", "\ud83d\ude0e", "|", "\ud83d\udc4c", "|", "\ud83e\udd73", "|", "\ud83d\udcaa", "|", "\u2728", "|", "\ud83d\udc49", "|", "\ud83d\udc40", "|", "\ud83d\udcaf", "|", "\ud83c\udf88", "|", "\ud83d\ude48", "|", "\ud83d\ude4c", "|", "\ud83d\udc80", "|", "\ud83d\udc47", "|", "\ud83d\udc4b", "|", "\u2705", "|", "\ud83c\udf81", "|", "\ud83c\udf1e", "|", "\ud83c\udf38", "|", "\ud83d\udcb0"],
      ids: [3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3],
      decoded: "<unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk>",
    },
    MULTIBYTE_EMOJIS: {
      text: BASE_TEST_STRINGS.MULTIBYTE_EMOJIS,
      tokens: ["\u2728", "|", "\ud83e\udd17", "|", "\ud83d\udc41", "\ufe0f", "|", "\ud83d\udc71", "\ud83c\udffb", "|", "\ud83d\udd75", "\u200d", "\u2642", "\ufe0f", "|", "\ud83e\uddd9", "\ud83c\udffb", "\u200d", "\u2642", "|", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\ud83c\udf3e", "|", "\ud83e\uddd1", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "|", "\ud83d\udc69", "\u200d", "\u2764", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "|", "\ud83d\udc69", "\u200d", "\ud83d\udc69", "\u200d", "\ud83d\udc67", "\u200d", "\ud83d\udc66", "|", "\ud83e\uddd1", "\ud83c\udffb", "\u200d", "\ud83e\udd1d", "\u200d", "\ud83e\uddd1", "\ud83c\udffb", "|", "\ud83c\udff4", "\udb40\udc67", "\udb40\udc62", "\udb40\udc65", "\udb40\udc6e", "\udb40\udc67", "\udb40\udc7f", "|", "\ud83d\udc68", "\ud83c\udffb", "\u200d", "\u2764", "\ufe0f", "\u200d", "\ud83d\udc8b", "\u200d", "\ud83d\udc68", "\ud83c\udffc"],
      ids: [3, 4, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3],
      decoded: "<unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk>",
    },
    ONLY_WHITESPACE: {
      text: BASE_TEST_STRINGS.ONLY_WHITESPACE,
      tokens: ["|", "\t", "\n"],
      ids: [4, 3, 3],
      decoded: "<unk>",
    },
    CHINESE_LATIN_MIXED: {
      text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED,
      tokens: ["a", "h", "\u535a", "\u63a8", "z", "z"],
      ids: [3, 3, 3, 3, 3, 3],
      decoded: "<unk>",
    },
    SIMPLE_WITH_ACCENTS: {
      text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS,
      tokens: ["H", "\u00e9", "l", "l", "o"],
      ids: [11, 3, 3, 3, 3],
      decoded: "H<unk>",
    },
    MIXED_CASE_WITHOUT_ACCENTS: {
      text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS,
      tokens: ["|", "\t", "H", "e", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"],
      ids: [4, 3, 11, 3, 15, 15, 3, 3, 3, 3, 3, 4, 4, 3, 4, 7, 3, 3, 4, 3, 3, 16, 3, 4, 4],
      decoded: "<unk>H<unk>L<unk> <unk> A<unk> <unk>U<unk>",
    },
    MIXED_CASE_WITH_ACCENTS: {
      text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS,
      tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"],
      ids: [4, 3, 11, 3, 15, 15, 3, 3, 3, 3, 3, 4, 4, 3, 4, 7, 3, 3, 4, 3, 3, 16, 3, 4, 4],
      decoded: "<unk>H<unk>L<unk> <unk> A<unk> <unk>U<unk>",
    },
  },
  "Xenova/wav2vec2-large-xlsr-53-english": {
    SIMPLE: {
      text: BASE_TEST_STRINGS.SIMPLE,
      tokens: ["H", "o", "w", "|", "a", "r", "e", "|", "y", "o", "u", "|", "d", "o", "i", "n", "g", "?"],
      ids: [3, 21, 29, 4, 7, 24, 11, 4, 31, 21, 27, 4, 10, 21, 15, 20, 13, 3],
      decoded: "<unk>ow are you doing<unk>",
    },
    SIMPLE_WITH_PUNCTUATION: {
      text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION,
      tokens: ["Y", "o", "u", "|", "s", "h", "o", "u", "l", "d", "'", "v", "e", "|", "d", "o", "n", "e", "|", "t", "h", "i", "s"],
      ids: [3, 21, 27, 4, 25, 14, 21, 27, 18, 10, 5, 28, 11, 4, 10, 21, 20, 11, 4, 26, 14, 15, 25],
      decoded: "<unk>ou should've done this",
    },
    TEXT_WITH_NUMBERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS,
      tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."],
      ids: [3, 14, 11, 4, 9, 21, 19, 22, 7, 20, 31, 4, 29, 7, 25, 4, 12, 21, 27, 20, 10, 11, 10, 4, 15, 20, 4, 3, 3, 3, 3, 3],
      decoded: "<unk>he company was founded in <unk>",
    },
    PUNCTUATION: {
      text: BASE_TEST_STRINGS.PUNCTUATION,
      tokens: ["A", "\n", "'", "l", "l", "|", "!", "!", "t", "o", "?", "'", "d", "'", "'", "d", "|", "o", "f", ",", "|", "c", "a", "n", "'", "t", "."],
      ids: [3, 3, 5, 18, 18, 4, 3, 3, 26, 21, 3, 5, 10, 5, 5, 10, 4, 21, 12, 3, 4, 9, 7, 20, 5, 26, 3],
      decoded: "<unk>'l <unk>to<unk>'d'd of<unk> can't<unk>",
    },
    PYTHON_CODE: {
      text: BASE_TEST_STRINGS.PYTHON_CODE,
      tokens: ["d", "e", "f", "|", "m", "a", "i", "n", "(", ")", ":", "\n", "\t", "p", "a", "s", "s"],
      ids: [10, 11, 12, 4, 19, 7, 15, 20, 3, 3, 3, 3, 3, 22, 7, 25, 25],
      decoded: "def main<unk>pas",
    },
    JAVASCRIPT_CODE: {
      text: BASE_TEST_STRINGS.JAVASCRIPT_CODE,
      tokens: ["l", "e", "t", "|", "a", "|", "=", "|", "o", "b", "j", ".", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";", "\n", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";"],
      ids: [18, 11, 26, 4, 7, 4, 3, 4, 21, 8, 16, 3, 26, 21, 3, 26, 24, 15, 20, 13, 3, 3, 3, 3, 26, 21, 3, 26, 24, 15, 20, 13, 3, 3, 3],
      decoded: "let a <unk> obj<unk>to<unk>tring<unk>to<unk>tring<unk>",
    },
    NEWLINES: {
      text: BASE_TEST_STRINGS.NEWLINES,
      tokens: ["T", "h", "i", "s", "\n", "\n", "i", "s", "\n", "a", "\n", "t", "e", "s", "t", "."],
      ids: [3, 14, 15, 25, 3, 3, 15, 25, 3, 7, 3, 26, 11, 25, 26, 3],
      decoded: "<unk>his<unk>is<unk>a<unk>test<unk>",
    },
    BASIC: {
      text: BASE_TEST_STRINGS.BASIC,
      tokens: ["U", "N", "w", "a", "n", "t", "\u00e9", "d", ",", "r", "u", "n", "n", "i", "n", "g"],
      ids: [3, 3, 29, 7, 20, 26, 3, 10, 3, 24, 27, 20, 20, 15, 20, 13],
      decoded: "<unk>want<unk>d<unk>runing",
    },
    HELLO_WORLD_TITLECASE: {
      text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE,
      tokens: ["H", "e", "l", "l", "o", "|", "W", "o", "r", "l", "d"],
      ids: [3, 11, 18, 18, 21, 4, 3, 21, 24, 18, 10],
      decoded: "<unk>elo <unk>orld",
    },
    HELLO_WORLD_LOWERCASE: {
      text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE,
      tokens: ["h", "e", "l", "l", "o", "|", "w", "o", "r", "l", "d"],
      ids: [14, 11, 18, 18, 21, 4, 29, 21, 24, 18, 10],
      decoded: "helo world",
    },
    LEADING_SPACE: {
      text: BASE_TEST_STRINGS.LEADING_SPACE,
      tokens: ["|", "|", "|", "l", "e", "a", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e"],
      ids: [4, 4, 4, 18, 11, 7, 10, 15, 20, 13, 4, 25, 22, 7, 9, 11],
      decoded: "leading space",
    },
    TRAILING_SPACE: {
      text: BASE_TEST_STRINGS.TRAILING_SPACE,
      tokens: ["t", "r", "a", "i", "l", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"],
      ids: [26, 24, 7, 15, 18, 15, 20, 13, 4, 25, 22, 7, 9, 11, 4, 4, 4],
      decoded: "trailing space",
    },
    SURROUNDING_SPACE: {
      text: BASE_TEST_STRINGS.SURROUNDING_SPACE,
      tokens: ["|", "|", "|", "s", "u", "r", "r", "o", "u", "n", "d", "i", "n", "g", "|", "s", "p", "a", "c", "e", "|", "|", "|"],
      ids: [4, 4, 4, 25, 27, 24, 24, 21, 27, 20, 10, 15, 20, 13, 4, 25, 22, 7, 9, 11, 4, 4, 4],
      decoded: "surounding space",
    },
    DOUBLE_SPACE: {
      text: BASE_TEST_STRINGS.DOUBLE_SPACE,
      tokens: ["H", "i", "|", "|", "H", "e", "l", "l", "o"],
      ids: [3, 15, 4, 4, 3, 11, 18, 18, 21],
      decoded: "<unk>i <unk>elo",
    },
    CURRENCY: {
      text: BASE_TEST_STRINGS.CURRENCY,
      tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"],
      ids: [26, 11, 25, 26, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 26, 11, 25, 26],
      decoded: "test <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> test",
    },
    CURRENCY_WITH_DECIMALS: {
      text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS,
      tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."],
      ids: [3, 4, 8, 21, 27, 13, 14, 26, 4, 7, 20, 4, 7, 22, 22, 18, 11, 4, 12, 21, 24, 4, 3, 3, 3, 3, 3, 4, 7, 26, 4, 26, 14, 11, 4, 25, 26, 21, 24, 11, 3],
      decoded: "<unk> bought an aple for <unk> at the store<unk>",
    },
    ELLIPSIS: {
      text: BASE_TEST_STRINGS.ELLIPSIS,
      tokens: ["y", "o", "u", "\u2026", "|", "|"],
      ids: [31, 21, 27, 3, 4, 4],
      decoded: "you<unk>",
    },
    TEXT_WITH_ESCAPE_CHARACTERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS,
      tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0"],
      ids: [31, 21, 27, 3, 3, 3],
      decoded: "you<unk>",
    },
    TEXT_WITH_ESCAPE_CHARACTERS_2: {
      text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2,
      tokens: ["y", "o", "u", "\u2026", "\u00a0", "\u00a0", "y", "o", "u", "\u2026", "\u00a0", "\u00a0"],
      ids: [31, 21, 27, 3, 3, 3, 31, 21, 27, 3, 3, 3],
      decoded: "you<unk>you<unk>",
    },
    TILDE_NORMALIZATION: {
      text: BASE_TEST_STRINGS.TILDE_NORMALIZATION,
      tokens: ["w", "e", "i", "r", "d", "|", "\uff5e", "|", "e", "d", "g", "e", "|", "\uff5e", "|", "c", "a", "s", "e"],
      ids: [29, 11, 15, 24, 10, 4, 3, 4, 11, 10, 13, 11, 4, 3, 4, 9, 7, 25, 11],
      decoded: "weird <unk> edge <unk> case",
    },
    SPIECE_UNDERSCORE: {
      text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE,
      tokens: ["\u2581", "T", "h", "i", "s", "|", "\u2581", "i", "s", "|", "\u2581", "a", "|", "\u2581", "t", "e", "s", "t", "|", "\u2581", "."],
      ids: [3, 3, 14, 15, 25, 4, 3, 15, 25, 4, 3, 7, 4, 3, 26, 11, 25, 26, 4, 3, 3],
      decoded: "<unk>his <unk>is <unk>a <unk>test <unk>",
    },
    CHINESE_LATIN_MIXED: {
      text: BERT_TEST_STRINGS.CHINESE_LATIN_MIXED,
      tokens: ["a", "h", "\u535a", "\u63a8", "z", "z"],
      ids: [7, 14, 3, 3, 32, 32],
      decoded: "ah<unk>z",
    },
    SIMPLE_WITH_ACCENTS: {
      text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS,
      tokens: ["H", "\u00e9", "l", "l", "o"],
      ids: [3, 3, 18, 18, 21],
      decoded: "<unk>lo",
    },
    MIXED_CASE_WITHOUT_ACCENTS: {
      text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS,
      tokens: ["|", "\t", "H", "e", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"],
      ids: [4, 3, 3, 11, 3, 3, 21, 3, 14, 21, 29, 4, 4, 3, 4, 3, 24, 11, 4, 31, 21, 3, 3, 4, 4],
      decoded: "<unk>e<unk>o<unk>how <unk> <unk>re yo<unk>",
    },
    MIXED_CASE_WITH_ACCENTS: {
      text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS,
      tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"],
      ids: [4, 3, 3, 3, 3, 3, 21, 3, 14, 21, 29, 4, 4, 3, 4, 3, 24, 11, 4, 31, 21, 3, 3, 4, 4],
      decoded: "<unk>o<unk>how <unk> <unk>re yo<unk>",
    },
  },
  "Xenova/mms-1b-all": {
    NUMBERS: {
      text: BASE_TEST_STRINGS.NUMBERS,
      tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "|", "0", "|", "1", "|", "2", "|", "3", "|", "4", "|", "5", "|", "6", "|", "7", "|", "8", "|", "9", "|", "1", "0", "|", "1", "0", "0", "|", "1", "0", "0", "0"],
      ids: [27, 30, 35, 41, 39, 38, 40, 43, 42, 36, 4, 27, 4, 30, 4, 35, 4, 41, 4, 39, 4, 38, 4, 40, 4, 43, 4, 42, 4, 36, 4, 30, 27, 4, 30, 27, 27, 4, 30, 27, 27, 27],
      decoded: "0123456789 0 1 2 3 4 5 6 7 8 9 10 10 10",
    },
    TEXT_WITH_NUMBERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS,
      tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."],
      ids: [3, 13, 5, 4, 16, 8, 18, 20, 7, 10, 22, 4, 23, 7, 11, 4, 19, 8, 17, 10, 15, 5, 15, 4, 9, 10, 4, 35, 27, 30, 40, 37],
      decoded: "<unk>he company was founded in 2016.",
    },
    PUNCTUATION: {
      text: BASE_TEST_STRINGS.PUNCTUATION,
      tokens: ["A", "\n", "'", "l", "l", "|", "!", "!", "t", "o", "?", "'", "d", "'", "'", "d", "|", "o", "f", ",", "|", "c", "a", "n", "'", "t", "."],
      ids: [3, 3, 31, 14, 14, 4, 75, 75, 6, 8, 3, 31, 15, 31, 31, 15, 4, 8, 19, 44, 4, 16, 7, 10, 31, 6, 37],
      decoded: "<unk>'l!to<unk>'d'd of, can't.",
    },
    PYTHON_CODE: {
      text: BASE_TEST_STRINGS.PYTHON_CODE,
      tokens: ["d", "e", "f", "|", "m", "a", "i", "n", "(", ")", ":", "\n", "\t", "p", "a", "s", "s"],
      ids: [15, 5, 19, 4, 18, 7, 9, 10, 3, 3, 46, 3, 3, 20, 7, 11, 11],
      decoded: "def main<unk>:<unk>pas",
    },
    JAVASCRIPT_CODE: {
      text: BASE_TEST_STRINGS.JAVASCRIPT_CODE,
      tokens: ["l", "e", "t", "|", "a", "|", "=", "|", "o", "b", "j", ".", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";", "\n", "t", "o", "S", "t", "r", "i", "n", "g", "(", ")", ";"],
      ids: [14, 5, 6, 4, 7, 4, 3, 4, 8, 24, 29, 37, 6, 8, 3, 6, 12, 9, 10, 21, 3, 3, 52, 3, 6, 8, 3, 6, 12, 9, 10, 21, 3, 3, 52],
      decoded: "let a <unk> obj.to<unk>tring<unk>;<unk>to<unk>tring<unk>;",
    },
    NEWLINES: {
      text: BASE_TEST_STRINGS.NEWLINES,
      tokens: ["T", "h", "i", "s", "\n", "\n", "i", "s", "\n", "a", "\n", "t", "e", "s", "t", "."],
      ids: [3, 13, 9, 11, 3, 3, 9, 11, 3, 7, 3, 6, 5, 11, 6, 37],
      decoded: "<unk>his<unk>is<unk>a<unk>test.",
    },
    BASIC: {
      text: BASE_TEST_STRINGS.BASIC,
      tokens: ["U", "N", "w", "a", "n", "t", "\u00e9", "d", ",", "r", "u", "n", "n", "i", "n", "g"],
      ids: [3, 3, 23, 7, 10, 6, 55, 15, 44, 12, 17, 10, 10, 9, 10, 21],
      decoded: "<unk>want\u00e9d,runing",
    },
    CONTROL_TOKENS: {
      text: BASE_TEST_STRINGS.CONTROL_TOKENS,
      tokens: ["1", "\u0000", "2", "\ufffd", "3"],
      ids: [30, 3, 35, 3, 41],
      decoded: "1<unk>2<unk>3",
    },
    CHINESE_ONLY: {
      text: BASE_TEST_STRINGS.CHINESE_ONLY,
      tokens: ["\u751f", "\u6d3b", "\u7684", "\u771f", "\u8c1b", "\u662f"],
      ids: [136, 3, 3, 3, 3, 3],
      decoded: "\u751f<unk>",
    },
    CURRENCY: {
      text: BASE_TEST_STRINGS.CURRENCY,
      tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"],
      ids: [6, 5, 11, 6, 4, 48, 30, 4, 3, 35, 4, 3, 41, 4, 3, 39, 4, 68, 38, 4, 53, 40, 4, 3, 43, 4, 3, 42, 4, 3, 36, 4, 6, 5, 11, 6],
      decoded: "test $1 <unk>2 <unk>3 <unk>4 \u00a35 \u00a56 <unk>7 <unk>8 <unk>9 test",
    },
    CURRENCY_WITH_DECIMALS: {
      text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS,
      tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."],
      ids: [3, 4, 24, 8, 17, 21, 13, 6, 4, 7, 10, 4, 7, 20, 20, 14, 5, 4, 19, 8, 12, 4, 48, 30, 37, 27, 27, 4, 7, 6, 4, 6, 13, 5, 4, 11, 6, 8, 12, 5, 37],
      decoded: "<unk> bought an aple for $1.0 at the store.",
    },
    SPIECE_UNDERSCORE: {
      text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE,
      tokens: ["\u2581", "T", "h", "i", "s", "|", "\u2581", "i", "s", "|", "\u2581", "a", "|", "\u2581", "t", "e", "s", "t", "|", "\u2581", "."],
      ids: [3, 3, 13, 9, 11, 4, 3, 9, 11, 4, 3, 7, 4, 3, 6, 5, 11, 6, 4, 3, 37],
      decoded: "<unk>his <unk>is <unk>a <unk>test <unk>.",
    },
    SIMPLE_WITH_ACCENTS: {
      text: BERT_TEST_STRINGS.SIMPLE_WITH_ACCENTS,
      tokens: ["H", "\u00e9", "l", "l", "o"],
      ids: [3, 55, 14, 14, 8],
      decoded: "<unk>\u00e9lo",
    },
    MIXED_CASE_WITHOUT_ACCENTS: {
      text: BERT_TEST_STRINGS.MIXED_CASE_WITHOUT_ACCENTS,
      tokens: ["|", "\t", "H", "e", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"],
      ids: [4, 3, 3, 5, 3, 3, 8, 75, 13, 8, 23, 4, 4, 3, 4, 3, 12, 5, 4, 22, 8, 3, 3, 4, 4],
      decoded: "<unk>e<unk>o!how <unk> <unk>re yo<unk>",
    },
    MIXED_CASE_WITH_ACCENTS: {
      text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS,
      tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"],
      ids: [4, 3, 3, 78, 3, 3, 8, 75, 13, 8, 23, 4, 4, 3, 4, 3, 12, 5, 4, 22, 8, 3, 3, 4, 4],
      decoded: "<unk>\u00e4<unk>o!how <unk> <unk>re yo<unk>",
    },
  },
  "Xenova/mms-1b-fl102": {
    MIXED_CASE_WITH_ACCENTS: {
      text: BERT_TEST_STRINGS.MIXED_CASE_WITH_ACCENTS,
      tokens: ["|", "\t", "H", "\u00e4", "L", "L", "o", "!", "h", "o", "w", "|", "|", "\n", "|", "A", "r", "e", "|", "y", "o", "U", "?", "|", "|"],
      ids: [4, 3, 3, 3, 3, 3, 8, 75, 13, 8, 23, 4, 4, 3, 4, 3, 12, 5, 4, 22, 8, 3, 3, 4, 4],
      decoded: "<unk>o!how <unk> <unk>re yo<unk>",
    },
  },
  "Xenova/mms-1b-l1107": {
    NUMBERS: {
      text: BASE_TEST_STRINGS.NUMBERS,
      tokens: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "|", "0", "|", "1", "|", "2", "|", "3", "|", "4", "|", "5", "|", "6", "|", "7", "|", "8", "|", "9", "|", "1", "0", "|", "1", "0", "0", "|", "1", "0", "0", "0"],
      ids: [34, 36, 37, 42, 38, 41, 39, 3, 3, 3, 4, 34, 4, 36, 4, 37, 4, 42, 4, 38, 4, 41, 4, 39, 4, 3, 4, 3, 4, 3, 4, 36, 34, 4, 36, 34, 34, 4, 36, 34, 34, 34],
      decoded: "0123456<unk> 0 1 2 3 4 5 6 <unk> <unk> <unk> 10 10 10",
    },
    TEXT_WITH_NUMBERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS,
      tokens: ["T", "h", "e", "|", "c", "o", "m", "p", "a", "n", "y", "|", "w", "a", "s", "|", "f", "o", "u", "n", "d", "e", "d", "|", "i", "n", "|", "2", "0", "1", "6", "."],
      ids: [3, 9, 5, 4, 21, 7, 18, 24, 8, 10, 20, 4, 17, 8, 12, 4, 19, 7, 16, 10, 14, 5, 14, 4, 11, 10, 4, 37, 34, 36, 39, 3],
      decoded: "<unk>he company was founded in 2016<unk>",
    },
    CURRENCY: {
      text: BASE_TEST_STRINGS.CURRENCY,
      tokens: ["t", "e", "s", "t", "|", "$", "1", "|", "R", "2", "|", "#", "3", "|", "\u20ac", "4", "|", "\u00a3", "5", "|", "\u00a5", "6", "|", "\u20a3", "7", "|", "\u20b9", "8", "|", "\u20b1", "9", "|", "t", "e", "s", "t"],
      ids: [6, 5, 12, 6, 4, 3, 36, 4, 3, 37, 4, 3, 42, 4, 3, 38, 4, 3, 41, 4, 3, 39, 4, 3, 3, 4, 3, 3, 4, 3, 3, 4, 6, 5, 12, 6],
      decoded: "test <unk>1 <unk>2 <unk>3 <unk>4 <unk>5 <unk>6 <unk> <unk> <unk> test",
    },
    CURRENCY_WITH_DECIMALS: {
      text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS,
      tokens: ["I", "|", "b", "o", "u", "g", "h", "t", "|", "a", "n", "|", "a", "p", "p", "l", "e", "|", "f", "o", "r", "|", "$", "1", ".", "0", "0", "|", "a", "t", "|", "t", "h", "e", "|", "s", "t", "o", "r", "e", "."],
      ids: [3, 4, 23, 7, 16, 22, 9, 6, 4, 8, 10, 4, 8, 24, 24, 15, 5, 4, 19, 7, 13, 4, 3, 36, 3, 34, 34, 4, 8, 6, 4, 6, 9, 5, 4, 12, 6, 7, 13, 5, 3],
      decoded: "<unk> bought an aple for <unk>1<unk>0 at the store<unk>",
    },
  },
};
