import { WhisperTokenizer } from "../../../src/tokenizers.js";
import { BASE_TEST_STRINGS, WHISPER_TEST_STRINGS } from "../test_strings.js";
import { compare } from "../../test_utils.js";

export const TOKENIZER_CLASS = WhisperTokenizer;
export const TEST_CONFIG = {
  "onnx-community/whisper-tiny.en": {
    SIMPLE: {
      text: BASE_TEST_STRINGS.SIMPLE,
      tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"],
      ids: [50257, 50362, 2437, 389, 345, 1804, 30, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>How are you doing?<|endoftext|>",
    },
    SIMPLE_WITH_PUNCTUATION: {
      text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION,
      tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120this"],
      ids: [50257, 50362, 1639, 815, 1053, 1760, 428, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>You should've done this<|endoftext|>",
    },
    NUMBERS: {
      text: BASE_TEST_STRINGS.NUMBERS,
      tokens: ["01", "23", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"],
      ids: [50257, 50362, 486, 1954, 2231, 3134, 4531, 657, 352, 362, 513, 604, 642, 718, 767, 807, 860, 838, 1802, 8576, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>",
    },
    TEXT_WITH_NUMBERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS,
      tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."],
      ids: [50257, 50362, 464, 1664, 373, 9393, 287, 1584, 13, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>The company was founded in 2016.<|endoftext|>",
    },
    PUNCTUATION: {
      text: BASE_TEST_STRINGS.PUNCTUATION,
      tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."],
      ids: [50257, 50362, 32, 198, 1183, 37867, 1462, 8348, 67, 7061, 67, 286, 11, 460, 470, 13, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>A\n'll!!to?'d''d of, can't.<|endoftext|>",
    },
    PYTHON_CODE: {
      text: BASE_TEST_STRINGS.PYTHON_CODE,
      tokens: ["def", "\u0120main", "():", "\u010a", "\u0109", "pass"],
      ids: [50257, 50362, 4299, 1388, 33529, 198, 197, 6603, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>def main():\n\tpass<|endoftext|>",
    },
    JAVASCRIPT_CODE: {
      text: BASE_TEST_STRINGS.JAVASCRIPT_CODE,
      tokens: ["let", "\u0120a", "\u0120=", "\u0120obj", ".", "to", "String", "();", "\u010a", "to", "String", "();"],
      ids: [50257, 50362, 1616, 257, 796, 26181, 13, 1462, 10100, 9783, 198, 1462, 10100, 9783, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>let a = obj.toString();\ntoString();<|endoftext|>",
    },
    NEWLINES: {
      text: BASE_TEST_STRINGS.NEWLINES,
      tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."],
      ids: [50257, 50362, 1212, 198, 198, 271, 198, 64, 198, 9288, 13, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>This\n\nis\na\ntest.<|endoftext|>",
    },
    BASIC: {
      text: BASE_TEST_STRINGS.BASIC,
      tokens: ["UN", "want", "\u00c3\u00a9", "d", ",", "running"],
      ids: [50257, 50362, 4944, 42949, 2634, 67, 11, 20270, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>UNwant\u00e9d,running<|endoftext|>",
    },
    CONTROL_TOKENS: {
      text: BASE_TEST_STRINGS.CONTROL_TOKENS,
      tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"],
      ids: [50257, 50362, 16, 188, 17, 4210, 18, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>1\u00002\ufffd3<|endoftext|>",
    },
    HELLO_WORLD_TITLECASE: {
      text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE,
      tokens: ["Hello", "\u0120World"],
      ids: [50257, 50362, 15496, 2159, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>Hello World<|endoftext|>",
    },
    HELLO_WORLD_LOWERCASE: {
      text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE,
      tokens: ["hello", "\u0120world"],
      ids: [50257, 50362, 31373, 995, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>hello world<|endoftext|>",
    },
    CHINESE_ONLY: {
      text: BASE_TEST_STRINGS.CHINESE_ONLY,
      tokens: ["\u00e7\u0136\u0141", "\u00e6", "\u00b4", "\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e", "\u0141", "\u00e8", "\u00b0", "\u013d", "\u00e6\u013a\u00af"],
      ids: [50257, 50362, 37955, 162, 112, 119, 21410, 40367, 253, 164, 108, 249, 42468, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>",
    },
    LEADING_SPACE: {
      text: BASE_TEST_STRINGS.LEADING_SPACE,
      tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"],
      ids: [50257, 50362, 220, 220, 3756, 2272, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>   leading space<|endoftext|>",
    },
    TRAILING_SPACE: {
      text: BASE_TEST_STRINGS.TRAILING_SPACE,
      tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"],
      ids: [50257, 50362, 9535, 4386, 2272, 220, 220, 220, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>trailing space   <|endoftext|>",
    },
    DOUBLE_SPACE: {
      text: BASE_TEST_STRINGS.DOUBLE_SPACE,
      tokens: ["Hi", "\u0120", "\u0120Hello"],
      ids: [50257, 50362, 17250, 220, 18435, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>Hi  Hello<|endoftext|>",
    },
    CURRENCY: {
      text: BASE_TEST_STRINGS.CURRENCY,
      tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"],
      ids: [50257, 50362, 9288, 720, 16, 371, 17, 1303, 18, 10432, 19, 4248, 20, 38221, 21, 2343, 224, 96, 22, 2343, 224, 117, 23, 2343, 224, 109, 24, 1332, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>",
    },
    CURRENCY_WITH_DECIMALS: {
      text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS,
      tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120the", "\u0120store", "."],
      ids: [50257, 50362, 40, 5839, 281, 17180, 329, 720, 16, 13, 405, 379, 262, 3650, 13, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>I bought an apple for $1.00 at the store.<|endoftext|>",
    },
    ELLIPSIS: {
      text: BASE_TEST_STRINGS.ELLIPSIS,
      tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"],
      ids: [50257, 50362, 5832, 1399, 220, 220, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>you\u2026  <|endoftext|>",
    },
    TEXT_WITH_ESCAPE_CHARACTERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS,
      tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"],
      ids: [50257, 50362, 5832, 1399, 4603, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0<|endoftext|>",
    },
    TEXT_WITH_ESCAPE_CHARACTERS_2: {
      text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2,
      tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2\u0142", "\u00c2\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2\u0142\u00c2\u0142"],
      ids: [50257, 50362, 5832, 1399, 1849, 1849, 5832, 1399, 4603, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>",
    },
    TILDE_NORMALIZATION: {
      text: BASE_TEST_STRINGS.TILDE_NORMALIZATION,
      tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"],
      ids: [50257, 50362, 732, 1447, 27332, 121, 252, 5743, 27332, 121, 252, 1339, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>weird \uff5e edge \uff5e case<|endoftext|>",
    },
    SPIECE_UNDERSCORE: {
      text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE,
      tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."],
      ids: [50257, 50362, 5008, 223, 1212, 11019, 223, 271, 11019, 223, 64, 11019, 223, 9288, 11019, 223, 13, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>\u2581This \u2581is \u2581a \u2581test \u2581.<|endoftext|>",
    },
    SPECIAL_TOKENS: {
      text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS,
      tokens: ["\u0120", "\u0120", "\u0120", "<|startoftranscript|>", "\u0120", "<|en|>", "\u0120", "\u0120", "\u0120"],
      ids: [50257, 50362, 220, 220, 220, 50257, 220, 50258, 220, 220, 220, 50256],
      decoded: "<|startoftranscript|><|notimestamps|>   <|startoftranscript|> <|en|>   <|endoftext|>",
    },
  },
  "distil-whisper/distil-large-v3": {
    NUMBERS: {
      text: BASE_TEST_STRINGS.NUMBERS,
      tokens: ["0", "12", "3", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"],
      ids: [50258, 50364, 15, 4762, 18, 8465, 22452, 21115, 1958, 502, 568, 805, 1017, 1025, 1386, 1614, 1649, 1722, 1266, 2319, 9714, 50257],
      decoded: "<|startoftranscript|><|notimestamps|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>",
    },
    PYTHON_CODE: {
      text: BASE_TEST_STRINGS.PYTHON_CODE,
      tokens: ["def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"],
      ids: [50258, 50364, 20595, 2135, 7, 4507, 198, 197, 9216, 50257],
      decoded: "<|startoftranscript|><|notimestamps|>def main():\n\tpass<|endoftext|>",
    },
    JAVASCRIPT_CODE: {
      text: BASE_TEST_STRINGS.JAVASCRIPT_CODE,
      tokens: ["let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "St", "ring", "(", ");", "\u010a", "to", "St", "ring", "(", ");"],
      ids: [50258, 50364, 2631, 257, 6585, 1111, 73, 13, 1353, 4520, 2937, 7, 34446, 198, 1353, 4520, 2937, 7, 34446, 50257],
      decoded: "<|startoftranscript|><|notimestamps|>let a = obj.toString();\ntoString();<|endoftext|>",
    },
    BASIC: {
      text: BASE_TEST_STRINGS.BASIC,
      tokens: ["UN", "w", "ant", "\u00c3\u00a9d", ",", "running"],
      ids: [50258, 50364, 3979, 86, 394, 7811, 11, 45482, 50257],
      decoded: "<|startoftranscript|><|notimestamps|>UNwant\u00e9d,running<|endoftext|>",
    },
    HELLO_WORLD_LOWERCASE: {
      text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE,
      tokens: ["he", "llo", "\u0120world"],
      ids: [50258, 50364, 675, 1913, 1002, 50257],
      decoded: "<|startoftranscript|><|notimestamps|>hello world<|endoftext|>",
    },
    CHINESE_ONLY: {
      text: BASE_TEST_STRINGS.CHINESE_ONLY,
      tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"],
      ids: [50258, 50364, 49958, 1546, 6303, 8897, 249, 1541, 50257],
      decoded: "<|startoftranscript|><|notimestamps|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>",
    },
    CURRENCY: {
      text: BASE_TEST_STRINGS.CURRENCY,
      tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120test"],
      ids: [50258, 50364, 31636, 1848, 16, 497, 17, 3536, 18, 17450, 19, 14378, 20, 1815, 98, 21, 672, 224, 96, 22, 672, 224, 117, 23, 672, 224, 109, 24, 1500, 50257],
      decoded: "<|startoftranscript|><|notimestamps|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>",
    },
    TEXT_WITH_ESCAPE_CHARACTERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS,
      tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"],
      ids: [50258, 50364, 5616, 1260, 126, 254, 126, 254, 50257],
      decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0<|endoftext|>",
    },
    TEXT_WITH_ESCAPE_CHARACTERS_2: {
      text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2,
      tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"],
      ids: [50258, 50364, 5616, 1260, 126, 254, 126, 254, 5616, 1260, 126, 254, 126, 254, 50257],
      decoded: "<|startoftranscript|><|notimestamps|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>",
    },
  },
  "distil-whisper/distil-large-v2": {
    SIMPLE: {
      text: BASE_TEST_STRINGS.SIMPLE,
      tokens: ["How", "\u0120are", "\u0120you", "\u0120doing", "?"],
      ids: [50258, 50259, 50359, 50363, 6462, 366, 291, 884, 30, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>How are you doing?<|endoftext|>",
    },
    SIMPLE_WITH_PUNCTUATION: {
      text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION,
      tokens: ["You", "\u0120should", "'ve", "\u0120done", "\u0120", "this"],
      ids: [50258, 50259, 50359, 50363, 3223, 820, 600, 1096, 220, 11176, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>You should've done this<|endoftext|>",
    },
    NUMBERS: {
      text: BASE_TEST_STRINGS.NUMBERS,
      tokens: ["0", "12", "3", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"],
      ids: [50258, 50259, 50359, 50363, 15, 4762, 18, 8465, 22452, 21115, 1958, 502, 568, 805, 1017, 1025, 1386, 1614, 1649, 1722, 1266, 2319, 9714, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>",
    },
    TEXT_WITH_NUMBERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS,
      tokens: ["The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."],
      ids: [50258, 50259, 50359, 50363, 2278, 2237, 390, 13234, 294, 6549, 13, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>The company was founded in 2016.<|endoftext|>",
    },
    PUNCTUATION: {
      text: BASE_TEST_STRINGS.PUNCTUATION,
      tokens: ["A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."],
      ids: [50258, 50259, 50359, 50363, 32, 198, 603, 15138, 1353, 8569, 67, 15025, 67, 295, 11, 393, 380, 13, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>A\n'll!!to?'d''d of, can't.<|endoftext|>",
    },
    PYTHON_CODE: {
      text: BASE_TEST_STRINGS.PYTHON_CODE,
      tokens: ["def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"],
      ids: [50258, 50259, 50359, 50363, 20595, 2135, 7, 4507, 198, 197, 9216, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>def main():\n\tpass<|endoftext|>",
    },
    JAVASCRIPT_CODE: {
      text: BASE_TEST_STRINGS.JAVASCRIPT_CODE,
      tokens: ["let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "St", "ring", "(", ");", "\u010a", "to", "St", "ring", "(", ");"],
      ids: [50258, 50259, 50359, 50363, 2631, 257, 6585, 1111, 73, 13, 1353, 4520, 2937, 7, 34446, 198, 1353, 4520, 2937, 7, 34446, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>let a = obj.toString();\ntoString();<|endoftext|>",
    },
    NEWLINES: {
      text: BASE_TEST_STRINGS.NEWLINES,
      tokens: ["This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."],
      ids: [50258, 50259, 50359, 50363, 5723, 198, 198, 271, 198, 64, 198, 31636, 13, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>This\n\nis\na\ntest.<|endoftext|>",
    },
    BASIC: {
      text: BASE_TEST_STRINGS.BASIC,
      tokens: ["UN", "w", "ant", "\u00c3\u00a9d", ",", "running"],
      ids: [50258, 50259, 50359, 50363, 3979, 86, 394, 7811, 11, 45482, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>UNwant\u00e9d,running<|endoftext|>",
    },
    CONTROL_TOKENS: {
      text: BASE_TEST_STRINGS.CONTROL_TOKENS,
      tokens: ["1", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"],
      ids: [50258, 50259, 50359, 50363, 16, 188, 17, 5342, 18, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>1\u00002\ufffd3<|endoftext|>",
    },
    HELLO_WORLD_TITLECASE: {
      text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE,
      tokens: ["Hello", "\u0120World"],
      ids: [50258, 50259, 50359, 50363, 15947, 3937, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>Hello World<|endoftext|>",
    },
    HELLO_WORLD_LOWERCASE: {
      text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE,
      tokens: ["he", "llo", "\u0120world"],
      ids: [50258, 50259, 50359, 50363, 675, 1913, 1002, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>hello world<|endoftext|>",
    },
    CHINESE_ONLY: {
      text: BASE_TEST_STRINGS.CHINESE_ONLY,
      tokens: ["\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"],
      ids: [50258, 50259, 50359, 50363, 49958, 1546, 6303, 8897, 249, 1541, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>\u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>",
    },
    LEADING_SPACE: {
      text: BASE_TEST_STRINGS.LEADING_SPACE,
      tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"],
      ids: [50258, 50259, 50359, 50363, 220, 220, 5775, 1901, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>   leading space<|endoftext|>",
    },
    TRAILING_SPACE: {
      text: BASE_TEST_STRINGS.TRAILING_SPACE,
      tokens: ["tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"],
      ids: [50258, 50259, 50359, 50363, 17227, 4883, 1901, 220, 220, 220, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>trailing space   <|endoftext|>",
    },
    DOUBLE_SPACE: {
      text: BASE_TEST_STRINGS.DOUBLE_SPACE,
      tokens: ["Hi", "\u0120", "\u0120Hello"],
      ids: [50258, 50259, 50359, 50363, 17155, 220, 2425, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>Hi  Hello<|endoftext|>",
    },
    CURRENCY: {
      text: BASE_TEST_STRINGS.CURRENCY,
      tokens: ["test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120", "test"],
      ids: [50258, 50259, 50359, 50363, 31636, 1848, 16, 497, 17, 3536, 18, 17450, 19, 14378, 20, 1815, 98, 21, 672, 224, 96, 22, 672, 224, 117, 23, 672, 224, 109, 24, 220, 31636, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>",
    },
    CURRENCY_WITH_DECIMALS: {
      text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS,
      tokens: ["I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120", "the", "\u0120store", "."],
      ids: [50258, 50259, 50359, 50363, 40, 4243, 364, 10606, 337, 1848, 16, 13, 628, 412, 220, 3322, 3531, 13, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>I bought an apple for $1.00 at the store.<|endoftext|>",
    },
    ELLIPSIS: {
      text: BASE_TEST_STRINGS.ELLIPSIS,
      tokens: ["you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"],
      ids: [50258, 50259, 50359, 50363, 5616, 1260, 220, 220, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>you\u2026  <|endoftext|>",
    },
    TEXT_WITH_ESCAPE_CHARACTERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS,
      tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"],
      ids: [50258, 50259, 50359, 50363, 5616, 1260, 126, 254, 126, 254, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>you\u2026\u00a0\u00a0<|endoftext|>",
    },
    TEXT_WITH_ESCAPE_CHARACTERS_2: {
      text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2,
      tokens: ["you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"],
      ids: [50258, 50259, 50359, 50363, 5616, 1260, 126, 254, 126, 254, 5616, 1260, 126, 254, 126, 254, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>",
    },
    TILDE_NORMALIZATION: {
      text: BASE_TEST_STRINGS.TILDE_NORMALIZATION,
      tokens: ["we", "ird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"],
      ids: [50258, 50259, 50359, 50363, 826, 1271, 25072, 121, 252, 4691, 25072, 121, 252, 1389, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>weird \uff5e edge \uff5e case<|endoftext|>",
    },
    SPIECE_UNDERSCORE: {
      text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE,
      tokens: ["\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."],
      ids: [50258, 50259, 50359, 50363, 39984, 223, 5723, 29405, 223, 271, 29405, 223, 64, 29405, 223, 31636, 29405, 223, 13, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>\u2581This \u2581is \u2581a \u2581test \u2581.<|endoftext|>",
    },
    SPECIAL_TOKENS: {
      text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS,
      tokens: ["\u0120", "\u0120", "\u0120", "<|startoftranscript|>", "\u0120", "<|en|>", "\u0120", "\u0120", "\u0120"],
      ids: [50258, 50259, 50359, 50363, 220, 220, 220, 50258, 220, 50259, 220, 220, 220, 50257],
      decoded: "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>   <|startoftranscript|> <|en|>   <|endoftext|>",
    },
  },
  "distil-whisper/distil-small.en": {
    SPECIAL_TOKENS: {
      text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS,

      // https://github.com/huggingface/transformers/issues/33371
      // tokens: ["   <|startoftranscript|> ", "<|en|>   "],
      tokens: ["<|startoftranscript|>", "<|en|>"],
      ids: [50257, 50362, 50257, 50258, 50256],
      decoded: "<|startoftranscript|><|notimestamps|><|startoftranscript|><|en|><|endoftext|>",
    },
  },
  "Xenova/nb-whisper-tiny-beta": {
    SIMPLE: {
      text: BASE_TEST_STRINGS.SIMPLE,
      tokens: ["\u0120How", "\u0120are", "\u0120you", "\u0120doing", "?"],
      ids: [50258, 50288, 50359, 50363, 1012, 366, 291, 884, 30, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> How are you doing?<|endoftext|>",
    },
    SIMPLE_WITH_PUNCTUATION: {
      text: BASE_TEST_STRINGS.SIMPLE_WITH_PUNCTUATION,
      tokens: ["\u0120You", "\u0120should", "'ve", "\u0120done", "\u0120", "this"],
      ids: [50258, 50288, 50359, 50363, 509, 820, 600, 1096, 220, 11176, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> You should've done this<|endoftext|>",
    },
    NUMBERS: {
      text: BASE_TEST_STRINGS.NUMBERS,
      tokens: ["\u01200", "12", "3", "45", "67", "89", "\u01200", "\u01201", "\u01202", "\u01203", "\u01204", "\u01205", "\u01206", "\u01207", "\u01208", "\u01209", "\u012010", "\u0120100", "\u01201000"],
      ids: [50258, 50288, 50359, 50363, 1958, 4762, 18, 8465, 22452, 21115, 1958, 502, 568, 805, 1017, 1025, 1386, 1614, 1649, 1722, 1266, 2319, 9714, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> 0123456789 0 1 2 3 4 5 6 7 8 9 10 100 1000<|endoftext|>",
    },
    TEXT_WITH_NUMBERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_NUMBERS,
      tokens: ["\u0120The", "\u0120company", "\u0120was", "\u0120founded", "\u0120in", "\u01202016", "."],
      ids: [50258, 50288, 50359, 50363, 440, 2237, 390, 13234, 294, 6549, 13, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> The company was founded in 2016.<|endoftext|>",
    },
    PUNCTUATION: {
      text: BASE_TEST_STRINGS.PUNCTUATION,
      tokens: ["\u0120A", "\u010a", "'ll", "\u0120!!", "to", "?'", "d", "''", "d", "\u0120of", ",", "\u0120can", "'t", "."],
      ids: [50258, 50288, 50359, 50363, 316, 198, 603, 15138, 1353, 8569, 67, 15025, 67, 295, 11, 393, 380, 13, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> A\n'll!!to?'d''d of, can't.<|endoftext|>",
    },
    PYTHON_CODE: {
      text: BASE_TEST_STRINGS.PYTHON_CODE,
      tokens: ["\u0120def", "\u0120main", "(", "):", "\u010a", "\u0109", "pass"],
      ids: [50258, 50288, 50359, 50363, 1060, 2135, 7, 4507, 198, 197, 9216, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> def main():\n\tpass<|endoftext|>",
    },
    JAVASCRIPT_CODE: {
      text: BASE_TEST_STRINGS.JAVASCRIPT_CODE,
      tokens: ["\u0120let", "\u0120a", "\u0120=", "\u0120ob", "j", ".", "to", "St", "ring", "(", ");", "\u010a", "to", "St", "ring", "(", ");"],
      ids: [50258, 50288, 50359, 50363, 718, 257, 6585, 1111, 73, 13, 1353, 4520, 2937, 7, 34446, 198, 1353, 4520, 2937, 7, 34446, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> let a = obj.toString();\ntoString();<|endoftext|>",
    },
    NEWLINES: {
      text: BASE_TEST_STRINGS.NEWLINES,
      tokens: ["\u0120This", "\u010a", "\u010a", "is", "\u010a", "a", "\u010a", "test", "."],
      ids: [50258, 50288, 50359, 50363, 639, 198, 198, 271, 198, 64, 198, 31636, 13, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> This\n\nis\na\ntest.<|endoftext|>",
    },
    BASIC: {
      text: BASE_TEST_STRINGS.BASIC,
      tokens: ["\u0120UN", "w", "ant", "\u00c3\u00a9d", ",", "running"],
      ids: [50258, 50288, 50359, 50363, 8229, 86, 394, 7811, 11, 45482, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> UNwant\u00e9d,running<|endoftext|>",
    },
    CONTROL_TOKENS: {
      text: BASE_TEST_STRINGS.CONTROL_TOKENS,
      tokens: ["\u01201", "\u0100", "2", "\u00ef\u00bf\u00bd", "3"],
      ids: [50258, 50288, 50359, 50363, 502, 188, 17, 5342, 18, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> 1\u00002\ufffd3<|endoftext|>",
    },
    HELLO_WORLD_TITLECASE: {
      text: BASE_TEST_STRINGS.HELLO_WORLD_TITLECASE,
      tokens: ["\u0120Hello", "\u0120World"],
      ids: [50258, 50288, 50359, 50363, 2425, 3937, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> Hello World<|endoftext|>",
    },
    HELLO_WORLD_LOWERCASE: {
      text: BASE_TEST_STRINGS.HELLO_WORLD_LOWERCASE,
      tokens: ["\u0120hello", "\u0120world"],
      ids: [50258, 50288, 50359, 50363, 7751, 1002, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> hello world<|endoftext|>",
    },
    CHINESE_ONLY: {
      text: BASE_TEST_STRINGS.CHINESE_ONLY,
      tokens: ["\u0120", "\u00e7\u0136\u0141\u00e6\u00b4\u00bb", "\u00e7\u013c\u0126", "\u00e7\u013e\u0141", "\u00e8\u00b0", "\u013d", "\u00e6\u013a\u00af"],
      ids: [50258, 50288, 50359, 50363, 220, 49958, 1546, 6303, 8897, 249, 1541, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> \u751f\u6d3b\u7684\u771f\u8c1b\u662f<|endoftext|>",
    },
    LEADING_SPACE: {
      text: BASE_TEST_STRINGS.LEADING_SPACE,
      tokens: ["\u0120", "\u0120", "\u0120leading", "\u0120space"],
      ids: [50258, 50288, 50359, 50363, 220, 220, 5775, 1901, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|>   leading space<|endoftext|>",
    },
    TRAILING_SPACE: {
      text: BASE_TEST_STRINGS.TRAILING_SPACE,
      tokens: ["\u0120", "tra", "iling", "\u0120space", "\u0120", "\u0120", "\u0120"],
      ids: [50258, 50288, 50359, 50363, 220, 17227, 4883, 1901, 220, 220, 220, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> trailing space   <|endoftext|>",
    },
    DOUBLE_SPACE: {
      text: BASE_TEST_STRINGS.DOUBLE_SPACE,
      tokens: ["\u0120Hi", "\u0120", "\u0120Hello"],
      ids: [50258, 50288, 50359, 50363, 2421, 220, 2425, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> Hi  Hello<|endoftext|>",
    },
    CURRENCY: {
      text: BASE_TEST_STRINGS.CURRENCY,
      tokens: ["\u0120", "test", "\u0120$", "1", "\u0120R", "2", "\u0120#", "3", "\u0120\u00e2\u0124\u00ac", "4", "\u0120\u00c2\u00a3", "5", "\u0120\u00c2", "\u00a5", "6", "\u0120\u00e2", "\u0124", "\u00a3", "7", "\u0120\u00e2", "\u0124", "\u00b9", "8", "\u0120\u00e2", "\u0124", "\u00b1", "9", "\u0120", "test"],
      ids: [50258, 50288, 50359, 50363, 220, 31636, 1848, 16, 497, 17, 3536, 18, 17450, 19, 14378, 20, 1815, 98, 21, 672, 224, 96, 22, 672, 224, 117, 23, 672, 224, 109, 24, 220, 31636, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> test $1 R2 #3 \u20ac4 \u00a35 \u00a56 \u20a37 \u20b98 \u20b19 test<|endoftext|>",
    },
    CURRENCY_WITH_DECIMALS: {
      text: BASE_TEST_STRINGS.CURRENCY_WITH_DECIMALS,
      tokens: ["\u0120I", "\u0120bought", "\u0120an", "\u0120apple", "\u0120for", "\u0120$", "1", ".", "00", "\u0120at", "\u0120", "the", "\u0120store", "."],
      ids: [50258, 50288, 50359, 50363, 286, 4243, 364, 10606, 337, 1848, 16, 13, 628, 412, 220, 3322, 3531, 13, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> I bought an apple for $1.00 at the store.<|endoftext|>",
    },
    ELLIPSIS: {
      text: BASE_TEST_STRINGS.ELLIPSIS,
      tokens: ["\u0120you", "\u00e2\u0122\u00a6", "\u0120", "\u0120"],
      ids: [50258, 50288, 50359, 50363, 291, 1260, 220, 220, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> you\u2026  <|endoftext|>",
    },
    TEXT_WITH_ESCAPE_CHARACTERS: {
      text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS,
      tokens: ["\u0120you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"],
      ids: [50258, 50288, 50359, 50363, 291, 1260, 126, 254, 126, 254, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> you\u2026\u00a0\u00a0<|endoftext|>",
    },
    TEXT_WITH_ESCAPE_CHARACTERS_2: {
      text: BASE_TEST_STRINGS.TEXT_WITH_ESCAPE_CHARACTERS_2,
      tokens: ["\u0120you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142", "you", "\u00e2\u0122\u00a6", "\u00c2", "\u0142", "\u00c2", "\u0142"],
      ids: [50258, 50288, 50359, 50363, 291, 1260, 126, 254, 126, 254, 5616, 1260, 126, 254, 126, 254, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> you\u2026\u00a0\u00a0you\u2026\u00a0\u00a0<|endoftext|>",
    },
    TILDE_NORMALIZATION: {
      text: BASE_TEST_STRINGS.TILDE_NORMALIZATION,
      tokens: ["\u0120weird", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120edge", "\u0120\u00ef", "\u00bd", "\u0140", "\u0120case"],
      ids: [50258, 50288, 50359, 50363, 3657, 25072, 121, 252, 4691, 25072, 121, 252, 1389, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> weird \uff5e edge \uff5e case<|endoftext|>",
    },
    SPIECE_UNDERSCORE: {
      text: BASE_TEST_STRINGS.SPIECE_UNDERSCORE,
      tokens: ["\u0120\u00e2\u0138", "\u0123", "This", "\u0120\u00e2\u0138", "\u0123", "is", "\u0120\u00e2\u0138", "\u0123", "a", "\u0120\u00e2\u0138", "\u0123", "test", "\u0120\u00e2\u0138", "\u0123", "."],
      ids: [50258, 50288, 50359, 50363, 29405, 223, 5723, 29405, 223, 271, 29405, 223, 64, 29405, 223, 31636, 29405, 223, 13, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|> \u2581This \u2581is \u2581a \u2581test \u2581.<|endoftext|>",
    },
    SPECIAL_TOKENS: {
      text: WHISPER_TEST_STRINGS.SPECIAL_TOKENS,
      tokens: ["\u0120", "\u0120", "\u0120", "<|startoftranscript|>", "\u0120", "<|en|>", "\u0120", "\u0120", "\u0120"],
      ids: [50258, 50288, 50359, 50363, 220, 220, 220, 50258, 220, 50259, 220, 220, 220, 50257],
      decoded: "<|startoftranscript|><|no|><|transcribe|><|notimestamps|>   <|startoftranscript|> <|en|>   <|endoftext|>",
    },
  },
};

const MAX_EXECUTION_TIME = 10_000;
export const CUSTOM_TESTS = () => {
  describe("Decode ASR", () => {
    it(
      "should decode ASR outputs",
      async () => {
        const tokenizer = await WhisperTokenizer.from_pretrained("onnx-community/whisper-tiny.en_timestamped");

        const model_outputs = [
          {
            stride: [30, 0, 5],
            tokens: [50257n, 50362n, 8410n, 7283n, 0n, 2329n, 8410n, 7283n, 0n, 2094n, 470n, 1309n, 534n, 10625n, 307n, 10625n, 13n, 34668n, 345n, 531n, 9439n, 11n, 523n, 655n, 8410n, 7283n, 0n, 39134n, 16592n, 10625n, 0n, 9440n, 36n, 26751n, 0n, 25848n, 8410n, 7283n, 0n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 0n, 50256n],
            token_timestamps: [0, 0, 0, 3.78, 4.22, 5.34, 6.04, 6.56, 7, 7.92, 8.58, 8.58, 8.88, 9.14, 9.54, 9.94, 10.58, 11.38, 11.88, 12.42, 12.62, 13, 13.36, 13.64, 14.26, 14.76, 15.12, 15.4, 15.74, 16.12, 16.66, 17.14, 17.24, 17.24, 17.72, 18.38, 18.6, 19.38, 19.92, 22.66, 22.9, 23.24, 23.5, 24.14, 24.56, 24.7, 24.72, 24.94, 25.18, 25.54, 25.72, 26.02, 26.34, 26.44, 26.84, 27.04, 27.16, 27.54, 28.06, 29.92],
          },
          {
            stride: [30, 5, 5],
            tokens: [50257n, 50362n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 13n, 921n, 815n, 651n, 284n, 262n, 966n, 810n, 2687n, 2073n, 561n, 11238n, 290n, 345n, 821n, 407n, 8066n, 2245n, 612n, 13n, 1400n, 11n, 644n, 389n, 345n, 4953n, 329n, 30n, 2141n, 340n, 0n, 2329n, 466n, 340n, 0n, 3363n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 50256n],
            token_timestamps: [0, 0, 0, 2.92, 3.24, 3.48, 4.14, 4.56, 4.7, 4.74, 4.92, 5.18, 5.54, 5.72, 6.04, 6.34, 6.46, 6.84, 7.04, 7.16, 7.54, 8.12, 10.16, 10.7, 10.9, 11.12, 11.24, 11.48, 11.84, 12.44, 12.82, 13.2, 13.46, 13.72, 14.06, 14.28, 14.34, 14.56, 14.8, 15.16, 15.9, 16.42, 16.82, 16.86, 17.02, 17.1, 17.22, 17.56, 18.06, 19.28, 19.62, 20.26, 21.96, 22.64, 24.28, 24.76, 25.18, 25.56, 25.78, 26.28, 27.12, 27.54, 27.82, 28.22, 29.48],
          },
          {
            stride: [23.7728125, 5, 0],
            tokens: [50257n, 50362n, 2329n, 466n, 340n, 0n, 3363n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 1002n, 345n, 821n, 10032n, 286n, 3599n, 625n, 11n, 2245n, 3501n, 510n, 13n, 50256n],
            token_timestamps: [0, 0, 0, 2.44, 4.3, 5.04, 5.06, 5.56, 5.8, 6.32, 7.12, 7.56, 7.8, 8.9, 10.92, 12.96, 13.28, 13.28, 13.44, 13.72, 13.96, 14.84, 15.5, 16.06, 16.86, 17.88, 20.92],
          },
        ];

        const target = [
          " DO IT! Just DO IT! Don't let your dreams be dreams. Yesterday you said tomorrow, so just DO IT! MAKE YOUR dreams! COME TRUE! JUST DO IT! Some people dream success while you're gonna wake up and work hard at it. Nothing is impossible. You should get to the point where anyone else would quit and you're not gonna stop there. No, what are you waiting for? Do it! Just do it! Yes you can! Just do it! If you're tired of starting over, stop giving up.",
          {
            chunks: [
              { text: " DO", timestamp: [0.0, 3.78] },
              { text: " IT!", timestamp: [3.78, 4.24 /* 5.34 */] },
              { text: " Just", timestamp: [5.34, 6.04] },
              { text: " DO", timestamp: [6.04, 6.56] },
              { text: " IT!", timestamp: [6.56, 7.02 /* 7.92 */] },
              { text: " Don't", timestamp: [7.92, 8.58] },
              { text: " let", timestamp: [8.58, 8.88] },
              { text: " your", timestamp: [8.88, 9.14] },
              { text: " dreams", timestamp: [9.14, 9.54] },
              { text: " be", timestamp: [9.54, 9.94] },
              { text: " dreams.", timestamp: [9.94, 10.6 /* 11.38 */] },
              { text: " Yesterday", timestamp: [11.38, 11.88] },
              { text: " you", timestamp: [11.88, 12.42] },
              { text: " said", timestamp: [12.42, 12.62] },
              { text: " tomorrow,", timestamp: [12.62, 13.02 /* 13.36 */] },
              { text: " so", timestamp: [13.36, 13.64] },
              { text: " just", timestamp: [13.64, 14.26] },
              { text: " DO", timestamp: [14.26, 14.76] },
              { text: " IT!", timestamp: [14.76, 15.14 /* 15.4 */] },
              { text: " MAKE", timestamp: [15.4, 15.74] },
              { text: " YOUR", timestamp: [15.74, 16.12] },
              { text: " dreams!", timestamp: [16.12, 16.68 /* 17.14 */] },
              { text: " COME", timestamp: [17.14, 17.24] },
              { text: " TRUE!", timestamp: [17.24, 17.74 /* 18.38 */] },
              { text: " JUST", timestamp: [18.38, 18.6] },
              { text: " DO", timestamp: [18.6, 19.38] },
              { text: " IT!", timestamp: [19.38, 19.94 /* 22.66 */] },
              { text: " Some", timestamp: [22.66, 22.9] },
              { text: " people", timestamp: [22.9, 23.24] },
              { text: " dream", timestamp: [23.24, 23.5] },
              { text: " success", timestamp: [23.5, 24.14] },
              { text: " while", timestamp: [24.14, 24.56] },
              { text: " you're", timestamp: [24.56, 24.72] },
              { text: " gonna", timestamp: [24.72, 24.94] },
              { text: " wake", timestamp: [24.94, 25.18] },
              { text: " up", timestamp: [25.18, 25.54] },
              { text: " and", timestamp: [25.54, 25.72] },
              { text: " work", timestamp: [25.72, 26.04] },
              { text: " hard", timestamp: [26.04, 26.34] },
              { text: " at", timestamp: [26.34, 26.46] },
              { text: " it.", timestamp: [26.46, 26.86 /* 27.04 */] },
              { text: " Nothing", timestamp: [27.04, 27.16] },
              { text: " is", timestamp: [27.16, 27.54] },
              { text: " impossible.", timestamp: [27.54, 28.14 /* 30.16 */] },
              { text: " You", timestamp: [30.16, 30.7] },
              { text: " should", timestamp: [30.7, 30.9] },
              { text: " get", timestamp: [30.9, 31.12] },
              { text: " to", timestamp: [31.12, 31.24] },
              { text: " the", timestamp: [31.24, 31.48] },
              { text: " point", timestamp: [31.48, 31.84] },
              { text: " where", timestamp: [31.84, 32.44] },
              { text: " anyone", timestamp: [32.44, 32.82] },
              { text: " else", timestamp: [32.82, 33.2] },
              { text: " would", timestamp: [33.2, 33.46] },
              { text: " quit", timestamp: [33.46, 33.72] },
              { text: " and", timestamp: [33.72, 34.06] },
              { text: " you're", timestamp: [34.06, 34.34] },
              { text: " not", timestamp: [34.34, 34.56] },
              { text: " gonna", timestamp: [34.56, 34.8] },
              { text: " stop", timestamp: [34.8, 35.16] },
              { text: " there.", timestamp: [35.16, 35.92 /* 36.42 */] },
              { text: " No,", timestamp: [36.42, 36.84 /* 36.86 */] },
              { text: " what", timestamp: [36.86, 37.02] },
              { text: " are", timestamp: [37.02, 37.1] },
              { text: " you", timestamp: [37.1, 37.22] },
              { text: " waiting", timestamp: [37.22, 37.56] },
              { text: " for?", timestamp: [37.56, 38.08 /* 39.28 */] },
              { text: " Do", timestamp: [39.28, 39.62] },
              { text: " it!", timestamp: [39.62, 40.28 /* 41.96 */] },
              { text: " Just", timestamp: [41.96, 42.64] },
              { text: " do", timestamp: [42.64, 44.28] },
              { text: " it!", timestamp: [44.28, 44.78 /* 45.18 */] },
              { text: " Yes", timestamp: [45.18, 45.56] },
              { text: " you", timestamp: [45.56, 45.78] },
              { text: " can!", timestamp: [45.8, 46.34 /* 47.12 */] },
              { text: " Just", timestamp: [47.12, 47.56] },
              { text: " do", timestamp: [47.56, 47.8] },
              { text: " it!", timestamp: [47.8, 48.92 /* 50.92 */] },
              { text: " If", timestamp: [50.92, 52.96] },
              { text: " you're", timestamp: [52.96, 53.28] },
              { text: " tired", timestamp: [53.28, 53.44] },
              { text: " of", timestamp: [53.44, 53.72] },
              { text: " starting", timestamp: [53.72, 53.96] },
              { text: " over,", timestamp: [53.96, 54.86 /* 55.5 */] },
              { text: " stop", timestamp: [55.5, 56.06] },
              { text: " giving", timestamp: [56.06, 56.86] },
              { text: " up.", timestamp: [56.86, 57.9 /* 60.92 */] },
            ],
          },
        ];

        compare(
          tokenizer._decode_asr(model_outputs, {
            return_timestamps: "word",
            time_precision: 0.02,
            force_full_sequences: false,
          }),
          target,
          1e-2,
        );
      },
      MAX_EXECUTION_TIME,
    );

    it(
      "should handle overlapping edge case",
      async () => {
        const tokenizer = await WhisperTokenizer.from_pretrained("onnx-community/whisper-tiny.en_timestamped");

        const model_outputs = [
          {
            stride: [30, 0, 5],
            tokens: [50257n, 50362n, 8410n, 7283n, 0n, 2329n, 8410n, 7283n, 0n, 2094n, 470n, 1309n, 534n, 10625n, 307n, 10625n, 13n, 34668n, 11n, 345n, 531n, 9439n, 11n, 523n, 655n, 8410n, 7283n, 0n, 39134n, 16592n, 10560n, 3955n, 50n, 0n, 7102n, 5446n, 46n, 0n, 25848n, 8410n, 7283n, 0n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 13n, 50256n],
            token_timestamps: [0, 0, 0, 3.78, 4.22, 5.26, 6.04, 6.54, 7, 7.94, 8.58, 8.58, 8.88, 9.16, 9.54, 9.94, 10.6, 11.38, 11.88, 12.38, 12.44, 12.62, 13, 13.36, 13.64, 14.24, 14.74, 15.12, 15.4, 15.74, 16.1, 16.54, 16.54, 16.78, 17.08, 17.2, 17.36, 17.56, 18.08, 18.58, 19.38, 19.88, 22.54, 22.9, 23.24, 23.5, 24.14, 24.56, 24.7, 24.94, 24.94, 25.18, 25.54, 25.72, 26.04, 26.34, 26.46, 26.84, 27.04, 27.14, 27.54, 28.06, 29.92],
          },
          {
            stride: [30, 5, 5],
            tokens: [50257n, 50362n, 2773n, 661n, 4320n, 1943n, 981n, 345n, 821n, 8066n, 7765n, 510n, 290n, 670n, 1327n, 379n, 340n, 13n, 10528n, 318n, 5340n, 13n, 921n, 815n, 651n, 284n, 262n, 966n, 810n, 2687n, 2073n, 561n, 11238n, 290n, 345n, 821n, 407n, 8066n, 2245n, 612n, 13n, 1400n, 11n, 644n, 389n, 345n, 4953n, 329n, 30n, 2141n, 340n, 0n, 2329n, 466n, 340n, 0n, 3363n, 11n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 50256n],
            token_timestamps: [0, 0, 0, 2.92, 3.24, 3.5, 4.14, 4.56, 4.7, 4.74, 4.92, 5.18, 5.54, 5.74, 6.04, 6.34, 6.46, 6.84, 7.04, 7.18, 7.56, 8.12, 9.68, 10.7, 10.88, 11.1, 11.24, 11.48, 11.82, 12.46, 12.82, 13.2, 13.46, 13.72, 14.08, 14.28, 14.34, 14.56, 14.82, 15.16, 15.72, 16.42, 16.82, 16.86, 17, 17.1, 17.2, 17.56, 18.06, 19.28, 19.6, 20.28, 21.96, 22.64, 24.28, 24.76, 25.18, 25.56, 25.56, 25.84, 26.36, 27.12, 27.54, 27.82, 28.16, 29.48],
          },
          {
            stride: [23.7728125, 5, 0],
            tokens: [50257n, 50362n, 2329n, 466n, 340n, 0n, 3363n, 345n, 460n, 0n, 2329n, 466n, 340n, 0n, 1002n, 534n, 15867n, 318n, 3599n, 625n, 11n, 2245n, 3501n, 510n, 13n, 50256n],
            token_timestamps: [0, 0, 0, 2.44, 4.3, 5.04, 5.06, 5.56, 5.8, 6.32, 7.12, 7.56, 7.8, 8.72, 10.04, 12.96, 13.3, 13.44, 13.72, 13.98, 14.86, 15.5, 16, 16.88, 17.76, 20.9],
          },
        ];

        const target = [
          " DO IT! Just DO IT! Don't let your dreams be dreams. Yesterday, you said tomorrow, so just DO IT! MAKE YOUR DRIMS! CONTRO! JUST DO IT! Some people dream success while you're gonna wake up and work hard at it. Nothing is impossible. You should get to the point where anyone else would quit and you're not gonna stop there. No, what are you waiting for? Do it! Just do it! Yes, you can! Just do it! If your tire is starting over, stop giving up.",
          {
            chunks: [
              { text: " DO", timestamp: [0, 3.78] },
              { text: " IT!", timestamp: [3.78, 4.24] },
              { text: " Just", timestamp: [5.26, 6.04] },
              { text: " DO", timestamp: [6.04, 6.54] },
              { text: " IT!", timestamp: [6.54, 7.02] },
              { text: " Don't", timestamp: [7.94, 8.58] },
              { text: " let", timestamp: [8.58, 8.88] },
              { text: " your", timestamp: [8.88, 9.16] },
              { text: " dreams", timestamp: [9.16, 9.54] },
              { text: " be", timestamp: [9.54, 9.94] },
              { text: " dreams.", timestamp: [9.94, 10.62] },
              { text: " Yesterday,", timestamp: [11.38, 11.9] },
              { text: " you", timestamp: [12.38, 12.44] },
              { text: " said", timestamp: [12.44, 12.62] },
              { text: " tomorrow,", timestamp: [12.62, 13.02] },
              { text: " so", timestamp: [13.36, 13.64] },
              { text: " just", timestamp: [13.64, 14.24] },
              { text: " DO", timestamp: [14.24, 14.74] },
              { text: " IT!", timestamp: [14.74, 15.14] },
              { text: " MAKE", timestamp: [15.4, 15.74] },
              { text: " YOUR", timestamp: [15.74, 16.1] },
              { text: " DRIMS!", timestamp: [16.1, 16.8] },
              { text: " CONTRO!", timestamp: [17.08, 17.58] },
              { text: " JUST", timestamp: [18.08, 18.58] },
              { text: " DO", timestamp: [18.58, 19.38] },
              { text: " IT!", timestamp: [19.38, 19.9] },
              { text: " Some", timestamp: [22.54, 22.9] },
              { text: " people", timestamp: [22.9, 23.24] },
              { text: " dream", timestamp: [23.24, 23.5] },
              { text: " success", timestamp: [23.5, 24.14] },
              { text: " while", timestamp: [24.14, 24.56] },
              { text: " you're", timestamp: [24.56, 24.94] },
              { text: " gonna", timestamp: [24.94, 24.94] },
              { text: " wake", timestamp: [24.94, 25.18] },
              { text: " up", timestamp: [25.18, 25.54] },
              { text: " and", timestamp: [25.54, 25.74] },
              { text: " work", timestamp: [25.74, 26.04] },
              { text: " hard", timestamp: [26.04, 26.34] },
              { text: " at", timestamp: [26.34, 26.46] },
              { text: " it.", timestamp: [26.46, 26.86] },
              { text: " Nothing", timestamp: [27.04, 27.18] },
              { text: " is", timestamp: [27.18, 27.56] },
              { text: " impossible.", timestamp: [27.56, 28.14] },
              { text: " You", timestamp: [29.68, 30.7] },
              { text: " should", timestamp: [30.7, 30.88] },
              { text: " get", timestamp: [30.88, 31.1] },
              { text: " to", timestamp: [31.1, 31.24] },
              { text: " the", timestamp: [31.24, 31.48] },
              { text: " point", timestamp: [31.48, 31.82] },
              { text: " where", timestamp: [31.82, 32.46] },
              { text: " anyone", timestamp: [32.46, 32.82] },
              { text: " else", timestamp: [32.82, 33.2] },
              { text: " would", timestamp: [33.2, 33.46] },
              { text: " quit", timestamp: [33.46, 33.72] },
              { text: " and", timestamp: [33.72, 34.08] },
              { text: " you're", timestamp: [34.08, 34.34] },
              { text: " not", timestamp: [34.34, 34.56] },
              { text: " gonna", timestamp: [34.56, 34.82] },
              { text: " stop", timestamp: [34.82, 35.16] },
              { text: " there.", timestamp: [35.16, 35.74] },
              { text: " No,", timestamp: [36.42, 36.84] },
              { text: " what", timestamp: [36.86, 37] },
              { text: " are", timestamp: [37, 37.1] },
              { text: " you", timestamp: [37.1, 37.2] },
              { text: " waiting", timestamp: [37.2, 37.56] },
              { text: " for?", timestamp: [37.56, 38.08] },
              { text: " Do", timestamp: [39.28, 39.6] },
              { text: " it!", timestamp: [39.6, 40.3] },
              { text: " Just", timestamp: [41.96, 42.64] },
              { text: " do", timestamp: [42.64, 44.28] },
              { text: " it!", timestamp: [44.28, 44.78] },
              { text: " Yes,", timestamp: [45.18, 45.56] },
              { text: " you", timestamp: [45.56, 45.84] },
              { text: " can!", timestamp: [45.8, 46.34] },
              { text: " Just", timestamp: [47.12, 47.56] },
              { text: " do", timestamp: [47.56, 47.8] },
              { text: " it!", timestamp: [47.8, 48.74] },
              { text: " If", timestamp: [50.04, 52.96] },
              { text: " your", timestamp: [52.96, 53.3] },
              { text: " tire", timestamp: [53.3, 53.44] },
              { text: " is", timestamp: [53.44, 53.72] },
              { text: " starting", timestamp: [53.72, 53.98] },
              { text: " over,", timestamp: [53.98, 54.88] },
              { text: " stop", timestamp: [55.5, 56] },
              { text: " giving", timestamp: [56, 56.88] },
              { text: " up.", timestamp: [56.88, 57.78] },
            ],
          },
        ];

        compare(
          tokenizer._decode_asr(model_outputs, {
            return_timestamps: "word",
            time_precision: 0.02,
            force_full_sequences: false,
          }),
          target,
          1e-2,
        );
      },
      MAX_EXECUTION_TIME,
    );
  });
};
