{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [], "normalizer": { "type": "Replace", "pattern": { "Regex": "[^$;:,.!?\u2014\u2026\"()\u201c\u201d \u0303\u02a3\u02a5\u02a6\u02a8\u1d5d\uab67AIOQSTWY\u1d4aabcdefhijklmnopqrstuvwxyz\u0251\u0250\u0252\u00e6\u03b2\u0254\u0255\u00e7\u0256\u00f0\u02a4\u0259\u025a\u025b\u025c\u025f\u0261\u0265\u0268\u026a\u029d\u026f\u0270\u014b\u0273\u0272\u0274\u00f8\u0278\u03b8\u0153\u0279\u027e\u027b\u0281\u027d\u0282\u0283\u0288\u02a7\u028a\u028b\u028c\u0263\u0264\u03c7\u028e\u0292\u0294\u02c8\u02cc\u02d0\u02b0\u02b2\u2193\u2192\u2197\u2198\u1d7b]" }, "content": "" }, "pre_tokenizer": { "type": "Split", "pattern": { "Regex": "" }, "behavior": "Isolated", "invert": false }, "post_processor": { "type": "TemplateProcessing", "single": [ { "SpecialToken": { "id": "$", "type_id": 0 } }, { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "$", "type_id": 0 } } ], "special_tokens": { "$": { "id": "$", "ids": [ 0 ], "tokens": [ "$" ] } } }, "decoder": null, "model": { "vocab": { "$": 0, ";": 1, ":": 2, ",": 3, ".": 4, "!": 5, "?": 6, "\u2014": 9, "\u2026": 10, "\"": 11, "(": 12, ")": 13, "\u201c": 14, "\u201d": 15, " ": 16, "\u0303": 17, "\u02a3": 18, "\u02a5": 19, "\u02a6": 20, "\u02a8": 21, "\u1d5d": 22, "\uab67": 23, "A": 24, "I": 25, "O": 31, "Q": 33, "S": 35, "T": 36, "W": 39, "Y": 41, "\u1d4a": 42, "a": 43, "b": 44, "c": 45, "d": 46, "e": 47, "f": 48, "h": 50, "i": 51, "j": 52, "k": 53, "l": 54, "m": 55, "n": 56, "o": 57, "p": 58, "q": 59, "r": 60, "s": 61, "t": 62, "u": 63, "v": 64, "w": 65, "x": 66, "y": 67, "z": 68, "\u0251": 69, "\u0250": 70, "\u0252": 71, "\u00e6": 72, "\u03b2": 75, "\u0254": 76, "\u0255": 77, "\u00e7": 78, "\u0256": 80, "\u00f0": 81, "\u02a4": 82, "\u0259": 83, "\u025a": 85, "\u025b": 86, "\u025c": 87, "\u025f": 90, "\u0261": 92, "\u0265": 99, "\u0268": 101, "\u026a": 102, "\u029d": 103, "\u026f": 110, "\u0270": 111, "\u014b": 112, "\u0273": 113, "\u0272": 114, "\u0274": 115, "\u00f8": 116, "\u0278": 118, "\u03b8": 119, "\u0153": 120, "\u0279": 123, "\u027e": 125, "\u027b": 126, "\u0281": 128, "\u027d": 129, "\u0282": 130, "\u0283": 131, "\u0288": 132, "\u02a7": 133, "\u028a": 135, "\u028b": 136, "\u028c": 138, "\u0263": 139, "\u0264": 140, "\u03c7": 142, "\u028e": 143, "\u0292": 147, "\u0294": 148, "\u02c8": 156, "\u02cc": 157, "\u02d0": 158, "\u02b0": 162, "\u02b2": 164, "\u2193": 169, "\u2192": 171, "\u2197": 172, "\u2198": 173, "\u1d7b": 177 } } }