ec40-64k / added_tokens.json
speedcell4's picture
Upload tokenizer
c48538d verified
raw
history blame
742 Bytes
{
"<af>": 64001,
"<am>": 64002,
"<ar>": 64003,
"<ast>": 64004,
"<be>": 64005,
"<bg>": 64006,
"<bn>": 64007,
"<bs>": 64008,
"<ca>": 64009,
"<cs>": 64010,
"<da>": 64011,
"<de>": 64012,
"<en>": 64013,
"<es>": 64014,
"<fr>": 64015,
"<gu>": 64016,
"<ha>": 64017,
"<he>": 64018,
"<hi>": 64019,
"<is>": 64020,
"<it>": 64021,
"<kab>": 64022,
"<kn>": 64023,
"<lb>": 64024,
"<model>": 64025,
"<mr>": 64026,
"<mt>": 64027,
"<ne>": 64028,
"<nl>": 64029,
"<no>": 64030,
"<oc>": 64031,
"<pl>": 64032,
"<pt>": 64033,
"<ro>": 64034,
"<ru>": 64035,
"<sd>": 64036,
"<so>": 64037,
"<sr>": 64038,
"<sv>": 64039,
"<ti>": 64040,
"<uk>": 64041,
"<ur>": 64042,
"<vocab>": 64043
}