Commit
·
61e5245
1
Parent(s):
af3de27
Upload tokenizer
Browse files- tokenizer.json +130 -2
tokenizer.json
CHANGED
|
@@ -1290,7 +1290,71 @@
|
|
| 1290 |
"Ġhoonii": 1228,
|
| 1291 |
"eiteh": 1229,
|
| 1292 |
"Ġniihen": 1230,
|
| 1293 |
-
"Ġdidn": 1231
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1294 |
},
|
| 1295 |
"merges": [
|
| 1296 |
"e n",
|
|
@@ -2438,7 +2502,71 @@
|
|
| 2438 |
"Ġhoon ii",
|
| 2439 |
"eit eh",
|
| 2440 |
"Ġniih en",
|
| 2441 |
-
"Ġdid n"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2442 |
]
|
| 2443 |
}
|
| 2444 |
}
|
|
|
|
| 1290 |
"Ġhoonii": 1228,
|
| 1291 |
"eiteh": 1229,
|
| 1292 |
"Ġniihen": 1230,
|
| 1293 |
+
"Ġdidn": 1231,
|
| 1294 |
+
"eciih": 1232,
|
| 1295 |
+
"ĠNiine": 1233,
|
| 1296 |
+
"yooniib": 1234,
|
| 1297 |
+
"uxot": 1235,
|
| 1298 |
+
"oninoo": 1236,
|
| 1299 |
+
"owoono": 1237,
|
| 1300 |
+
"um": 1238,
|
| 1301 |
+
"Ġhad": 1239,
|
| 1302 |
+
"Ġwooh": 1240,
|
| 1303 |
+
"iihei": 1241,
|
| 1304 |
+
"Ġhiix": 1242,
|
| 1305 |
+
"iten": 1243,
|
| 1306 |
+
"iibis": 1244,
|
| 1307 |
+
"ĠU": 1245,
|
| 1308 |
+
"ool": 1246,
|
| 1309 |
+
"chool": 1247,
|
| 1310 |
+
"oobe": 1248,
|
| 1311 |
+
"iitei": 1249,
|
| 1312 |
+
"Ġneeneisiini": 1250,
|
| 1313 |
+
"Ġhiiwo": 1251,
|
| 1314 |
+
"eenetiitoonin": 1252,
|
| 1315 |
+
"ĠHiiwoonhehe": 1253,
|
| 1316 |
+
"Ġhiseih": 1254,
|
| 1317 |
+
"ĠJohn": 1255,
|
| 1318 |
+
"if": 1256,
|
| 1319 |
+
"ihb": 1257,
|
| 1320 |
+
"Ġhiniisih": 1258,
|
| 1321 |
+
"ebine": 1259,
|
| 1322 |
+
"Ġhinent": 1260,
|
| 1323 |
+
"el": 1261,
|
| 1324 |
+
"yein": 1262,
|
| 1325 |
+
"Ġhiisiis": 1263,
|
| 1326 |
+
"eestoot": 1264,
|
| 1327 |
+
"xoon": 1265,
|
| 1328 |
+
"owob": 1266,
|
| 1329 |
+
"ake": 1267,
|
| 1330 |
+
"Ġhinii": 1268,
|
| 1331 |
+
"oowP": 1269,
|
| 1332 |
+
"etini": 1270,
|
| 1333 |
+
"niiteheib": 1271,
|
| 1334 |
+
"coono": 1272,
|
| 1335 |
+
"Ġnihce": 1273,
|
| 1336 |
+
"Ġbes": 1274,
|
| 1337 |
+
"Ġheetc": 1275,
|
| 1338 |
+
"oteihiit": 1276,
|
| 1339 |
+
"Ġheetnoo": 1277,
|
| 1340 |
+
"Ġtotoos": 1278,
|
| 1341 |
+
"'m": 1279,
|
| 1342 |
+
"Ġbeb": 1280,
|
| 1343 |
+
"Ġhere": 1281,
|
| 1344 |
+
"Ġhiniito": 1282,
|
| 1345 |
+
"ĠHenee": 1283,
|
| 1346 |
+
"okut": 1284,
|
| 1347 |
+
"ĠHei": 1285,
|
| 1348 |
+
"Ġway": 1286,
|
| 1349 |
+
"Ġhoowuni": 1287,
|
| 1350 |
+
"Ġgood": 1288,
|
| 1351 |
+
"woono": 1289,
|
| 1352 |
+
"Ġyear": 1290,
|
| 1353 |
+
"Ġheniis": 1291,
|
| 1354 |
+
"hh": 1292,
|
| 1355 |
+
"Ġwohoe": 1293,
|
| 1356 |
+
"owoon": 1294,
|
| 1357 |
+
"eiseet": 1295
|
| 1358 |
},
|
| 1359 |
"merges": [
|
| 1360 |
"e n",
|
|
|
|
| 2502 |
"Ġhoon ii",
|
| 2503 |
"eit eh",
|
| 2504 |
"Ġniih en",
|
| 2505 |
+
"Ġdid n",
|
| 2506 |
+
"ec iih",
|
| 2507 |
+
"ĠN iine",
|
| 2508 |
+
"yoon iib",
|
| 2509 |
+
"ux ot",
|
| 2510 |
+
"on inoo",
|
| 2511 |
+
"ow oono",
|
| 2512 |
+
"u m",
|
| 2513 |
+
"Ġh ad",
|
| 2514 |
+
"Ġw ooh",
|
| 2515 |
+
"iih ei",
|
| 2516 |
+
"Ġhii x",
|
| 2517 |
+
"it en",
|
| 2518 |
+
"iib is",
|
| 2519 |
+
"Ġ U",
|
| 2520 |
+
"oo l",
|
| 2521 |
+
"ch ool",
|
| 2522 |
+
"oo be",
|
| 2523 |
+
"iit ei",
|
| 2524 |
+
"Ġneeneis iini",
|
| 2525 |
+
"Ġhii wo",
|
| 2526 |
+
"eenetiit oonin",
|
| 2527 |
+
"ĠHii woonhehe",
|
| 2528 |
+
"Ġhis eih",
|
| 2529 |
+
"ĠJ ohn",
|
| 2530 |
+
"i f",
|
| 2531 |
+
"ih b",
|
| 2532 |
+
"Ġhin iisih",
|
| 2533 |
+
"eb ine",
|
| 2534 |
+
"Ġhinen t",
|
| 2535 |
+
"e l",
|
| 2536 |
+
"y ein",
|
| 2537 |
+
"Ġhiis iis",
|
| 2538 |
+
"eest oot",
|
| 2539 |
+
"x oon",
|
| 2540 |
+
"ow ob",
|
| 2541 |
+
"ak e",
|
| 2542 |
+
"Ġhin ii",
|
| 2543 |
+
"oow P",
|
| 2544 |
+
"et ini",
|
| 2545 |
+
"niit eheib",
|
| 2546 |
+
"c oono",
|
| 2547 |
+
"Ġnih ce",
|
| 2548 |
+
"Ġb es",
|
| 2549 |
+
"Ġheet c",
|
| 2550 |
+
"ot eihiit",
|
| 2551 |
+
"Ġheetn oo",
|
| 2552 |
+
"Ġtot oos",
|
| 2553 |
+
"' m",
|
| 2554 |
+
"Ġb eb",
|
| 2555 |
+
"Ġhe re",
|
| 2556 |
+
"Ġhin iito",
|
| 2557 |
+
"ĠH enee",
|
| 2558 |
+
"ok ut",
|
| 2559 |
+
"ĠH ei",
|
| 2560 |
+
"Ġw ay",
|
| 2561 |
+
"Ġhoow uni",
|
| 2562 |
+
"Ġg ood",
|
| 2563 |
+
"w oono",
|
| 2564 |
+
"Ġy ear",
|
| 2565 |
+
"Ġhen iis",
|
| 2566 |
+
"h h",
|
| 2567 |
+
"Ġw ohoe",
|
| 2568 |
+
"ow oon",
|
| 2569 |
+
"eis eet"
|
| 2570 |
]
|
| 2571 |
}
|
| 2572 |
}
|