hyojin99 commited on
Commit
a198bab
Β·
verified Β·
1 Parent(s): e8d72f2

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +70 -70
vocab.json CHANGED
@@ -1,75 +1,75 @@
1
  {
2
- "0": 67,
3
- "1": 63,
4
- "2": 35,
5
- "3": 46,
6
- "4": 23,
7
- "8": 53,
8
  "[PAD]": 72,
9
  "[UNK]": 71,
10
- "a": 18,
11
- "c": 48,
12
- "d": 68,
13
- "e": 29,
14
- "f": 39,
15
- "i": 58,
16
- "j": 61,
17
- "m": 21,
18
- "n": 12,
19
- "o": 42,
20
- "p": 6,
21
- "r": 31,
22
- "s": 20,
23
  "t": 47,
24
- "u": 28,
25
- "v": 60,
26
- "|": 44,
27
- "γ„±": 16,
28
- "γ„²": 8,
29
- "γ„΄": 52,
30
- "γ„΅": 24,
31
- "γ„Ά": 41,
32
- "γ„·": 14,
33
- "γ„Έ": 70,
34
- "γ„Ή": 57,
35
- "γ„Ί": 11,
36
- "γ„»": 2,
37
- "γ„Ό": 3,
38
- "γ„Ύ": 26,
39
- "γ…€": 4,
40
- "ㅁ": 1,
41
- "γ…‚": 27,
42
- "γ…ƒ": 54,
43
- "γ…„": 25,
44
- "γ……": 36,
45
- "γ…†": 17,
46
- "γ…‡": 9,
47
- "γ…ˆ": 30,
48
- "γ…‰": 66,
49
- "γ…Š": 33,
50
- "γ…‹": 37,
51
- "γ…Œ": 22,
52
- "ㅍ": 19,
53
- "γ…Ž": 49,
54
- "ㅏ": 10,
55
- "ㅐ": 32,
56
- "γ…‘": 56,
57
- "γ…’": 69,
58
- "γ…“": 43,
59
- "γ…”": 62,
60
- "γ…•": 55,
61
- "γ…–": 64,
62
- "γ…—": 59,
63
- "γ…˜": 13,
64
- "γ…™": 51,
65
- "γ…š": 38,
66
- "γ…›": 15,
67
- "γ…œ": 40,
68
- "ㅝ": 50,
69
- "γ…ž": 65,
70
- "γ…Ÿ": 5,
71
- "γ… ": 34,
72
- "γ…‘": 45,
73
- "γ…’": 0,
74
- "γ…£": 7
75
  }
 
1
  {
2
+ "0": 48,
3
+ "1": 49,
4
+ "2": 55,
5
+ "3": 15,
6
+ "4": 35,
7
+ "8": 66,
8
  "[PAD]": 72,
9
  "[UNK]": 71,
10
+ "a": 24,
11
+ "c": 67,
12
+ "d": 26,
13
+ "e": 20,
14
+ "f": 41,
15
+ "i": 19,
16
+ "j": 62,
17
+ "m": 0,
18
+ "n": 6,
19
+ "o": 56,
20
+ "p": 23,
21
+ "r": 5,
22
+ "s": 40,
23
  "t": 47,
24
+ "u": 39,
25
+ "v": 44,
26
+ "|": 21,
27
+ "γ„±": 11,
28
+ "γ„²": 43,
29
+ "γ„΄": 8,
30
+ "γ„΅": 45,
31
+ "γ„Ά": 38,
32
+ "γ„·": 63,
33
+ "γ„Έ": 65,
34
+ "γ„Ή": 29,
35
+ "γ„Ί": 69,
36
+ "γ„»": 25,
37
+ "γ„Ό": 70,
38
+ "γ„Ύ": 52,
39
+ "γ…€": 59,
40
+ "ㅁ": 16,
41
+ "γ…‚": 4,
42
+ "γ…ƒ": 61,
43
+ "γ…„": 36,
44
+ "γ……": 10,
45
+ "γ…†": 7,
46
+ "γ…‡": 64,
47
+ "γ…ˆ": 68,
48
+ "γ…‰": 2,
49
+ "γ…Š": 34,
50
+ "γ…‹": 3,
51
+ "γ…Œ": 37,
52
+ "ㅍ": 30,
53
+ "γ…Ž": 12,
54
+ "ㅏ": 13,
55
+ "ㅐ": 42,
56
+ "γ…‘": 22,
57
+ "γ…’": 54,
58
+ "γ…“": 17,
59
+ "γ…”": 57,
60
+ "γ…•": 1,
61
+ "γ…–": 50,
62
+ "γ…—": 51,
63
+ "γ…˜": 14,
64
+ "γ…™": 46,
65
+ "γ…š": 27,
66
+ "γ…›": 32,
67
+ "γ…œ": 31,
68
+ "ㅝ": 9,
69
+ "γ…ž": 18,
70
+ "γ…Ÿ": 28,
71
+ "γ… ": 58,
72
+ "γ…‘": 33,
73
+ "γ…’": 53,
74
+ "γ…£": 60
75
  }