minseong commited on
Commit
076a128
ยท
1 Parent(s): 75c64d9

add tokens

Browse files
Files changed (2) hide show
  1. added_tokens.json +7 -5
  2. tokenizer.json +18 -0
added_tokens.json CHANGED
@@ -1,9 +1,11 @@
1
  {
2
- "์žฌ๋ฏบ๋‚˜์š”": 50140,
3
- "์ฐธ์ง€์•Š๊ธฌ": 50141,
4
- "์ด๋ผ๋Š”๋‘ฅ": 50136,
5
- "๊ป“๋ฑ€": 50135,
6
  "๊ฝ†๊ณ ": 50139,
7
  "๋ฎซ์œผ๋ฉด": 50138,
8
- "๋™๊ตฌ๋ ‡๊ฒŒ": 50137
 
 
 
 
 
 
9
  }
 
1
  {
 
 
 
 
2
  "๊ฝ†๊ณ ": 50139,
3
  "๋ฎซ์œผ๋ฉด": 50138,
4
+ "๊ฐ ํƒ†์•—์Œ": 50142,
5
+ "๋™๊ตฌ๋ ‡๊ฒŒ": 50137,
6
+ "์ด๋ผ๋Š”๋‘ฅ": 50136,
7
+ "์–ผ๋งˆ์ •๋„แ„‡แ†ž๋Š”๋ฐ": 50143,
8
+ "๊ป“๋ฑ€": 50135,
9
+ "์ฐธ์ง€์•Š๊ธฌ": 50141,
10
+ "์žฌ๋ฏบ๋‚˜์š”": 50140
11
  }
tokenizer.json CHANGED
@@ -110,6 +110,24 @@
110
  "lstrip": false,
111
  "rstrip": false,
112
  "normalized": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
113
  }
114
  ],
115
  "normalizer": {
 
110
  "lstrip": false,
111
  "rstrip": false,
112
  "normalized": true
113
+ },
114
+ {
115
+ "id": 50142,
116
+ "special": false,
117
+ "content": "๊ฐ ํƒ†์•—์Œ",
118
+ "single_word": false,
119
+ "lstrip": false,
120
+ "rstrip": false,
121
+ "normalized": true
122
+ },
123
+ {
124
+ "id": 50143,
125
+ "special": false,
126
+ "content": "์–ผ๋งˆ์ •๋„แ„‡แ†ž๋Š”๋ฐ",
127
+ "single_word": false,
128
+ "lstrip": false,
129
+ "rstrip": false,
130
+ "normalized": true
131
  }
132
  ],
133
  "normalizer": {