add tokens
Browse files- added_tokens.json +7 -5
- tokenizer.json +18 -0
added_tokens.json
CHANGED
@@ -1,9 +1,11 @@
|
|
1 |
{
|
2 |
-
"์ฌ๋ฏบ๋์": 50140,
|
3 |
-
"์ฐธ์ง์๊ธฌ": 50141,
|
4 |
-
"์ด๋ผ๋๋ฅ": 50136,
|
5 |
-
"๊ป๋ฑ": 50135,
|
6 |
"๊ฝ๊ณ ": 50139,
|
7 |
"๋ฎซ์ผ๋ฉด": 50138,
|
8 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
9 |
}
|
|
|
1 |
{
|
|
|
|
|
|
|
|
|
2 |
"๊ฝ๊ณ ": 50139,
|
3 |
"๋ฎซ์ผ๋ฉด": 50138,
|
4 |
+
"๊ฐ ํ์์": 50142,
|
5 |
+
"๋๊ตฌ๋ ๊ฒ": 50137,
|
6 |
+
"์ด๋ผ๋๋ฅ": 50136,
|
7 |
+
"์ผ๋ง์ ๋แแ๋๋ฐ": 50143,
|
8 |
+
"๊ป๋ฑ": 50135,
|
9 |
+
"์ฐธ์ง์๊ธฌ": 50141,
|
10 |
+
"์ฌ๋ฏบ๋์": 50140
|
11 |
}
|
tokenizer.json
CHANGED
@@ -110,6 +110,24 @@
|
|
110 |
"lstrip": false,
|
111 |
"rstrip": false,
|
112 |
"normalized": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
113 |
}
|
114 |
],
|
115 |
"normalizer": {
|
|
|
110 |
"lstrip": false,
|
111 |
"rstrip": false,
|
112 |
"normalized": true
|
113 |
+
},
|
114 |
+
{
|
115 |
+
"id": 50142,
|
116 |
+
"special": false,
|
117 |
+
"content": "๊ฐ ํ์์",
|
118 |
+
"single_word": false,
|
119 |
+
"lstrip": false,
|
120 |
+
"rstrip": false,
|
121 |
+
"normalized": true
|
122 |
+
},
|
123 |
+
{
|
124 |
+
"id": 50143,
|
125 |
+
"special": false,
|
126 |
+
"content": "์ผ๋ง์ ๋แแ๋๋ฐ",
|
127 |
+
"single_word": false,
|
128 |
+
"lstrip": false,
|
129 |
+
"rstrip": false,
|
130 |
+
"normalized": true
|
131 |
}
|
132 |
],
|
133 |
"normalizer": {
|