Upload tokenizer.json with huggingface_hub
Browse files- tokenizer.json +1 -3
tokenizer.json
CHANGED
|
@@ -991,7 +991,6 @@
|
|
| 991 |
{
|
| 992 |
"type": "Metaspace",
|
| 993 |
"replacement": "▁",
|
| 994 |
-
"add_prefix_space": true,
|
| 995 |
"prepend_scheme": "always"
|
| 996 |
},
|
| 997 |
{
|
|
@@ -1105,8 +1104,7 @@
|
|
| 1105 |
{
|
| 1106 |
"type": "Metaspace",
|
| 1107 |
"replacement": "▁",
|
| 1108 |
-
"
|
| 1109 |
-
"prepend_scheme": "always"
|
| 1110 |
},
|
| 1111 |
{
|
| 1112 |
"type": "Strip",
|
|
|
|
| 991 |
{
|
| 992 |
"type": "Metaspace",
|
| 993 |
"replacement": "▁",
|
|
|
|
| 994 |
"prepend_scheme": "always"
|
| 995 |
},
|
| 996 |
{
|
|
|
|
| 1104 |
{
|
| 1105 |
"type": "Metaspace",
|
| 1106 |
"replacement": "▁",
|
| 1107 |
+
"prepend_scheme": "never"
|
|
|
|
| 1108 |
},
|
| 1109 |
{
|
| 1110 |
"type": "Strip",
|