Hack90
/

virus_pythia_small

Text Generation

text-generation-inference

Model card Files Files and versions Community

Hack90 commited on Mar 1, 2024

Commit

dda1b6a

·

verified ·

1 Parent(s): a70aa14

Update tokenizer.json

Files changed (1) hide show

tokenizer.json +2 -2

tokenizer.json CHANGED Viewed

@@ -51,9 +51,9 @@
   "model": {
     "type": "BPE",
     "dropout": null,
-    "unk_token": null,
     "continuing_subword_prefix": null,
-    "end_of_word_suffix": null,
     "fuse_unk": false,
     "byte_fallback": false,
     "vocab": {

   "model": {
     "type": "BPE",
     "dropout": null,
+    "unk_token": true,
     "continuing_subword_prefix": null,
+    "end_of_word_suffix": true,
     "fuse_unk": false,
     "byte_fallback": false,
     "vocab": {