Hack90 commited on
Commit
4cbb22a
·
verified ·
1 Parent(s): dda1b6a

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -0
tokenizer.json CHANGED
@@ -59,6 +59,7 @@
59
  "vocab": {
60
  "<|endoftext|>": 0,
61
  "<|padding|>": 1,
 
62
  "a": 2,
63
  "c": 3,
64
  "g": 4,
 
59
  "vocab": {
60
  "<|endoftext|>": 0,
61
  "<|padding|>": 1,
62
+ "unk_token":8,
63
  "a": 2,
64
  "c": 3,
65
  "g": 4,