Hack90 commited on
Commit
4fbbbc8
·
verified ·
1 Parent(s): 3929aa1

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +4 -3
tokenizer.json CHANGED
@@ -51,7 +51,7 @@
51
  "model": {
52
  "type": "BPE",
53
  "dropout": null,
54
- "unk_token": true,
55
  "continuing_subword_prefix": null,
56
  "end_of_word_suffix": true,
57
  "fuse_unk": false,
@@ -59,13 +59,14 @@
59
  "vocab": {
60
  "<|endoftext|>": 0,
61
  "<|padding|>": 1,
62
- "<|unk_token|>":8,
63
  "a": 2,
64
  "c": 3,
65
  "g": 4,
66
  "t": 5,
67
  "n": 6,
68
- "y":7
 
69
  },
70
  "merges": []
71
  }
 
51
  "model": {
52
  "type": "BPE",
53
  "dropout": null,
54
+ "unk_token": null,
55
  "continuing_subword_prefix": null,
56
  "end_of_word_suffix": true,
57
  "fuse_unk": false,
 
59
  "vocab": {
60
  "<|endoftext|>": 0,
61
  "<|padding|>": 1,
62
+ "q":8,
63
  "a": 2,
64
  "c": 3,
65
  "g": 4,
66
  "t": 5,
67
  "n": 6,
68
+ "y":7,
69
+
70
  },
71
  "merges": []
72
  }