MohamedAhmedAE commited on
Commit
3d61427
·
verified ·
1 Parent(s): 788e29e

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -2064,7 +2064,7 @@
2064
  "pad_token": "<|eot_id|>",
2065
  "padding_side": "left",
2066
  "stride": 0,
2067
- "tokenizer_class": "PreTrainedTokenizerFast",
2068
  "truncation_side": "right",
2069
  "truncation_strategy": "longest_first"
2070
  }
 
2064
  "pad_token": "<|eot_id|>",
2065
  "padding_side": "left",
2066
  "stride": 0,
2067
+ "tokenizer_class": "PreTrainedTokenizer",
2068
  "truncation_side": "right",
2069
  "truncation_strategy": "longest_first"
2070
  }