scampion commited on
Commit
2735e1b
·
verified ·
1 Parent(s): 0d8f8be
Files changed (1) hide show
  1. tokenizer_config.json +4 -6
tokenizer_config.json CHANGED
@@ -1,5 +1,4 @@
1
  {
2
- "add_prefix_space": true,
3
  "added_tokens_decoder": {
4
  "0": {
5
  "content": "|||IP_ADDRESS|||",
@@ -932,15 +931,14 @@
932
  },
933
  "clean_up_tokenization_spaces": true,
934
  "cls_token": "[CLS]",
935
- "extra_special_tokens": {},
936
  "mask_token": "[MASK]",
937
- "model_input_names": [
938
- "input_ids",
939
- "attention_mask"
940
- ],
941
  "model_max_length": 8192,
942
  "pad_token": "[PAD]",
943
  "sep_token": "[SEP]",
944
  "tokenizer_class": "PreTrainedTokenizerFast",
 
 
 
 
945
  "unk_token": "[UNK]"
946
  }
 
1
  {
 
2
  "added_tokens_decoder": {
3
  "0": {
4
  "content": "|||IP_ADDRESS|||",
 
931
  },
932
  "clean_up_tokenization_spaces": true,
933
  "cls_token": "[CLS]",
 
934
  "mask_token": "[MASK]",
 
 
 
 
935
  "model_max_length": 8192,
936
  "pad_token": "[PAD]",
937
  "sep_token": "[SEP]",
938
  "tokenizer_class": "PreTrainedTokenizerFast",
939
+ "model_input_names": [
940
+ "input_ids",
941
+ "attention_mask"
942
+ ],
943
  "unk_token": "[UNK]"
944
  }