Goekdeniz-Guelmez commited on
Commit
22904b4
·
verified ·
1 Parent(s): 9347a4b

Upload 2 files

Browse files
Files changed (2) hide show
  1. generation_config.json +10 -0
  2. tokenizer_config.json +1 -1
generation_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 0,
4
+ "eos_token_id": [
5
+ 10,
6
+ 11
7
+ ],
8
+ "pad_token_id": 0,
9
+ "transformers_version": "4.47.0"
10
+ }
tokenizer_config.json CHANGED
@@ -114,7 +114,7 @@
114
  "bos_token": "<|begin_of_text|>",
115
  "chat_template": "{{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
116
  "clean_up_tokenization_spaces": true,
117
- "eos_token": "<|im_end|>",
118
  "extra_special_tokens": {},
119
  "max_length": null,
120
  "model_input_names": [
 
114
  "bos_token": "<|begin_of_text|>",
115
  "chat_template": "{{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
116
  "clean_up_tokenization_spaces": true,
117
+ "eos_token": "<|end_of_text|>",
118
  "extra_special_tokens": {},
119
  "max_length": null,
120
  "model_input_names": [