Upload processor

tokenizer_config.json

@@ -125,7 +125,7 @@
     }
   },
   "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "chat_template": "{% set is_splitted = index is defined and length is defined %}\n{% for message in messages %}\n    {% set content = message['content'] + '<|end|>\\n' %}\n    {% if message['role'] != 'assistant' or not is_splitted %}\n        {% set content = '<|' + message['role'] + '|>\\n' + content %}\n    {% endif %}\n    {% if (is_splitted and index == 0 and loop.index0 == 0) or (not is_splitted and loop.index0 == 0) %}\n        {% set content = bos_token + content %}\n    {% endif %}\n    {{- content -}}\n{% endfor %}\n{% if add_generation_prompt %}\n    {{- '<|assistant|>\\n' -}}\n{% endif %}\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "legacy": false,