Spaces:
Runtime error
Runtime error
Update train.py
Browse files
train.py
CHANGED
@@ -111,6 +111,8 @@ def configure_tokenizer(tokenizer):
|
|
111 |
tokenizer.add_special_tokens(special_tokens)
|
112 |
|
113 |
tokenizer.pad_token_id = MAX_SEQ_LENGTH - 1
|
|
|
|
|
114 |
|
115 |
if INSTRUCT_FINETUNE_BOOL:
|
116 |
tokenizer.user_token_id = tokenizer.convert_tokens_to_ids("<|user|>")
|
|
|
111 |
tokenizer.add_special_tokens(special_tokens)
|
112 |
|
113 |
tokenizer.pad_token_id = MAX_SEQ_LENGTH - 1
|
114 |
+
tokenizer.bos_token_id = MAX_SEQ_LENGTH - 2
|
115 |
+
tokenizer.eos_token_id = MAX_SEQ_LENGTH - 3
|
116 |
|
117 |
if INSTRUCT_FINETUNE_BOOL:
|
118 |
tokenizer.user_token_id = tokenizer.convert_tokens_to_ids("<|user|>")
|