nroggendorff commited on
Commit
d94939a
·
verified ·
1 Parent(s): 5123979

Update train.py

Browse files
Files changed (1) hide show
  1. train.py +2 -0
train.py CHANGED
@@ -111,6 +111,8 @@ def configure_tokenizer(tokenizer):
111
  tokenizer.add_special_tokens(special_tokens)
112
 
113
  tokenizer.pad_token_id = MAX_SEQ_LENGTH - 1
 
 
114
 
115
  if INSTRUCT_FINETUNE_BOOL:
116
  tokenizer.user_token_id = tokenizer.convert_tokens_to_ids("<|user|>")
 
111
  tokenizer.add_special_tokens(special_tokens)
112
 
113
  tokenizer.pad_token_id = MAX_SEQ_LENGTH - 1
114
+ tokenizer.bos_token_id = MAX_SEQ_LENGTH - 2
115
+ tokenizer.eos_token_id = MAX_SEQ_LENGTH - 3
116
 
117
  if INSTRUCT_FINETUNE_BOOL:
118
  tokenizer.user_token_id = tokenizer.convert_tokens_to_ids("<|user|>")