nroggendorff commited on
Commit
fdff0a0
·
verified ·
1 Parent(s): 0737b38

Update train.py

Browse files
Files changed (1) hide show
  1. train.py +4 -1
train.py CHANGED
@@ -176,7 +176,10 @@ def main(push_to_hub=True, is_inst_finetune=False):
176
  dataset = load_data()
177
  if not is_inst_finetune:
178
  training_corpus = get_training_corpus(dataset)
179
- tokenizer = create_tokenizer(training_corpus)
 
 
 
180
  else:
181
  tokenizer = load_tokenizer()
182
  configure_tokenizer(tokenizer)
 
176
  dataset = load_data()
177
  if not is_inst_finetune:
178
  training_corpus = get_training_corpus(dataset)
179
+ if INIT == 0:
180
+ tokenizer = create_tokenizer(training_corpus)
181
+ else:
182
+ tokenizer = load_tokenizer()
183
  else:
184
  tokenizer = load_tokenizer()
185
  configure_tokenizer(tokenizer)