Spaces:
Runtime error
Runtime error
Update train.py
Browse files
train.py
CHANGED
@@ -176,7 +176,10 @@ def main(push_to_hub=True, is_inst_finetune=False):
|
|
176 |
dataset = load_data()
|
177 |
if not is_inst_finetune:
|
178 |
training_corpus = get_training_corpus(dataset)
|
179 |
-
|
|
|
|
|
|
|
180 |
else:
|
181 |
tokenizer = load_tokenizer()
|
182 |
configure_tokenizer(tokenizer)
|
|
|
176 |
dataset = load_data()
|
177 |
if not is_inst_finetune:
|
178 |
training_corpus = get_training_corpus(dataset)
|
179 |
+
if INIT == 0:
|
180 |
+
tokenizer = create_tokenizer(training_corpus)
|
181 |
+
else:
|
182 |
+
tokenizer = load_tokenizer()
|
183 |
else:
|
184 |
tokenizer = load_tokenizer()
|
185 |
configure_tokenizer(tokenizer)
|