Spaces:
Runtime error
Runtime error
Update train.py
Browse files
train.py
CHANGED
@@ -214,9 +214,9 @@ def main(push_to_hub=True, is_inst_finetune=False):
|
|
214 |
tokenizer = load_tokenizer()
|
215 |
print("Loaded Tokenizer.")
|
216 |
|
217 |
-
print("Adding Tokens..")
|
218 |
-
num_new_tokens = update_tokenizer(tokenizer, dataset)
|
219 |
-
print(f"Added {num_new_tokens} new tokens to the vocabulary")
|
220 |
|
221 |
|
222 |
if INIT == 0:
|
|
|
214 |
tokenizer = load_tokenizer()
|
215 |
print("Loaded Tokenizer.")
|
216 |
|
217 |
+
# print("Adding Tokens..")
|
218 |
+
# num_new_tokens = update_tokenizer(tokenizer, dataset)
|
219 |
+
# print(f"Added {num_new_tokens} new tokens to the vocabulary")
|
220 |
|
221 |
|
222 |
if INIT == 0:
|