Upload tokenizer
Browse files- tokenizer.py +0 -1
- tokenizer_config.json +1 -2
tokenizer.py
CHANGED
@@ -73,7 +73,6 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
|
|
73 |
ambiguous_token_ids=ambiguous_token_ids,
|
74 |
model_input_names=["input_ids", "attention_mask", "special_tokens_mask"],
|
75 |
tokenizer_object=tokenizer_object,
|
76 |
-
**kwargs,
|
77 |
)
|
78 |
|
79 |
if other_special_token_ids is not None:
|
|
|
73 |
ambiguous_token_ids=ambiguous_token_ids,
|
74 |
model_input_names=["input_ids", "attention_mask", "special_tokens_mask"],
|
75 |
tokenizer_object=tokenizer_object,
|
|
|
76 |
)
|
77 |
|
78 |
if other_special_token_ids is not None:
|
tokenizer_config.json
CHANGED
@@ -69,6 +69,5 @@
|
|
69 |
"truncation_side": "right",
|
70 |
"unk_token": "<unk>",
|
71 |
"unk_token_id": 1,
|
72 |
-
"vocab_path": "/home/mila/l/lola.lebreton/AMPLIFY/conf/tokenizer/amplify_vocab.txt"
|
73 |
-
"vocab_size": 27
|
74 |
}
|
|
|
69 |
"truncation_side": "right",
|
70 |
"unk_token": "<unk>",
|
71 |
"unk_token_id": 1,
|
72 |
+
"vocab_path": "/home/mila/l/lola.lebreton/AMPLIFY/conf/tokenizer/amplify_vocab.txt"
|
|
|
73 |
}
|