Upload tokenizer

tokenizer.py

@@ -1,6 +1,7 @@
 import torch
 from typing import List, Optional, Union, Dict
 from torch import Tensor
+import copy
 
 from itertools import compress
 
@@ -21,7 +22,7 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
         bos_token_id: int,
         eos_token_id: int,
         unk_token_id: int,
-        max_length: int,
+        model_max_length: int,
         other_special_token_ids: Optional[List[int]] = None,
         ambiguous_token_ids: Optional[List[int]] = None,  # str = "XBOUZJ"
         **kwargs,
@@ -54,16 +55,25 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
         tokenizer_object.pre_tokenizer = Split("", behavior="removed")
 
         super().__init__(
-            model_max_length=max_length,
+            vocab_path=vocab_path,
+            model_max_length=model_max_length,
             padding_side="right",
             truncation_side="right",
+            pad_token_id=pad_token_id,
             pad_token=id_to_token.get(pad_token_id),
+            mask_token_id=mask_token_id,
+            mask_token=id_to_token.get(mask_token_id),
+            bos_token_id=bos_token_id,
             bos_token=id_to_token.get(bos_token_id),
+            eos_token_id=eos_token_id,
             eos_token=id_to_token.get(eos_token_id),
+            unk_token_id=unk_token_id,
             unk_token=id_to_token.get(unk_token_id),
-            mask_token=id_to_token.get(mask_token_id),
+            other_special_token_ids=other_special_token_ids,
+            ambiguous_token_ids=ambiguous_token_ids,
             model_input_names=["input_ids", "attention_mask", "special_tokens_mask"],
             tokenizer_object=tokenizer_object,
+            **kwargs,
         )
 
         if other_special_token_ids is not None:

tokenizer_config.json

@@ -41,6 +41,7 @@
       "special": true
     }
   },
+  "ambiguous_token_ids": null,
   "auto_map": {
     "AutoTokenizer": [
       "tokenizer.ProteinTokenizer",
@@ -48,18 +49,26 @@
     ]
   },
   "bos_token": "<bos>",
+  "bos_token_id": 3,
   "clean_up_tokenization_spaces": false,
   "eos_token": "<eos>",
+  "eos_token_id": 4,
   "mask_token": "<mask>",
+  "mask_token_id": 2,
   "model_input_names": [
     "input_ids",
     "attention_mask",
     "special_tokens_mask"
   ],
   "model_max_length": 2048,
+  "other_special_token_ids": null,
   "pad_token": "<pad>",
+  "pad_token_id": 0,
   "padding_side": "right",
   "tokenizer_class": "ProteinTokenizer",
   "truncation_side": "right",
-  "unk_token": "<unk>"
+  "unk_token": "<unk>",
+  "unk_token_id": 1,
+  "vocab_path": "/home/mila/l/lola.lebreton/AMPLIFY/conf/tokenizer/amplify_vocab.txt",
+  "vocab_size": 27
 }