Upload tokenizer

tokenizer.py

@@ -16,7 +16,7 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
 
     def __init__(
         self,
-        vocab_path: str,
+        vocab: dict,
         pad_token_id: int,
         mask_token_id: int,
         bos_token_id: int,
@@ -42,11 +42,10 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
         token_to_id = dict()
         id_to_token = dict()
 
-        with open(vocab_path, "r") as vocab_file:
-            for i, token in enumerate(vocab_file):
-                token = token.strip()
-                token_to_id[token] = i
-                id_to_token[i] = token
+        for token, token_id in vocab.items():
+            token = token.strip()
+            token_to_id[token] = token_id
+            id_to_token[token_id] = token
 
         # Define tokenizer and model
         tokenizer_object = Tokenizer(WordPiece(vocab=token_to_id, unk_token=id_to_token.get(unk_token_id)))
@@ -55,7 +54,7 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
         tokenizer_object.pre_tokenizer = Split("", behavior="removed")
 
         super().__init__(
-            vocab_path=vocab_path,
+            vocab=vocab,
             model_max_length=model_max_length,
             padding_side="right",
             truncation_side="right",

tokenizer_config.json

@@ -77,5 +77,33 @@
   "truncation_side": "right",
   "unk_token": "<unk>",
   "unk_token_id": 1,
-  "vocab_path": "conf/tokenizer/amplify_vocab.txt"
+  "vocab": {
+    "<bos>": 3,
+    "<eos>": 4,
+    "<mask>": 2,
+    "<pad>": 0,
+    "<unk>": 1,
+    "A": 7,
+    "B": 26,
+    "C": 25,
+    "D": 15,
+    "E": 11,
+    "F": 20,
+    "G": 8,
+    "H": 23,
+    "I": 14,
+    "K": 17,
+    "L": 6,
+    "M": 22,
+    "N": 19,
+    "P": 16,
+    "Q": 18,
+    "R": 12,
+    "S": 10,
+    "T": 13,
+    "V": 9,
+    "W": 24,
+    "Y": 21,
+    "|": 5
+  }
 }