Upload tokenizer

tokenizer.py

@@ -24,7 +24,6 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
         unk_token_id: int,
         model_max_length: int,
         other_special_token_ids: Optional[List[int]] = None,
-        ambiguous_token_ids: Optional[List[int]] = None,  # str = "XBOUZJ"
         **kwargs,
     ):
         """Vocabulary comprising the amino acids, and the special tokens <unk>, <bos>, <eos>, <pad> and <mask>.
@@ -69,7 +68,6 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
             unk_token_id=unk_token_id,
             unk_token=id_to_token.get(unk_token_id),
             other_special_token_ids=other_special_token_ids,
-            ambiguous_token_ids=ambiguous_token_ids,
             model_input_names=["input_ids", "attention_mask", "special_tokens_mask"],
             tokenizer_object=tokenizer_object,
         )
@@ -77,8 +75,6 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
         if other_special_token_ids is not None:
             self.add_special_tokens({"additional_special_tokens": list(id_to_token.get(i) for i in other_special_token_ids)})
 
-        self.ambiguous_token_ids = ambiguous_token_ids
-
         self.key_to_padding = {"input_ids": self.pad_token_id, "attention_mask": 0, "special_tokens_mask": 1, "position_ids": 0}
         self.key_to_dtype = {
             "input_ids": torch.long,
@@ -130,7 +126,7 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
         """
 
         for i, sequence in enumerate(encoded_inputs["input_ids"]):
-            mask = [token not in self.ambiguous_token_ids for token in sequence]
+            mask = [token_id != self.unk_token_id for token_id in sequence]
             for key in encoded_inputs:
                 encoded_inputs[key][i] = list(compress(encoded_inputs[key][i], mask))
         return encoded_inputs
@@ -143,15 +139,6 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
         pad_to_multiple_of: int = 8,
         **kwargs,
     ) -> Dict[str, List[List[int]]]:
-        """
-        Remove ambiguous amino acids from the input sequences.
-
-        Args:
-            encoded_inputs (Dict[str, List[List[int]]): Tokenized inputs with keys like 'input_ids' as tensors.
-
-        Returns:
-            Dict[str, List[List[int]]]: Tokenized inputs without ambiguous amino acids.
-        """
 
         if isinstance(encoded_inputs, list):
             tmp = dict()
@@ -185,15 +172,6 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
         return_tensors: str = "pt",
         **kwargs,
     ) -> Dict[str, List[List[int]]]:
-        """
-        Remove ambiguous amino acids from the input sequences.
-
-        Args:
-            encoded_inputs (Dict[str, List[List[int]]): Tokenized inputs with keys like 'input_ids' as tensors.
-
-        Returns:
-            Dict[str, List[List[int]]]: Tokenized inputs without ambiguous amino acids.
-        """
 
         encoded_inputs = self._pad(
             encoded_inputs,

tokenizer_config.json

@@ -41,15 +41,6 @@
       "special": true
     }
   },
-  "ambiguous_token_ids": [
-    1,
-    6,
-    7,
-    8,
-    9,
-    10,
-    11
-  ],
   "auto_map": {
     "AutoTokenizer": [
       "tokenizer.ProteinTokenizer",