class_docstr: Constructs an ESM tokenizer.
class_docstr_tok: ["Constructs","an","ESM","tokenizer","."]
code: def __init__(self, vocab_file, **kwargs):
      super().__init__(**kwargs)
      self.all_tokens = load_vocab_file(vocab_file)
      self._id_to_token = {
          ind: tok for ind, tok in enumerate(self.all_tokens)
      }
      self._token_to_id = {
          tok: ind for ind, tok in enumerate(self.all_tokens)
      }
      self.unk_token = "<unk>"
      self.cls_token = "<cls>"
      self.pad_token = "<pad>"
      self.mask_token = "<mask>"
      self.eos_token = "<eos>"
      self.unique_no_split_tokens = self.all_tokens
      self._create_trie(self.unique_no_split_tokens)
  
imports: import os
  from typing import List, Optional, Union
  from ...tokenization_utils import PreTrainedTokenizer
  from ...tokenization_utils_base import AddedToken
  from ...utils import logging
label_desc: Use image node_id 1 to create a new EsmTokenizer object from inherited base classes: PreTrainedTokenizer with example: obj = EsmTokenizer(vocab_file)