class_docstr: Constructs a Blenderbot-90M tokenizer based on BPE (Byte-Pair-Encoding)
  
  This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
  the superclass for more information regarding methods.
  
  Args:
      vocab_file (`str`):
          File containing the vocabulary.
      merges_file (`str`):
          Path to the merges file.
      bos_token (`str`, *optional*, defaults to `"__start__"`):
          The beginning of sentence token.
      eos_token (`str`, *optional*, defaults to `"__end__"`):
          The end of sentence token.
      unk_token (`str`, *optional*, defaults to `"__unk__"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      pad_token (`str`, *optional*, defaults to `"__null__"`):
          The token used for padding, for example when batching sequences of different lengths.
      kwargs (*optional*):
          Additional keyword arguments passed along to [`PreTrainedTokenizer`]
class_docstr_tok: ["Constructs","a","Blenderbot-90M","tokenizer","based","on","BPE","(","Byte-Pair-Encoding",")","This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","the","superclass","for","more","information","regarding","methods",".","Args",":","vocab_file","(","`","str","`",")",":","File","containing","the","vocabulary",".","merges_file","(","`","str","`",")",":","Path","to","the","merges","file",".","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","__start__","''","`",")",":","The","beginning","of","sentence","token",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","__end__","''","`",")",":","The","end","of","sentence","token",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","__unk__","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","__null__","''","`",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","kwargs","(","*","optional","*",")",":","Additional","keyword","arguments","passed","along","to","[","`","PreTrainedTokenizer","`","]"]
docstr: Converts a token to an id using the vocab.
docstr_tok: ["Converts","a","token","to","an","id","using","the","vocab","."]
code: def _convert_token_to_id(self, token: str) -> int:
      """Converts a token to an id using the vocab."""
      token = token.lower()
      return self.encoder.get(token, self.encoder.get(self.unk_token))
  
file_path: transformers/src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py
imports: import json
  import os
  from typing import Dict, List, Optional, Tuple
  import regex
  from ...tokenization_utils import PreTrainedTokenizer
  from ...utils import logging
label_desc: Use image node_id 6 for calling the BlenderbotSmallTokenizer obj's underlying member method code with example usage: obj._convert_token_to_id(token) and returns: self