class_docstr: Allow to use the Torch Agent with the wordpiece dictionary of Hugging Face.
class_docstr_tok: ["Allow","to","use","the","Torch","Agent","with","the","wordpiece","dictionary","of","Hugging","Face","."]
code: def __init__(self, opt):
      super().__init__(opt)
      # initialize from vocab path
      warn_once(
          "WARNING: BERT uses a Hugging Face tokenizer; ParlAI dictionary args are ignored"
      )
      download(opt["datapath"])
      vocab_path = PathManager.get_local_path(
          os.path.join(
              opt["datapath"], "models", "bert_models", VOCAB_PATH
          )
      )
      self.tokenizer = BertTokenizer.from_pretrained(vocab_path)
  
      self.start_token = "[CLS]"
      self.end_token = "[SEP]"
      self.null_token = "[PAD]"
      self.start_idx = self.tokenizer.convert_tokens_to_ids(["[CLS]"])[
          0
      ]  # should be 101
      self.end_idx = self.tokenizer.convert_tokens_to_ids(["[SEP]"])[
          0
      ]  # should be 102
      self.pad_idx = self.tokenizer.convert_tokens_to_ids(["[PAD]"])[
          0
      ]  # should be 0
      # set tok2ind for special tokens
      self.tok2ind[self.start_token] = self.start_idx
      self.tok2ind[self.end_token] = self.end_idx
      self.tok2ind[self.null_token] = self.pad_idx
      # set ind2tok for special tokens
      self.ind2tok[self.start_idx] = self.start_token
      self.ind2tok[self.end_idx] = self.end_token
      self.ind2tok[self.pad_idx] = self.null_token
  
file_path: ParlAI/parlai/agents/bert_ranker/bert_dictionary.py
imports: from parlai.core.dict import DictionaryAgent
  from parlai.utils.io import PathManager
  from parlai.utils.misc import warn_once
  from parlai.zoo.bert.build import download
  import os
  from .helpers import VOCAB_PATH
label_desc: Use image node_id 2 to create a new BertDictionaryAgent object from inherited base classes: DictionaryAgent with example: obj = BertDictionaryAgent(opt)