docstr: BERT encodes
  - single sequence: ``[CLS] X [SEP]``
  - pair of sequences: ``[CLS] A [SEP] B [SEP]``
docstr_tok: ["BERT","encodes","-","single","sequence",":","``","[","CLS","]","X","[","SEP","]","``","-","pair","of","sequences",":","``","[","CLS","]","A","[","SEP","]","B","[","SEP","]","``"]
code: def _convert_to_indices(
      self,
      tokens_a: List[str],
      tokens_b: Optional[List[str]] = None,
      probability: float = 0.15,
  ) -> Dict[str, torch.Tensor]:
      """
      BERT encodes
      - single sequence: ``[CLS] X [SEP]``
      - pair of sequences: ``[CLS] A [SEP] B [SEP]``
      """
      tokens_a, label_a = self._random_word(
          tokens_a, probability=probability
      )
      tokens = [self._CLS_TOKEN] + tokens_a + [self._SEP_TOKEN]
      segment_ids = [0] + [0] * len(tokens_a) + [0]
  
      if tokens_b:
          tokens_b, label_b = self._random_word(
              tokens_b, probability=probability
          )
          lm_label_ids = [-1] + label_a + [-1] + label_b + [-1]
          assert len(tokens_b) > 0
          tokens += tokens_b + [self._SEP_TOKEN]
          segment_ids += [1] * len(tokens_b) + [1]
      else:
          lm_label_ids = [-1] + label_a + [-1]
  
      input_ids = self._convert_tokens_to_ids(tokens)
      input_mask = [1] * len(input_ids)
  
      # Zero-pad up to the sequence length.
      while len(input_ids) < self._max_seq_length:
          input_ids.append(self._PAD_TOKEN_ID)
          input_mask.append(0)
          segment_ids.append(0)
          lm_label_ids.append(-1)
  
      assert len(input_ids) == self._max_seq_length
      assert len(input_mask) == self._max_seq_length
      assert len(segment_ids) == self._max_seq_length
      assert len(lm_label_ids) == self._max_seq_length
  
      input_ids = torch.tensor(input_ids, dtype=torch.long)
      input_mask = torch.tensor(input_mask, dtype=torch.long)
      segment_ids = torch.tensor(segment_ids, dtype=torch.long)
      lm_label_ids = torch.tensor(lm_label_ids, dtype=torch.long)
      return {
          "input_ids": input_ids,
          "input_mask": input_mask,
          "segment_ids": segment_ids,
          "lm_label_ids": lm_label_ids,
          "tokens": tokens,
      }
  
imports: import copy
  import random
  from typing import Any, Dict, List, Optional, Tuple, Union
  import torch
  from mmf.common.registry import registry
  from mmf.common.sample import Sample, SampleList
  from mmf.datasets.processors.processors import BaseProcessor
label_desc: Use image node_id 8 for calling the MaskedTokenProcessor obj's underlying member method code with example usage: obj._convert_to_indices(tokens_a, tokens_b, probability) and returns: dict