class_docstr: Masked language model for BERT style pre-training.
class_docstr_tok: ["Masked","language","model","for","BERT","style","pre-training","."]
code: def __init__(
      self,
      encoder: TransformerSentenceEncoderBase,
      decoder: MLPDecoder,
      output_layer: LMOutputLayer,
      token_tensorizer: BERTTensorizerBase,
      vocab: Vocabulary,
      mask_prob: float = Config.mask_prob,
      mask_bos: float = Config.mask_bos,
      masking_strategy: MaskingStrategy = Config.masking_strategy,
      stage: Stage = Stage.TRAIN,
  ) -> None:
      super().__init__(stage=stage)
      self.encoder = encoder
      self.decoder = decoder
      self.module_list = [encoder, decoder]
      self.output_layer = output_layer
      self.token_tensorizer = token_tensorizer
      self.vocab = vocab
      self.mask_prob = mask_prob
      self.mask_bos = mask_bos
      self.stage = stage
      self.masking_strategy = masking_strategy
  
      # initialize the frequency based sampling weights if these will be used
      self.token_sampling_weights = None
      if self.masking_strategy == MaskingStrategy.FREQUENCY:
          self.token_sampling_weights = [
              x**-0.5 for x in self.vocab.counts
          ]
  
          # Set probability of masking special tokens to be very low, since it doesn't
          # make sense to use them for MLM (unless there are no other tokens in the
          # the batch).
          tokens_to_avoid_masking = [
              SpecialTokens.PAD,
              SpecialTokens.UNK,
              SpecialTokens.MASK,
          ]
          if not self.mask_bos:
              tokens_to_avoid_masking.extend(
                  [SpecialTokens.BOS, SpecialTokens.EOS]
              )
          for token in tokens_to_avoid_masking:
              token_idx = self.vocab.idx.get(token)
              if token_idx is not None:
                  self.token_sampling_weights[token_idx] = 1e-20
      log_class_usage(__class__)
  
imports: from typing import Dict, List
  import torch
  from fairseq.modules.transformer_sentence_encoder import init_bert_params
  from pytext.common.constants import SpecialTokens, Stage
  from pytext.config import ConfigBase
  from pytext.data.bert_tensorizer import BERTTensorizerBase
  from pytext.data.tensorizers import Tensorizer
  from pytext.data.utils import Vocabulary
  from pytext.models.decoders.mlp_decoder import MLPDecoder
  from pytext.models.masking_utils import frequency_based_masking, MaskingStrategy, random_masking
  from pytext.models.model import BaseModel
  from pytext.models.module import create_module
  from pytext.models.output_layers.lm_output_layer import LMOutputLayer
  from pytext.models.representations.transformer_sentence_encoder import TransformerSentenceEncoder
  from pytext.models.representations.transformer_sentence_encoder_base import TransformerSentenceEncoderBase
  from pytext.utils.usage import log_class_usage
label_desc: Use image node_id 2 to create a new MaskedLanguageModel object from inherited base classes: BaseModel with example: obj = MaskedLanguageModel(encoder, decoder, output_layer, token_tensorizer, vocab, mask_prob, mask_bos, masking_strategy, stage)