class_docstr: BERT model ("Bidirectional Encoder Representations from Transformers").
class_docstr_tok: ["BERT","model","(","``","Bidirectional","Encoder","Representations","from","Transformers","''",")","."]
docstr: Get loss and logits for the masked LM.
code: def get_masked_lm_output(self, positions, label_ids, label_weights):
      """Get loss and logits for the masked LM."""
      input_tensor = self.get_sequence_output()
      output_weights = self.get_embedding_table()
  
      # [batch_size, num_position, hidden]
      input_tensor = mtf.gather(input_tensor, positions, self.seq_dim)
      with tf.variable_scope("cls/predictions"):
          # We apply one more non-linear transformation before the output layer.
          # This matrix is not used after pre-training.
          with tf.variable_scope("transform"):
              input_tensor = mtf.layers.dense(
                  input_tensor,
                  reduced_dims=[self.model_dim],
                  new_dims=[self.model_dim],
                  activation=get_activation(
                      self.config.feedforward_intermediate_act
                  ),
                  kernel_initializer=self.dense_initializer,
                  use_bias=self.config.use_bias,
              )
              input_tensor = self.normalize(input_tensor)
          # The output weights are the same as the input embeddings, but there is
          # an output-only bias for each token.
          output_bias = mtf.get_variable(
              input_tensor.mesh,
              name="output_bias",
              shape=[self.vocab_dim],
              initializer=tf.zeros_initializer(),
          )
          logits = (
              mtf.einsum(
                  [input_tensor, output_weights],
                  reduced_dims=[self.model_dim],
              )
              + output_bias
          )
          per_example_loss = (
              mtf.layers.softmax_cross_entropy_with_logits(
                  logits, label_ids, self.vocab_dim, z_loss=1e-4
              )
          )
          # The `positions` tensor might be zero-padded (if the sequence is too
          # short to have the maximum number of predictions). The `label_weights`
          # tensor has a value of 1.0 for every real prediction and 0.0 for the
          # padding predictions.
          numerator = mtf.reduce_sum(label_weights * per_example_loss)
          denominator = mtf.reduce_sum(label_weights) + mtf.constant(
              input_tensor.mesh, 1e-5, dtype=tf.float32
          )
          loss = numerator / denominator
      return (loss, per_example_loss, logits)
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import copy
  import json
  import math
  import re
  import mesh_tensorflow
  import mesh_tensorflow.transformer.moe
  import six
  import tensorflow.compat.v1
label_desc: Use image node_id 5 for calling the BertModel obj's underlying member method code with example usage: obj.get_masked_lm_output(positions, label_ids, label_weights) and returns: loss,  per_example_loss,  logits