class_docstr: BERT model ("Bidirectional Encoder Representations from Transformers").
  
  Example usage:
  
  ```python
  # Already been converted into WordPiece token ids
  input_ids = tf.constant([[31, 51, 99], [15, 5, 0]])
  input_mask = tf.constant([[1, 1, 1], [1, 1, 0]])
  token_type_ids = tf.constant([[0, 0, 1], [0, 2, 0]])
  
  config = modeling.BertConfig(vocab_size=32000, hidden_size=512,
    num_hidden_layers=8, num_attention_heads=6, intermediate_size=1024)
  
  model = modeling.BertModel(config=config, is_training=True,
    input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids)
  
  label_embeddings = tf.get_variable(...)
  pooled_output = model.get_pooled_output()
  logits = tf.matmul(pooled_output, label_embeddings)
  ...
  ```
class_docstr_tok: ["BERT","model","(","``","Bidirectional","Encoder","Representations","from","Transformers","''",")",".","Example","usage",":","``","`","python","#","Already","been","converted","into","WordPiece","token","ids","input_ids","=","tf.constant","(","[","[","31",",","51",",","99","]",",","[","15",",","5",",","0","]","]",")","input_mask","=","tf.constant","(","[","[","1",",","1",",","1","]",",","[","1",",","1",",","0","]","]",")","token_type_ids","=","tf.constant","(","[","[","0",",","0",",","1","]",",","[","0",",","2",",","0","]","]",")","config","=","modeling.BertConfig","(","vocab_size=32000",",","hidden_size=512",",","num_hidden_layers=8",",","num_attention_heads=6",",","intermediate_size=1024",")","model","=","modeling.BertModel","(","config=config",",","is_training=True",",","input_ids=input_ids",",","input_mask=input_mask",",","token_type_ids=token_type_ids",")","label_embeddings","=","tf.get_variable","(","...",")","pooled_output","=","model.get_pooled_output","(",")","logits","=","tf.matmul","(","pooled_output",",","label_embeddings",")","...","``","`"]
docstr: Constructor for BertModel.
  
  Args:
    config: `BertConfig` instance.
    is_training: bool. true for training model, false for eval model. Controls
      whether dropout will be applied.
    input_ids: int32 Tensor of shape [batch_size, seq_length].
    input_mask: (optional) int32 Tensor of shape [batch_size, seq_length].
    token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length].
    use_one_hot_embeddings: (optional) bool. Whether to use one-hot word
      embeddings or tf.embedding_lookup() for the word embeddings.
    scope: (optional) variable scope. Defaults to "bert".
  
  Raises:
    ValueError: The config is invalid or one of the input tensor shapes
      is invalid.
docstr_tok: ["Constructor","for","BertModel",".","Args",":","config",":","`","BertConfig","`","instance",".","is_training",":","bool",".","true","for","training","model",",","false","for","eval","model",".","Controls","whether","dropout","will","be","applied",".","input_ids",":","int32","Tensor","of","shape","[","batch_size",",","seq_length","]",".","input_mask",":","(","optional",")","int32","Tensor","of","shape","[","batch_size",",","seq_length","]",".","token_type_ids",":","(","optional",")","int32","Tensor","of","shape","[","batch_size",",","seq_length","]",".","use_one_hot_embeddings",":","(","optional",")","bool",".","Whether","to","use","one-hot","word","embeddings","or","tf.embedding_lookup","(",")","for","the","word","embeddings",".","scope",":","(","optional",")","variable","scope",".","Defaults","to","``","bert","''",".","Raises",":","ValueError",":","The","config","is","invalid","or","one","of","the","input","tensor","shapes","is","invalid","."]
code: def __init__(
      self,
      config,
      is_training,
      input_ids,
      input_mask=None,
      token_type_ids=None,
      use_one_hot_embeddings=False,
      scope=None,
  ):
      """Constructor for BertModel.
  
      Args:
        config: `BertConfig` instance.
        is_training: bool. true for training model, false for eval model. Controls
          whether dropout will be applied.
        input_ids: int32 Tensor of shape [batch_size, seq_length].
        input_mask: (optional) int32 Tensor of shape [batch_size, seq_length].
        token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length].
        use_one_hot_embeddings: (optional) bool. Whether to use one-hot word
          embeddings or tf.embedding_lookup() for the word embeddings.
        scope: (optional) variable scope. Defaults to "bert".
  
      Raises:
        ValueError: The config is invalid or one of the input tensor shapes
          is invalid.
      """
      config = copy.deepcopy(config)
      if not is_training:
          config.hidden_dropout_prob = 0.0
          config.attention_probs_dropout_prob = 0.0
  
      input_shape = get_shape_list(input_ids, expected_rank=2)
      batch_size = input_shape[0]
      seq_length = input_shape[1]
  
      if input_mask is None:
          input_mask = tf.ones(
              shape=[batch_size, seq_length], dtype=tf.int32
          )
  
      if token_type_ids is None:
          token_type_ids = tf.zeros(
              shape=[batch_size, seq_length], dtype=tf.int32
          )
  
      with tf.variable_scope(scope, default_name="bert"):
          with tf.variable_scope("embeddings"):
              # Perform embedding lookup on the word ids.
              (
                  self.word_embedding_output,
                  self.embedding_table,
              ) = embedding_lookup(
                  input_ids=input_ids,
                  vocab_size=config.vocab_size,
                  embedding_size=config.hidden_size,
                  initializer_range=config.initializer_range,
                  word_embedding_name="word_embeddings",
                  use_one_hot_embeddings=use_one_hot_embeddings,
              )
  
              # Add positional embeddings and token type embeddings, then layer
              # normalize and perform dropout.
              self.embedding_output = embedding_postprocessor(
                  input_tensor=self.word_embedding_output,
                  use_token_type=True,
                  token_type_ids=token_type_ids,
                  token_type_vocab_size=config.type_vocab_size,
                  token_type_embedding_name="token_type_embeddings",
                  use_position_embeddings=True,
                  position_embedding_name="position_embeddings",
                  initializer_range=config.initializer_range,
                  max_position_embeddings=config.max_position_embeddings,
                  dropout_prob=config.hidden_dropout_prob,
              )
  
          with tf.variable_scope("encoder"):
              # This converts a 2D mask of shape [batch_size, seq_length] to a 3D
              # mask of shape [batch_size, seq_length, seq_length] which is used
              # for the attention scores.
              attention_mask = create_attention_mask_from_input_mask(
                  input_ids, input_mask
              )
  
              # Run the stacked transformer.
              # `sequence_output` shape = [batch_size, seq_length, hidden_size].
              self.all_encoder_layers = transformer_model(
                  input_tensor=self.embedding_output,
                  attention_mask=attention_mask,
                  hidden_size=config.hidden_size,
                  num_hidden_layers=config.num_hidden_layers,
                  num_attention_heads=config.num_attention_heads,
                  intermediate_size=config.intermediate_size,
                  intermediate_act_fn=get_activation(config.hidden_act),
                  hidden_dropout_prob=config.hidden_dropout_prob,
                  attention_probs_dropout_prob=config.attention_probs_dropout_prob,
                  initializer_range=config.initializer_range,
                  do_return_all_layers=True,
              )
  
          self.sequence_output = self.all_encoder_layers[-1]
          # The "pooler" converts the encoded sequence tensor of shape
          # [batch_size, seq_length, hidden_size] to a tensor of shape
          # [batch_size, hidden_size]. This is necessary for segment-level
          # (or segment-pair-level) classification tasks where we need a fixed
          # dimensional representation of the segment.
          with tf.variable_scope("pooler"):
              # We "pool" the model by simply taking the hidden state corresponding
              # to the first token. We assume that this has been pre-trained
              first_token_tensor = tf.squeeze(
                  self.sequence_output[:, 0:1, :], axis=1
              )
              self.pooled_output = tf.layers.dense(
                  first_token_tensor,
                  config.hidden_size,
                  activation=tf.tanh,
                  kernel_initializer=create_initializer(
                      config.initializer_range
                  ),
              )
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import copy
  import json
  import math
  import re
  import numpy
  import six
  import tensorflow.compat.v1
label_desc: Use image node_id 1 to create a new BertModel object from inherited base classes: object with example: obj = BertModel(config, is_training, input_ids, input_mask, token_type_ids, use_one_hot_embeddings, scope)