code: def transformer_model(
      input_tensor,
      attention_mask=None,
      hidden_size=768,
      num_hidden_layers=12,
      num_attention_heads=12,
      intermediate_size=3072,
      intermediate_act_fn=gelu,
      hidden_dropout_prob=0.1,
      attention_probs_dropout_prob=0.1,
      initializer_range=0.02,
      do_return_all_layers=False,
  ):
      """Multi-headed, multi-layer Transformer from "Attention is All You Need".
  
      This is almost an exact implementation of the original Transformer encoder.
  
      See the original paper:
      https://arxiv.org/abs/1706.03762
  
      Also see:
      https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/models/transformer.py
  
      Args:
        input_tensor: float Tensor of shape [batch_size, seq_length, hidden_size].
        attention_mask: (optional) int32 Tensor of shape [batch_size, seq_length,
          seq_length], with 1 for positions that can be attended to and 0 in
          positions that should not be.
        hidden_size: int. Hidden size of the Transformer.
        num_hidden_layers: int. Number of layers (blocks) in the Transformer.
        num_attention_heads: int. Number of attention heads in the Transformer.
        intermediate_size: int. The size of the "intermediate" (a.k.a., feed
          forward) layer.
        intermediate_act_fn: function. The non-linear activation function to apply
          to the output of the intermediate/feed-forward layer.
        hidden_dropout_prob: float. Dropout probability for the hidden layers.
        attention_probs_dropout_prob: float. Dropout probability of the attention
          probabilities.
        initializer_range: float. Range of the initializer (stddev of truncated
          normal).
        do_return_all_layers: Whether to also return all layers or just the final
          layer.
  
      Returns:
        float Tensor of shape [batch_size, seq_length, hidden_size], the final
        hidden layer of the Transformer.
  
      Raises:
        ValueError: A Tensor shape or parameter is invalid.
      """
      if hidden_size % num_attention_heads != 0:
          raise ValueError(
              "The hidden size (%d) is not a multiple of the number of attention "
              "heads (%d)" % (hidden_size, num_attention_heads)
          )
  
      attention_head_size = int(hidden_size / num_attention_heads)
      input_shape = get_shape_list(input_tensor, expected_rank=3)
      input_width = input_shape[2]
  
      # The Transformer performs sum residuals on all layers so the input needs
      # to be the same as the hidden size.
      if input_width != hidden_size:
          raise ValueError(
              "The width of the input tensor (%d) != hidden size (%d)"
              % (input_width, hidden_size)
          )
  
      prev_output = input_tensor
      all_layer_outputs = []
      for layer_idx in range(num_hidden_layers):
          with tf.variable_scope("layer_%d" % layer_idx):
              layer_input = prev_output
  
              with tf.variable_scope("attention"):
                  with tf.variable_scope("self"):
                      attention_output = attention_layer(
                          from_tensor=layer_input,
                          to_tensor=layer_input,
                          attention_mask=attention_mask,
                          num_attention_heads=num_attention_heads,
                          size_per_head=attention_head_size,
                          attention_probs_dropout_prob=attention_probs_dropout_prob,
                          initializer_range=initializer_range,
                      )
  
                  # Run a linear projection of `hidden_size` then add a residual
                  # with `layer_input`.
                  with tf.variable_scope("output"):
                      attention_output = dense_layer_3d_proj(
                          attention_output,
                          hidden_size,
                          num_attention_heads,
                          attention_head_size,
                          create_initializer(initializer_range),
                          None,
                          "dense",
                      )
                      attention_output = dropout(
                          attention_output, hidden_dropout_prob
                      )
                      attention_output = layer_norm(
                          attention_output + layer_input
                      )
  
              # The activation is only applied to the "intermediate" hidden layer.
              with tf.variable_scope("intermediate"):
                  intermediate_output = dense_layer_2d(
                      attention_output,
                      intermediate_size,
                      create_initializer(initializer_range),
                      intermediate_act_fn,
                      "dense",
                  )
  
              # Down-project back to `hidden_size` then add the residual.
              with tf.variable_scope("output"):
                  layer_output = dense_layer_2d(
                      intermediate_output,
                      hidden_size,
                      create_initializer(initializer_range),
                      None,
                      "dense",
                  )
                  layer_output = dropout(
                      layer_output, hidden_dropout_prob
                  )
                  layer_output = layer_norm(
                      layer_output + attention_output
                  )
                  prev_output = layer_output
                  all_layer_outputs.append(layer_output)
  
      if do_return_all_layers:
          return all_layer_outputs
      else:
          return all_layer_outputs[-1]
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import copy
  import json
  import math
  import re
  import numpy
  import six
  import tensorflow.compat.v1
label_desc: Use image node_id 15 for calling a global function with example usage: transformer_model(input_tensor, attention_mask, hidden_size, num_hidden_layers, num_attention_heads, intermediate_size, intermediate_act_fn, hidden_dropout_prob, attention_probs_dropout_prob, initializer_range, do_return_all_layers) and returns: all_layer_outputs, all_layer_outputs