code: def attention_decoder(
      decoder_inputs,
      initial_state,
      attention_states,
      cell,
      output_size=None,
      num_heads=1,
      loop_function=None,
      dtype=tf.float32,
      scope=None,
      initial_state_attention=False,
      attn_num_hidden=128,
  ):
      """RNN decoder with attention for the sequence-to-sequence model.
  
      In this context "attention" means that, during decoding, the RNN can look up
      information in the additional tensor attention_states, and it does this by
      focusing on a few entries from the tensor. This model has proven to yield
      especially good results in a number of sequence-to-sequence tasks. This
      implementation is based on http://arxiv.org/abs/1412.7449 (see below for
      details). It is recommended for complex sequence-to-sequence tasks.
  
      Args:
          decoder_inputs: A list of 2D Tensors [batch_size x input_size].
          initial_state: 2D Tensor [batch_size x cell.state_size].
          attention_states: 3D Tensor [batch_size x attn_length x attn_size].
          cell: rnn_cell.RNNCell defining the cell function and size.
          output_size: Size of the output vectors; if None, we use cell.output_size.
          num_heads: Number of attention heads that read from attention_states.
          loop_function: If not None, this function will be applied to i-th output
              in order to generate i+1-th input, and decoder_inputs will be ignored,
              except for the first element ("GO" symbol). This can be used for decoding,
              but also for training to emulate http://arxiv.org/abs/1506.03099.
              Signature -- loop_function(prev, i) = next
                  * prev is a 2D Tensor of shape [batch_size x output_size],
                  * i is an integer, the step number (when advanced control is needed),
                  * next is a 2D Tensor of shape [batch_size x input_size].
          dtype: The dtype to use for the RNN initial state (default: tf.float32).
          scope: VariableScope for the created subgraph; default: "attention_decoder".
          initial_state_attention: If False (default), initial attentions are zero.
              If True, initialize the attentions from the initial state and attention
              states -- useful when we wish to resume decoding from a previously
              stored decoder state and attention states.
  
      Returns:
          A tuple of the form (outputs, state), where:
              outputs: A list of the same length as decoder_inputs of 2D Tensors of
                  shape [batch_size x output_size]. These represent the generated outputs.
                  Output i is computed from input i (which is either the i-th element
                  of decoder_inputs or loop_function(output {i-1}, i)) as follows.
                  First, we run the cell on a combination of the input and previous
                  attention masks:
                      cell_output, new_state = cell(linear(input, prev_attn), prev_state).
                  Then, we calculate new attention masks:
                      new_attn = softmax(V^T * tanh(W * attention_states + U * new_state))
                  and then we calculate the output:
                      output = linear(cell_output, new_attn).
              state: The state of each decoder cell the final time-step.
                  It is a 2D Tensor of shape [batch_size x cell.state_size].
  
      Raises:
          ValueError: when num_heads is not positive, there are no inputs, or shapes
              of attention_states are not set.
      """
      # MODIFIED ADD START
      assert (
          num_heads == 1
      ), "We only consider the case where num_heads=1!"
      # MODIFIED ADD END
      if not decoder_inputs:
          raise ValueError(
              "Must provide at least 1 input to attention decoder."
          )
      if num_heads < 1:
          raise ValueError(
              "With less than 1 heads, use a non-attention decoder."
          )
      if not attention_states.get_shape()[1:2].is_fully_defined():
          raise ValueError(
              "Shape[1] and [2] of attention_states must be known: %s"
              % attention_states.get_shape()
          )
      if output_size is None:
          output_size = cell.output_size
  
      with tf.compat.v1.variable_scope(scope or "attention_decoder"):
          batch_size = tf.shape(decoder_inputs[0])[
              0
          ]  # Needed for reshaping.
          attn_length = attention_states.get_shape()[1]
          attn_size = attention_states.get_shape()[2]
  
          # To calculate W1 * h_t we use a 1-by-1 convolution, need to reshape before.
          hidden = tf.reshape(
              attention_states, [-1, attn_length, 1, attn_size]
          )
          hidden_features = []
          v = []
          attention_vec_size = (
              attn_size  # Size of query vectors for attention.
          )
          for a in xrange(num_heads):
              k = tf.compat.v1.get_variable(
                  "AttnW_%d" % a, [1, 1, attn_size, attention_vec_size]
              )
              hidden_features.append(
                  tf.nn.conv2d(
                      hidden,
                      filters=k,
                      strides=[1, 1, 1, 1],
                      padding="SAME",
                  )
              )
              v.append(
                  tf.compat.v1.get_variable(
                      "AttnV_%d" % a, [attention_vec_size]
                  )
              )
  
          state = initial_state
  
          # MODIFIED: return both context vector and attention weights
          def attention(query):
              """Put attention masks on hidden using hidden_features and query."""
              # MODIFIED ADD START
              ss = None  # record attention weights
              # MODIFIED ADD END
              ds = []  # Results of attention reads will be stored here.
              for a in xrange(num_heads):
                  with tf.compat.v1.variable_scope("Attention_%d" % a):
                      y = linear(query, attention_vec_size, True)
                      y = tf.reshape(y, [-1, 1, 1, attention_vec_size])
                      # Attention mask is a softmax of v^T * tanh(...).
                      s = tf.reduce_sum(
                          v[a] * tf.tanh(hidden_features[a] + y), [2, 3]
                      )
                      a = tf.nn.softmax(s)
                      ss = a
                      # a = tf.Print(a, [a], message="a: ",summarize=30)
                      # Now calculate the attention-weighted vector d.
                      d = tf.reduce_sum(
                          tf.reshape(a, [-1, attn_length, 1, 1])
                          * hidden,
                          [1, 2],
                      )
                      ds.append(tf.reshape(d, [-1, attn_size]))
              # MODIFIED DELETED return ds
              # MODIFIED ADD START
              return ds, ss
              # MODIFIED ADD END
  
          outputs = []
          # MODIFIED ADD START
          attention_weights_history = []
          # MODIFIED ADD END
          prev = None
          batch_attn_size = tf.stack([batch_size, attn_size])
          attns = [
              tf.zeros(batch_attn_size, dtype=dtype)
              for _ in xrange(num_heads)
          ]
          for (
              a
          ) in (
              attns
          ):  # Ensure the second shape of attention vectors is set.
              a.set_shape([None, attn_size])
          if initial_state_attention:
              # MODIFIED DELETED attns = attention(initial_state)
              # MODIFIED ADD START
              attns, attn_weights = attention(initial_state)
              attention_weights_history.append(attn_weights)
              # MODIFIED ADD END
          for i, inp in enumerate(decoder_inputs):
              if i > 0:
                  tf.compat.v1.get_variable_scope().reuse_variables()
              # If loop_function is set, we use it instead of decoder_inputs.
              if loop_function is not None and prev is not None:
                  with tf.compat.v1.variable_scope(
                      "loop_function", reuse=True
                  ):
                      inp = loop_function(prev, i)
              # Merge input and previous attentions into one vector of the right size.
              # input_size = inp.get_shape().with_rank(2)[1]
              # todo: use input_size
              input_size = attn_num_hidden
              x = linear([inp] + attns, input_size, True)
              # Run the RNN.
              cell_output, state = cell(x, state)
              # Run the attention mechanism.
              if i == 0 and initial_state_attention:
                  with tf.compat.v1.variable_scope(
                      tf.compat.v1.get_variable_scope(), reuse=True
                  ):
                      # MODIFIED DELETED attns = attention(state)
                      # MODIFIED ADD START
                      attns, attn_weights = attention(state)
                      # MODIFIED ADD END
              else:
                  # MODIFIED DELETED attns = attention(state)
                  # MODIFIED ADD START
                  attns, attn_weights = attention(state)
                  attention_weights_history.append(attn_weights)
                  # MODIFIED ADD END
  
              with tf.compat.v1.variable_scope("AttnOutputProjection"):
                  output = linear(
                      [cell_output] + attns, output_size, True
                  )
              if loop_function is not None:
                  prev = output
              outputs.append(output)
  
      # MODIFIED DELETED return outputs, state
      # MODIFIED ADD START
      return outputs, state, attention_weights_history
  
file_path: attention-ocr/aocr/model/seq2seq.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from six.moves import xrange
  from six.moves import zip
  import tensorflow
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import init_ops
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops import nn_ops
  from tensorflow.python.ops import variable_scope
  from tensorflow.python.util import nest
label_desc: Use image node_id 3 for calling a global function with example usage: attention_decoder(decoder_inputs, initial_state, attention_states, cell, output_size, num_heads, loop_function, dtype, scope, initial_state_attention, attn_num_hidden) and returns: outputs,  state,  attention_weights_history, ds,  ss