# SPDX-License-Identifier: Apache-2.0


"""Unit Tests for custom rnns."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf

from tensorflow.python.ops import init_ops
from backend_test_base import Tf2OnnxBackendTestBase
from common import *  # pylint: disable=wildcard-import, unused-wildcard-import
from tf2onnx.tf_loader import is_tf2


# pylint: disable=missing-docstring,invalid-name,unused-argument,using-constant-test
# pylint: disable=abstract-method,arguments-differ

if is_tf2():
    BasicLSTMCell = tf.compat.v1.nn.rnn_cell.BasicLSTMCell
    LSTMCell = tf.compat.v1.nn.rnn_cell.LSTMCell
    GRUCell = tf.compat.v1.nn.rnn_cell.GRUCell
    RNNCell = tf.compat.v1.nn.rnn_cell.RNNCell
    MultiRNNCell = tf.compat.v1.nn.rnn_cell.MultiRNNCell
    dynamic_rnn = tf.compat.v1.nn.dynamic_rnn
    bidirectional_dynamic_rnn = tf.compat.v1.nn.bidirectional_dynamic_rnn
else:
    LSTMBlockCell = tf.contrib.rnn.LSTMBlockCell
    LSTMCell = tf.nn.rnn_cell.LSTMCell
    GRUCell = tf.nn.rnn_cell.LSTMCell
    RNNCell = tf.nn.rnn_cell.RNNCell
    MultiRNNCell = tf.contrib.rnn.MultiRNNCell
    dynamic_rnn = tf.nn.dynamic_rnn
    bidirectional_dynamic_rnn = tf.nn.bidirectional_dynamic_rnn


class CustomRnnCellTests(Tf2OnnxBackendTestBase):
    @check_opset_min_version(8, "Scan")
    @skip_tf2()
    def test_single_dynamic_custom_rnn(self):
        size = 5  # size of each model layer.
        batch_size = 1
        cell = GatedGRUCell(size)

        x_val = np.array([[1., 1.], [2., 2.], [3., 3.]], dtype=np.float32)
        x_val = np.stack([x_val] * batch_size)
        def func(x):
            xs, s = dynamic_rnn(cell=cell, dtype=tf.float32, inputs=x, time_major=False)
            return tf.identity(xs, name="output"), tf.identity(s, name="final_state")

        feed_dict = {"input_1:0": x_val}
        input_names_with_port = ["input_1:0"]
        output_names_with_port = ["output:0", "final_state:0"]
        self.run_test_case(func, feed_dict, input_names_with_port, output_names_with_port, 0.1)

    @check_opset_min_version(8, "Scan")
    @skip_tf2()
    def test_single_dynamic_custom_rnn_time_major(self):
        size = 5  # size of each model layer.
        batch_size = 1
        x_val = np.array([[1., 1.], [2., 2.], [3., 3.]], dtype=np.float32)
        x_val = np.stack([x_val] * batch_size)
        def func(x):
            cell = GatedGRUCell(size)
            xs, s = dynamic_rnn(cell=cell, dtype=tf.float32, inputs=x, time_major=True)
            return tf.identity(xs, name="output"), tf.identity(s, name="final_state")

        feed_dict = {"input_1:0": x_val}
        input_names_with_port = ["input_1:0"]
        output_names_with_port = ["output:0", "final_state:0"]
        self.run_test_case(func, feed_dict, input_names_with_port, output_names_with_port, 0.1)

    @check_opset_min_version(8, "Scan")
    @skip_tf2()
    def test_single_dynamic_custom_rnn_with_seq_length(self):
        units = 5
        batch_size = 6
        x_val = np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.], [5., 5.]], dtype=np.float32)
        x_val = np.stack([x_val] * batch_size)
        def func(x):
            # no scope
            cell = GatedGRUCell(units)
            outputs, cell_state = dynamic_rnn(
                cell,
                x,
                dtype=tf.float32,
                sequence_length=[4, 3, 4, 5, 2, 1])
            return tf.identity(outputs, name="output"), tf.identity(cell_state, name="cell_state")

        feed_dict = {"input_1:0": x_val}
        input_names_with_port = ["input_1:0"]
        output_names_with_port = ["output:0", "cell_state:0"]
        self.run_test_case(func, feed_dict, input_names_with_port, output_names_with_port, rtol=1e-06)

    @check_opset_min_version(8, "Scan")
    @skip_tf2()
    def test_single_dynamic_custom_rnn_with_non_const_seq_length(self):
        units = 5
        batch_size = 6
        x_val = np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.], [5., 5.]], dtype=np.float32)
        x_val = np.stack([x_val] * batch_size)
        y_val = np.array([4, 3, 4, 5, 2, 1], dtype=np.int32)
        def func(x, seq_length):
            # no scope
            cell = GatedGRUCell(units)
            outputs, cell_state = dynamic_rnn(
                cell,
                x,
                dtype=tf.float32,
                sequence_length=tf.identity(seq_length))
            return tf.identity(outputs, name="output"), tf.identity(cell_state, name="cell_state")

        feed_dict = {"input_1:0": x_val, "input_2:0": y_val}
        input_names_with_port = ["input_1:0", "input_2:0"]
        output_names_with_port = ["output:0", "cell_state:0"]
        self.run_test_case(func, feed_dict, input_names_with_port, output_names_with_port, rtol=1e-06)

    @check_opset_min_version(8, "Scan")
    @check_tf_min_version("1.8")
    @skip_tf2()
    def test_attention_wrapper_const_encoder(self):
        size = 5
        time_step = 3
        input_size = 4
        attn_size = size
        batch_size = 9
        # shape  [batch size, time step, size]
        # attention_state: usually the output of an RNN encoder.
        # This tensor should be shaped `[batch_size, max_time, ...]`.
        decoder_time_step = 6
        x_val = np.random.randn(decoder_time_step, input_size).astype('f')
        x_val = np.stack([x_val] * batch_size)

        attention_states = np.random.randn(batch_size, time_step, attn_size).astype('f')
        def func(x):
            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(attn_size, attention_states)

            match_input_fn = lambda curr_input, state: tf.concat([curr_input, state], axis=-1)
            cell = LSTMCell(size)
            match_cell_fw = tf.contrib.seq2seq.AttentionWrapper(cell,
                                                                attention_mechanism,
                                                                attention_layer_size=attn_size,
                                                                cell_input_fn=match_input_fn,
                                                                output_attention=False)
            output, attr_state = dynamic_rnn(match_cell_fw, x, dtype=tf.float32)

            return tf.identity(output, name="output"), tf.identity(attr_state.cell_state, name="final_state")

        feed_dict = {"input_1:0": x_val}
        input_names_with_port = ["input_1:0"]
        output_names_with_port = ["output:0"]
        output_names_with_port = ["output:0", "final_state:0"]
        self.run_test_case(func, feed_dict, input_names_with_port, output_names_with_port, 0.1)

    @check_opset_min_version(8, "Scan")
    @check_tf_min_version("1.8")
    @skip_tf2()
    def test_attention_wrapper_lstm_encoder(self):
        size = 5
        time_step = 3
        input_size = 4
        attn_size = size
        batch_size = 9

        # shape  [batch size, time step, size]
        # attention_state: usually the output of an RNN encoder.
        # This tensor should be shaped `[batch_size, max_time, ...]`
        encoder_time_step = time_step
        encoder_x_val = np.random.randn(encoder_time_step, input_size).astype('f')
        encoder_x_val = np.stack([encoder_x_val] * batch_size)
        decoder_time_step = 6
        decoder_x_val = np.random.randn(decoder_time_step, input_size).astype('f')
        decoder_x_val = np.stack([decoder_x_val] * batch_size)

        def func(encoder_x, decoder_x):
            encoder_cell = LSTMCell(size)
            output, attr_state = dynamic_rnn(encoder_cell, encoder_x, dtype=tf.float32)
            output_0 = tf.identity(output, name="output_0")
            attention_states = output
            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(attn_size,
                                                                       attention_states)

            match_input_fn = lambda curr_input, state: tf.concat([curr_input, state], axis=-1)
            cell = LSTMCell(size)
            match_cell_fw = tf.contrib.seq2seq.AttentionWrapper(cell,
                                                                attention_mechanism,
                                                                attention_layer_size=attn_size,
                                                                cell_input_fn=match_input_fn,
                                                                output_attention=False)

            output, attr_state = dynamic_rnn(match_cell_fw, decoder_x, dtype=tf.float32)

            return output_0, tf.identity(output, name="output"), tf.identity(attr_state.cell_state, name="final_state")

        feed_dict = {"input_1:0": encoder_x_val, "input_2:0": decoder_x_val}
        input_names_with_port = ["input_1:0", "input_2:0"]
        output_names_with_port = ["output_0:0", "output:0", "final_state:0"]
        self.run_test_case(func, feed_dict, input_names_with_port, output_names_with_port, 0.1)

    @check_opset_min_version(8, "Scan")
    @check_tf_min_version("1.8")
    @skip_tf2()
    def test_attention_wrapper_gru_encoder(self):
        size = 5
        time_step = 3
        input_size = 4
        attn_size = size
        batch_size = 9

        # shape  [batch size, time step, size]
        # attention_state: usually the output of an RNN encoder.
        # This tensor should be shaped `[batch_size, max_time, ...]`
        encoder_time_step = time_step
        encoder_x_val = np.random.randn(encoder_time_step, input_size).astype('f')
        encoder_x_val = np.stack([encoder_x_val] * batch_size)
        decoder_time_step = 6
        decoder_x_val = np.random.randn(decoder_time_step, input_size).astype('f')
        decoder_x_val = np.stack([decoder_x_val] * batch_size)

        def func(encoder_x, decoder_x):
            encoder_cell = GRUCell(size)
            output, attr_state = dynamic_rnn(encoder_cell, encoder_x, dtype=tf.float32)
            _ = tf.identity(output, name="output_0")
            attention_states = output
            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(attn_size,
                                                                       attention_states)

            match_input_fn = lambda curr_input, state: tf.concat([curr_input, state], axis=-1)
            cell = GRUCell(size)
            match_cell_fw = tf.contrib.seq2seq.AttentionWrapper(cell,
                                                                attention_mechanism,
                                                                attention_layer_size=attn_size,
                                                                cell_input_fn=match_input_fn,
                                                                output_attention=False)
            output, attr_state = dynamic_rnn(match_cell_fw, decoder_x, dtype=tf.float32)
            return tf.identity(output, name="output"), tf.identity(attr_state.cell_state, name="final_state")

        feed_dict = {"input_1:0": encoder_x_val, "input_2:0": decoder_x_val}
        input_names_with_port = ["input_1:0", "input_2:0"]
        output_names_with_port = ["output_0:0", "output:0", "final_state:0"]
        self.run_test_case(func, feed_dict, input_names_with_port, output_names_with_port, 0.1)

    @check_opset_min_version(8, "Scan")
    @check_tf_min_version("1.8")
    @skip_tf2()
    def test_attention_wrapper_lstm_encoder_input_has_none_dim(self):
        size = 5
        time_step = 3
        input_size = 4
        attn_size = size
        batch_size = 9

        # shape  [batch size, time step, size]
        # attention_state: usually the output of an RNN encoder.
        # This tensor should be shaped `[batch_size, max_time, ...]`
        encoder_time_step = time_step
        encoder_x_val = np.random.randn(encoder_time_step, input_size).astype('f')
        encoder_x_val = np.stack([encoder_x_val] * batch_size)
        decoder_time_step = 6
        decoder_x_val = np.random.randn(decoder_time_step, input_size).astype('f')
        decoder_x_val = np.stack([decoder_x_val] * batch_size)

        def func(encoder_x, decoder_x):
            encoder_cell = LSTMCell(size)
            output, attr_state = dynamic_rnn(encoder_cell, encoder_x, dtype=tf.float32)
            _ = tf.identity(output, name="output_0")
            attention_states = output
            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(attn_size,
                                                                       attention_states)

            match_input_fn = lambda curr_input, state: tf.concat([curr_input, state], axis=-1)
            cell = LSTMCell(size)
            match_cell_fw = tf.contrib.seq2seq.AttentionWrapper(cell,
                                                                attention_mechanism,
                                                                attention_layer_size=attn_size,
                                                                cell_input_fn=match_input_fn,
                                                                output_attention=False)

            output, attr_state = dynamic_rnn(match_cell_fw, decoder_x, dtype=tf.float32)

            return tf.identity(output, name="output"), tf.identity(attr_state.cell_state, name="final_state")

        feed_dict = {"input_1:0": encoder_x_val, "input_2:0": decoder_x_val}
        input_names_with_port = ["input_1:0", "input_2:0"]
        output_names_with_port = ["output_0:0", "output:0", "final_state:0"]

        self.run_test_case(func, feed_dict, input_names_with_port, output_names_with_port, 0.1)

    @check_opset_min_version(8, "Scan")
    @skip_tf2()
    def test_multi_rnn_lstm(self, state_is_tuple=True):
        units = 5
        batch_size = 6
        x_val = np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]], dtype=np.float32)
        x_val = np.stack([x_val] * batch_size)
        def func(x):
            initializer = init_ops.constant_initializer(0.5)

            cell_0 = LSTMCell(units,
                              initializer=initializer,
                              state_is_tuple=state_is_tuple)

            cell_1 = LSTMCell(units,
                              initializer=initializer,
                              state_is_tuple=state_is_tuple)

            cell_2 = LSTMCell(units,
                              initializer=initializer,
                              state_is_tuple=state_is_tuple)

            cells = MultiRNNCell([cell_0, cell_1, cell_2], state_is_tuple=state_is_tuple)
            outputs, cell_state = dynamic_rnn(cells, x, dtype=tf.float32)
            return tf.identity(outputs, name="output"), tf.identity(cell_state, name="cell_state")

        input_names_with_port = ["input_1:0"]
        feed_dict = {"input_1:0": x_val}

        output_names_with_port = ["output:0", "cell_state:0"]
        self.run_test_case(func, feed_dict, input_names_with_port, output_names_with_port, rtol=1e-06)

    @check_opset_min_version(8, "Scan")
    @check_tf_min_version("1.8")
    @skip_opset(9, "ReverseSequence")
    @skip_tf2()
    @allow_missing_shapes("Missing RNN shape")
    def test_bidrectional_attention_wrapper_lstm_encoder(self):
        size = 30
        time_step = 3
        input_size = 4
        attn_size = size
        batch_size = 9

        # shape  [batch size, time step, size]
        # attention_state: usually the output of an RNN encoder.
        # This tensor should be shaped `[batch_size, max_time, ...]`
        encoder_time_step = time_step
        encoder_x_val = np.random.randn(encoder_time_step, input_size).astype('f')
        encoder_x_val = np.stack([encoder_x_val] * batch_size)
        decoder_time_step = 6
        decoder_x_val = np.random.randn(decoder_time_step, batch_size, input_size).astype('f')

        def func(encoder_x, decoder_x, seq_length):
            encoder_cell = LSTMCell(size)
            attention_states, _ = dynamic_rnn(encoder_cell, encoder_x, dtype=tf.float32)
            # [9, 3, 30], [9, 30]
            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(attn_size,
                                                                       attention_states)

            match_input_fn = lambda curr_input, state: tf.concat([curr_input, state], axis=-1)
            cell = LSTMCell(size)
            match_cell_fw = tf.contrib.seq2seq.AttentionWrapper(cell,
                                                                attention_mechanism,
                                                                attention_layer_size=attn_size,
                                                                cell_input_fn=match_input_fn,
                                                                output_attention=False)
            match_cell_bk = tf.contrib.seq2seq.AttentionWrapper(cell,
                                                                attention_mechanism,
                                                                attention_layer_size=attn_size,
                                                                cell_input_fn=match_input_fn,
                                                                output_attention=False)
            (match_output_fw, match_output_bk), (match_state_fw, match_state_bk) = \
                bidirectional_dynamic_rnn(cell_fw=match_cell_fw,
                                          cell_bw=match_cell_bk,
                                          inputs=decoder_x,
                                          sequence_length=tf.identity(seq_length),
                                          dtype=tf.float32,
                                          time_major=True)

            matched_output = tf.concat([match_output_fw, match_output_bk], axis=-1)
            matched_state = tf.concat([match_state_fw.cell_state, match_state_bk.cell_state], -1)
            return tf.identity(matched_output, name="output_0"), tf.identity(matched_state, name="final_state")

        feed_dict = {"input_1:0": encoder_x_val, "input_2:0": decoder_x_val,
                     "input_3:0": np.array([6, 5, 4, 3, 2, 1, 2, 3, 6], dtype=np.int32)}
        input_names_with_port = ["input_1:0", "input_2:0", "input_3:0"]
        output_names_with_port = ["output_0:0", "final_state:0"]
        self.run_test_case(func, feed_dict, input_names_with_port, output_names_with_port, 0.1)


class GatedGRUCell(RNNCell):
    def __init__(self, hidden_dim, reuse=None):
        super().__init__(self, _reuse=reuse)
        self._num_units = hidden_dim
        self._activation = tf.tanh

    @property
    def state_size(self):
        return self._num_units

    @property
    def output_size(self):
        return self._num_units

    def call(self, inputs, state):
        # inputs shape: [batch size, time step, input size] = [1, 3, 2]
        # num_units: 5
        # W shape: [2, 3 * 5] = [2, 15]
        # U shape: [5, 3 * 5] = [5, 15]
        # b shape: [1, 3 * 5] = [1, 15]
        # state shape: [batch size, state size] = [1, 5]

        input_dim = inputs.get_shape()[-1]
        assert input_dim is not None, "input dimension must be defined"
        # W = tf.get_variable(name="W", shape=[input_dim, 3 * self._num_units], dtype=tf.float32)
        W = np.arange(30.0, dtype=np.float32).reshape((2, 15))
        # U = tf.get_variable(name='U', shape=[self._num_units, 3 * self._num_units], dtype=tf.float32)
        U = np.arange(75.0, dtype=np.float32).reshape((5, 15))
        # b = tf.get_variable(name='b', shape=[1, 3 * self._num_units], dtype=tf.float32)
        b = np.arange(15.0, dtype=np.float32).reshape((1, 15))

        xw = tf.split(tf.matmul(inputs, W) + b, 3, 1)
        hu = tf.split(tf.matmul(state, U), 3, 1)
        r = tf.sigmoid(xw[0] + hu[0])
        z = tf.sigmoid(xw[1] + hu[1])
        h1 = self._activation(xw[2] + r * hu[2])
        next_h = h1 * (1 - z) + state * z
        return next_h, next_h


if __name__ == '__main__':
    unittest_main()

[((584, 592), 'tf2onnx.tf_loader.is_tf2', 'is_tf2', ([], {}), '()\n', (590, 592), False, 'from tf2onnx.tf_loader import is_tf2\n'), ((1563, 1627), 'numpy.array', 'np.array', (['[[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]]'], {'dtype': 'np.float32'}), '([[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]], dtype=np.float32)\n', (1571, 1627), True, 'import numpy as np\n'), ((1638, 1668), 'numpy.stack', 'np.stack', (['([x_val] * batch_size)'], {}), '([x_val] * batch_size)\n', (1646, 1668), True, 'import numpy as np\n'), ((2311, 2375), 'numpy.array', 'np.array', (['[[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]]'], {'dtype': 'np.float32'}), '([[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]], dtype=np.float32)\n', (2319, 2375), True, 'import numpy as np\n'), ((2386, 2416), 'numpy.stack', 'np.stack', (['([x_val] * batch_size)'], {}), '([x_val] * batch_size)\n', (2394, 2416), True, 'import numpy as np\n'), ((3073, 3165), 'numpy.array', 'np.array', (['[[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0], [5.0, 5.0]]'], {'dtype': 'np.float32'}), '([[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0], [5.0, 5.0]],\n    dtype=np.float32)\n', (3081, 3165), True, 'import numpy as np\n'), ((3168, 3198), 'numpy.stack', 'np.stack', (['([x_val] * batch_size)'], {}), '([x_val] * batch_size)\n', (3176, 3198), True, 'import numpy as np\n'), ((3994, 4086), 'numpy.array', 'np.array', (['[[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0], [5.0, 5.0]]'], {'dtype': 'np.float32'}), '([[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0], [5.0, 5.0]],\n    dtype=np.float32)\n', (4002, 4086), True, 'import numpy as np\n'), ((4089, 4119), 'numpy.stack', 'np.stack', (['([x_val] * batch_size)'], {}), '([x_val] * batch_size)\n', (4097, 4119), True, 'import numpy as np\n'), ((4136, 4180), 'numpy.array', 'np.array', (['[4, 3, 4, 5, 2, 1]'], {'dtype': 'np.int32'}), '([4, 3, 4, 5, 2, 1], dtype=np.int32)\n', (4144, 4180), True, 'import numpy as np\n'), ((5395, 5425), 'numpy.stack', 'np.stack', (['([x_val] * batch_size)'], {}), '([x_val] * batch_size)\n', (5403, 5425), True, 'import numpy as np\n'), ((7263, 7301), 'numpy.stack', 'np.stack', (['([encoder_x_val] * batch_size)'], {}), '([encoder_x_val] * batch_size)\n', (7271, 7301), True, 'import numpy as np\n'), ((7439, 7477), 'numpy.stack', 'np.stack', (['([decoder_x_val] * batch_size)'], {}), '([decoder_x_val] * batch_size)\n', (7447, 7477), True, 'import numpy as np\n'), ((9579, 9617), 'numpy.stack', 'np.stack', (['([encoder_x_val] * batch_size)'], {}), '([encoder_x_val] * batch_size)\n', (9587, 9617), True, 'import numpy as np\n'), ((9755, 9793), 'numpy.stack', 'np.stack', (['([decoder_x_val] * batch_size)'], {}), '([decoder_x_val] * batch_size)\n', (9763, 9793), True, 'import numpy as np\n'), ((11894, 11932), 'numpy.stack', 'np.stack', (['([encoder_x_val] * batch_size)'], {}), '([encoder_x_val] * batch_size)\n', (11902, 11932), True, 'import numpy as np\n'), ((12070, 12108), 'numpy.stack', 'np.stack', (['([decoder_x_val] * batch_size)'], {}), '([decoder_x_val] * batch_size)\n', (12078, 12108), True, 'import numpy as np\n'), ((13787, 13863), 'numpy.array', 'np.array', (['[[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0]]'], {'dtype': 'np.float32'}), '([[1.0, 1.0], [2.0, 2.0], [3.0, 3.0], [4.0, 4.0]], dtype=np.float32)\n', (13795, 13863), True, 'import numpy as np\n'), ((13872, 13902), 'numpy.stack', 'np.stack', (['([x_val] * batch_size)'], {}), '([x_val] * batch_size)\n', (13880, 13902), True, 'import numpy as np\n'), ((15642, 15680), 'numpy.stack', 'np.stack', (['([encoder_x_val] * batch_size)'], {}), '([encoder_x_val] * batch_size)\n', (15650, 15680), True, 'import numpy as np\n'), ((19725, 19750), 'tensorflow.sigmoid', 'tf.sigmoid', (['(xw[0] + hu[0])'], {}), '(xw[0] + hu[0])\n', (19735, 19750), True, 'import tensorflow as tf\n'), ((19763, 19788), 'tensorflow.sigmoid', 'tf.sigmoid', (['(xw[1] + hu[1])'], {}), '(xw[1] + hu[1])\n', (19773, 19788), True, 'import tensorflow as tf\n'), ((5571, 5636), 'tensorflow.contrib.seq2seq.BahdanauAttention', 'tf.contrib.seq2seq.BahdanauAttention', (['attn_size', 'attention_states'], {}), '(attn_size, attention_states)\n', (5607, 5636), True, 'import tensorflow as tf\n'), ((5795, 5951), 'tensorflow.contrib.seq2seq.AttentionWrapper', 'tf.contrib.seq2seq.AttentionWrapper', (['cell', 'attention_mechanism'], {'attention_layer_size': 'attn_size', 'cell_input_fn': 'match_input_fn', 'output_attention': '(False)'}), '(cell, attention_mechanism,\n    attention_layer_size=attn_size, cell_input_fn=match_input_fn,\n    output_attention=False)\n', (5830, 5951), True, 'import tensorflow as tf\n'), ((7672, 7708), 'tensorflow.identity', 'tf.identity', (['output'], {'name': '"""output_0"""'}), "(output, name='output_0')\n", (7683, 7708), True, 'import tensorflow as tf\n'), ((7781, 7846), 'tensorflow.contrib.seq2seq.BahdanauAttention', 'tf.contrib.seq2seq.BahdanauAttention', (['attn_size', 'attention_states'], {}), '(attn_size, attention_states)\n', (7817, 7846), True, 'import tensorflow as tf\n'), ((8076, 8232), 'tensorflow.contrib.seq2seq.AttentionWrapper', 'tf.contrib.seq2seq.AttentionWrapper', (['cell', 'attention_mechanism'], {'attention_layer_size': 'attn_size', 'cell_input_fn': 'match_input_fn', 'output_attention': '(False)'}), '(cell, attention_mechanism,\n    attention_layer_size=attn_size, cell_input_fn=match_input_fn,\n    output_attention=False)\n', (8111, 8232), True, 'import tensorflow as tf\n'), ((9980, 10016), 'tensorflow.identity', 'tf.identity', (['output'], {'name': '"""output_0"""'}), "(output, name='output_0')\n", (9991, 10016), True, 'import tensorflow as tf\n'), ((10089, 10154), 'tensorflow.contrib.seq2seq.BahdanauAttention', 'tf.contrib.seq2seq.BahdanauAttention', (['attn_size', 'attention_states'], {}), '(attn_size, attention_states)\n', (10125, 10154), True, 'import tensorflow as tf\n'), ((10383, 10539), 'tensorflow.contrib.seq2seq.AttentionWrapper', 'tf.contrib.seq2seq.AttentionWrapper', (['cell', 'attention_mechanism'], {'attention_layer_size': 'attn_size', 'cell_input_fn': 'match_input_fn', 'output_attention': '(False)'}), '(cell, attention_mechanism,\n    attention_layer_size=attn_size, cell_input_fn=match_input_fn,\n    output_attention=False)\n', (10418, 10539), True, 'import tensorflow as tf\n'), ((12296, 12332), 'tensorflow.identity', 'tf.identity', (['output'], {'name': '"""output_0"""'}), "(output, name='output_0')\n", (12307, 12332), True, 'import tensorflow as tf\n'), ((12405, 12470), 'tensorflow.contrib.seq2seq.BahdanauAttention', 'tf.contrib.seq2seq.BahdanauAttention', (['attn_size', 'attention_states'], {}), '(attn_size, attention_states)\n', (12441, 12470), True, 'import tensorflow as tf\n'), ((12700, 12856), 'tensorflow.contrib.seq2seq.AttentionWrapper', 'tf.contrib.seq2seq.AttentionWrapper', (['cell', 'attention_mechanism'], {'attention_layer_size': 'attn_size', 'cell_input_fn': 'match_input_fn', 'output_attention': '(False)'}), '(cell, attention_mechanism,\n    attention_layer_size=attn_size, cell_input_fn=match_input_fn,\n    output_attention=False)\n', (12735, 12856), True, 'import tensorflow as tf\n'), ((13950, 13984), 'tensorflow.python.ops.init_ops.constant_initializer', 'init_ops.constant_initializer', (['(0.5)'], {}), '(0.5)\n', (13979, 13984), False, 'from tensorflow.python.ops import init_ops\n'), ((16058, 16123), 'tensorflow.contrib.seq2seq.BahdanauAttention', 'tf.contrib.seq2seq.BahdanauAttention', (['attn_size', 'attention_states'], {}), '(attn_size, attention_states)\n', (16094, 16123), True, 'import tensorflow as tf\n'), ((16353, 16509), 'tensorflow.contrib.seq2seq.AttentionWrapper', 'tf.contrib.seq2seq.AttentionWrapper', (['cell', 'attention_mechanism'], {'attention_layer_size': 'attn_size', 'cell_input_fn': 'match_input_fn', 'output_attention': '(False)'}), '(cell, attention_mechanism,\n    attention_layer_size=attn_size, cell_input_fn=match_input_fn,\n    output_attention=False)\n', (16388, 16509), True, 'import tensorflow as tf\n'), ((16786, 16942), 'tensorflow.contrib.seq2seq.AttentionWrapper', 'tf.contrib.seq2seq.AttentionWrapper', (['cell', 'attention_mechanism'], {'attention_layer_size': 'attn_size', 'cell_input_fn': 'match_input_fn', 'output_attention': '(False)'}), '(cell, attention_mechanism,\n    attention_layer_size=attn_size, cell_input_fn=match_input_fn,\n    output_attention=False)\n', (16821, 16942), True, 'import tensorflow as tf\n'), ((17698, 17752), 'tensorflow.concat', 'tf.concat', (['[match_output_fw, match_output_bk]'], {'axis': '(-1)'}), '([match_output_fw, match_output_bk], axis=-1)\n', (17707, 17752), True, 'import tensorflow as tf\n'), ((17781, 17850), 'tensorflow.concat', 'tf.concat', (['[match_state_fw.cell_state, match_state_bk.cell_state]', '(-1)'], {}), '([match_state_fw.cell_state, match_state_bk.cell_state], -1)\n', (17790, 17850), True, 'import tensorflow as tf\n'), ((18075, 18128), 'numpy.array', 'np.array', (['[6, 5, 4, 3, 2, 1, 2, 3, 6]'], {'dtype': 'np.int32'}), '([6, 5, 4, 3, 2, 1, 2, 3, 6], dtype=np.int32)\n', (18083, 18128), True, 'import numpy as np\n'), ((19686, 19705), 'tensorflow.matmul', 'tf.matmul', (['state', 'U'], {}), '(state, U)\n', (19695, 19705), True, 'import tensorflow as tf\n'), ((1798, 1828), 'tensorflow.identity', 'tf.identity', (['xs'], {'name': '"""output"""'}), "(xs, name='output')\n", (1809, 1828), True, 'import tensorflow as tf\n'), ((1830, 1864), 'tensorflow.identity', 'tf.identity', (['s'], {'name': '"""final_state"""'}), "(s, name='final_state')\n", (1841, 1864), True, 'import tensorflow as tf\n'), ((2583, 2613), 'tensorflow.identity', 'tf.identity', (['xs'], {'name': '"""output"""'}), "(xs, name='output')\n", (2594, 2613), True, 'import tensorflow as tf\n'), ((2615, 2649), 'tensorflow.identity', 'tf.identity', (['s'], {'name': '"""final_state"""'}), "(s, name='final_state')\n", (2626, 2649), True, 'import tensorflow as tf\n'), ((3475, 3510), 'tensorflow.identity', 'tf.identity', (['outputs'], {'name': '"""output"""'}), "(outputs, name='output')\n", (3486, 3510), True, 'import tensorflow as tf\n'), ((3512, 3554), 'tensorflow.identity', 'tf.identity', (['cell_state'], {'name': '"""cell_state"""'}), "(cell_state, name='cell_state')\n", (3523, 3554), True, 'import tensorflow as tf\n'), ((4474, 4509), 'tensorflow.identity', 'tf.identity', (['outputs'], {'name': '"""output"""'}), "(outputs, name='output')\n", (4485, 4509), True, 'import tensorflow as tf\n'), ((4511, 4553), 'tensorflow.identity', 'tf.identity', (['cell_state'], {'name': '"""cell_state"""'}), "(cell_state, name='cell_state')\n", (4522, 4553), True, 'import tensorflow as tf\n'), ((5320, 5366), 'numpy.random.randn', 'np.random.randn', (['decoder_time_step', 'input_size'], {}), '(decoder_time_step, input_size)\n', (5335, 5366), True, 'import numpy as np\n'), ((5454, 5503), 'numpy.random.randn', 'np.random.randn', (['batch_size', 'time_step', 'attn_size'], {}), '(batch_size, time_step, attn_size)\n', (5469, 5503), True, 'import numpy as np\n'), ((5693, 5732), 'tensorflow.concat', 'tf.concat', (['[curr_input, state]'], {'axis': '(-1)'}), '([curr_input, state], axis=-1)\n', (5702, 5732), True, 'import tensorflow as tf\n'), ((6301, 6335), 'tensorflow.identity', 'tf.identity', (['output'], {'name': '"""output"""'}), "(output, name='output')\n", (6312, 6335), True, 'import tensorflow as tf\n'), ((6337, 6391), 'tensorflow.identity', 'tf.identity', (['attr_state.cell_state'], {'name': '"""final_state"""'}), "(attr_state.cell_state, name='final_state')\n", (6348, 6391), True, 'import tensorflow as tf\n'), ((7180, 7226), 'numpy.random.randn', 'np.random.randn', (['encoder_time_step', 'input_size'], {}), '(encoder_time_step, input_size)\n', (7195, 7226), True, 'import numpy as np\n'), ((7356, 7402), 'numpy.random.randn', 'np.random.randn', (['decoder_time_step', 'input_size'], {}), '(decoder_time_step, input_size)\n', (7371, 7402), True, 'import numpy as np\n'), ((7974, 8013), 'tensorflow.concat', 'tf.concat', (['[curr_input, state]'], {'axis': '(-1)'}), '([curr_input, state], axis=-1)\n', (7983, 8013), True, 'import tensorflow as tf\n'), ((8601, 8635), 'tensorflow.identity', 'tf.identity', (['output'], {'name': '"""output"""'}), "(output, name='output')\n", (8612, 8635), True, 'import tensorflow as tf\n'), ((8637, 8691), 'tensorflow.identity', 'tf.identity', (['attr_state.cell_state'], {'name': '"""final_state"""'}), "(attr_state.cell_state, name='final_state')\n", (8648, 8691), True, 'import tensorflow as tf\n'), ((9496, 9542), 'numpy.random.randn', 'np.random.randn', (['encoder_time_step', 'input_size'], {}), '(encoder_time_step, input_size)\n', (9511, 9542), True, 'import numpy as np\n'), ((9672, 9718), 'numpy.random.randn', 'np.random.randn', (['decoder_time_step', 'input_size'], {}), '(decoder_time_step, input_size)\n', (9687, 9718), True, 'import numpy as np\n'), ((10282, 10321), 'tensorflow.concat', 'tf.concat', (['[curr_input, state]'], {'axis': '(-1)'}), '([curr_input, state], axis=-1)\n', (10291, 10321), True, 'import tensorflow as tf\n'), ((10896, 10930), 'tensorflow.identity', 'tf.identity', (['output'], {'name': '"""output"""'}), "(output, name='output')\n", (10907, 10930), True, 'import tensorflow as tf\n'), ((10932, 10986), 'tensorflow.identity', 'tf.identity', (['attr_state.cell_state'], {'name': '"""final_state"""'}), "(attr_state.cell_state, name='final_state')\n", (10943, 10986), True, 'import tensorflow as tf\n'), ((11811, 11857), 'numpy.random.randn', 'np.random.randn', (['encoder_time_step', 'input_size'], {}), '(encoder_time_step, input_size)\n', (11826, 11857), True, 'import numpy as np\n'), ((11987, 12033), 'numpy.random.randn', 'np.random.randn', (['decoder_time_step', 'input_size'], {}), '(decoder_time_step, input_size)\n', (12002, 12033), True, 'import numpy as np\n'), ((12598, 12637), 'tensorflow.concat', 'tf.concat', (['[curr_input, state]'], {'axis': '(-1)'}), '([curr_input, state], axis=-1)\n', (12607, 12637), True, 'import tensorflow as tf\n'), ((13215, 13249), 'tensorflow.identity', 'tf.identity', (['output'], {'name': '"""output"""'}), "(output, name='output')\n", (13226, 13249), True, 'import tensorflow as tf\n'), ((13251, 13305), 'tensorflow.identity', 'tf.identity', (['attr_state.cell_state'], {'name': '"""final_state"""'}), "(attr_state.cell_state, name='final_state')\n", (13262, 13305), True, 'import tensorflow as tf\n'), ((14631, 14666), 'tensorflow.identity', 'tf.identity', (['outputs'], {'name': '"""output"""'}), "(outputs, name='output')\n", (14642, 14666), True, 'import tensorflow as tf\n'), ((14668, 14710), 'tensorflow.identity', 'tf.identity', (['cell_state'], {'name': '"""cell_state"""'}), "(cell_state, name='cell_state')\n", (14679, 14710), True, 'import tensorflow as tf\n'), ((15559, 15605), 'numpy.random.randn', 'np.random.randn', (['encoder_time_step', 'input_size'], {}), '(encoder_time_step, input_size)\n', (15574, 15605), True, 'import numpy as np\n'), ((15735, 15793), 'numpy.random.randn', 'np.random.randn', (['decoder_time_step', 'batch_size', 'input_size'], {}), '(decoder_time_step, batch_size, input_size)\n', (15750, 15793), True, 'import numpy as np\n'), ((16251, 16290), 'tensorflow.concat', 'tf.concat', (['[curr_input, state]'], {'axis': '(-1)'}), '([curr_input, state], axis=-1)\n', (16260, 16290), True, 'import tensorflow as tf\n'), ((17870, 17914), 'tensorflow.identity', 'tf.identity', (['matched_output'], {'name': '"""output_0"""'}), "(matched_output, name='output_0')\n", (17881, 17914), True, 'import tensorflow as tf\n'), ((17916, 17962), 'tensorflow.identity', 'tf.identity', (['matched_state'], {'name': '"""final_state"""'}), "(matched_state, name='final_state')\n", (17927, 17962), True, 'import tensorflow as tf\n'), ((19238, 19271), 'numpy.arange', 'np.arange', (['(30.0)'], {'dtype': 'np.float32'}), '(30.0, dtype=np.float32)\n', (19247, 19271), True, 'import numpy as np\n'), ((19405, 19438), 'numpy.arange', 'np.arange', (['(75.0)'], {'dtype': 'np.float32'}), '(75.0, dtype=np.float32)\n', (19414, 19438), True, 'import numpy as np\n'), ((19558, 19591), 'numpy.arange', 'np.arange', (['(15.0)'], {'dtype': 'np.float32'}), '(15.0, dtype=np.float32)\n', (19567, 19591), True, 'import numpy as np\n'), ((19632, 19652), 'tensorflow.matmul', 'tf.matmul', (['inputs', 'W'], {}), '(inputs, W)\n', (19641, 19652), True, 'import tensorflow as tf\n'), ((4430, 4453), 'tensorflow.identity', 'tf.identity', (['seq_length'], {}), '(seq_length)\n', (4441, 4453), True, 'import tensorflow as tf\n'), ((17524, 17547), 'tensorflow.identity', 'tf.identity', (['seq_length'], {}), '(seq_length)\n', (17535, 17547), True, 'import tensorflow as tf\n')]