import tensorflow as tf


    if decoder.cell_type.lower() == 'lstm' and decoder.use_lstm_full_state:
        initial_output = initial_state
    else:
        # Last layer's state is the right-most part. Output is the left-most part of an LSTM's state.
        initial_output = initial_state[:, -cell_output_size:]

    time = tf.constant(0, dtype=tf.int32, name='time')
    outputs = tf.TensorArray(dtype=tf.float32, size=time_steps)
    samples = tf.TensorArray(dtype=tf.int64, size=time_steps)
    inputs = tf.TensorArray(dtype=tf.int64, size=time_steps).unstack(tf.to_int64(tf.transpose(decoder_inputs)))

    states = tf.TensorArray(dtype=tf.float32, size=time_steps)
    weights = tf.TensorArray(dtype=tf.float32, size=time_steps)
    attns = tf.TensorArray(dtype=tf.float32, size=time_steps)

    initial_symbol = inputs.read(0)  # first symbol is BOS
    initial_input = embed(initial_symbol)
    initial_pos = tf.zeros([batch_size], tf.float32)
    initial_weights = tf.zeros(tf.shape(attention_states[align_encoder_id])[:2])
    zero_context = tf.zeros(shape=tf.shape(attention_states[align_encoder_id][:,0]))  # FIXME

    with tf.variable_scope('decoder_{}'.format(decoder.name)):
        initial_context, _ = look(0, initial_output, initial_input, pos=initial_pos, prev_weights=initial_weights,
                                  context=zero_context)
    initial_data = tf.concat([initial_state, initial_context, tf.expand_dims(initial_pos, axis=1), initial_weights],
