import tensorflow as tf

                # this is useful to reduce the number of parameters, and
                # to use the output embeddings for output projection (tie_embeddings parameter)
                output_ = dense(output_, decoder.embedding_size, use_bias=False, name='softmax0')

        if decoder.tie_embeddings and (decoder.pred_embed_proj or decoder.pred_deep_layer):
            bias = get_variable('softmax1/bias', shape=[decoder.vocab_size])
            output_ = tf.matmul(output_, tf.transpose(embedding)) + bias
        else:
            output_ = dense(output_, decoder.vocab_size, use_bias=True, name='softmax1')
        return output_

    if decoder.use_dropout:   # FIXME: why no pervasive dropout here?
        initial_state = tf.nn.dropout(initial_state, keep_prob=decoder.initial_state_keep_prob)

    with tf.variable_scope(scope_name):
        activation_fn = None if decoder.initial_state == 'linear' else tf.nn.tanh
        if decoder.initial_state == 'trained':
            initial_state = get_variable(shape=[cell_state_size], name='initial_state')
            initial_state = tf.tile(tf.expand_dims(initial_state, axis=0), [batch_size, 1])
        elif decoder.initial_state == 'zero':
            initial_state = tf.zeros(shape=[batch_size, cell_state_size])
        elif decoder.layer_norm:
            initial_state = dense(initial_state, cell_state_size, use_bias=False, name='initial_state_projection')
            initial_state = tf.contrib.layers.layer_norm(initial_state, activation_fn=activation_fn,
                                                         scope='initial_state_layer_norm')
        else:
            initial_state = dense(initial_state, cell_state_size, use_bias=True, name='initial_state_projection',
                                  activation=activation_fn)
