def cs_bahdanau_attention(key, context, hidden_size, depth, projected_align=False):
    """ It is a implementation of the Bahdanau et al. attention mechanism. Based on the papers:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        depth: Number of csoftmax usages
        projected_align: Using bidirectional lstm for hidden representation of context.
        If true, beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [hidden_size].
        If false, bidirectional lstm is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, depth * hidden_size]
    """
    if hidden_size % 2 != 0:
        raise ValueError("hidden size must be dividable by two")
    batch_size = tf.shape(context)[0]
    max_num_tokens, token_size = context.get_shape().as_list()[-2:]

    r_context = tf.reshape(context, shape=[-1, max_num_tokens, token_size])
    # projected context: [None, max_num_tokens, token_size]
    projected_context = tf.layers.dense(r_context, token_size,
                                        kernel_initializer=xav(),
                                        name='projected_context')

    # projected_key: [None, None, hidden_size]
    projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav(),
                                    name='projected_key')
    r_projected_key = \
        tf.tile(tf.reshape(projected_key, shape=[-1, 1, hidden_size]),
                [1, max_num_tokens, 1])

    lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2)
    lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2)
    (output_fw, output_bw), states = \
        tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,
                                        cell_bw=lstm_bw_cell,
                                        inputs=projected_context,
                                        dtype=tf.float32)

    # bilstm_output: [-1, max_num_tokens, hidden_size]
    bilstm_output = tf.concat([output_fw, output_bw], -1)
    concat_h_state = tf.concat([r_projected_key, output_fw, output_bw], -1)

    if projected_align:
        log.info("Using projected attention alignment")
        h_state_for_attn_alignment = bilstm_output
        aligned_h_state = csoftmax_attention.attention_bah_block(
            concat_h_state, h_state_for_attn_alignment, depth)
        output = \
            tf.reshape(aligned_h_state, shape=[batch_size, -1, depth * hidden_size])
    else:
        log.info("Using without projected attention alignment")
        h_state_for_attn_alignment = projected_context
        aligned_h_state = csoftmax_attention.attention_bah_block(
            concat_h_state, h_state_for_attn_alignment, depth)
        output = \
            tf.reshape(aligned_h_state, shape=[batch_size, -1, depth * token_size])
    return output
def cs_bahdanau_attention(key, context, hidden_size, depth, projected_align=False):
    """ It is a implementation of the Bahdanau et al. attention mechanism. Based on the papers:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        depth: Number of csoftmax usages
        projected_align: Using bidirectional lstm for hidden representation of context.
        If true, beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [hidden_size].
        If false, bidirectional lstm is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, depth * hidden_size]
    """
    if hidden_size % 2 != 0:
        raise ValueError("hidden size must be dividable by two")
    batch_size = tf.shape(context)[0]
    max_num_tokens, token_size = context.get_shape().as_list()[-2:]

    r_context = tf.reshape(context, shape=[-1, max_num_tokens, token_size])
    # projected context: [None, max_num_tokens, token_size]
    projected_context = tf.layers.dense(r_context, token_size,
                                        kernel_initializer=xav(),
                                        name='projected_context')

    # projected_key: [None, None, hidden_size]
    projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav(),
                                    name='projected_key')
    r_projected_key = \
        tf.tile(tf.reshape(projected_key, shape=[-1, 1, hidden_size]),
                [1, max_num_tokens, 1])

    lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2)
    lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//2)
    (output_fw, output_bw), states = \
        tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,
                                        cell_bw=lstm_bw_cell,
                                        inputs=projected_context,
                                        dtype=tf.float32)

    # bilstm_output: [-1, max_num_tokens, hidden_size]
    bilstm_output = tf.concat([output_fw, output_bw], -1)
    concat_h_state = tf.concat([r_projected_key, output_fw, output_bw], -1)

    if projected_align:
        log.info("Using projected attention alignment")
        h_state_for_attn_alignment = bilstm_output
        aligned_h_state = csoftmax_attention.attention_bah_block(
            concat_h_state, h_state_for_attn_alignment, depth)
        output = \
            tf.reshape(aligned_h_state, shape=[batch_size, -1, depth * hidden_size])
    else:
        log.info("Using without projected attention alignment")
        h_state_for_attn_alignment = projected_context
        aligned_h_state = csoftmax_attention.attention_bah_block(
            concat_h_state, h_state_for_attn_alignment, depth)
        output = \
            tf.reshape(aligned_h_state, shape=[batch_size, -1, depth * token_size])
    return output
def cs_bahdanau_attention ( key , context , hidden_size , depth , projected_align = False ) : if hidden_size % 2 != 0 : raise ValueError ( "hidden size must be dividable by two" ) batch_size = tf . shape ( context ) [ 0 ] max_num_tokens , token_size = context . get_shape ( ) . as_list ( ) [ - 2 : ] r_context = tf . reshape ( context , shape = [ - 1 , max_num_tokens , token_size ] ) # projected context: [None, max_num_tokens, token_size] projected_context = tf . layers . dense ( r_context , token_size , kernel_initializer = xav ( ) , name = 'projected_context' ) # projected_key: [None, None, hidden_size] projected_key = tf . layers . dense ( key , hidden_size , kernel_initializer = xav ( ) , name = 'projected_key' ) r_projected_key = tf . tile ( tf . reshape ( projected_key , shape = [ - 1 , 1 , hidden_size ] ) , [ 1 , max_num_tokens , 1 ] ) lstm_fw_cell = tf . nn . rnn_cell . LSTMCell ( hidden_size // 2 ) lstm_bw_cell = tf . nn . rnn_cell . LSTMCell ( hidden_size // 2 ) ( output_fw , output_bw ) , states = tf . nn . bidirectional_dynamic_rnn ( cell_fw = lstm_fw_cell , cell_bw = lstm_bw_cell , inputs = projected_context , dtype = tf . float32 ) # bilstm_output: [-1, max_num_tokens, hidden_size] bilstm_output = tf . concat ( [ output_fw , output_bw ] , - 1 ) concat_h_state = tf . concat ( [ r_projected_key , output_fw , output_bw ] , - 1 ) if projected_align : log . info ( "Using projected attention alignment" ) h_state_for_attn_alignment = bilstm_output aligned_h_state = csoftmax_attention . attention_bah_block ( concat_h_state , h_state_for_attn_alignment , depth ) output = tf . reshape ( aligned_h_state , shape = [ batch_size , - 1 , depth * hidden_size ] ) else : log . info ( "Using without projected attention alignment" ) h_state_for_attn_alignment = projected_context aligned_h_state = csoftmax_attention . attention_bah_block ( concat_h_state , h_state_for_attn_alignment , depth ) output = tf . reshape ( aligned_h_state , shape = [ batch_size , - 1 , depth * token_size ] ) return output
It is a implementation of the Bahdanau et al. attention mechanism. Based on the papers:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        depth: Number of csoftmax usages
        projected_align: Using bidirectional lstm for hidden representation of context.
        If true, beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [hidden_size].
        If false, bidirectional lstm is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, depth * hidden_size]
It is a implementation of the Bahdanau et al . attention mechanism . Based on the papers : https : // arxiv . org / abs / 1409 . 0473 Neural Machine Translation by Jointly Learning to Align and Translate https : // andre - martins . github . io / docs / emnlp2017_final . pdf Learning What s Easy : Fully Differentiable Neural Easy - First Taggers Args : key : A tensorflow tensor with dimensionality [ None None key_size ] context : A tensorflow tensor with dimensionality [ None None max_num_tokens token_size ] hidden_size : Number of units in hidden representation depth : Number of csoftmax usages projected_align : Using bidirectional lstm for hidden representation of context . If true beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [ hidden_size ] . If false bidirectional lstm is not used . Returns : output : Tensor at the output with dimensionality [ None None depth * hidden_size ]