def additive_self_attention(units, n_hidden=None, n_output_features=None, activation=None):
    """ Computes additive self attention for time series of vectors (with batch dimension)
        the formula: score(h_i, h_j) = <v, tanh(W_1 h_i + W_2 h_j)>
        v is a learnable vector of n_hidden dimensionality,
        W_1 and W_2 are learnable [n_hidden, n_input_features] matrices

    Args:
        units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
        n_hidden: number of units in hidden representation of similarity measure
        n_output_features: number of features in output dense layer
        activation: activation at the output

    Returns:
        output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]
    """
    n_input_features = units.get_shape().as_list()[2]
    if n_hidden is None:
        n_hidden = n_input_features
    if n_output_features is None:
        n_output_features = n_input_features
    units_pairs = tf.concat([expand_tile(units, 1), expand_tile(units, 2)], 3)
    query = tf.layers.dense(units_pairs, n_hidden, activation=tf.tanh, kernel_initializer=INITIALIZER())
    attention = tf.nn.softmax(tf.layers.dense(query, 1), dim=2)
    attended_units = tf.reduce_sum(attention * expand_tile(units, 1), axis=2)
    output = tf.layers.dense(attended_units, n_output_features, activation, kernel_initializer=INITIALIZER())
    return output
def additive_self_attention(units, n_hidden=None, n_output_features=None, activation=None):
    """ Computes additive self attention for time series of vectors (with batch dimension)
        the formula: score(h_i, h_j) = <v, tanh(W_1 h_i + W_2 h_j)>
        v is a learnable vector of n_hidden dimensionality,
        W_1 and W_2 are learnable [n_hidden, n_input_features] matrices

    Args:
        units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
        n_hidden: number of units in hidden representation of similarity measure
        n_output_features: number of features in output dense layer
        activation: activation at the output

    Returns:
        output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]
    """
    n_input_features = units.get_shape().as_list()[2]
    if n_hidden is None:
        n_hidden = n_input_features
    if n_output_features is None:
        n_output_features = n_input_features
    units_pairs = tf.concat([expand_tile(units, 1), expand_tile(units, 2)], 3)
    query = tf.layers.dense(units_pairs, n_hidden, activation=tf.tanh, kernel_initializer=INITIALIZER())
    attention = tf.nn.softmax(tf.layers.dense(query, 1), dim=2)
    attended_units = tf.reduce_sum(attention * expand_tile(units, 1), axis=2)
    output = tf.layers.dense(attended_units, n_output_features, activation, kernel_initializer=INITIALIZER())
    return output
def additive_self_attention ( units , n_hidden = None , n_output_features = None , activation = None ) : n_input_features = units . get_shape ( ) . as_list ( ) [ 2 ] if n_hidden is None : n_hidden = n_input_features if n_output_features is None : n_output_features = n_input_features units_pairs = tf . concat ( [ expand_tile ( units , 1 ) , expand_tile ( units , 2 ) ] , 3 ) query = tf . layers . dense ( units_pairs , n_hidden , activation = tf . tanh , kernel_initializer = INITIALIZER ( ) ) attention = tf . nn . softmax ( tf . layers . dense ( query , 1 ) , dim = 2 ) attended_units = tf . reduce_sum ( attention * expand_tile ( units , 1 ) , axis = 2 ) output = tf . layers . dense ( attended_units , n_output_features , activation , kernel_initializer = INITIALIZER ( ) ) return output
Computes additive self attention for time series of vectors (with batch dimension)
        the formula: score(h_i, h_j) = <v, tanh(W_1 h_i + W_2 h_j)>
        v is a learnable vector of n_hidden dimensionality,
        W_1 and W_2 are learnable [n_hidden, n_input_features] matrices

    Args:
        units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
        n_hidden: number of units in hidden representation of similarity measure
        n_output_features: number of features in output dense layer
        activation: activation at the output

    Returns:
        output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]
Computes additive self attention for time series of vectors ( with batch dimension ) the formula : score ( h_i h_j ) = <v tanh ( W_1 h_i + W_2 h_j ) > v is a learnable vector of n_hidden dimensionality W_1 and W_2 are learnable [ n_hidden n_input_features ] matrices