code/rnet_ptrnet_modules.py
def build_graph(self, values, values_mask, keys):
        """
        Keys attend to values.
        For each key, return an attention distribution and an attention output vector.

        Inputs:
          values: Tensor shape (batch_size, num_values, value_vec_size).
          values_mask: Tensor shape (batch_size, num_values).
            1s where there's real input, 0s where there's padding
          keys: Tensor shape (batch_size, num_keys, key_vec_size)

        Outputs:
          attn_dist: Tensor shape (batch_size, num_keys, num_values).
            For each key, the distribution should sum to 1,
            and should be 0 in the value locations that correspond to padding.
          output: Tensor shape (batch_size, num_keys, hidden_size).
            This is the attention output; the weighted sum of the values
            (using the attention distribution as weights).
        """
        with vs.variable_scope("MultiplicativeAttn"):
            keys_shape = keys.get_shape().as_list() # (batch_size, num_keys, key_vec_size)
            values_shape = values.get_shape().as_list() # (batch_size, num_values, value_vec_size)

            # Calculate attention distribution
            W = tf.get_variable('W_mul_attn', shape=(self.key_vec_size, self.value_vec_size),
                                initializer=tf.contrib.layers.xavier_initializer())
            keys_r = tf.reshape(keys, [-1, keys_shape[2]]) # (batch_size * num_keys, key_vec_size)
            attn_logits = tf.matmul(keys_r, W) # (batch_size * num_keys, value_vec_size)
            attn_logits = tf.reshape(attn_logits, [-1, keys_shape[1], values_shape[2]]) # (batch_size, num_keys, value_vec_size)
            values_t = tf.transpose(values, perm=[0, 2, 1]) # (batch_size, value_vec_size, num_values)
            attn_logits = tf.matmul(attn_logits, values_t) # (batch_size, num_keys, num_values)
            attn_logits_mask = tf.expand_dims(values_mask, 1) # shape (batch_size, 1, num_values)
            attn_masked_logits, attn_prob_dist = masked_softmax(attn_logits, attn_logits_mask, 2) # shape (batch_size, num_keys, num_values). take softmax over values

            # Use attention distribution to take weighted sum of values
            output = tf.matmul(attn_prob_dist, values) # shape (batch_size, num_keys, value_vec_size)

            # Apply dropout
            output = tf.nn.dropout(output, self.keep_prob)

            return attn_masked_logits, attn_prob_dist, output

        Keys attend to values.
        For each key, return an attention distribution and an attention output vector.

        Inputs:
          values: Tensor shape (batch_size, num_values, value_vec_size).
          values_mask: Tensor shape (batch_size, num_values).
            1s where there's real input, 0s where there's padding
          keys: Tensor shape (batch_size, num_keys, key_vec_size)

        Outputs:
          attn_dist: Tensor shape (batch_size, num_keys, num_values).
            For each key, the distribution should sum to 1,
            and should be 0 in the value locations that correspond to padding.
          output: Tensor shape (batch_size, num_keys, hidden_size).
            This is the attention output; the weighted sum of the values
            (using the attention distribution as weights).
        
Keys attend to values.
For each key, return an attention distribution and an attention output vector.




Keys attend to values . For each key return an attention distribution and an attention output vector .
def build_graph(self, values, values_mask, keys):
        with vs.variable_scope("MultiplicativeAttn"):
            keys_shape = keys.get_shape().as_list() 
            values_shape = values.get_shape().as_list() 
            W = tf.get_variable('W_mul_attn', shape=(self.key_vec_size, self.value_vec_size),
                                initializer=tf.contrib.layers.xavier_initializer())
            keys_r = tf.reshape(keys, [-1, keys_shape[2]]) 
            attn_logits = tf.matmul(keys_r, W) 
            attn_logits = tf.reshape(attn_logits, [-1, keys_shape[1], values_shape[2]]) 
            values_t = tf.transpose(values, perm=[0, 2, 1]) 
            attn_logits = tf.matmul(attn_logits, values_t) 
            attn_logits_mask = tf.expand_dims(values_mask, 1) 
            attn_masked_logits, attn_prob_dist = masked_softmax(attn_logits, attn_logits_mask, 2) 
            output = tf.matmul(attn_prob_dist, values) 
            output = tf.nn.dropout(output, self.keep_prob)
            return attn_masked_logits, attn_prob_dist, output
def build_graph ( self , values , values_mask , keys ) : with vs . variable_scope ( "MultiplicativeAttn" ) : keys_shape = keys . get_shape ( ) . as_list ( ) values_shape = values . get_shape ( ) . as_list ( ) W = tf . get_variable ( 'W_mul_attn' , shape = ( self . key_vec_size , self . value_vec_size ) , initializer = tf . contrib . layers . xavier_initializer ( ) ) keys_r = tf . reshape ( keys , [ - 1 , keys_shape [ 2 ] ] ) attn_logits = tf . matmul ( keys_r , W ) attn_logits = tf . reshape ( attn_logits , [ - 1 , keys_shape [ 1 ] , values_shape [ 2 ] ] ) values_t = tf . transpose ( values , perm = [ 0 , 2 , 1 ] ) attn_logits = tf . matmul ( attn_logits , values_t ) attn_logits_mask = tf . expand_dims ( values_mask , 1 ) attn_masked_logits , attn_prob_dist = masked_softmax ( attn_logits , attn_logits_mask , 2 ) output = tf . matmul ( attn_prob_dist , values ) output = tf . nn . dropout ( output , self . keep_prob ) return attn_masked_logits , attn_prob_dist , output
Keys attend to values.
Keys attend to values .
"""
        Keys attend to values.
        For each key, return an attention distribution and an attention output vector.

        Inputs:
          values: Tensor shape (batch_size, num_values, value_vec_size).
          values_mask: Tensor shape (batch_size, num_values).
            1s where there's real input, 0s where there's padding
          keys: Tensor shape (batch_size, num_keys, key_vec_size)

        Outputs:
          attn_dist: Tensor shape (batch_size, num_keys, num_values).
            For each key, the distribution should sum to 1,
            and should be 0 in the value locations that correspond to padding.
          output: Tensor shape (batch_size, num_keys, hidden_size).
            This is the attention output; the weighted sum of the values
            (using the attention distribution as weights).
        """ # (batch_size, num_keys, key_vec_size) # (batch_size, num_values, value_vec_size) # Calculate attention distribution # (batch_size * num_keys, key_vec_size) # (batch_size * num_keys, value_vec_size) # (batch_size, num_keys, value_vec_size) # (batch_size, value_vec_size, num_values) # (batch_size, num_keys, num_values) # shape (batch_size, 1, num_values) # shape (batch_size, num_keys, num_values). take softmax over values # Use attention distribution to take weighted sum of values # shape (batch_size, num_keys, value_vec_size) # Apply dropout
self
values
values_mask
keys