code/char_model.py
def build_graph(self):
        """Builds the main part of the graph for the model, starting from the input embeddings to the final distributions for the answer span.

        Defines:
          self.logits_start, self.logits_end: Both tensors shape (batch_size, context_len).
            These are the logits (i.e. values that are fed into the softmax function) for the start and end distribution.
            Important: these are -large in the pad locations. Necessary for when we feed into the cross entropy function.
          self.probdist_start, self.probdist_end: Both shape (batch_size, context_len). Each row sums to 1.
            These are the result of taking (masked) softmax of logits_start and logits_end.
        """

        # Use a RNN to get hidden states for the context and the question
        # Note: here the RNNEncoder is shared (i.e. the weights are the same)
        # between the context and the question.
        context_input_lens = tf.reshape(tf.reduce_sum(tf.cast(tf.cast(self.context_char_ids, tf.bool), tf.int32), axis=2), [-1])
        qn_input_lens = tf.reshape(tf.reduce_sum(tf.cast(tf.cast(self.qn_char_ids, tf.bool), tf.int32), axis=2), [-1])
        cell_fw = rnn_cell.GRUCell(self.FLAGS.hidden_size)
        cell_bw = rnn_cell.GRUCell(self.FLAGS.hidden_size)
        _, (state_fw, state_bw) = tf.nn.bidirectional_dynamic_rnn(
            cell_fw, cell_bw, self.context_char_embs, context_input_lens, dtype=tf.float32)
        ch_emb = tf.reshape(tf.concat([state_fw, state_bw], axis=1), [-1, self.FLAGS.context_len, 2 * self.FLAGS.hidden_size])
        self.context_embs = tf.concat([self.context_embs, ch_emb], axis=2)

        _, (state_fw, state_bw) = tf.nn.bidirectional_dynamic_rnn(
            cell_fw, cell_bw, self.qn_char_embs, qn_input_lens, dtype=tf.float32)
        qh_emb = tf.reshape(tf.concat([state_fw, state_bw], axis=1), [-1, self.FLAGS.question_len, 2 * self.FLAGS.hidden_size])
        self.qn_embs = tf.concat([self.qn_embs, qh_emb], axis=2)


        # ToDo Deep encoder
        encoder = RNNEncoder(self.FLAGS.hidden_size, self.keep_prob)
        context_hiddens = encoder.build_graph(self.context_embs, self.context_mask) # (batch_size, context_len, hidden_size*2)
        question_hiddens = encoder.build_graph(self.qn_embs, self.qn_mask) # (batch_size, question_len, hidden_size*2)

        # Use context hidden states to attend to question hidden states
        attn_layer = BasicAttn(self.keep_prob, self.FLAGS.hidden_size*2, self.FLAGS.hidden_size*2)
        _, attn_output = attn_layer.build_graph(question_hiddens, self.qn_mask, context_hiddens) # attn_output is shape (batch_size, context_len, hidden_size*2)

        # Concat attn_output to context_hiddens to get blended_reps
        blended_reps = tf.concat([context_hiddens, attn_output], axis=2) # (batch_size, context_len, hidden_size*4)

        # Apply fully connected layer to each blended representation
        # Note, blended_reps_final corresponds to b' in the handout
        # Note, tf.contrib.layers.fully_connected applies a ReLU non-linarity here by default
        blended_reps_final = tf.contrib.layers.fully_connected(blended_reps, num_outputs=self.FLAGS.hidden_size) # blended_reps_final is shape (batch_size, context_len, hidden_size)

        # Use softmax layer to compute probability distribution for start location
        # Note this produces self.logits_start and self.probdist_start, both of which have shape (batch_size, context_len)
        with vs.variable_scope("StartDist"):
            softmax_layer_start = SimpleSoftmaxLayer()
            self.logits_start, self.probdist_start = softmax_layer_start.build_graph(blended_reps_final, self.context_mask)

        # Use softmax layer to compute probability distribution for end location
        # Note this produces self.logits_end and self.probdist_end, both of which have shape (batch_size, context_len)
        with vs.variable_scope("EndDist"):
            softmax_layer_end = SimpleSoftmaxLayer()
            self.logits_end, self.probdist_end = softmax_layer_end.build_graph(blended_reps_final, self.context_mask)
Builds the main part of the graph for the model, starting from the input embeddings to the final distributions for the answer span.

        Defines:
          self.logits_start, self.logits_end: Both tensors shape (batch_size, context_len).
            These are the logits (i.e. values that are fed into the softmax function) for the start and end distribution.
            Important: these are -large in the pad locations. Necessary for when we feed into the cross entropy function.
          self.probdist_start, self.probdist_end: Both shape (batch_size, context_len). Each row sums to 1.
            These are the result of taking (masked) softmax of logits_start and logits_end.
        
Builds the main part of the graph for the model, starting from the input embeddings to the final distributions for the answer span.
Builds the main part of the graph for the model starting from the input embeddings to the final distributions for the answer span .
def build_graph(self):
        context_input_lens = tf.reshape(tf.reduce_sum(tf.cast(tf.cast(self.context_char_ids, tf.bool), tf.int32), axis=2), [-1])
        qn_input_lens = tf.reshape(tf.reduce_sum(tf.cast(tf.cast(self.qn_char_ids, tf.bool), tf.int32), axis=2), [-1])
        cell_fw = rnn_cell.GRUCell(self.FLAGS.hidden_size)
        cell_bw = rnn_cell.GRUCell(self.FLAGS.hidden_size)
        _, (state_fw, state_bw) = tf.nn.bidirectional_dynamic_rnn(
            cell_fw, cell_bw, self.context_char_embs, context_input_lens, dtype=tf.float32)
        ch_emb = tf.reshape(tf.concat([state_fw, state_bw], axis=1), [-1, self.FLAGS.context_len, 2 * self.FLAGS.hidden_size])
        self.context_embs = tf.concat([self.context_embs, ch_emb], axis=2)
        _, (state_fw, state_bw) = tf.nn.bidirectional_dynamic_rnn(
            cell_fw, cell_bw, self.qn_char_embs, qn_input_lens, dtype=tf.float32)
        qh_emb = tf.reshape(tf.concat([state_fw, state_bw], axis=1), [-1, self.FLAGS.question_len, 2 * self.FLAGS.hidden_size])
        self.qn_embs = tf.concat([self.qn_embs, qh_emb], axis=2)
        encoder = RNNEncoder(self.FLAGS.hidden_size, self.keep_prob)
        context_hiddens = encoder.build_graph(self.context_embs, self.context_mask) 
        question_hiddens = encoder.build_graph(self.qn_embs, self.qn_mask) 
        attn_layer = BasicAttn(self.keep_prob, self.FLAGS.hidden_size*2, self.FLAGS.hidden_size*2)
        _, attn_output = attn_layer.build_graph(question_hiddens, self.qn_mask, context_hiddens) 
        blended_reps = tf.concat([context_hiddens, attn_output], axis=2) 
        blended_reps_final = tf.contrib.layers.fully_connected(blended_reps, num_outputs=self.FLAGS.hidden_size) 
        with vs.variable_scope("StartDist"):
            softmax_layer_start = SimpleSoftmaxLayer()
            self.logits_start, self.probdist_start = softmax_layer_start.build_graph(blended_reps_final, self.context_mask)
        with vs.variable_scope("EndDist"):
            softmax_layer_end = SimpleSoftmaxLayer()
            self.logits_end, self.probdist_end = softmax_layer_end.build_graph(blended_reps_final, self.context_mask)
def build_graph ( self ) : context_input_lens = tf . reshape ( tf . reduce_sum ( tf . cast ( tf . cast ( self . context_char_ids , tf . bool ) , tf . int32 ) , axis = 2 ) , [ - 1 ] ) qn_input_lens = tf . reshape ( tf . reduce_sum ( tf . cast ( tf . cast ( self . qn_char_ids , tf . bool ) , tf . int32 ) , axis = 2 ) , [ - 1 ] ) cell_fw = rnn_cell . GRUCell ( self . FLAGS . hidden_size ) cell_bw = rnn_cell . GRUCell ( self . FLAGS . hidden_size ) _ , ( state_fw , state_bw ) = tf . nn . bidirectional_dynamic_rnn ( cell_fw , cell_bw , self . context_char_embs , context_input_lens , dtype = tf . float32 ) ch_emb = tf . reshape ( tf . concat ( [ state_fw , state_bw ] , axis = 1 ) , [ - 1 , self . FLAGS . context_len , 2 * self . FLAGS . hidden_size ] ) self . context_embs = tf . concat ( [ self . context_embs , ch_emb ] , axis = 2 ) _ , ( state_fw , state_bw ) = tf . nn . bidirectional_dynamic_rnn ( cell_fw , cell_bw , self . qn_char_embs , qn_input_lens , dtype = tf . float32 ) qh_emb = tf . reshape ( tf . concat ( [ state_fw , state_bw ] , axis = 1 ) , [ - 1 , self . FLAGS . question_len , 2 * self . FLAGS . hidden_size ] ) self . qn_embs = tf . concat ( [ self . qn_embs , qh_emb ] , axis = 2 ) encoder = RNNEncoder ( self . FLAGS . hidden_size , self . keep_prob ) context_hiddens = encoder . build_graph ( self . context_embs , self . context_mask ) question_hiddens = encoder . build_graph ( self . qn_embs , self . qn_mask ) attn_layer = BasicAttn ( self . keep_prob , self . FLAGS . hidden_size * 2 , self . FLAGS . hidden_size * 2 ) _ , attn_output = attn_layer . build_graph ( question_hiddens , self . qn_mask , context_hiddens ) blended_reps = tf . concat ( [ context_hiddens , attn_output ] , axis = 2 ) blended_reps_final = tf . contrib . layers . fully_connected ( blended_reps , num_outputs = self . FLAGS . hidden_size ) with vs . variable_scope ( "StartDist" ) : softmax_layer_start = SimpleSoftmaxLayer ( ) self . logits_start , self . probdist_start = softmax_layer_start . build_graph ( blended_reps_final , self . context_mask ) with vs . variable_scope ( "EndDist" ) : softmax_layer_end = SimpleSoftmaxLayer ( ) self . logits_end , self . probdist_end = softmax_layer_end . build_graph ( blended_reps_final , self . context_mask )
Builds the main part of the graph for the model, starting from the input embeddings to the final distributions for the answer span.
Builds the main part of the graph for the model starting from the input embeddings to the final distributions for the answer span .
"""Builds the main part of the graph for the model, starting from the input embeddings to the final distributions for the answer span.

        Defines:
          self.logits_start, self.logits_end: Both tensors shape (batch_size, context_len).
            These are the logits (i.e. values that are fed into the softmax function) for the start and end distribution.
            Important: these are -large in the pad locations. Necessary for when we feed into the cross entropy function.
          self.probdist_start, self.probdist_end: Both shape (batch_size, context_len). Each row sums to 1.
            These are the result of taking (masked) softmax of logits_start and logits_end.
        """ # Use a RNN to get hidden states for the context and the question # Note: here the RNNEncoder is shared (i.e. the weights are the same) # between the context and the question. # ToDo Deep encoder # (batch_size, context_len, hidden_size*2) # (batch_size, question_len, hidden_size*2) # Use context hidden states to attend to question hidden states # attn_output is shape (batch_size, context_len, hidden_size*2) # Concat attn_output to context_hiddens to get blended_reps # (batch_size, context_len, hidden_size*4) # Apply fully connected layer to each blended representation # Note, blended_reps_final corresponds to b' in the handout # Note, tf.contrib.layers.fully_connected applies a ReLU non-linarity here by default # blended_reps_final is shape (batch_size, context_len, hidden_size) # Use softmax layer to compute probability distribution for start location # Note this produces self.logits_start and self.probdist_start, both of which have shape (batch_size, context_len) # Use softmax layer to compute probability distribution for end location # Note this produces self.logits_end and self.probdist_end, both of which have shape (batch_size, context_len)
self