deeppavlov/models/elmo/bilm_model.py
def _build_loss(self, lstm_outputs):
        """
        Create:
            self.total_loss: total loss op for training
            self.softmax_W, softmax_b: the softmax variables
            self.next_token_id / _reverse: placeholders for gold input

        """
        batch_size = self.options['batch_size']
        unroll_steps = self.options['unroll_steps']

        n_tokens_vocab = self.options['n_tokens_vocab']

        # DEFINE next_token_id and *_reverse placeholders for the gold input
        def _get_next_token_placeholders(suffix):
            name = 'next_token_id' + suffix
            id_placeholder = tf.placeholder(DTYPE_INT,
                                            shape=(batch_size, unroll_steps),
                                            name=name)
            return id_placeholder

        # get the window and weight placeholders
        self.next_token_id = _get_next_token_placeholders('')
        if self.bidirectional:
            self.next_token_id_reverse = _get_next_token_placeholders(
                '_reverse')

        # DEFINE THE SOFTMAX VARIABLES
        # get the dimension of the softmax weights
        # softmax dimension is the size of the output projection_dim
        softmax_dim = self.options['lstm']['projection_dim']

        # the output softmax variables -- they are shared if bidirectional
        if self.share_embedding_softmax:
            # softmax_W is just the embedding layer
            self.softmax_W = self.embedding_weights

        with tf.variable_scope('softmax'), tf.device('/cpu:0'):
            # Glorit init (std=(1.0 / sqrt(fan_in))
            softmax_init = tf.random_normal_initializer(0.0, 1.0 / np.sqrt(softmax_dim))
            if not self.share_embedding_softmax:
                self.softmax_W = tf.get_variable(
                    'W', [n_tokens_vocab, softmax_dim],
                    dtype=DTYPE,
                    initializer=softmax_init
                )
            self.softmax_b = tf.get_variable(
                'b', [n_tokens_vocab],
                dtype=DTYPE,
                initializer=tf.constant_initializer(0.0))

        # now calculate losses
        # loss for each direction of the LSTM
        self.individual_train_losses = []
        self.individual_eval_losses = []

        if self.bidirectional:
            next_ids = [self.next_token_id, self.next_token_id_reverse]
        else:
            next_ids = [self.next_token_id]

        for id_placeholder, lstm_output_flat in zip(next_ids, lstm_outputs):
            # flatten the LSTM output and next token id gold to shape:
            # (batch_size * unroll_steps, softmax_dim)
            # Flatten and reshape the token_id placeholders
            next_token_id_flat = tf.reshape(id_placeholder, [-1, 1])

            with tf.control_dependencies([lstm_output_flat]):
                sampled_losses = tf.nn.sampled_softmax_loss(self.softmax_W, self.softmax_b,
                                                            next_token_id_flat, lstm_output_flat,
                                                            self.options['n_negative_samples_batch'],
                                                            self.options['n_tokens_vocab'],
                                                            num_true=1)

                # get the full softmax loss
                output_scores = tf.matmul(
                    lstm_output_flat,
                    tf.transpose(self.softmax_W)
                ) + self.softmax_b
                # NOTE: tf.nn.sparse_softmax_cross_entropy_with_logits
                #   expects unnormalized output since it performs the
                #   softmax internally
                losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
                    logits=output_scores,
                    labels=tf.squeeze(next_token_id_flat, squeeze_dims=[1])
                )
            sampled_losses = tf.reshape(sampled_losses, [self.options['batch_size'], -1])
            losses = tf.reshape(losses, [self.options['batch_size'], -1])
            self.individual_train_losses.append(tf.reduce_mean(sampled_losses, axis=1))
            self.individual_eval_losses.append(tf.reduce_mean(losses, axis=1))

        # now make the total loss -- it's the train of the individual losses
        if self.bidirectional:
            self.total_train_loss = 0.5 * (self.individual_train_losses[0] + self.individual_train_losses[1])
            self.total_eval_loss = 0.5 * (self.individual_eval_losses[0] + self.individual_eval_losses[1])
        else:
            self.total_train_loss = self.individual_train_losses[0]
            self.total_eval_loss = self.individual_eval_losses[0]
def _build_loss(self, lstm_outputs):
        """
        Create:
            self.total_loss: total loss op for training
            self.softmax_W, softmax_b: the softmax variables
            self.next_token_id / _reverse: placeholders for gold input

        """
        batch_size = self.options['batch_size']
        unroll_steps = self.options['unroll_steps']

        n_tokens_vocab = self.options['n_tokens_vocab']

        # DEFINE next_token_id and *_reverse placeholders for the gold input
        def _get_next_token_placeholders(suffix):
            name = 'next_token_id' + suffix
            id_placeholder = tf.placeholder(DTYPE_INT,
                                            shape=(batch_size, unroll_steps),
                                            name=name)
            return id_placeholder

        # get the window and weight placeholders
        self.next_token_id = _get_next_token_placeholders('')
        if self.bidirectional:
            self.next_token_id_reverse = _get_next_token_placeholders(
                '_reverse')

        # DEFINE THE SOFTMAX VARIABLES
        # get the dimension of the softmax weights
        # softmax dimension is the size of the output projection_dim
        softmax_dim = self.options['lstm']['projection_dim']

        # the output softmax variables -- they are shared if bidirectional
        if self.share_embedding_softmax:
            # softmax_W is just the embedding layer
            self.softmax_W = self.embedding_weights

        with tf.variable_scope('softmax'), tf.device('/cpu:0'):
            # Glorit init (std=(1.0 / sqrt(fan_in))
            softmax_init = tf.random_normal_initializer(0.0, 1.0 / np.sqrt(softmax_dim))
            if not self.share_embedding_softmax:
                self.softmax_W = tf.get_variable(
                    'W', [n_tokens_vocab, softmax_dim],
                    dtype=DTYPE,
                    initializer=softmax_init
                )
            self.softmax_b = tf.get_variable(
                'b', [n_tokens_vocab],
                dtype=DTYPE,
                initializer=tf.constant_initializer(0.0))

        # now calculate losses
        # loss for each direction of the LSTM
        self.individual_train_losses = []
        self.individual_eval_losses = []

        if self.bidirectional:
            next_ids = [self.next_token_id, self.next_token_id_reverse]
        else:
            next_ids = [self.next_token_id]

        for id_placeholder, lstm_output_flat in zip(next_ids, lstm_outputs):
            # flatten the LSTM output and next token id gold to shape:
            # (batch_size * unroll_steps, softmax_dim)
            # Flatten and reshape the token_id placeholders
            next_token_id_flat = tf.reshape(id_placeholder, [-1, 1])

            with tf.control_dependencies([lstm_output_flat]):
                sampled_losses = tf.nn.sampled_softmax_loss(self.softmax_W, self.softmax_b,
                                                            next_token_id_flat, lstm_output_flat,
                                                            self.options['n_negative_samples_batch'],
                                                            self.options['n_tokens_vocab'],
                                                            num_true=1)

                # get the full softmax loss
                output_scores = tf.matmul(
                    lstm_output_flat,
                    tf.transpose(self.softmax_W)
                ) + self.softmax_b
                # NOTE: tf.nn.sparse_softmax_cross_entropy_with_logits
                #   expects unnormalized output since it performs the
                #   softmax internally
                losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
                    logits=output_scores,
                    labels=tf.squeeze(next_token_id_flat, squeeze_dims=[1])
                )
            sampled_losses = tf.reshape(sampled_losses, [self.options['batch_size'], -1])
            losses = tf.reshape(losses, [self.options['batch_size'], -1])
            self.individual_train_losses.append(tf.reduce_mean(sampled_losses, axis=1))
            self.individual_eval_losses.append(tf.reduce_mean(losses, axis=1))

        # now make the total loss -- it's the train of the individual losses
        if self.bidirectional:
            self.total_train_loss = 0.5 * (self.individual_train_losses[0] + self.individual_train_losses[1])
            self.total_eval_loss = 0.5 * (self.individual_eval_losses[0] + self.individual_eval_losses[1])
        else:
            self.total_train_loss = self.individual_train_losses[0]
            self.total_eval_loss = self.individual_eval_losses[0]
def _build_loss ( self , lstm_outputs ) : batch_size = self . options [ 'batch_size' ] unroll_steps = self . options [ 'unroll_steps' ] n_tokens_vocab = self . options [ 'n_tokens_vocab' ] # DEFINE next_token_id and *_reverse placeholders for the gold input def _get_next_token_placeholders ( suffix ) : name = 'next_token_id' + suffix id_placeholder = tf . placeholder ( DTYPE_INT , shape = ( batch_size , unroll_steps ) , name = name ) return id_placeholder # get the window and weight placeholders self . next_token_id = _get_next_token_placeholders ( '' ) if self . bidirectional : self . next_token_id_reverse = _get_next_token_placeholders ( '_reverse' ) # DEFINE THE SOFTMAX VARIABLES # get the dimension of the softmax weights # softmax dimension is the size of the output projection_dim softmax_dim = self . options [ 'lstm' ] [ 'projection_dim' ] # the output softmax variables -- they are shared if bidirectional if self . share_embedding_softmax : # softmax_W is just the embedding layer self . softmax_W = self . embedding_weights with tf . variable_scope ( 'softmax' ) , tf . device ( '/cpu:0' ) : # Glorit init (std=(1.0 / sqrt(fan_in)) softmax_init = tf . random_normal_initializer ( 0.0 , 1.0 / np . sqrt ( softmax_dim ) ) if not self . share_embedding_softmax : self . softmax_W = tf . get_variable ( 'W' , [ n_tokens_vocab , softmax_dim ] , dtype = DTYPE , initializer = softmax_init ) self . softmax_b = tf . get_variable ( 'b' , [ n_tokens_vocab ] , dtype = DTYPE , initializer = tf . constant_initializer ( 0.0 ) ) # now calculate losses # loss for each direction of the LSTM self . individual_train_losses = [ ] self . individual_eval_losses = [ ] if self . bidirectional : next_ids = [ self . next_token_id , self . next_token_id_reverse ] else : next_ids = [ self . next_token_id ] for id_placeholder , lstm_output_flat in zip ( next_ids , lstm_outputs ) : # flatten the LSTM output and next token id gold to shape: # (batch_size * unroll_steps, softmax_dim) # Flatten and reshape the token_id placeholders next_token_id_flat = tf . reshape ( id_placeholder , [ - 1 , 1 ] ) with tf . control_dependencies ( [ lstm_output_flat ] ) : sampled_losses = tf . nn . sampled_softmax_loss ( self . softmax_W , self . softmax_b , next_token_id_flat , lstm_output_flat , self . options [ 'n_negative_samples_batch' ] , self . options [ 'n_tokens_vocab' ] , num_true = 1 ) # get the full softmax loss output_scores = tf . matmul ( lstm_output_flat , tf . transpose ( self . softmax_W ) ) + self . softmax_b # NOTE: tf.nn.sparse_softmax_cross_entropy_with_logits #   expects unnormalized output since it performs the #   softmax internally losses = tf . nn . sparse_softmax_cross_entropy_with_logits ( logits = output_scores , labels = tf . squeeze ( next_token_id_flat , squeeze_dims = [ 1 ] ) ) sampled_losses = tf . reshape ( sampled_losses , [ self . options [ 'batch_size' ] , - 1 ] ) losses = tf . reshape ( losses , [ self . options [ 'batch_size' ] , - 1 ] ) self . individual_train_losses . append ( tf . reduce_mean ( sampled_losses , axis = 1 ) ) self . individual_eval_losses . append ( tf . reduce_mean ( losses , axis = 1 ) ) # now make the total loss -- it's the train of the individual losses if self . bidirectional : self . total_train_loss = 0.5 * ( self . individual_train_losses [ 0 ] + self . individual_train_losses [ 1 ] ) self . total_eval_loss = 0.5 * ( self . individual_eval_losses [ 0 ] + self . individual_eval_losses [ 1 ] ) else : self . total_train_loss = self . individual_train_losses [ 0 ] self . total_eval_loss = self . individual_eval_losses [ 0 ]
Create:
            self.total_loss: total loss op for training
            self.softmax_W, softmax_b: the softmax variables
            self.next_token_id / _reverse: placeholders for gold input
Create : self . total_loss : total loss op for training self . softmax_W softmax_b : the softmax variables self . next_token_id / _reverse : placeholders for gold input