nlpmodels/models/word2vec.py
def forward(self, data: tuple) -> torch.Tensor:
        """
        The main call function of the model, but we output loss not prediction.

        Args:
           data (tuple): batches of (input_word, context_word)
        Returns:
            0-d tensor with loss
        """

        # unpack of input, context vectors
        input_word, context_word = data

        batch_size = context_word.size(0)

        # (1) Grab negative sample indices (batch_size, k)
        # Rather than negative sample in the construction of the dataset,
        # we find negative samples here directly from embedding matrix
        # Note: the word_frequencies are set to 0 for special tokens and therefore
        # will not be sampled.
        negative_sample_indices = torch.multinomial(self._word_frequency,
                                                    batch_size * self._negative_sample_size,
                                                    replacement=True).view(batch_size, -1)

        negative_sample_indices = negative_sample_indices.to(self._device)

        # Alternative: If no information on word frequencies, sample uniformly.
        # negative_sample_indices = torch.randint(0, self._vocab_size - 1,
        #                                         size=(batch_size, self._negative_sample_size))

        # (2) Look-up input, context word embeddings
        # (batch_size, 1) -> (batch_size, 1, embedding_size) ->
        # (batch_size, embedding_size)
        input_vectors = self._input_embedding(input_word).unsqueeze(2)
        output_vectors = self._output_embedding(context_word).unsqueeze(2)

        # (3) Look-up negative context word embeddings
        # (batch_size, k) -> (batch_size, k, embedding_size)
        neg_output_vectors = self._output_embedding(negative_sample_indices).neg()

        # (4) Calculate loss function
        # 1 target=1 binary classification + k target=0 binary classifications
        pos_loss = F.logsigmoid(torch.mul(output_vectors, input_vectors).squeeze()).mean(1)
        neg_loss = F.logsigmoid(torch.bmm(neg_output_vectors, input_vectors).squeeze())\
            .view(-1, 1, self._negative_sample_size).sum(2).mean(1)

        # (5) Return collapsed loss functions
        return -torch.mean(pos_loss + neg_loss)

        The main call function of the model, but we output loss not prediction.

        Args:
           data (tuple): batches of (input_word, context_word)
        Returns:
            0-d tensor with loss
        
The main call function of the model, but we output loss not prediction.
The main call function of the model but we output loss not prediction .
def forward(self, data: tuple) -> torch.Tensor:
        input_word, context_word = data
        batch_size = context_word.size(0)
        negative_sample_indices = torch.multinomial(self._word_frequency,
                                                    batch_size * self._negative_sample_size,
                                                    replacement=True).view(batch_size, -1)
        negative_sample_indices = negative_sample_indices.to(self._device)
        input_vectors = self._input_embedding(input_word).unsqueeze(2)
        output_vectors = self._output_embedding(context_word).unsqueeze(2)
        neg_output_vectors = self._output_embedding(negative_sample_indices).neg()
        pos_loss = F.logsigmoid(torch.mul(output_vectors, input_vectors).squeeze()).mean(1)
        neg_loss = F.logsigmoid(torch.bmm(neg_output_vectors, input_vectors).squeeze())\
            .view(-1, 1, self._negative_sample_size).sum(2).mean(1)
        return -torch.mean(pos_loss + neg_loss)
def forward ( self , data : tuple ) -> torch . Tensor : input_word , context_word = data batch_size = context_word . size ( 0 ) negative_sample_indices = torch . multinomial ( self . _word_frequency , batch_size * self . _negative_sample_size , replacement = True ) . view ( batch_size , - 1 ) negative_sample_indices = negative_sample_indices . to ( self . _device ) input_vectors = self . _input_embedding ( input_word ) . unsqueeze ( 2 ) output_vectors = self . _output_embedding ( context_word ) . unsqueeze ( 2 ) neg_output_vectors = self . _output_embedding ( negative_sample_indices ) . neg ( ) pos_loss = F . logsigmoid ( torch . mul ( output_vectors , input_vectors ) . squeeze ( ) ) . mean ( 1 ) neg_loss = F . logsigmoid ( torch . bmm ( neg_output_vectors , input_vectors ) . squeeze ( ) ) . view ( - 1 , 1 , self . _negative_sample_size ) . sum ( 2 ) . mean ( 1 ) return - torch . mean ( pos_loss + neg_loss )
The main call function of the model, but we output loss not prediction.
The main call function of the model but we output loss not prediction .
"""
        The main call function of the model, but we output loss not prediction.

        Args:
           data (tuple): batches of (input_word, context_word)
        Returns:
            0-d tensor with loss
        """ # unpack of input, context vectors # (1) Grab negative sample indices (batch_size, k) # Rather than negative sample in the construction of the dataset, # we find negative samples here directly from embedding matrix # Note: the word_frequencies are set to 0 for special tokens and therefore # will not be sampled. # Alternative: If no information on word frequencies, sample uniformly. # negative_sample_indices = torch.randint(0, self._vocab_size - 1, #                                         size=(batch_size, self._negative_sample_size)) # (2) Look-up input, context word embeddings # (batch_size, 1) -> (batch_size, 1, embedding_size) -> # (batch_size, embedding_size) # (3) Look-up negative context word embeddings # (batch_size, k) -> (batch_size, k, embedding_size) # (4) Calculate loss function # 1 target=1 binary classification + k target=0 binary classifications # (5) Return collapsed loss functions
0-d tensor with loss
0 - d tensor with loss
self
data
tuple
batches of (input_word, context_word)
batches of ( input_word context_word )