tests/transformer/modeling.py
def forward(self, predict, label):
        r"""
        Computes cross entropy loss with or without label smoothing. 

        Args:
            predict (Tensor):
                The predict results of `TransformerModel` with shape
                `[batch_size, sequence_length, vocab_size]` whose data type can
                be float32 or float64.
            label (Tensor):
                The label for correspoding results with shape
                `[batch_size, sequence_length, 1]`.

        Returns:
            tuple:
                A tuple with items: (`sum_cost`, `avg_cost`, `token_num`).

                With the corresponding fields:

                - `sum_cost` (Tensor):
                    The sum of loss of current batch whose data type can be float32, float64.
                - `avg_cost` (Tensor):
                    The average loss of current batch whose data type can be float32, float64.
                    The relation between `sum_cost` and `avg_cost` can be described as:

                    .. math:

                        avg_cost = sum_cost / token_num

                - `token_num` (Tensor):
                    The number of tokens of current batch. 

        Example:
            .. code-block::

                import paddle
                from paddlenlp.transformers import CrossEntropyCriterion

                criterion = CrossEntropyCriterion(label_smooth_eps=0.1, pad_idx=0)
                batch_size = 1
                seq_len = 2
                vocab_size = 30000
                predict = paddle.rand(shape=[batch_size, seq_len, vocab_size])
                label = paddle.randint(
                    low=3,
                    high=vocab_size,
                    shape=[batch_size, seq_len, 1])

                criterion(predict, label)
        """
        weights = paddle.cast(
            label != self.pad_idx, dtype=paddle.get_default_dtype())
        if self.label_smooth_eps:
            label = paddle.squeeze(label, axis=[2])
            label = F.label_smooth(
                label=F.one_hot(
                    x=label, num_classes=predict.shape[-1]),
                epsilon=self.label_smooth_eps)
            if paddle.get_default_dtype() != "float32":
                label = paddle.cast(label, dtype=paddle.get_default_dtype())

        cost = F.cross_entropy(
            input=predict,
            label=label,
            reduction='none',
            soft_label=True if self.label_smooth_eps else False)
        weighted_cost = cost * weights
        sum_cost = paddle.sum(weighted_cost)
        token_num = paddle.sum(weights)
        token_num.stop_gradient = True
        avg_cost = sum_cost / token_num
        return sum_cost, avg_cost, token_num
r"""
        Computes cross entropy loss with or without label smoothing. 

        Args:
            predict (Tensor):
                The predict results of `TransformerModel` with shape
                `[batch_size, sequence_length, vocab_size]` whose data type can
                be float32 or float64.
            label (Tensor):
                The label for correspoding results with shape
                `[batch_size, sequence_length, 1]`.

        Returns:
            tuple:
                A tuple with items: (`sum_cost`, `avg_cost`, `token_num`).

                With the corresponding fields:

                - `sum_cost` (Tensor):
                    The sum of loss of current batch whose data type can be float32, float64.
                - `avg_cost` (Tensor):
                    The average loss of current batch whose data type can be float32, float64.
                    The relation between `sum_cost` and `avg_cost` can be described as:

                    .. math:

                        avg_cost = sum_cost / token_num

                - `token_num` (Tensor):
                    The number of tokens of current batch. 

        Example:
            .. code-block::

                import paddle
                from paddlenlp.transformers import CrossEntropyCriterion

                criterion = CrossEntropyCriterion(label_smooth_eps=0.1, pad_idx=0)
                batch_size = 1
                seq_len = 2
                vocab_size = 30000
                predict = paddle.rand(shape=[batch_size, seq_len, vocab_size])
                label = paddle.randint(
                    low=3,
                    high=vocab_size,
                    shape=[batch_size, seq_len, 1])

                criterion(predict, label)
        
r"""
Computes cross entropy loss with or without label smoothing.
r " " " Computes cross entropy loss with or without label smoothing .
def forward(self, predict, label):
        weights = paddle.cast(
            label != self.pad_idx, dtype=paddle.get_default_dtype())
        if self.label_smooth_eps:
            label = paddle.squeeze(label, axis=[2])
            label = F.label_smooth(
                label=F.one_hot(
                    x=label, num_classes=predict.shape[-1]),
                epsilon=self.label_smooth_eps)
            if paddle.get_default_dtype() != "float32":
                label = paddle.cast(label, dtype=paddle.get_default_dtype())
        cost = F.cross_entropy(
            input=predict,
            label=label,
            reduction='none',
            soft_label=True if self.label_smooth_eps else False)
        weighted_cost = cost * weights
        sum_cost = paddle.sum(weighted_cost)
        token_num = paddle.sum(weights)
        token_num.stop_gradient = True
        avg_cost = sum_cost / token_num
        return sum_cost, avg_cost, token_num
def forward ( self , predict , label ) : weights = paddle . cast ( label != self . pad_idx , dtype = paddle . get_default_dtype ( ) ) if self . label_smooth_eps : label = paddle . squeeze ( label , axis = [ 2 ] ) label = F . label_smooth ( label = F . one_hot ( x = label , num_classes = predict . shape [ - 1 ] ) , epsilon = self . label_smooth_eps ) if paddle . get_default_dtype ( ) != "float32" : label = paddle . cast ( label , dtype = paddle . get_default_dtype ( ) ) cost = F . cross_entropy ( input = predict , label = label , reduction = 'none' , soft_label = True if self . label_smooth_eps else False ) weighted_cost = cost * weights sum_cost = paddle . sum ( weighted_cost ) token_num = paddle . sum ( weights ) token_num . stop_gradient = True avg_cost = sum_cost / token_num return sum_cost , avg_cost , token_num
r"""
Computes cross entropy loss with or without label smoothing.
r " " " Computes cross entropy loss with or without label smoothing .
r"""
        Computes cross entropy loss with or without label smoothing. 

        Args:
            predict (Tensor):
                The predict results of `TransformerModel` with shape
                `[batch_size, sequence_length, vocab_size]` whose data type can
                be float32 or float64.
            label (Tensor):
                The label for correspoding results with shape
                `[batch_size, sequence_length, 1]`.

        Returns:
            tuple:
                A tuple with items: (`sum_cost`, `avg_cost`, `token_num`).

                With the corresponding fields:

                - `sum_cost` (Tensor):
                    The sum of loss of current batch whose data type can be float32, float64.
                - `avg_cost` (Tensor):
                    The average loss of current batch whose data type can be float32, float64.
                    The relation between `sum_cost` and `avg_cost` can be described as:

                    .. math:

                        avg_cost = sum_cost / token_num

                - `token_num` (Tensor):
                    The number of tokens of current batch. 

        Example:
            .. code-block::

                import paddle
                from paddlenlp.transformers import CrossEntropyCriterion

                criterion = CrossEntropyCriterion(label_smooth_eps=0.1, pad_idx=0)
                batch_size = 1
                seq_len = 2
                vocab_size = 30000
                predict = paddle.rand(shape=[batch_size, seq_len, vocab_size])
                label = paddle.randint(
                    low=3,
                    high=vocab_size,
                    shape=[batch_size, seq_len, 1])

                criterion(predict, label)
        """
A tuple with items: (`sum_cost`, `avg_cost`, `token_num`).

With the corresponding fields.

`sum_cost` (Tensor):
The sum of loss of current batch whose data type can be float32, float64.
`avg_cost` (Tensor):
The average loss of current batch whose data type can be float32, float64.
The relation between `sum_cost` and `avg_cost` can be described as.





`token_num` (Tensor):
The number of tokens of current batch.
A tuple with items : ( ` sum_cost ` ` avg_cost ` ` token_num ` ) . With the corresponding fields . ` sum_cost ` ( Tensor ) : The sum of loss of current batch whose data type can be float32 float64 . ` avg_cost ` ( Tensor ) : The average loss of current batch whose data type can be float32 float64 . The relation between ` sum_cost ` and ` avg_cost ` can be described as . ` token_num ` ( Tensor ) : The number of tokens of current batch .
tuple
self
predict
The predict results of `TransformerModel` with shape
`[batch_size, sequence_length, vocab_size]` whose data type can
be float32 or float64.
The predict results of ` TransformerModel ` with shape ` [ batch_size sequence_length vocab_size ] ` whose data type can be float32 or float64 .
label
The label for correspoding results with shape
`[batch_size, sequence_length, 1]`.
The label for correspoding results with shape ` [ batch_size sequence_length 1 ] ` .
examples
code-block:.

import paddle
from paddlenlp.transformers import CrossEntropyCriterion

criterion = CrossEntropyCriterion(label_smooth_eps=0.1, pad_idx=0)
batch_size = 1
seq_len = 2
vocab_size = 30000
predict = paddle.rand(shape=[batch_size, seq_len, vocab_size])
label = paddle.randint(
low=3,
high=vocab_size,
shape=[batch_size, seq_len, 1])

criterion(predict, label)
code - block : . import paddle from paddlenlp . transformers import CrossEntropyCriterion criterion = CrossEntropyCriterion ( label_smooth_eps = 0 . 1 pad_idx = 0 ) batch_size = 1 seq_len = 2 vocab_size = 30000 predict = paddle . rand ( shape = [ batch_size seq_len vocab_size ] ) label = paddle . randint ( low = 3 high = vocab_size shape = [ batch_size seq_len 1 ] ) criterion ( predict label )