tests/transformer/modeling.py
def position_encoding_init(n_position, d_pos_vec, dtype="float64"):
    """
    Generates the initial values for the sinusoidal position encoding table.
    This method follows the implementation in tensor2tensor, but is slightly
    different from the description in "Attention Is All You Need".

    Args:
        n_position (int): 
            The largest position for sequences, that is, the maximum length
            of source or target sequences.
        d_pos_vec (int): 
            The size of positional embedding vector. 
        dtype (str, optional): 
            The output `numpy.array`'s data type. Defaults to "float32".

    Returns:
        numpy.array: 
            The embedding table of sinusoidal position encoding with shape
            `[n_position, d_pos_vec]`.

    Example:
        .. code-block::

            from paddlenlp.transformers import position_encoding_init

            max_length = 256
            emb_dim = 512
            pos_table = position_encoding_init(max_length, emb_dim)
    """
    channels = d_pos_vec
    position = np.arange(n_position)
    num_timescales = channels // 2
    log_timescale_increment = (np.log(float(1e4) / float(1)) /
                               (num_timescales - 1))
    inv_timescales = np.exp(
        np.arange(num_timescales) * -log_timescale_increment)
    scaled_time = np.expand_dims(position, 1) * np.expand_dims(inv_timescales,
                                                               0)
    signal = np.concatenate([np.sin(scaled_time), np.cos(scaled_time)], axis=1)
    signal = np.pad(signal, [[0, 0], [0, np.mod(channels, 2)]], 'constant')
    position_enc = signal
    return position_enc.astype(dtype)

    Generates the initial values for the sinusoidal position encoding table.
    This method follows the implementation in tensor2tensor, but is slightly
    different from the description in "Attention Is All You Need".

    Args:
        n_position (int): 
            The largest position for sequences, that is, the maximum length
            of source or target sequences.
        d_pos_vec (int): 
            The size of positional embedding vector. 
        dtype (str, optional): 
            The output `numpy.array`'s data type. Defaults to "float32".

    Returns:
        numpy.array: 
            The embedding table of sinusoidal position encoding with shape
            `[n_position, d_pos_vec]`.

    Example:
        .. code-block::

            from paddlenlp.transformers import position_encoding_init

            max_length = 256
            emb_dim = 512
            pos_table = position_encoding_init(max_length, emb_dim)
    
Generates the initial values for the sinusoidal position encoding table.
This method follows the implementation in tensor2tensor, but is slightly
different from the description in "Attention Is All You Need".
Generates the initial values for the sinusoidal position encoding table . This method follows the implementation in tensor2tensor but is slightly different from the description in " Attention Is All You Need " .
def position_encoding_init(n_position, d_pos_vec, dtype="float64"):
    channels = d_pos_vec
    position = np.arange(n_position)
    num_timescales = channels // 2
    log_timescale_increment = (np.log(float(1e4) / float(1)) /
                               (num_timescales - 1))
    inv_timescales = np.exp(
        np.arange(num_timescales) * -log_timescale_increment)
    scaled_time = np.expand_dims(position, 1) * np.expand_dims(inv_timescales,
                                                               0)
    signal = np.concatenate([np.sin(scaled_time), np.cos(scaled_time)], axis=1)
    signal = np.pad(signal, [[0, 0], [0, np.mod(channels, 2)]], 'constant')
    position_enc = signal
    return position_enc.astype(dtype)
def position_encoding_init ( n_position , d_pos_vec , dtype = "float64" ) : channels = d_pos_vec position = np . arange ( n_position ) num_timescales = channels // 2 log_timescale_increment = ( np . log ( float ( 1e4 ) / float ( 1 ) ) / ( num_timescales - 1 ) ) inv_timescales = np . exp ( np . arange ( num_timescales ) * - log_timescale_increment ) scaled_time = np . expand_dims ( position , 1 ) * np . expand_dims ( inv_timescales , 0 ) signal = np . concatenate ( [ np . sin ( scaled_time ) , np . cos ( scaled_time ) ] , axis = 1 ) signal = np . pad ( signal , [ [ 0 , 0 ] , [ 0 , np . mod ( channels , 2 ) ] ] , 'constant' ) position_enc = signal return position_enc . astype ( dtype )
Generates the initial values for the sinusoidal position encoding table.
Generates the initial values for the sinusoidal position encoding table .
"""
    Generates the initial values for the sinusoidal position encoding table.
    This method follows the implementation in tensor2tensor, but is slightly
    different from the description in "Attention Is All You Need".

    Args:
        n_position (int): 
            The largest position for sequences, that is, the maximum length
            of source or target sequences.
        d_pos_vec (int): 
            The size of positional embedding vector. 
        dtype (str, optional): 
            The output `numpy.array`'s data type. Defaults to "float32".

    Returns:
        numpy.array: 
            The embedding table of sinusoidal position encoding with shape
            `[n_position, d_pos_vec]`.

    Example:
        .. code-block::

            from paddlenlp.transformers import position_encoding_init

            max_length = 256
            emb_dim = 512
            pos_table = position_encoding_init(max_length, emb_dim)
    """
The embedding table of sinusoidal position encoding with shape
`[n_position, d_pos_vec]`.
The embedding table of sinusoidal position encoding with shape ` [ n_position d_pos_vec ] ` .
numpy.array
n_position
The largest position for sequences, that is, the maximum length
of source or target sequences.
The largest position for sequences that is the maximum length of source or target sequences .
d_pos_vec
The size of positional embedding vector.
The size of positional embedding vector .
dtype
The output `numpy.array`'s data type. Defaults to "float32".
The output ` numpy . array ` ' s data type . Defaults to " float32 " .
"float32"
examples
code-block:.

from paddlenlp.transformers import position_encoding_init

max_length = 256
emb_dim = 512
pos_table = position_encoding_init(max_length, emb_dim)
code - block : . from paddlenlp . transformers import position_encoding_init max_length = 256 emb_dim = 512 pos_table = position_encoding_init ( max_length emb_dim )