examples/text_matching/sentence_transformers/train.py
def convert_example(example, tokenizer, max_seq_length=512, is_test=False):
    """
    Builds model inputs from a sequence or a pair of sequence for sequence classification tasks
    by concatenating and adding special tokens. And creates a mask from the two sequences passed 
    to be used in a sequence-pair classification task.
        
    A BERT sequence has the following format:

    - single sequence: ``[CLS] X [SEP]``
    - pair of sequences: ``[CLS] A [SEP] B [SEP]``

    A BERT sequence pair mask has the following format:
    ::
        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
        | first sequence    | second sequence |

    If only one sequence, only returns the first portion of the mask (0's).


    Args:
        example(obj:`list[str]`): List of input data, containing query, title and label if it have label.
        tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer` 
            which contains most of the methods. Users should refer to the superclass for more information regarding methods.
        max_seq_len(obj:`int`): The maximum total input sequence length after tokenization. 
            Sequences longer than this will be truncated, sequences shorter will be padded.
        is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.

    Returns:
        query_input_ids(obj:`list[int]`): The list of query token ids.
        query_token_type_ids(obj: `list[int]`): List of query sequence pair mask.
        title_input_ids(obj:`list[int]`): The list of title token ids.
        title_token_type_ids(obj: `list[int]`): List of title sequence pair mask.
        label(obj:`numpy.array`, data type of int64, optional): The input label if not is_test.
    """
    query, title = example["query"], example["title"]

    query_encoded_inputs = tokenizer(text=query, max_seq_len=max_seq_length)
    query_input_ids = query_encoded_inputs["input_ids"]
    query_token_type_ids = query_encoded_inputs["token_type_ids"]

    title_encoded_inputs = tokenizer(text=title, max_seq_len=max_seq_length)
    title_input_ids = title_encoded_inputs["input_ids"]
    title_token_type_ids = title_encoded_inputs["token_type_ids"]

    if not is_test:
        label = np.array([example["label"]], dtype="int64")
        return query_input_ids, query_token_type_ids, title_input_ids, title_token_type_ids, label
    else:
        return query_input_ids, query_token_type_ids, title_input_ids, title_token_type_ids

    Builds model inputs from a sequence or a pair of sequence for sequence classification tasks
    by concatenating and adding special tokens. And creates a mask from the two sequences passed 
    to be used in a sequence-pair classification task.
        
    A BERT sequence has the following format:

    - single sequence: ``[CLS] X [SEP]``
    - pair of sequences: ``[CLS] A [SEP] B [SEP]``

    A BERT sequence pair mask has the following format:
    ::
        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
        | first sequence    | second sequence |

    If only one sequence, only returns the first portion of the mask (0's).


    Args:
        example(obj:`list[str]`): List of input data, containing query, title and label if it have label.
        tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer` 
            which contains most of the methods. Users should refer to the superclass for more information regarding methods.
        max_seq_len(obj:`int`): The maximum total input sequence length after tokenization. 
            Sequences longer than this will be truncated, sequences shorter will be padded.
        is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.

    Returns:
        query_input_ids(obj:`list[int]`): The list of query token ids.
        query_token_type_ids(obj: `list[int]`): List of query sequence pair mask.
        title_input_ids(obj:`list[int]`): The list of title token ids.
        title_token_type_ids(obj: `list[int]`): List of title sequence pair mask.
        label(obj:`numpy.array`, data type of int64, optional): The input label if not is_test.
    
Builds model inputs from a sequence or a pair of sequence for sequence classification tasks
by concatenating and adding special tokens. And creates a mask from the two sequences passed
to be used in a sequence-pair classification task.

A BERT sequence has the following format.





If only one sequence, only returns the first portion of the mask (0's).
Builds model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens . And creates a mask from the two sequences passed to be used in a sequence - pair classification task . A BERT sequence has the following format . If only one sequence only returns the first portion of the mask ( 0 ' s ) .
def convert_example(example, tokenizer, max_seq_length=512, is_test=False):
    query, title = example["query"], example["title"]
    query_encoded_inputs = tokenizer(text=query, max_seq_len=max_seq_length)
    query_input_ids = query_encoded_inputs["input_ids"]
    query_token_type_ids = query_encoded_inputs["token_type_ids"]
    title_encoded_inputs = tokenizer(text=title, max_seq_len=max_seq_length)
    title_input_ids = title_encoded_inputs["input_ids"]
    title_token_type_ids = title_encoded_inputs["token_type_ids"]
    if not is_test:
        label = np.array([example["label"]], dtype="int64")
        return query_input_ids, query_token_type_ids, title_input_ids, title_token_type_ids, label
    else:
        return query_input_ids, query_token_type_ids, title_input_ids, title_token_type_ids
def convert_example ( example , tokenizer , max_seq_length = 512 , is_test = False ) : query , title = example [ "query" ] , example [ "title" ] query_encoded_inputs = tokenizer ( text = query , max_seq_len = max_seq_length ) query_input_ids = query_encoded_inputs [ "input_ids" ] query_token_type_ids = query_encoded_inputs [ "token_type_ids" ] title_encoded_inputs = tokenizer ( text = title , max_seq_len = max_seq_length ) title_input_ids = title_encoded_inputs [ "input_ids" ] title_token_type_ids = title_encoded_inputs [ "token_type_ids" ] if not is_test : label = np . array ( [ example [ "label" ] ] , dtype = "int64" ) return query_input_ids , query_token_type_ids , title_input_ids , title_token_type_ids , label else : return query_input_ids , query_token_type_ids , title_input_ids , title_token_type_ids
Builds model inputs from a sequence or a pair of sequence for sequence classification tasks
by concatenating and adding special tokens.
Builds model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens .
"""
    Builds model inputs from a sequence or a pair of sequence for sequence classification tasks
    by concatenating and adding special tokens. And creates a mask from the two sequences passed 
    to be used in a sequence-pair classification task.
        
    A BERT sequence has the following format:

    - single sequence: ``[CLS] X [SEP]``
    - pair of sequences: ``[CLS] A [SEP] B [SEP]``

    A BERT sequence pair mask has the following format:
    ::
        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
        | first sequence    | second sequence |

    If only one sequence, only returns the first portion of the mask (0's).


    Args:
        example(obj:`list[str]`): List of input data, containing query, title and label if it have label.
        tokenizer(obj:`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer` 
            which contains most of the methods. Users should refer to the superclass for more information regarding methods.
        max_seq_len(obj:`int`): The maximum total input sequence length after tokenization. 
            Sequences longer than this will be truncated, sequences shorter will be padded.
        is_test(obj:`False`, defaults to `False`): Whether the example contains label or not.

    Returns:
        query_input_ids(obj:`list[int]`): The list of query token ids.
        query_token_type_ids(obj: `list[int]`): List of query sequence pair mask.
        title_input_ids(obj:`list[int]`): The list of title token ids.
        title_token_type_ids(obj: `list[int]`): List of title sequence pair mask.
        label(obj:`numpy.array`, data type of int64, optional): The input label if not is_test.
    """
`list[int]`): The list of query token ids.
query_token_type_ids(obj: `list[int]`): List of query sequence pair mask.
title_input_ids(obj:`list[int]`): The list of title token ids.
title_token_type_ids(obj: `list[int]`): List of title sequence pair mask.
label(obj:`numpy.array`, data type of int64, optional): The input label if not is_test.
` list [ int ] ` ) : The list of query token ids . query_token_type_ids ( obj : ` list [ int ] ` ) : List of query sequence pair mask . title_input_ids ( obj : ` list [ int ] ` ) : The list of title token ids . title_token_type_ids ( obj : ` list [ int ] ` ) : List of title sequence pair mask . label ( obj : ` numpy . array ` data type of int64 optional ) : The input label if not is_test .
query_input_ids(obj
example
tokenizer
max_seq_length
is_test
example(obj
`list[str]`): List of input data, containing query, title and label if it have label.
` list [ str ] ` ) : List of input data containing query title and label if it have label .
tokenizer(obj
`PretrainedTokenizer`): This tokenizer inherits from :class:`~paddlenlp.transformers.PretrainedTokenizer`
which contains most of the methods. Users should refer to the superclass for more information regarding methods.
` PretrainedTokenizer ` ) : This tokenizer inherits from : class : ` ~paddlenlp . transformers . PretrainedTokenizer ` which contains most of the methods . Users should refer to the superclass for more information regarding methods .
max_seq_len(obj
`int`): The maximum total input sequence length after tokenization.
Sequences longer than this will be truncated, sequences shorter will be padded.
` int ` ) : The maximum total input sequence length after tokenization . Sequences longer than this will be truncated sequences shorter will be padded .
is_test(obj
`False`, defaults to `False`): Whether the example contains label or not.
` False ` defaults to ` False ` ) : Whether the example contains label or not .