examples/text_summarization/bart/utils.py
def convert_example(example,
                    text_column,
                    summary_column,
                    tokenizer,
                    decoder_start_token_id,
                    max_source_length,
                    max_target_length,
                    ignore_pad_token_for_loss=True,
                    is_train=True):
    """
    Convert a example into necessary features.
    """
    inputs = example[text_column]
    targets = example[summary_column]
    labels = tokenizer(
        targets, max_seq_len=max_target_length, pad_to_max_seq_len=True)
    decoder_input_ids = [decoder_start_token_id] + labels["input_ids"][:-1]
    if ignore_pad_token_for_loss:
        labels["input_ids"] = [(l if l != tokenizer.pad_token_id else -100)
                               for l in labels["input_ids"]]
    if is_train:
        model_inputs = tokenizer(
            inputs,
            max_seq_len=max_source_length,
            pad_to_max_seq_len=True,
            return_length=False)
        return model_inputs["input_ids"], model_inputs[
            "attention_mask"], decoder_input_ids, labels["input_ids"]
    else:
        model_inputs = tokenizer(
            inputs,
            max_seq_len=max_source_length,
            pad_to_max_seq_len=True,
            return_length=True)
        return model_inputs["input_ids"], model_inputs["attention_mask"], model_inputs["seq_len"], decoder_input_ids, \
               labels["input_ids"]

    Convert a example into necessary features.
    
Convert a example into necessary features.
Convert a example into necessary features .
def convert_example(example,
                    text_column,
                    summary_column,
                    tokenizer,
                    decoder_start_token_id,
                    max_source_length,
                    max_target_length,
                    ignore_pad_token_for_loss=True,
                    is_train=True):
    inputs = example[text_column]
    targets = example[summary_column]
    labels = tokenizer(
        targets, max_seq_len=max_target_length, pad_to_max_seq_len=True)
    decoder_input_ids = [decoder_start_token_id] + labels["input_ids"][:-1]
    if ignore_pad_token_for_loss:
        labels["input_ids"] = [(l if l != tokenizer.pad_token_id else -100)
                               for l in labels["input_ids"]]
    if is_train:
        model_inputs = tokenizer(
            inputs,
            max_seq_len=max_source_length,
            pad_to_max_seq_len=True,
            return_length=False)
        return model_inputs["input_ids"], model_inputs[
            "attention_mask"], decoder_input_ids, labels["input_ids"]
    else:
        model_inputs = tokenizer(
            inputs,
            max_seq_len=max_source_length,
            pad_to_max_seq_len=True,
            return_length=True)
        return model_inputs["input_ids"], model_inputs["attention_mask"], model_inputs["seq_len"], decoder_input_ids, \
               labels["input_ids"]
def convert_example ( example , text_column , summary_column , tokenizer , decoder_start_token_id , max_source_length , max_target_length , ignore_pad_token_for_loss = True , is_train = True ) : inputs = example [ text_column ] targets = example [ summary_column ] labels = tokenizer ( targets , max_seq_len = max_target_length , pad_to_max_seq_len = True ) decoder_input_ids = [ decoder_start_token_id ] + labels [ "input_ids" ] [ : - 1 ] if ignore_pad_token_for_loss : labels [ "input_ids" ] = [ ( l if l != tokenizer . pad_token_id else - 100 ) for l in labels [ "input_ids" ] ] if is_train : model_inputs = tokenizer ( inputs , max_seq_len = max_source_length , pad_to_max_seq_len = True , return_length = False ) return model_inputs [ "input_ids" ] , model_inputs [ "attention_mask" ] , decoder_input_ids , labels [ "input_ids" ] else : model_inputs = tokenizer ( inputs , max_seq_len = max_source_length , pad_to_max_seq_len = True , return_length = True ) return model_inputs [ "input_ids" ] , model_inputs [ "attention_mask" ] , model_inputs [ "seq_len" ] , decoder_input_ids , labels [ "input_ids" ]
Convert a example into necessary features.
Convert a example into necessary features .
"""
    Convert a example into necessary features.
    """
example
text_column
summary_column
tokenizer
decoder_start_token_id
max_source_length
max_target_length
ignore_pad_token_for_loss
is_train