examples/text_generation/unimo-text/gen_utils.py
def convert_example(example,
                    tokenizer,
                    max_seq_len=512,
                    max_target_len=128,
                    max_title_len=256,
                    mode='train'):
    """Convert all examples into necessary features."""
    source = example['source']
    title = None
    if 'title' in example.keys():
        title = example['title']

    if mode != 'test':
        tokenized_example = tokenizer.gen_encode(
            source,
            title=title,
            target=example['target'],
            max_seq_len=max_seq_len,
            max_target_len=max_target_len,
            max_title_len=max_title_len,
            return_position_ids=True,
            return_length=True)
        target_start = tokenized_example['input_ids'].index(
            tokenizer.cls_token_id, 1)
        target_end = tokenized_example['seq_len']
        # Use to gather the logits corresponding to the labels during training
        tokenized_example['masked_positions'] = list(
            range(target_start, target_end - 1))
        tokenized_example['labels'] = tokenized_example['input_ids'][
            target_start + 1:target_end]

        return tokenized_example
    else:
        tokenized_example = tokenizer.gen_encode(
            source,
            title=title,
            max_seq_len=max_seq_len,
            max_title_len=max_title_len,
            add_start_token_for_decoding=True,
            return_position_ids=True)

        if 'target' in example and example['target']:
            tokenized_example['target'] = example['target']
        return tokenized_example
Convert all examples into necessary features.
Convert all examples into necessary features.
Convert all examples into necessary features .
def convert_example(example,
                    tokenizer,
                    max_seq_len=512,
                    max_target_len=128,
                    max_title_len=256,
                    mode='train'):
    source = example['source']
    title = None
    if 'title' in example.keys():
        title = example['title']
    if mode != 'test':
        tokenized_example = tokenizer.gen_encode(
            source,
            title=title,
            target=example['target'],
            max_seq_len=max_seq_len,
            max_target_len=max_target_len,
            max_title_len=max_title_len,
            return_position_ids=True,
            return_length=True)
        target_start = tokenized_example['input_ids'].index(
            tokenizer.cls_token_id, 1)
        target_end = tokenized_example['seq_len']
        tokenized_example['masked_positions'] = list(
            range(target_start, target_end - 1))
        tokenized_example['labels'] = tokenized_example['input_ids'][
            target_start + 1:target_end]
        return tokenized_example
    else:
        tokenized_example = tokenizer.gen_encode(
            source,
            title=title,
            max_seq_len=max_seq_len,
            max_title_len=max_title_len,
            add_start_token_for_decoding=True,
            return_position_ids=True)
        if 'target' in example and example['target']:
            tokenized_example['target'] = example['target']
        return tokenized_example
def convert_example ( example , tokenizer , max_seq_len = 512 , max_target_len = 128 , max_title_len = 256 , mode = 'train' ) : source = example [ 'source' ] title = None if 'title' in example . keys ( ) : title = example [ 'title' ] if mode != 'test' : tokenized_example = tokenizer . gen_encode ( source , title = title , target = example [ 'target' ] , max_seq_len = max_seq_len , max_target_len = max_target_len , max_title_len = max_title_len , return_position_ids = True , return_length = True ) target_start = tokenized_example [ 'input_ids' ] . index ( tokenizer . cls_token_id , 1 ) target_end = tokenized_example [ 'seq_len' ] tokenized_example [ 'masked_positions' ] = list ( range ( target_start , target_end - 1 ) ) tokenized_example [ 'labels' ] = tokenized_example [ 'input_ids' ] [ target_start + 1 : target_end ] return tokenized_example else : tokenized_example = tokenizer . gen_encode ( source , title = title , max_seq_len = max_seq_len , max_title_len = max_title_len , add_start_token_for_decoding = True , return_position_ids = True ) if 'target' in example and example [ 'target' ] : tokenized_example [ 'target' ] = example [ 'target' ] return tokenized_example
Convert all examples into necessary features.
Convert all examples into necessary features .
"""Convert all examples into necessary features.""" # Use to gather the logits corresponding to the labels during training
example
tokenizer
max_seq_len
max_target_len
max_title_len
mode