examples/dialogue/unified_transformer/utils.py
def convert_example(example,
                    tokenizer,
                    max_seq_len=512,
                    max_response_len=128,
                    max_knowledge_len=256,
                    mode='train'):
    """Convert all examples into necessary features."""
    goal = example['goal']
    knowledge = example['knowledge']
    goal_knowledge = ' '.join([' '.join(lst) for lst in goal + knowledge])

    if mode != 'test':
        tokenized_example = tokenizer.dialogue_encode(
            example['history'],
            response=example['response'],
            knowledge=goal_knowledge,
            task_type='knowledge',
            max_seq_len=max_seq_len,
            max_response_len=max_response_len,
            max_knowledge_len=max_knowledge_len,
            return_length=True)
        response_start = tokenized_example['input_ids'].index(
            tokenizer.cls_token_id, 1)
        response_end = tokenized_example['seq_len']
        # Use to gather the logits corresponding to the labels during training
        tokenized_example['masked_positions'] = list(
            range(response_start, response_end - 1))
        tokenized_example['labels'] = tokenized_example['input_ids'][
            response_start + 1:response_end]
        return tokenized_example
    else:
        tokenized_example = tokenizer.dialogue_encode(
            example['history'],
            knowledge=goal_knowledge,
            task_type='knowledge',
            max_seq_len=max_seq_len,
            max_knowledge_len=max_knowledge_len,
            add_start_token_as_response=True,
            return_length=True)

        if 'response' in example:
            tokenized_example['response'] = example['response']
        return tokenized_example
Convert all examples into necessary features.
Convert all examples into necessary features.
Convert all examples into necessary features .
def convert_example(example,
                    tokenizer,
                    max_seq_len=512,
                    max_response_len=128,
                    max_knowledge_len=256,
                    mode='train'):
    goal = example['goal']
    knowledge = example['knowledge']
    goal_knowledge = ' '.join([' '.join(lst) for lst in goal + knowledge])
    if mode != 'test':
        tokenized_example = tokenizer.dialogue_encode(
            example['history'],
            response=example['response'],
            knowledge=goal_knowledge,
            task_type='knowledge',
            max_seq_len=max_seq_len,
            max_response_len=max_response_len,
            max_knowledge_len=max_knowledge_len,
            return_length=True)
        response_start = tokenized_example['input_ids'].index(
            tokenizer.cls_token_id, 1)
        response_end = tokenized_example['seq_len']
        tokenized_example['masked_positions'] = list(
            range(response_start, response_end - 1))
        tokenized_example['labels'] = tokenized_example['input_ids'][
            response_start + 1:response_end]
        return tokenized_example
    else:
        tokenized_example = tokenizer.dialogue_encode(
            example['history'],
            knowledge=goal_knowledge,
            task_type='knowledge',
            max_seq_len=max_seq_len,
            max_knowledge_len=max_knowledge_len,
            add_start_token_as_response=True,
            return_length=True)
        if 'response' in example:
            tokenized_example['response'] = example['response']
        return tokenized_example
def convert_example ( example , tokenizer , max_seq_len = 512 , max_response_len = 128 , max_knowledge_len = 256 , mode = 'train' ) : goal = example [ 'goal' ] knowledge = example [ 'knowledge' ] goal_knowledge = ' ' . join ( [ ' ' . join ( lst ) for lst in goal + knowledge ] ) if mode != 'test' : tokenized_example = tokenizer . dialogue_encode ( example [ 'history' ] , response = example [ 'response' ] , knowledge = goal_knowledge , task_type = 'knowledge' , max_seq_len = max_seq_len , max_response_len = max_response_len , max_knowledge_len = max_knowledge_len , return_length = True ) response_start = tokenized_example [ 'input_ids' ] . index ( tokenizer . cls_token_id , 1 ) response_end = tokenized_example [ 'seq_len' ] tokenized_example [ 'masked_positions' ] = list ( range ( response_start , response_end - 1 ) ) tokenized_example [ 'labels' ] = tokenized_example [ 'input_ids' ] [ response_start + 1 : response_end ] return tokenized_example else : tokenized_example = tokenizer . dialogue_encode ( example [ 'history' ] , knowledge = goal_knowledge , task_type = 'knowledge' , max_seq_len = max_seq_len , max_knowledge_len = max_knowledge_len , add_start_token_as_response = True , return_length = True ) if 'response' in example : tokenized_example [ 'response' ] = example [ 'response' ] return tokenized_example
Convert all examples into necessary features.
Convert all examples into necessary features .
"""Convert all examples into necessary features.""" # Use to gather the logits corresponding to the labels during training
example
tokenizer
max_seq_len
max_response_len
max_knowledge_len
mode