examples/language_model/gpt/run_glue.py
def convert_example(example,
                    tokenizer,
                    label_list,
                    max_seq_length=512,
                    is_test=False):
    """convert a glue example into necessary features"""
    if not is_test:
        # `label_list == None` is for regression task
        label_dtype = "int64" if label_list else "float32"
        # Get the label
        label = example['labels']
        label = np.array([label], dtype=label_dtype)
    # Convert raw text to feature
    if (int(is_test) + len(example)) == 2:
        example = tokenizer(
            example['sentence'],
            max_seq_len=max_seq_length,
            return_token_type_ids=False)
    else:
        example = tokenizer(
            example['sentence1'],
            text_pair=example['sentence2'],
            max_seq_len=max_seq_length,
            return_token_type_ids=False)

    if not is_test:
        return example['input_ids'], label
    else:
        return example['input_ids']
convert a glue example into necessary features
convert a glue example into necessary features
convert a glue example into necessary features
def convert_example(example,
                    tokenizer,
                    label_list,
                    max_seq_length=512,
                    is_test=False):
    if not is_test:
        label_dtype = "int64" if label_list else "float32"
        label = example['labels']
        label = np.array([label], dtype=label_dtype)
    if (int(is_test) + len(example)) == 2:
        example = tokenizer(
            example['sentence'],
            max_seq_len=max_seq_length,
            return_token_type_ids=False)
    else:
        example = tokenizer(
            example['sentence1'],
            text_pair=example['sentence2'],
            max_seq_len=max_seq_length,
            return_token_type_ids=False)
    if not is_test:
        return example['input_ids'], label
    else:
        return example['input_ids']
def convert_example ( example , tokenizer , label_list , max_seq_length = 512 , is_test = False ) : if not is_test : label_dtype = "int64" if label_list else "float32" label = example [ 'labels' ] label = np . array ( [ label ] , dtype = label_dtype ) if ( int ( is_test ) + len ( example ) ) == 2 : example = tokenizer ( example [ 'sentence' ] , max_seq_len = max_seq_length , return_token_type_ids = False ) else : example = tokenizer ( example [ 'sentence1' ] , text_pair = example [ 'sentence2' ] , max_seq_len = max_seq_length , return_token_type_ids = False ) if not is_test : return example [ 'input_ids' ] , label else : return example [ 'input_ids' ]
convert a glue example into necessary features
convert a glue example into necessary features
"""convert a glue example into necessary features""" # `label_list == None` is for regression task # Get the label # Convert raw text to feature
example
tokenizer
label_list
max_seq_length
is_test