kb-albert-char-v1/examples/utils_nsmc.py
def convert_examples_to_features(
    examples: List[InputExample],
    max_seq_len: int,
    tokenizer: PreTrainedTokenizer,
    cls_token_segment_id=0,
    pad_token_segment_id=0,
    sequence_a_segment_id=0,
    mask_padding_with_zero=True,
) -> List[InputFeatures]:
    """Loads a data file into a list of `InputFeatures`"""
    # Setting based on the current model type
    cls_token = tokenizer.cls_token
    sep_token = tokenizer.sep_token
    pad_token_id = tokenizer.pad_token_id

    features = []
    for (ex_index, example) in enumerate(examples):
        tokens = tokenizer.tokenize(example.text_a)

        # Account for [CLS] and [SEP]
        special_tokens_count = 2
        if len(tokens) > max_seq_len - special_tokens_count:
            tokens = tokens[: (max_seq_len - special_tokens_count)]

        # Add [SEP] token
        tokens += [sep_token]
        token_type_ids = [sequence_a_segment_id] * len(tokens)

        # Add [CLS] token
        tokens = [cls_token] + tokens
        token_type_ids = [cls_token_segment_id] + token_type_ids

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)

        # Zero-pad up to the sequence length.
        padding_length = max_seq_len - len(input_ids)
        input_ids = input_ids + ([pad_token_id] * padding_length)
        attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)
        token_type_ids = token_type_ids + ([pad_token_segment_id] * padding_length)

        assert len(input_ids) == max_seq_len, "Error with input length {} vs {}".format(len(input_ids), max_seq_len)
        assert len(attention_mask) == max_seq_len, "Error with attention mask length {} vs {}".format(
            len(attention_mask), max_seq_len
        )
        assert len(token_type_ids) == max_seq_len, "Error with token type length {} vs {}".format(
            len(token_type_ids), max_seq_len
        )

        label_id = example.label

        features.append(
            InputFeatures(
                input_ids=input_ids,
                attention_mask=attention_mask,
                token_type_ids=token_type_ids,
                label=label_id,
            )
        )

    return features
Loads a data file into a list of `InputFeatures`
Loads a data file into a list of `InputFeatures`
Loads a data file into a list of ` InputFeatures `
def convert_examples_to_features(
    examples: List[InputExample],
    max_seq_len: int,
    tokenizer: PreTrainedTokenizer,
    cls_token_segment_id=0,
    pad_token_segment_id=0,
    sequence_a_segment_id=0,
    mask_padding_with_zero=True,
) -> List[InputFeatures]:
    cls_token = tokenizer.cls_token
    sep_token = tokenizer.sep_token
    pad_token_id = tokenizer.pad_token_id
    features = []
    for (ex_index, example) in enumerate(examples):
        tokens = tokenizer.tokenize(example.text_a)
        special_tokens_count = 2
        if len(tokens) > max_seq_len - special_tokens_count:
            tokens = tokens[: (max_seq_len - special_tokens_count)]
        tokens += [sep_token]
        token_type_ids = [sequence_a_segment_id] * len(tokens)
        tokens = [cls_token] + tokens
        token_type_ids = [cls_token_segment_id] + token_type_ids
        input_ids = tokenizer.convert_tokens_to_ids(tokens)
        attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
        padding_length = max_seq_len - len(input_ids)
        input_ids = input_ids + ([pad_token_id] * padding_length)
        attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)
        token_type_ids = token_type_ids + ([pad_token_segment_id] * padding_length)
        assert len(input_ids) == max_seq_len, "Error with input length {} vs {}".format(len(input_ids), max_seq_len)
        assert len(attention_mask) == max_seq_len, "Error with attention mask length {} vs {}".format(
            len(attention_mask), max_seq_len
        )
        assert len(token_type_ids) == max_seq_len, "Error with token type length {} vs {}".format(
            len(token_type_ids), max_seq_len
        )
        label_id = example.label
        features.append(
            InputFeatures(
                input_ids=input_ids,
                attention_mask=attention_mask,
                token_type_ids=token_type_ids,
                label=label_id,
            )
        )
    return features
def convert_examples_to_features ( examples : List [ InputExample ] , max_seq_len : int , tokenizer : PreTrainedTokenizer , cls_token_segment_id = 0 , pad_token_segment_id = 0 , sequence_a_segment_id = 0 , mask_padding_with_zero = True , ) -> List [ InputFeatures ] : cls_token = tokenizer . cls_token sep_token = tokenizer . sep_token pad_token_id = tokenizer . pad_token_id features = [ ] for ( ex_index , example ) in enumerate ( examples ) : tokens = tokenizer . tokenize ( example . text_a ) special_tokens_count = 2 if len ( tokens ) > max_seq_len - special_tokens_count : tokens = tokens [ : ( max_seq_len - special_tokens_count ) ] tokens += [ sep_token ] token_type_ids = [ sequence_a_segment_id ] * len ( tokens ) tokens = [ cls_token ] + tokens token_type_ids = [ cls_token_segment_id ] + token_type_ids input_ids = tokenizer . convert_tokens_to_ids ( tokens ) attention_mask = [ 1 if mask_padding_with_zero else 0 ] * len ( input_ids ) padding_length = max_seq_len - len ( input_ids ) input_ids = input_ids + ( [ pad_token_id ] * padding_length ) attention_mask = attention_mask + ( [ 0 if mask_padding_with_zero else 1 ] * padding_length ) token_type_ids = token_type_ids + ( [ pad_token_segment_id ] * padding_length ) assert len ( input_ids ) == max_seq_len , "Error with input length {} vs {}" . format ( len ( input_ids ) , max_seq_len ) assert len ( attention_mask ) == max_seq_len , "Error with attention mask length {} vs {}" . format ( len ( attention_mask ) , max_seq_len ) assert len ( token_type_ids ) == max_seq_len , "Error with token type length {} vs {}" . format ( len ( token_type_ids ) , max_seq_len ) label_id = example . label features . append ( InputFeatures ( input_ids = input_ids , attention_mask = attention_mask , token_type_ids = token_type_ids , label = label_id , ) ) return features
Loads a data file into a list of `InputFeatures`
Loads a data file into a list of ` InputFeatures `
"""Loads a data file into a list of `InputFeatures`""" # Setting based on the current model type # Account for [CLS] and [SEP] # Add [SEP] token # Add [CLS] token # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. # Zero-pad up to the sequence length.
examples
List[InputExample]
max_seq_len
int
tokenizer
PreTrainedTokenizer
cls_token_segment_id
pad_token_segment_id
sequence_a_segment_id
mask_padding_with_zero