examples/language_model/bert/create_pretraining_data.py
def write_instance_to_example_file(instances, tokenizer, max_seq_length,
                                   max_predictions_per_seq, output_file):
    """Create example files from `TrainingInstance`s."""

    total_written = 0
    features = collections.OrderedDict()

    num_instances = len(instances)
    features["input_ids"] = np.zeros(
        [num_instances, max_seq_length], dtype="int32")
    features["input_mask"] = np.zeros(
        [num_instances, max_seq_length], dtype="int32")
    features["segment_ids"] = np.zeros(
        [num_instances, max_seq_length], dtype="int32")
    features["masked_lm_positions"] = np.zeros(
        [num_instances, max_predictions_per_seq], dtype="int32")
    features["masked_lm_ids"] = np.zeros(
        [num_instances, max_predictions_per_seq], dtype="int32")
    features["next_sentence_labels"] = np.zeros(num_instances, dtype="int32")

    for inst_index, instance in enumerate(tqdm(instances)):
        input_ids = tokenizer.convert_tokens_to_ids(instance.tokens)
        input_mask = [1] * len(input_ids)
        segment_ids = list(instance.segment_ids)
        assert len(input_ids) <= max_seq_length

        while len(input_ids) < max_seq_length:
            input_ids.append(0)
            input_mask.append(0)
            segment_ids.append(0)

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        masked_lm_positions = list(instance.masked_lm_positions)
        masked_lm_ids = tokenizer.convert_tokens_to_ids(
            instance.masked_lm_labels)
        masked_lm_weights = [1.0] * len(masked_lm_ids)

        while len(masked_lm_positions) < max_predictions_per_seq:
            masked_lm_positions.append(0)
            masked_lm_ids.append(0)
            masked_lm_weights.append(0.0)

        next_sentence_label = 1 if instance.is_random_next else 0

        features["input_ids"][inst_index] = input_ids
        features["input_mask"][inst_index] = input_mask
        features["segment_ids"][inst_index] = segment_ids
        features["masked_lm_positions"][inst_index] = masked_lm_positions
        features["masked_lm_ids"][inst_index] = masked_lm_ids
        features["next_sentence_labels"][inst_index] = next_sentence_label

        total_written += 1

    print("saving data")
    f = h5py.File(output_file, 'w')
    f.create_dataset(
        "input_ids", data=features["input_ids"], dtype='i4', compression='gzip')
    f.create_dataset(
        "input_mask",
        data=features["input_mask"],
        dtype='i1',
        compression='gzip')
    f.create_dataset(
        "segment_ids",
        data=features["segment_ids"],
        dtype='i1',
        compression='gzip')
    f.create_dataset(
        "masked_lm_positions",
        data=features["masked_lm_positions"],
        dtype='i4',
        compression='gzip')
    f.create_dataset(
        "masked_lm_ids",
        data=features["masked_lm_ids"],
        dtype='i4',
        compression='gzip')
    f.create_dataset(
        "next_sentence_labels",
        data=features["next_sentence_labels"],
        dtype='i1',
        compression='gzip')
    f.flush()
    f.close()
Create example files from `TrainingInstance`s.
Create example files from `TrainingInstance`s.
Create example files from ` TrainingInstance ` s .
def write_instance_to_example_file(instances, tokenizer, max_seq_length,
                                   max_predictions_per_seq, output_file):
    total_written = 0
    features = collections.OrderedDict()
    num_instances = len(instances)
    features["input_ids"] = np.zeros(
        [num_instances, max_seq_length], dtype="int32")
    features["input_mask"] = np.zeros(
        [num_instances, max_seq_length], dtype="int32")
    features["segment_ids"] = np.zeros(
        [num_instances, max_seq_length], dtype="int32")
    features["masked_lm_positions"] = np.zeros(
        [num_instances, max_predictions_per_seq], dtype="int32")
    features["masked_lm_ids"] = np.zeros(
        [num_instances, max_predictions_per_seq], dtype="int32")
    features["next_sentence_labels"] = np.zeros(num_instances, dtype="int32")
    for inst_index, instance in enumerate(tqdm(instances)):
        input_ids = tokenizer.convert_tokens_to_ids(instance.tokens)
        input_mask = [1] * len(input_ids)
        segment_ids = list(instance.segment_ids)
        assert len(input_ids) <= max_seq_length
        while len(input_ids) < max_seq_length:
            input_ids.append(0)
            input_mask.append(0)
            segment_ids.append(0)
        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length
        masked_lm_positions = list(instance.masked_lm_positions)
        masked_lm_ids = tokenizer.convert_tokens_to_ids(
            instance.masked_lm_labels)
        masked_lm_weights = [1.0] * len(masked_lm_ids)
        while len(masked_lm_positions) < max_predictions_per_seq:
            masked_lm_positions.append(0)
            masked_lm_ids.append(0)
            masked_lm_weights.append(0.0)
        next_sentence_label = 1 if instance.is_random_next else 0
        features["input_ids"][inst_index] = input_ids
        features["input_mask"][inst_index] = input_mask
        features["segment_ids"][inst_index] = segment_ids
        features["masked_lm_positions"][inst_index] = masked_lm_positions
        features["masked_lm_ids"][inst_index] = masked_lm_ids
        features["next_sentence_labels"][inst_index] = next_sentence_label
        total_written += 1
    print("saving data")
    f = h5py.File(output_file, 'w')
    f.create_dataset(
        "input_ids", data=features["input_ids"], dtype='i4', compression='gzip')
    f.create_dataset(
        "input_mask",
        data=features["input_mask"],
        dtype='i1',
        compression='gzip')
    f.create_dataset(
        "segment_ids",
        data=features["segment_ids"],
        dtype='i1',
        compression='gzip')
    f.create_dataset(
        "masked_lm_positions",
        data=features["masked_lm_positions"],
        dtype='i4',
        compression='gzip')
    f.create_dataset(
        "masked_lm_ids",
        data=features["masked_lm_ids"],
        dtype='i4',
        compression='gzip')
    f.create_dataset(
        "next_sentence_labels",
        data=features["next_sentence_labels"],
        dtype='i1',
        compression='gzip')
    f.flush()
    f.close()
def write_instance_to_example_file ( instances , tokenizer , max_seq_length , max_predictions_per_seq , output_file ) : total_written = 0 features = collections . OrderedDict ( ) num_instances = len ( instances ) features [ "input_ids" ] = np . zeros ( [ num_instances , max_seq_length ] , dtype = "int32" ) features [ "input_mask" ] = np . zeros ( [ num_instances , max_seq_length ] , dtype = "int32" ) features [ "segment_ids" ] = np . zeros ( [ num_instances , max_seq_length ] , dtype = "int32" ) features [ "masked_lm_positions" ] = np . zeros ( [ num_instances , max_predictions_per_seq ] , dtype = "int32" ) features [ "masked_lm_ids" ] = np . zeros ( [ num_instances , max_predictions_per_seq ] , dtype = "int32" ) features [ "next_sentence_labels" ] = np . zeros ( num_instances , dtype = "int32" ) for inst_index , instance in enumerate ( tqdm ( instances ) ) : input_ids = tokenizer . convert_tokens_to_ids ( instance . tokens ) input_mask = [ 1 ] * len ( input_ids ) segment_ids = list ( instance . segment_ids ) assert len ( input_ids ) <= max_seq_length while len ( input_ids ) < max_seq_length : input_ids . append ( 0 ) input_mask . append ( 0 ) segment_ids . append ( 0 ) assert len ( input_ids ) == max_seq_length assert len ( input_mask ) == max_seq_length assert len ( segment_ids ) == max_seq_length masked_lm_positions = list ( instance . masked_lm_positions ) masked_lm_ids = tokenizer . convert_tokens_to_ids ( instance . masked_lm_labels ) masked_lm_weights = [ 1.0 ] * len ( masked_lm_ids ) while len ( masked_lm_positions ) < max_predictions_per_seq : masked_lm_positions . append ( 0 ) masked_lm_ids . append ( 0 ) masked_lm_weights . append ( 0.0 ) next_sentence_label = 1 if instance . is_random_next else 0 features [ "input_ids" ] [ inst_index ] = input_ids features [ "input_mask" ] [ inst_index ] = input_mask features [ "segment_ids" ] [ inst_index ] = segment_ids features [ "masked_lm_positions" ] [ inst_index ] = masked_lm_positions features [ "masked_lm_ids" ] [ inst_index ] = masked_lm_ids features [ "next_sentence_labels" ] [ inst_index ] = next_sentence_label total_written += 1 print ( "saving data" ) f = h5py . File ( output_file , 'w' ) f . create_dataset ( "input_ids" , data = features [ "input_ids" ] , dtype = 'i4' , compression = 'gzip' ) f . create_dataset ( "input_mask" , data = features [ "input_mask" ] , dtype = 'i1' , compression = 'gzip' ) f . create_dataset ( "segment_ids" , data = features [ "segment_ids" ] , dtype = 'i1' , compression = 'gzip' ) f . create_dataset ( "masked_lm_positions" , data = features [ "masked_lm_positions" ] , dtype = 'i4' , compression = 'gzip' ) f . create_dataset ( "masked_lm_ids" , data = features [ "masked_lm_ids" ] , dtype = 'i4' , compression = 'gzip' ) f . create_dataset ( "next_sentence_labels" , data = features [ "next_sentence_labels" ] , dtype = 'i1' , compression = 'gzip' ) f . flush ( ) f . close ( )
Create example files from `TrainingInstance`s.
Create example files from ` TrainingInstance ` s .
"""Create example files from `TrainingInstance`s."""
instances
tokenizer
max_seq_length
max_predictions_per_seq
output_file