BERT Pretraining and Preprocessing/Preprocessing Code/create_pretraining_data_ENS_with_diff_tokenizer.py
def write_instance_to_example_files(instances, tokenizer, max_seq_length,
                                    max_predictions_per_seq, output_files):
  """Create TF example files from `TrainingInstance`s."""
  writers = []
  for output_file in output_files:
    writers.append(tf.io.TFRecordWriter(output_file))

  writer_index = 0

  total_written = 0
  for (inst_index, instance) in enumerate(instances):
    if(inst_index%10000==0):
      logging.info("Writing: "+str(inst_index)+" instance")
    input_ids = [tokenizer.token_to_id(x) for x in instance.tokens]
#     input_ids = tokenizer.convert_tokens_to_ids(instance.tokens)
    input_mask = [1] * len(input_ids)
    segment_ids = list(instance.segment_ids)
    assert len(input_ids) <= max_seq_length

    while len(input_ids) < max_seq_length:
      input_ids.append(0)
      input_mask.append(0)
      segment_ids.append(0)

    assert len(input_ids) == max_seq_length
    assert len(input_mask) == max_seq_length
    assert len(segment_ids) == max_seq_length

    masked_lm_positions = list(instance.masked_lm_positions)
    masked_lm_ids = [tokenizer.token_to_id(x) for x in instance.masked_lm_labels]
#     masked_lm_ids = tokenizer.convert_tokens_to_ids(instance.masked_lm_labels)
    masked_lm_weights = [1.0] * len(masked_lm_ids)

    while len(masked_lm_positions) < max_predictions_per_seq:
      masked_lm_positions.append(0)
      masked_lm_ids.append(0)
      masked_lm_weights.append(0.0)

    next_sentence_label = 1 if instance.is_random_next else 0

    features = collections.OrderedDict()
    features["input_ids"] = create_int_feature(input_ids)
    features["input_mask"] = create_int_feature(input_mask)
    features["segment_ids"] = create_int_feature(segment_ids)
    features["masked_lm_positions"] = create_int_feature(masked_lm_positions)
    features["masked_lm_ids"] = create_int_feature(masked_lm_ids)
    features["masked_lm_weights"] = create_float_feature(masked_lm_weights)
    features["next_sentence_labels"] = create_int_feature([next_sentence_label])

    tf_example = tf.train.Example(features=tf.train.Features(feature=features))

    writers[writer_index].write(tf_example.SerializeToString())
    writer_index = (writer_index + 1) % len(writers)

    total_written += 1

    if inst_index < 20:
      logging.info("*** Example ***")
      logging.info("tokens: %s" % " ".join(
          [tokenization.printable_text(x) for x in instance.tokens]))

      for feature_name in features.keys():
        feature = features[feature_name]
        values = []
        if feature.int64_list.value:
          values = feature.int64_list.value
        elif feature.float_list.value:
          values = feature.float_list.value
        logging.info(
            "%s: %s" % (feature_name, " ".join([str(x) for x in values])))

  for writer in writers:
    writer.close()

  logging.info("Wrote %d total instances" % total_written)
Create TF example files from `TrainingInstance`s.
Create TF example files from `TrainingInstance`s.
Create TF example files from ` TrainingInstance ` s .
def write_instance_to_example_files(instances, tokenizer, max_seq_length,
                                    max_predictions_per_seq, output_files):
  writers = []
  for output_file in output_files:
    writers.append(tf.io.TFRecordWriter(output_file))
  writer_index = 0
  total_written = 0
  for (inst_index, instance) in enumerate(instances):
    if(inst_index%10000==0):
      logging.info("Writing: "+str(inst_index)+" instance")
    input_ids = [tokenizer.token_to_id(x) for x in instance.tokens]
    input_mask = [1] * len(input_ids)
    segment_ids = list(instance.segment_ids)
    assert len(input_ids) <= max_seq_length
    while len(input_ids) < max_seq_length:
      input_ids.append(0)
      input_mask.append(0)
      segment_ids.append(0)
    assert len(input_ids) == max_seq_length
    assert len(input_mask) == max_seq_length
    assert len(segment_ids) == max_seq_length
    masked_lm_positions = list(instance.masked_lm_positions)
    masked_lm_ids = [tokenizer.token_to_id(x) for x in instance.masked_lm_labels]
    masked_lm_weights = [1.0] * len(masked_lm_ids)
    while len(masked_lm_positions) < max_predictions_per_seq:
      masked_lm_positions.append(0)
      masked_lm_ids.append(0)
      masked_lm_weights.append(0.0)
    next_sentence_label = 1 if instance.is_random_next else 0
    features = collections.OrderedDict()
    features["input_ids"] = create_int_feature(input_ids)
    features["input_mask"] = create_int_feature(input_mask)
    features["segment_ids"] = create_int_feature(segment_ids)
    features["masked_lm_positions"] = create_int_feature(masked_lm_positions)
    features["masked_lm_ids"] = create_int_feature(masked_lm_ids)
    features["masked_lm_weights"] = create_float_feature(masked_lm_weights)
    features["next_sentence_labels"] = create_int_feature([next_sentence_label])
    tf_example = tf.train.Example(features=tf.train.Features(feature=features))
    writers[writer_index].write(tf_example.SerializeToString())
    writer_index = (writer_index + 1) % len(writers)
    total_written += 1
    if inst_index < 20:
      logging.info("*** Example ***")
      logging.info("tokens: %s" % " ".join(
          [tokenization.printable_text(x) for x in instance.tokens]))
      for feature_name in features.keys():
        feature = features[feature_name]
        values = []
        if feature.int64_list.value:
          values = feature.int64_list.value
        elif feature.float_list.value:
          values = feature.float_list.value
        logging.info(
            "%s: %s" % (feature_name, " ".join([str(x) for x in values])))
  for writer in writers:
    writer.close()
  logging.info("Wrote %d total instances" % total_written)
def write_instance_to_example_files ( instances , tokenizer , max_seq_length , max_predictions_per_seq , output_files ) : writers = [ ] for output_file in output_files : writers . append ( tf . io . TFRecordWriter ( output_file ) ) writer_index = 0 total_written = 0 for ( inst_index , instance ) in enumerate ( instances ) : if ( inst_index % 10000 == 0 ) : logging . info ( "Writing: " + str ( inst_index ) + " instance" ) input_ids = [ tokenizer . token_to_id ( x ) for x in instance . tokens ] input_mask = [ 1 ] * len ( input_ids ) segment_ids = list ( instance . segment_ids ) assert len ( input_ids ) <= max_seq_length while len ( input_ids ) < max_seq_length : input_ids . append ( 0 ) input_mask . append ( 0 ) segment_ids . append ( 0 ) assert len ( input_ids ) == max_seq_length assert len ( input_mask ) == max_seq_length assert len ( segment_ids ) == max_seq_length masked_lm_positions = list ( instance . masked_lm_positions ) masked_lm_ids = [ tokenizer . token_to_id ( x ) for x in instance . masked_lm_labels ] masked_lm_weights = [ 1.0 ] * len ( masked_lm_ids ) while len ( masked_lm_positions ) < max_predictions_per_seq : masked_lm_positions . append ( 0 ) masked_lm_ids . append ( 0 ) masked_lm_weights . append ( 0.0 ) next_sentence_label = 1 if instance . is_random_next else 0 features = collections . OrderedDict ( ) features [ "input_ids" ] = create_int_feature ( input_ids ) features [ "input_mask" ] = create_int_feature ( input_mask ) features [ "segment_ids" ] = create_int_feature ( segment_ids ) features [ "masked_lm_positions" ] = create_int_feature ( masked_lm_positions ) features [ "masked_lm_ids" ] = create_int_feature ( masked_lm_ids ) features [ "masked_lm_weights" ] = create_float_feature ( masked_lm_weights ) features [ "next_sentence_labels" ] = create_int_feature ( [ next_sentence_label ] ) tf_example = tf . train . Example ( features = tf . train . Features ( feature = features ) ) writers [ writer_index ] . write ( tf_example . SerializeToString ( ) ) writer_index = ( writer_index + 1 ) % len ( writers ) total_written += 1 if inst_index < 20 : logging . info ( "*** Example ***" ) logging . info ( "tokens: %s" % " " . join ( [ tokenization . printable_text ( x ) for x in instance . tokens ] ) ) for feature_name in features . keys ( ) : feature = features [ feature_name ] values = [ ] if feature . int64_list . value : values = feature . int64_list . value elif feature . float_list . value : values = feature . float_list . value logging . info ( "%s: %s" % ( feature_name , " " . join ( [ str ( x ) for x in values ] ) ) ) for writer in writers : writer . close ( ) logging . info ( "Wrote %d total instances" % total_written )
Create TF example files from `TrainingInstance`s.
Create TF example files from ` TrainingInstance ` s .
"""Create TF example files from `TrainingInstance`s.""" #     input_ids = tokenizer.convert_tokens_to_ids(instance.tokens) #     masked_lm_ids = tokenizer.convert_tokens_to_ids(instance.masked_lm_labels)
instances
tokenizer
max_seq_length
max_predictions_per_seq
output_files