BERT Pretraining and Preprocessing/Preprocessing Code/create_pretraining_data_ENS_with_diff_tokenizer.py
def create_instances_from_document(
    all_documents, document_index, max_seq_length, short_seq_prob,
    masked_lm_prob, max_predictions_per_seq, vocab_words, rng):
  """Creates `TrainingInstance`s for a single document."""
  document = all_documents[document_index]

  # Account for [CLS], [SEP], [SEP]
  max_num_tokens = max_seq_length - 3

  # We *usually* want to fill up the entire sequence since we are padding
  # to `max_seq_length` anyways, so short sequences are generally wasted
  # computation. However, we *sometimes*
  # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter
  # sequences to minimize the mismatch between pre-training and fine-tuning.
  # The `target_seq_length` is just a rough target however, whereas
  # `max_seq_length` is a hard limit.
  target_seq_length = max_num_tokens
  if rng.random() < short_seq_prob:
    target_seq_length = rng.randint(2, max_num_tokens)

  # We DON'T just concatenate all of the tokens from a document into a long
  # sequence and choose an arbitrary split point because this would make the
  # next sentence prediction task too easy. Instead, we split the input into
  # segments "A" and "B" based on the actual "sentences" provided by the user
  # input.
  instances = []
  current_chunk = []
  current_length = 0
  i = 0
  while i < len(document):
    segment = document[i]
    current_chunk.append(segment)
    current_length += len(segment)
    if i == len(document) - 1 or current_length >= target_seq_length:
      if current_chunk:
        # `a_end` is how many segments from `current_chunk` go into the `A`
        # (first) sentence.
        # a_end is set to the complete current_chunk as the
        # 'B' has to be kept empty and hence the whole 
        # current_chunk goes into 'A' (before swapping)
        a_end = len(current_chunk)
        tokens_a = []
        for j in range(a_end):
          tokens_a.extend(current_chunk[j])

        tokens_b = []
        # tokens_b will be kept empty        
        # Random next. Just being kept because code requires it to.
        # Not used as such.        
        is_random_next = False
        truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng)
        
        assert len(tokens_a) >= 1
        tokens_b=[]  # tokens_b has been made empty
        # Swapping tokens_a and tokens_b to have either empty_first_sentence
        # or empty_next_sentence
        if(rng.random() < 0.5):
          tokens_a,tokens_b=tokens_b,tokens_a
        
        tokens = []
        segment_ids = []
        tokens.append("[CLS]")
        segment_ids.append(0)
        for token in tokens_a:
          tokens.append(token)
          segment_ids.append(0)

        tokens.append("[SEP]")
        segment_ids.append(0)

        for token in tokens_b:
          tokens.append(token)
          segment_ids.append(1)
        tokens.append("[SEP]")
        segment_ids.append(1)

        (tokens, masked_lm_positions,
         masked_lm_labels) = create_masked_lm_predictions(
             tokens, masked_lm_prob, max_predictions_per_seq, vocab_words, rng)
        instance = TrainingInstance(
            tokens=tokens,
            segment_ids=segment_ids,
            is_random_next=is_random_next,
            masked_lm_positions=masked_lm_positions,
            masked_lm_labels=masked_lm_labels)
        instances.append(instance)
      current_chunk = []
      current_length = 0
    i += 1

  return instances
Creates `TrainingInstance`s for a single document.
Creates `TrainingInstance`s for a single document.
Creates ` TrainingInstance ` s for a single document .
def create_instances_from_document(
    all_documents, document_index, max_seq_length, short_seq_prob,
    masked_lm_prob, max_predictions_per_seq, vocab_words, rng):
  document = all_documents[document_index]
  max_num_tokens = max_seq_length - 3
  target_seq_length = max_num_tokens
  if rng.random() < short_seq_prob:
    target_seq_length = rng.randint(2, max_num_tokens)
  instances = []
  current_chunk = []
  current_length = 0
  i = 0
  while i < len(document):
    segment = document[i]
    current_chunk.append(segment)
    current_length += len(segment)
    if i == len(document) - 1 or current_length >= target_seq_length:
      if current_chunk:
        a_end = len(current_chunk)
        tokens_a = []
        for j in range(a_end):
          tokens_a.extend(current_chunk[j])
        tokens_b = []
        is_random_next = False
        truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng)
        assert len(tokens_a) >= 1
        tokens_b=[]  
        if(rng.random() < 0.5):
          tokens_a,tokens_b=tokens_b,tokens_a
        tokens = []
        segment_ids = []
        tokens.append("[CLS]")
        segment_ids.append(0)
        for token in tokens_a:
          tokens.append(token)
          segment_ids.append(0)
        tokens.append("[SEP]")
        segment_ids.append(0)
        for token in tokens_b:
          tokens.append(token)
          segment_ids.append(1)
        tokens.append("[SEP]")
        segment_ids.append(1)
        (tokens, masked_lm_positions,
         masked_lm_labels) = create_masked_lm_predictions(
             tokens, masked_lm_prob, max_predictions_per_seq, vocab_words, rng)
        instance = TrainingInstance(
            tokens=tokens,
            segment_ids=segment_ids,
            is_random_next=is_random_next,
            masked_lm_positions=masked_lm_positions,
            masked_lm_labels=masked_lm_labels)
        instances.append(instance)
      current_chunk = []
      current_length = 0
    i += 1
  return instances
def create_instances_from_document ( all_documents , document_index , max_seq_length , short_seq_prob , masked_lm_prob , max_predictions_per_seq , vocab_words , rng ) : document = all_documents [ document_index ] max_num_tokens = max_seq_length - 3 target_seq_length = max_num_tokens if rng . random ( ) < short_seq_prob : target_seq_length = rng . randint ( 2 , max_num_tokens ) instances = [ ] current_chunk = [ ] current_length = 0 i = 0 while i < len ( document ) : segment = document [ i ] current_chunk . append ( segment ) current_length += len ( segment ) if i == len ( document ) - 1 or current_length >= target_seq_length : if current_chunk : a_end = len ( current_chunk ) tokens_a = [ ] for j in range ( a_end ) : tokens_a . extend ( current_chunk [ j ] ) tokens_b = [ ] is_random_next = False truncate_seq_pair ( tokens_a , tokens_b , max_num_tokens , rng ) assert len ( tokens_a ) >= 1 tokens_b = [ ] if ( rng . random ( ) < 0.5 ) : tokens_a , tokens_b = tokens_b , tokens_a tokens = [ ] segment_ids = [ ] tokens . append ( "[CLS]" ) segment_ids . append ( 0 ) for token in tokens_a : tokens . append ( token ) segment_ids . append ( 0 ) tokens . append ( "[SEP]" ) segment_ids . append ( 0 ) for token in tokens_b : tokens . append ( token ) segment_ids . append ( 1 ) tokens . append ( "[SEP]" ) segment_ids . append ( 1 ) ( tokens , masked_lm_positions , masked_lm_labels ) = create_masked_lm_predictions ( tokens , masked_lm_prob , max_predictions_per_seq , vocab_words , rng ) instance = TrainingInstance ( tokens = tokens , segment_ids = segment_ids , is_random_next = is_random_next , masked_lm_positions = masked_lm_positions , masked_lm_labels = masked_lm_labels ) instances . append ( instance ) current_chunk = [ ] current_length = 0 i += 1 return instances
Creates `TrainingInstance`s for a single document.
Creates ` TrainingInstance ` s for a single document .
"""Creates `TrainingInstance`s for a single document.""" # Account for [CLS], [SEP], [SEP] # We *usually* want to fill up the entire sequence since we are padding # to `max_seq_length` anyways, so short sequences are generally wasted # computation. However, we *sometimes* # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter # sequences to minimize the mismatch between pre-training and fine-tuning. # The `target_seq_length` is just a rough target however, whereas # `max_seq_length` is a hard limit. # We DON'T just concatenate all of the tokens from a document into a long # sequence and choose an arbitrary split point because this would make the # next sentence prediction task too easy. Instead, we split the input into # segments "A" and "B" based on the actual "sentences" provided by the user # input. # `a_end` is how many segments from `current_chunk` go into the `A` # (first) sentence. # a_end is set to the complete current_chunk as the # 'B' has to be kept empty and hence the whole  # current_chunk goes into 'A' (before swapping) # tokens_b will be kept empty         # Random next. Just being kept because code requires it to. # Not used as such.         # tokens_b has been made empty # Swapping tokens_a and tokens_b to have either empty_first_sentence # or empty_next_sentence
all_documents
document_index
max_seq_length
short_seq_prob
masked_lm_prob
max_predictions_per_seq
vocab_words
rng