BERT Pretraining and Preprocessing/Preprocessing Code/create_pretraining_data_ENS_with_diff_tokenizer.py
def create_training_instances(input_files, tokenizer, max_seq_length,
                              dupe_factor, short_seq_prob, masked_lm_prob,
                              max_predictions_per_seq, rng):
  """Create `TrainingInstance`s from raw text."""
  all_documents = [[]]

  # Input file format:
  # (1) One sentence per line. These should ideally be actual sentences, not
  # entire paragraphs or arbitrary spans of text. (Because we use the
  # sentence boundaries for the "next sentence prediction" task).
  # (2) Blank lines between documents. Document boundaries are needed so
  # that the "next sentence prediction" task doesn't span between documents.
  for input_file in input_files:
    with tf.io.gfile.GFile(input_file, "r") as reader:
      count=0
      while True:
        
        if(count%50000==0):
          logging.info("Reading document number: %d" % count)
        line = tokenization.convert_to_unicode(reader.readline())
        if not line:
          break
        line = line.strip()

        # Empty lines are used as document delimiters
        if not line:
          count+=1
          all_documents.append([])
        tokens = tokenizer.encode(line).tokens  
#         tokens = tokenizer.tokenize(line)
        if tokens:
          all_documents[-1].append(tokens)

  # Remove empty documents
  all_documents = [x for x in all_documents if x]
  rng.shuffle(all_documents)

  vocab_words = list(tokenizer.get_vocab().keys())
#   vocab_words = list(tokenizer.vocab.keys())
    
  instances = []

  logging.info("Number of documents="+str(len(all_documents)))
  for dff in range(dupe_factor):
    for document_index in range(len(all_documents)):
      if(document_index%10000==0):
        logging.info("Creating for document index,dupe_factor: %d , %d" %(document_index,dff))
      instances.extend(
          create_instances_from_document(
              all_documents, document_index, max_seq_length, short_seq_prob,
              masked_lm_prob, max_predictions_per_seq, vocab_words, rng))

  rng.shuffle(instances)
  return instances
Create `TrainingInstance`s from raw text.
Create `TrainingInstance`s from raw text.
Create ` TrainingInstance ` s from raw text .
def create_training_instances(input_files, tokenizer, max_seq_length,
                              dupe_factor, short_seq_prob, masked_lm_prob,
                              max_predictions_per_seq, rng):
  all_documents = [[]]
  for input_file in input_files:
    with tf.io.gfile.GFile(input_file, "r") as reader:
      count=0
      while True:
        if(count%50000==0):
          logging.info("Reading document number: %d" % count)
        line = tokenization.convert_to_unicode(reader.readline())
        if not line:
          break
        line = line.strip()
        if not line:
          count+=1
          all_documents.append([])
        tokens = tokenizer.encode(line).tokens  
        if tokens:
          all_documents[-1].append(tokens)
  all_documents = [x for x in all_documents if x]
  rng.shuffle(all_documents)
  vocab_words = list(tokenizer.get_vocab().keys())
  instances = []
  logging.info("Number of documents="+str(len(all_documents)))
  for dff in range(dupe_factor):
    for document_index in range(len(all_documents)):
      if(document_index%10000==0):
        logging.info("Creating for document index,dupe_factor: %d , %d" %(document_index,dff))
      instances.extend(
          create_instances_from_document(
              all_documents, document_index, max_seq_length, short_seq_prob,
              masked_lm_prob, max_predictions_per_seq, vocab_words, rng))
  rng.shuffle(instances)
  return instances
def create_training_instances ( input_files , tokenizer , max_seq_length , dupe_factor , short_seq_prob , masked_lm_prob , max_predictions_per_seq , rng ) : all_documents = [ [ ] ] for input_file in input_files : with tf . io . gfile . GFile ( input_file , "r" ) as reader : count = 0 while True : if ( count % 50000 == 0 ) : logging . info ( "Reading document number: %d" % count ) line = tokenization . convert_to_unicode ( reader . readline ( ) ) if not line : break line = line . strip ( ) if not line : count += 1 all_documents . append ( [ ] ) tokens = tokenizer . encode ( line ) . tokens if tokens : all_documents [ - 1 ] . append ( tokens ) all_documents = [ x for x in all_documents if x ] rng . shuffle ( all_documents ) vocab_words = list ( tokenizer . get_vocab ( ) . keys ( ) ) instances = [ ] logging . info ( "Number of documents=" + str ( len ( all_documents ) ) ) for dff in range ( dupe_factor ) : for document_index in range ( len ( all_documents ) ) : if ( document_index % 10000 == 0 ) : logging . info ( "Creating for document index,dupe_factor: %d , %d" % ( document_index , dff ) ) instances . extend ( create_instances_from_document ( all_documents , document_index , max_seq_length , short_seq_prob , masked_lm_prob , max_predictions_per_seq , vocab_words , rng ) ) rng . shuffle ( instances ) return instances
Create `TrainingInstance`s from raw text.
Create ` TrainingInstance ` s from raw text .
"""Create `TrainingInstance`s from raw text.""" # Input file format: # (1) One sentence per line. These should ideally be actual sentences, not # entire paragraphs or arbitrary spans of text. (Because we use the # sentence boundaries for the "next sentence prediction" task). # (2) Blank lines between documents. Document boundaries are needed so # that the "next sentence prediction" task doesn't span between documents. # Empty lines are used as document delimiters #         tokens = tokenizer.tokenize(line) # Remove empty documents #   vocab_words = list(tokenizer.vocab.keys())
input_files
tokenizer
max_seq_length
dupe_factor
short_seq_prob
masked_lm_prob
max_predictions_per_seq
rng