examples/language_model/bert/create_pretraining_data.py
def create_training_instances(input_files, tokenizer, max_seq_length,
                              dupe_factor, short_seq_prob, masked_lm_prob,
                              max_predictions_per_seq, rng):
    """Create `TrainingInstance`s from raw text."""
    all_documents = [[]]

    # Input file format:
    # (1) One sentence per line. These should ideally be actual sentences, not
    # entire paragraphs or arbitrary spans of text. (Because we use the
    # sentence boundaries for the "next sentence prediction" task).
    # (2) Blank lines between documents. Document boundaries are needed so
    # that the "next sentence prediction" task doesn't span between documents.
    for input_file in input_files:
        print("creating instance from {}".format(input_file))
        with open(input_file, "r", encoding="UTF-8") as reader:
            while True:
                line = convert_to_unicode(reader.readline())
                if not line:
                    break
                line = line.strip()

                # Empty lines are used as document delimiters
                if not line:
                    all_documents.append([])
                tokens = tokenizer.tokenize(line)
                if tokens:
                    all_documents[-1].append(tokens)

    # Remove empty documents
    all_documents = [x for x in all_documents if x]
    rng.shuffle(all_documents)

    # vocab_words = list(tokenizer.vocab.keys())
    vocab_words = list(tokenizer.vocab.token_to_idx.keys())
    instances = []
    for _ in range(dupe_factor):
        for document_index in range(len(all_documents)):
            instances.extend(
                create_instances_from_document(
                    all_documents, document_index, max_seq_length,
                    short_seq_prob, masked_lm_prob, max_predictions_per_seq,
                    vocab_words, rng))

    rng.shuffle(instances)
    return instances
Create `TrainingInstance`s from raw text.
Create `TrainingInstance`s from raw text.
Create ` TrainingInstance ` s from raw text .
def create_training_instances(input_files, tokenizer, max_seq_length,
                              dupe_factor, short_seq_prob, masked_lm_prob,
                              max_predictions_per_seq, rng):
    all_documents = [[]]
    for input_file in input_files:
        print("creating instance from {}".format(input_file))
        with open(input_file, "r", encoding="UTF-8") as reader:
            while True:
                line = convert_to_unicode(reader.readline())
                if not line:
                    break
                line = line.strip()
                if not line:
                    all_documents.append([])
                tokens = tokenizer.tokenize(line)
                if tokens:
                    all_documents[-1].append(tokens)
    all_documents = [x for x in all_documents if x]
    rng.shuffle(all_documents)
    vocab_words = list(tokenizer.vocab.token_to_idx.keys())
    instances = []
    for _ in range(dupe_factor):
        for document_index in range(len(all_documents)):
            instances.extend(
                create_instances_from_document(
                    all_documents, document_index, max_seq_length,
                    short_seq_prob, masked_lm_prob, max_predictions_per_seq,
                    vocab_words, rng))
    rng.shuffle(instances)
    return instances
def create_training_instances ( input_files , tokenizer , max_seq_length , dupe_factor , short_seq_prob , masked_lm_prob , max_predictions_per_seq , rng ) : all_documents = [ [ ] ] for input_file in input_files : print ( "creating instance from {}" . format ( input_file ) ) with open ( input_file , "r" , encoding = "UTF-8" ) as reader : while True : line = convert_to_unicode ( reader . readline ( ) ) if not line : break line = line . strip ( ) if not line : all_documents . append ( [ ] ) tokens = tokenizer . tokenize ( line ) if tokens : all_documents [ - 1 ] . append ( tokens ) all_documents = [ x for x in all_documents if x ] rng . shuffle ( all_documents ) vocab_words = list ( tokenizer . vocab . token_to_idx . keys ( ) ) instances = [ ] for _ in range ( dupe_factor ) : for document_index in range ( len ( all_documents ) ) : instances . extend ( create_instances_from_document ( all_documents , document_index , max_seq_length , short_seq_prob , masked_lm_prob , max_predictions_per_seq , vocab_words , rng ) ) rng . shuffle ( instances ) return instances
Create `TrainingInstance`s from raw text.
Create ` TrainingInstance ` s from raw text .
"""Create `TrainingInstance`s from raw text.""" # Input file format: # (1) One sentence per line. These should ideally be actual sentences, not # entire paragraphs or arbitrary spans of text. (Because we use the # sentence boundaries for the "next sentence prediction" task). # (2) Blank lines between documents. Document boundaries are needed so # that the "next sentence prediction" task doesn't span between documents. # Empty lines are used as document delimiters # Remove empty documents # vocab_words = list(tokenizer.vocab.keys())
input_files
tokenizer
max_seq_length
dupe_factor
short_seq_prob
masked_lm_prob
max_predictions_per_seq
rng