examples/language_model/pretraining_data_prepare/create_pretraining_data.py
def create_masked_lm_predictions(model_name,
                                 tokens,
                                 masked_lm_prob,
                                 max_predictions_per_seq,
                                 vocab_words,
                                 rng,
                                 check=False):
    """Creates the predictions for the masked LM objective."""
    cand_indexes = []
    for (i, token) in enumerate(tokens):
        if token == "[CLS]" or token == "[SEP]":
            continue
        # Whole Word Masking means that if we mask all of the wordpieces
        # corresponding to an original word. When a word has been split into
        # WordPieces, the first token does not have any marker and any subsequence
        # tokens are prefixed with ##. So whenever we see the ## token, we
        # append it to the previous set of word indexes.
        #
        # Note that Whole Word Masking does *not* change the training code
        # at all -- we still predict each WordPiece independently, softmaxed
        # over the entire vocabulary.
        if model_name in ["bert-wwm-chinese", "ernie-1.0"] and len(
                cand_indexes) >= 1 and token.startswith("##"):
            cand_indexes[-1].append(i)
        else:
            cand_indexes.append([i])

    rng.shuffle(cand_indexes)
    # drop the additiona;l "##" flag in Chinse word piece
    output_tokens = [
        t[2:] if len(re.findall('##[\u4E00-\u9FA5]', t)) > 0 else t
        for t in tokens
    ]

    num_to_predict = min(max_predictions_per_seq,
                         max(1, int(round(len(tokens) * masked_lm_prob))))

    masked_lms = []
    covered_indexes = set()
    for index_set in cand_indexes:
        if len(masked_lms) >= num_to_predict:
            break
        # If adding a whole-word mask would exceed the maximum number of
        # predictions, then just skip this candidate.
        if len(masked_lms) + len(index_set) > num_to_predict:
            continue
        is_any_index_covered = False
        for index in index_set:
            if index in covered_indexes:
                is_any_index_covered = True
                break
        if is_any_index_covered:
            continue
        for index in index_set:
            covered_indexes.add(index)

            masked_token = None
            # 80% of the time, replace with [MASK]
            if rng.random() < 0.8:
                masked_token = "[MASK]"
            else:
                # 10% of the time, keep original
                if rng.random() < 0.5:
                    # drop the additiona;l "##" flag in Chinse word piece
                    masked_token = tokens[index][2:] if len(
                        re.findall('##[\u4E00-\u9FA5]', tokens[
                            index])) > 0 else tokens[index]
                # 10% of the time, replace with random word
                else:
                    masked_token = vocab_words[rng.randint(
                        0, len(vocab_words) - 1)]

            output_tokens[index] = masked_token

            masked_lms.append(
                MaskedLmInstance(
                    index=index, label=tokens[index]))
    assert len(masked_lms) <= num_to_predict and len(masked_lms) > 0
    masked_lms = sorted(masked_lms, key=lambda x: x.index)

    masked_lm_positions = []
    masked_lm_labels = []
    for p in masked_lms:
        masked_lm_positions.append(p.index)
        masked_lm_labels.append(p.label)

    # Check some data. 
    if check and rng.random() < 0.005:
        print("%" * 100)
        print("raw tokens: ", tokens)
        print("output tokens: ", output_tokens)
    return (output_tokens, masked_lm_positions, masked_lm_labels)
Creates the predictions for the masked LM objective.
Creates the predictions for the masked LM objective.
Creates the predictions for the masked LM objective .
def create_masked_lm_predictions(model_name,
                                 tokens,
                                 masked_lm_prob,
                                 max_predictions_per_seq,
                                 vocab_words,
                                 rng,
                                 check=False):
    cand_indexes = []
    for (i, token) in enumerate(tokens):
        if token == "[CLS]" or token == "[SEP]":
            continue
         Note that Whole Word Masking does *not* change the training code
         at all -- we still predict each WordPiece independently, softmaxed
         over the entire vocabulary.
        if model_name in ["bert-wwm-chinese", "ernie-1.0"] and len(
                cand_indexes) >= 1 and token.startswith(""):
            cand_indexes[-1].append(i)
        else:
            cand_indexes.append([i])
    rng.shuffle(cand_indexes)
     drop the additiona;l "" flag in Chinse word piece
    output_tokens = [
        t[2:] if len(re.findall('[\u4E00-\u9FA5]', t)) > 0 else t
        for t in tokens
    ]
    num_to_predict = min(max_predictions_per_seq,
                         max(1, int(round(len(tokens) * masked_lm_prob))))
    masked_lms = []
    covered_indexes = set()
    for index_set in cand_indexes:
        if len(masked_lms) >= num_to_predict:
            break
         If adding a whole-word mask would exceed the maximum number of
         predictions, then just skip this candidate.
        if len(masked_lms) + len(index_set) > num_to_predict:
            continue
        is_any_index_covered = False
        for index in index_set:
            if index in covered_indexes:
                is_any_index_covered = True
                break
        if is_any_index_covered:
            continue
        for index in index_set:
            covered_indexes.add(index)
            masked_token = None
             80% of the time, replace with [MASK]
            if rng.random() < 0.8:
                masked_token = "[MASK]"
            else:
                 10% of the time, keep original
                if rng.random() < 0.5:
                     drop the additiona;l "" flag in Chinse word piece
                    masked_token = tokens[index][2:] if len(
                        re.findall('[\u4E00-\u9FA5]', tokens[
                            index])) > 0 else tokens[index]
                 10% of the time, replace with random word
                else:
                    masked_token = vocab_words[rng.randint(
                        0, len(vocab_words) - 1)]
            output_tokens[index] = masked_token
            masked_lms.append(
                MaskedLmInstance(
                    index=index, label=tokens[index]))
    assert len(masked_lms) <= num_to_predict and len(masked_lms) > 0
    masked_lms = sorted(masked_lms, key=lambda x: x.index)
    masked_lm_positions = []
    masked_lm_labels = []
    for p in masked_lms:
        masked_lm_positions.append(p.index)
        masked_lm_labels.append(p.label)
     Check some data. 
    if check and rng.random() < 0.005:
        print("%" * 100)
        print("raw tokens: ", tokens)
        print("output tokens: ", output_tokens)
    return (output_tokens, masked_lm_positions, masked_lm_labels)
def create_masked_lm_predictions ( model_name , tokens , masked_lm_prob , max_predictions_per_seq , vocab_words , rng , check = False ) : cand_indexes = [ ] for ( i , token ) in enumerate ( tokens ) : if token == "[CLS]" or token == "[SEP]" : continue if model_name in [ "bert-wwm-chinese" , "ernie-1.0" ] and len ( cand_indexes ) >= 1 and token . startswith ( "##" ) : cand_indexes [ - 1 ] . append ( i ) else : cand_indexes . append ( [ i ] ) rng . shuffle ( cand_indexes ) output_tokens = [ t [ 2 : ] if len ( re . findall ( '##[\u4E00-\u9FA5]' , t ) ) > 0 else t for t in tokens ] num_to_predict = min ( max_predictions_per_seq , max ( 1 , int ( round ( len ( tokens ) * masked_lm_prob ) ) ) ) masked_lms = [ ] covered_indexes = set ( ) for index_set in cand_indexes : if len ( masked_lms ) >= num_to_predict : break if len ( masked_lms ) + len ( index_set ) > num_to_predict : continue is_any_index_covered = False for index in index_set : if index in covered_indexes : is_any_index_covered = True break if is_any_index_covered : continue for index in index_set : covered_indexes . add ( index ) masked_token = None if rng . random ( ) < 0.8 : masked_token = "[MASK]" else : if rng . random ( ) < 0.5 : masked_token = tokens [ index ] [ 2 : ] if len ( re . findall ( '##[\u4E00-\u9FA5]' , tokens [ index ] ) ) > 0 else tokens [ index ] else : masked_token = vocab_words [ rng . randint ( 0 , len ( vocab_words ) - 1 ) ] output_tokens [ index ] = masked_token masked_lms . append ( MaskedLmInstance ( index = index , label = tokens [ index ] ) ) assert len ( masked_lms ) <= num_to_predict and len ( masked_lms ) > 0 masked_lms = sorted ( masked_lms , key = lambda x : x . index ) masked_lm_positions = [ ] masked_lm_labels = [ ] for p in masked_lms : masked_lm_positions . append ( p . index ) masked_lm_labels . append ( p . label ) if check and rng . random ( ) < 0.005 : print ( "%" * 100 ) print ( "raw tokens: " , tokens ) print ( "output tokens: " , output_tokens ) return ( output_tokens , masked_lm_positions , masked_lm_labels )
Creates the predictions for the masked LM objective.
Creates the predictions for the masked LM objective .
"""Creates the predictions for the masked LM objective.""" # Whole Word Masking means that if we mask all of the wordpieces # corresponding to an original word. When a word has been split into # WordPieces, the first token does not have any marker and any subsequence # tokens are prefixed with ##. So whenever we see the ## token, we # append it to the previous set of word indexes. # # Note that Whole Word Masking does *not* change the training code # at all -- we still predict each WordPiece independently, softmaxed # over the entire vocabulary. # drop the additiona;l "##" flag in Chinse word piece # If adding a whole-word mask would exceed the maximum number of # predictions, then just skip this candidate. # 80% of the time, replace with [MASK] # 10% of the time, keep original # drop the additiona;l "##" flag in Chinse word piece # 10% of the time, replace with random word # Check some data. 
model_name
tokens
masked_lm_prob
max_predictions_per_seq
vocab_words
rng
check