examples/text_generation/unimo-text/gen_utils.py
def post_process_sum(token_ids, tokenizer):
    """Post-process the decoded sequence. Truncate from the first <eos>."""
    eos_pos = len(token_ids)
    for i, tok_id in enumerate(token_ids):
        if tok_id == tokenizer.mask_token_id:
            eos_pos = i
            break
    token_ids = token_ids[:eos_pos]
    tokens = tokenizer.convert_ids_to_tokens(token_ids)
    tokens = tokenizer.merge_subword(tokens)
    special_tokens = ['[UNK]']
    tokens = [token for token in tokens if token not in special_tokens]
    return token_ids, tokens
Post-process the decoded sequence. Truncate from the first <eos>.
Post-process the decoded sequence. Truncate from the first .
Post - process the decoded sequence . Truncate from the first .
def post_process_sum(token_ids, tokenizer):
    eos_pos = len(token_ids)
    for i, tok_id in enumerate(token_ids):
        if tok_id == tokenizer.mask_token_id:
            eos_pos = i
            break
    token_ids = token_ids[:eos_pos]
    tokens = tokenizer.convert_ids_to_tokens(token_ids)
    tokens = tokenizer.merge_subword(tokens)
    special_tokens = ['[UNK]']
    tokens = [token for token in tokens if token not in special_tokens]
    return token_ids, tokens
def post_process_sum ( token_ids , tokenizer ) : eos_pos = len ( token_ids ) for i , tok_id in enumerate ( token_ids ) : if tok_id == tokenizer . mask_token_id : eos_pos = i break token_ids = token_ids [ : eos_pos ] tokens = tokenizer . convert_ids_to_tokens ( token_ids ) tokens = tokenizer . merge_subword ( tokens ) special_tokens = [ '[UNK]' ] tokens = [ token for token in tokens if token not in special_tokens ] return token_ids , tokens
Post-process the decoded sequence.
Post - process the decoded sequence .
"""Post-process the decoded sequence. Truncate from the first <eos>."""
token_ids
tokenizer