class_docstr: Runs end-to-end tokenziation.
class_docstr_tok: ["Runs","end-to-end","tokenziation","."]
docstr: Merge subword.
docstr_tok: ["Merge","subword","."]
code: def merge_subword(self, tokens):
      """Merge subword."""
      ret = []
      for token in tokens:
          if token.startswith("▁"):
              ret.append(token[1:])
          else:
              if len(ret):
                  ret[-1] += token
              else:
                  ret.append(token)
  
      ret = [token for token in ret if token]
      return ret
  
imports: import collections
  import sentencepiece
  import unicodedata
  from .args import str2bool
label_desc: Use image node_id 6 for calling the SentencePieceTokenizer obj's underlying member method code with example usage: obj.merge_subword(tokens) and returns: ret