class_docstr: Version of parlai.utils.bpe.SubwordBPEHelper that can be TorchScripted.
class_docstr_tok: ["Version","of","parlai.utils.bpe.SubwordBPEHelper","that","can","be","TorchScripted","."]
docstr: segment a sequence of tokens with BPE encoding.
docstr_tok: ["segment","a","sequence","of","tokens","with","BPE","encoding","."]
code: def segment_tokens(self, tokens: List[str]) -> List[str]:
      """
      segment a sequence of tokens with BPE encoding.
      """
      output: List[str] = []
      for word in tokens:
          # eliminate double spaces
          if not word:
              continue
  
          new_word: List[str] = []
          for out in self.bpe(word):
              new_word.append(out)
  
          for item in new_word[:-1]:
              output.append(item + self.separator)
          output.append(new_word[-1])
  
      return output
  
imports: from typing import Dict, List, Tuple
  import torch.jit
label_desc: Use image node_id 6 for calling the ScriptableSubwordBpeHelper obj's underlying member method code with example usage: obj.segment_tokens(tokens) and returns: output