class_docstr: SentencePiece tokenizer for `lang`
class_docstr_tok: ["SentencePiece","tokenizer","for","`","lang","`"]
code: def setup(self, items, rules=None):
      from sentencepiece import SentencePieceProcessor
  
      if rules is None:
          rules = []
      if self.tok is not None:
          return {"sp_model": self.sp_model}
      raw_text_path = self.cache_dir / "texts.out"
      with open(raw_text_path, "w") as f:
          for t in progress_bar(
              maps(*rules, items), total=len(items), leave=False
          ):
              f.write(f"{t}\n")
      sp_model = self.train(raw_text_path)
      self.tok = SentencePieceProcessor()
      self.tok.Load(str(sp_model))
      return {"sp_model": sp_model}
  
imports: from __future__ import annotations
  from ..torch_basics import *
  from ..data.all import *
  import html
label_desc: Use image node_id 4 for calling the SentencePieceTokenizer obj's underlying member method code with example usage: obj.setup(items, rules) and returns: dict, dict