class_docstr: SentencePiece tokenizer for `lang`
class_docstr_tok: ["SentencePiece","tokenizer","for","`","lang","`"]
docstr: Train a sentencepiece tokenizer on `texts` and save it in `path/tmp_dir`
docstr_tok: ["Train","a","sentencepiece","tokenizer","on","`","texts","`","and","save","it","in","`","path\/tmp_dir","`"]
code: def train(self, raw_text_path):
      "Train a sentencepiece tokenizer on `texts` and save it in `path/tmp_dir`"
      from sentencepiece import SentencePieceTrainer
  
      vocab_sz = (
          self._get_vocab_sz(raw_text_path)
          if self.vocab_sz is None
          else self.vocab_sz
      )
      spec_tokens = ["\u2581" + s for s in self.special_toks]
      SentencePieceTrainer.Train(
          " ".join(
              [
                  f"--input={raw_text_path} --vocab_size={vocab_sz} --model_prefix={self.cache_dir/'spm'}",
                  f"--character_coverage={self.char_coverage} --model_type={self.model_type}",
                  f"--unk_id={len(spec_tokens)} --pad_id=-1 --bos_id=-1 --eos_id=-1 --minloglevel=2",
                  f"--user_defined_symbols={','.join(spec_tokens)} --hard_vocab_limit=false",
              ]
          )
      )
      raw_text_path.unlink()
      return self.cache_dir / "spm.model"
  
imports: from __future__ import annotations
  from ..torch_basics import *
  from ..data.all import *
  import html
label_desc: Use image node_id 3 for calling the SentencePieceTokenizer obj's underlying member method code with example usage: obj.train(raw_text_path) and returns: unknown