class_docstr: This class is a copy of `DeDLOC's tokenizer implementation <https://github.com/yandex-research/DeDLOC/blob/main/sahajbert/tokenizer/tokenizer_model.py>`__ .
  
  Custom SentencePiece Unigram Tokenizer with NMT, NKFC, spaces and lower-casing characters normalization
  Represents the Unigram algorithm, with the pretokenization used by SentencePiece
class_docstr_tok: ["This","class","is","a","copy","of","`","DeDLOC","'s","tokenizer","implementation","<","https",":","\/\/github.com\/yandex-research\/DeDLOC\/blob\/main\/sahajbert\/tokenizer\/tokenizer_model.py",">","`","__",".","Custom","SentencePiece","Unigram","Tokenizer","with","NMT",",","NKFC",",","spaces","and","lower-casing","characters","normalization","Represents","the","Unigram","algorithm",",","with","the","pretokenization","used","by","SentencePiece"]
docstr: Train the model using the given files
docstr_tok: ["Train","the","model","using","the","given","files"]
code: def train(
      self,
      files: Union[str, List[str]],
      vocab_size: int = 8000,
      show_progress: bool = True,
  ):
      """Train the model using the given files"""
  
      trainer = trainers.UnigramTrainer(
          vocab_size=vocab_size,
          special_tokens=self.special_tokens_list,
          show_progress=show_progress,
      )
  
      if isinstance(files, str):
          files = [files]
      self._tokenizer.train(files, trainer=trainer)
  
      self.add_unk_id()
  
file_path: H2O/h2o_flexgen/benchmark/third_party/transformers/examples/flax/language-modeling/t5_tokenizer_model.py
imports: import json
  from typing import Iterator, List, Union
  from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
  from tokenizers.implementations.base_tokenizer import BaseTokenizer
  from tokenizers.models import Unigram
  from tokenizers.processors import TemplateProcessing
label_desc: Use image node_id 2 for calling the SentencePieceUnigramTokenizer obj's underlying member method code with example usage: obj.train(files, vocab_size, show_progress) without return types