code: def train_tokenizer(filename, vocab_size=128000):
      spm.SentencePieceTrainer.train(
          (
              "--input={} --model_prefix={} --user_defined_symbols=<_start_>,<_delimiter_>,<_classify_> --unk_id=0 "
              "--vocab_size={} --input_sentence_size=10000000 --shuffle_input_sentence=true"
              " --max_sentence_length=10000000 --character_coverage=0.9999"
          ).format(filename, ENCODER_PATH, vocab_size)
      )
  
file_path: finetune/finetune/base_models/oscar/encoder.py
imports: import os
  import json
  import regex
  import logging
  from functools import lru_cache
  import numpy
  import finetune
  from finetune.encoding.input_encoder import BaseEncoder, EncodedOutput, get_pairs
  from finetune.util.tokenization import normalize_nfkc, WEIRD_SPM_CHAR
  import sentencepiece
  import unicodedata
label_desc: Use image node_id 1 for calling a global function with example usage: train_tokenizer(filename, vocab_size) without return types