code: def __process_data(
      text_path: str,
      dst_folder: str,
      vocab_size: int,
      tokenizer_type: str,
      spe_type: str,
      spe_character_coverage: float,
      spe_train_extremely_large_corpus: bool,
      spe_sample_size: int,
      spe_max_sentencepiece_length: int,
      spe_split_by_unicode_script: bool,
      spe_bos: bool,
      spe_eos: bool,
      spe_pad: bool,
      spe_control_symbols: Optional[List[str]],
      spe_user_defined_symbols: Optional[List[str]],
      spe_byte_fallback: bool,
      spe_split_digits: bool,
      lower_case: bool,
  ):
      """
      Converts flac to wav and build manifests's json
      Args:
          text_path: source with text lines
          dst_folder: where wav files will be stored
          vocab_size: vocabular size used in encoding the text
          tokenizer_type: type of tokenization to perform - wpe or spe
          spe_type: type of tokenization model used for spe.
          spe_character_coverage: float value between 0 and 1 (as a percentage). For languages with a vast charset,
              can be < 1.0, but for all other languages, it should be set as 1.0
          spe_sample_size: int, default of -1. If positive integer is used, samples the dataset
              by given sample size.
          spe_train_extremely_large_corpus: bool. If dataset is too large, and user has sufficient RAM,
              this flag can be set to try to trained the tokenizer. Will silently fail if it runs out of RAM.
          spe_max_sentencepiece_length: Limits the maximum length of the SentencePiece subword that can be constructed.
              By default, no limit is placed.
          spe_bos: Bool flag, whether to add <s> to SentencePiece tokenizer vocabulary.
          spe_eos: Bool flag, whether to add </s> to SentencePiece tokenizer vocabulary.
          spe_pad: Bool flag, whether to add <pad> to SentencePiece tokenizer vocabulary.
          spe_control_symbols: control symbols to add to tokenizer, as defined by sentencepiece.
              These tokens get removed at decode time and are not encoded from the text - can only be added to the input programatically.
          spe_user_defined_symbols: user symbols to add to tokenizer, as defined by sentencepiece.
              These tokens remain in the decoded text and are encoded automatically when present in the input text.
          spe_byte_fallback: If <unk>, fallback to a byte sequence of the character.
          spe_split_digits: If true, digits are split into individual tokens.
          lower_case: whether to tokenize with lower case character set only (for english)
  
      Returns:
      """
      if tokenizer_type == "spe":
          # Prepare directory of tokenizer
          if spe_max_sentencepiece_length > 0:
              tokenizer_dir = os.path.join(
                  dst_folder, "tokenizer_{}_{}_v{}_max_{}"
              ).format(
                  tokenizer_type,
                  spe_type,
                  vocab_size,
                  spe_max_sentencepiece_length,
              )
          else:
              tokenizer_dir = os.path.join(
                  dst_folder, "tokenizer_{}_{}_v{}"
              ).format(tokenizer_type, spe_type, vocab_size)
  
          if spe_pad:
              tokenizer_dir = f"{tokenizer_dir}_pad"
          if spe_bos:
              tokenizer_dir = f"{tokenizer_dir}_bos"
          if spe_eos:
              tokenizer_dir = f"{tokenizer_dir}_eos"
  
          if not os.path.exists(tokenizer_dir):
              os.makedirs(tokenizer_dir)
  
          if os.path.exists(
              os.path.join(tokenizer_dir, "tokenizer.model")
          ):
              logging.warning(
                  "Model file already exists, overriding old model file !"
              )
              os.remove(os.path.join(tokenizer_dir, "tokenizer.model"))
  
          # Build tokenizer
          tokenizer_path, vocab_path = create_spt_model(
              data_file=text_path,
              vocab_size=vocab_size,
              sample_size=spe_sample_size,
              do_lower_case=lower_case,
              output_dir=tokenizer_dir,
              tokenizer_type=spe_type,
              character_coverage=spe_character_coverage,
              train_extremely_large_corpus=spe_train_extremely_large_corpus,
              max_sentencepiece_length=spe_max_sentencepiece_length,
              split_by_unicode_script=spe_split_by_unicode_script,
              bos=spe_bos,
              eos=spe_eos,
              pad=spe_pad,
              control_symbols=spe_control_symbols,
              user_defined_symbols=spe_user_defined_symbols,
              byte_fallback=spe_byte_fallback,
              split_digits=spe_split_digits,
          )
  
      else:
          tokenizer_dir = os.path.join(
              dst_folder, "tokenizer_{}_v{}"
          ).format(tokenizer_type, vocab_size)
  
          if not os.path.exists(tokenizer_dir):
              os.makedirs(tokenizer_dir)
  
          tokenizer = tokenizers.BertWordPieceTokenizer(
              lowercase=lower_case
          )
  
          tokenizer.train(text_path, vocab_size=vocab_size)
          tokenizer.save_model(tokenizer_dir)
  
      return tokenizer_dir
  
file_path: NeMo/scripts/tokenizers/process_asr_text_tokenizer.py
imports: import argparse
  import json
  import logging
  import os
  from typing import List, Optional
  import tokenizers
  from nemo.collections.common.tokenizers.sentencepiece_tokenizer import create_spt_model
  from nemo.utils.data_utils import DataStoreObject
label_desc: Use image node_id 2 for calling a global function with example usage: __process_data(text_path, dst_folder, vocab_size, tokenizer_type, spe_type, spe_character_coverage, spe_train_extremely_large_corpus, spe_sample_size, spe_max_sentencepiece_length, spe_split_by_unicode_script, spe_bos, spe_eos, spe_pad, spe_control_symbols, spe_user_defined_symbols, spe_byte_fallback, spe_split_digits, lower_case) and returns: tokenizer_dir