code: def tokenize(
      input: str,
      output: str,
      field: Optional[str],
      delimiter: Optional[str],
      token_type: str,
      space_symbol: str,
      non_linguistic_symbols: Optional[str],
      bpemodel: Optional[str],
      log_level: str,
      write_vocabulary: bool,
      vocabulary_size: int,
      remove_non_linguistic_symbols: bool,
      cutoff: int,
      add_symbol: List[str],
      cleaner: Optional[str],
      g2p: Optional[str],
      add_nonsplit_symbol: List[str],
  ):
      assert check_argument_types()
  
      logging.basicConfig(
          level=log_level,
          format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
      )
      if input == "-":
          fin = sys.stdin
      else:
          fin = Path(input).open("r", encoding="utf-8")
      if output == "-":
          fout = sys.stdout
      else:
          p = Path(output)
          p.parent.mkdir(parents=True, exist_ok=True)
          fout = p.open("w", encoding="utf-8")
  
      cleaner = TextCleaner(cleaner)
      tokenizer = build_tokenizer(
          token_type=token_type,
          bpemodel=bpemodel,
          delimiter=delimiter,
          space_symbol=space_symbol,
          non_linguistic_symbols=non_linguistic_symbols,
          remove_non_linguistic_symbols=remove_non_linguistic_symbols,
          g2p_type=g2p,
          nonsplit_symbol=add_nonsplit_symbol,
      )
  
      counter = Counter()
      if field is not None:
          field = field2slice(field)
  
      for line in fin:
          line = line.rstrip()
          if field is not None:
              # e.g. field="2-"
              # uttidA hello world!! -> hello world!!
              tokens = line.split(delimiter)
              tokens = tokens[field]
              if delimiter is None:
                  line = " ".join(tokens)
              else:
                  line = delimiter.join(tokens)
  
          line = cleaner(line)
          tokens = tokenizer.text2tokens(line)
          if not write_vocabulary:
              fout.write(" ".join(tokens) + "\n")
          else:
              for t in tokens:
                  counter[t] += 1
  
      if not write_vocabulary:
          return
  
      # ======= write_vocabulary mode from here =======
      # Sort by the number of occurrences in descending order
      # and filter lower frequency words than cutoff value
      words_and_counts = list(
          filter(
              lambda x: x[1] > cutoff,
              sorted(counter.items(), key=lambda x: -x[1]),
          )
      )
      # Restrict the vocabulary size
      if vocabulary_size > 0:
          if vocabulary_size < len(add_symbol):
              raise RuntimeError(
                  f"vocabulary_size is too small: {vocabulary_size}"
              )
          words_and_counts = words_and_counts[
              : vocabulary_size - len(add_symbol)
          ]
  
      # Parse the values of --add_symbol and --add_nonsplit_symbol
      for symbol_and_id in add_symbol + add_nonsplit_symbol:
          # e.g symbol="<blank>:0"
          try:
              symbol, idx = symbol_and_id.split(":")
              idx = int(idx)
          except ValueError:
              raise RuntimeError(
                  f"Format error: e.g. '<blank>:0': {symbol_and_id}"
              )
          symbol = symbol.strip()
  
          # e.g. idx=0  -> append as the first symbol
          # e.g. idx=-1 -> append as the last symbol
          if idx < 0:
              idx = len(words_and_counts) + 1 + idx
          words_and_counts.insert(idx, (symbol, None))
  
      # Write words
      for w, c in words_and_counts:
          fout.write(w + "\n")
  
      # Logging
      total_count = sum(counter.values())
      invocab_count = sum(
          c for w, c in words_and_counts if c is not None
      )
      logging.info(
          f"OOV rate = {(total_count - invocab_count) / total_count * 100} %"
      )
  
file_path: espnet/espnet2/bin/tokenize_text.py
imports: import argparse
  import logging
  import sys
  from collections import Counter
  from pathlib import Path
  from typing import List, Optional
  from typeguard import check_argument_types
  from espnet2.text.build_tokenizer import build_tokenizer
  from espnet2.text.cleaner import TextCleaner
  from espnet2.text.phoneme_tokenizer import g2p_choices
  from espnet2.utils.types import str2bool, str_or_none
  from espnet.utils.cli_utils import get_commandline_args
label_desc: Use image node_id 2 for calling a global function with example usage: tokenize(input, output, field, delimiter, token_type, space_symbol, non_linguistic_symbols, bpemodel, log_level, write_vocabulary, vocabulary_size, remove_non_linguistic_symbols, cutoff, add_symbol, cleaner, g2p, add_nonsplit_symbol) without return types