code: def main(args=None):
      parser = argparse.ArgumentParser()
      parser.add_argument(
          "--lang",
          type=str,
          default="sd",
          help="Which language to use for tokenization",
      )
      parser.add_argument(
          "--tokenize_model_path",
          type=str,
          default=None,
          help="Specific tokenizer model to use",
      )
      parser.add_argument(
          "input_files",
          type=str,
          nargs="+",
          help="Which input files to tokenize",
      )
      parser.add_argument(
          "--output_file",
          type=str,
          default="glove.txt",
          help="Where to write the tokenized output",
      )
      parser.add_argument(
          "--model_dir",
          type=str,
          default=None,
          help="Where to get models for a Pipeline (None => default models dir)",
      )
      parser.add_argument(
          "--chunk_size",
          type=int,
          default=500,
          help="How many 'documents' to use in a chunk when tokenizing.  This is separate from the tokenizer batching - this limits how much memory gets used at once, since we don't need to store an entire file in memory at once",
      )
      args = parser.parse_args(args=args)
  
      if os.path.exists(args.output_file):
          print(
              "Cowardly refusing to overwrite existing output file %s"
              % args.output_file
          )
          return
  
      if args.tokenize_model_path:
          config = {
              "model_path": args.tokenize_model_path,
              "check_requirements": False,
          }
          tokenizer = TokenizeProcessor(
              config, pipeline=None, device=default_device()
          )
      else:
          pipe = stanza.Pipeline(
              lang=args.lang,
              processors="tokenize",
              model_dir=args.model_dir,
          )
          tokenizer = pipe.processors["tokenize"]
  
      with open(args.output_file, "w", encoding="utf-8") as fout:
          for filename in tqdm(args.input_files):
              if filename.endswith(".zip"):
                  with zipfile.ZipFile(filename) as zin:
                      input_names = zin.namelist()
                      for input_name in tqdm(input_names, leave=False):
                          with zin.open(input_names[0]) as fin:
                              fin = io.TextIOWrapper(
                                  fin, encoding="utf-8"
                              )
                              tokenize_to_file(tokenizer, fin, fout)
              else:
                  with open_read_text(
                      filename, encoding="utf-8"
                  ) as fin:
                      tokenize_to_file(tokenizer, fin, fout)
  
file_path: stanza/stanza/models/tokenization/tokenize_files.py
imports: import argparse
  import io
  import os
  import time
  import re
  import zipfile
  import torch
  import stanza
  from stanza.models.common.utils import open_read_text, default_device
  from stanza.models.tokenization.data import TokenizationDataset
  from stanza.models.tokenization.utils import output_predictions
  from stanza.pipeline.tokenize_processor import TokenizeProcessor
  from stanza.utils.get_tqdm import get_tqdm
label_desc: Use image node_id 2 for calling a global function with example usage: main(args) without return types