code: def main():
      parser = argparse.ArgumentParser()
      parser.add_argument(
          "src_root",
          default="src",
          help="Root directory with all source files.  Expected structure is root dir -> language dirs -> package dirs -> text files to process",
      )
      parser.add_argument(
          "tgt_root",
          default="tgt",
          help="Root directory with all target files.",
      )
      parser.add_argument(
          "--langs",
          default="",
          help="A list of language codes to process.  If not set, all languages under src_root will be processed.",
      )
      parser.add_argument(
          "--packages",
          default="",
          help="A list of packages to process.  If not set, all packages under the languages found will be processed.",
      )
      parser.add_argument(
          "--no_xz_output",
          default=True,
          dest="xz_output",
          action="store_false",
          help="Output compressed xz files",
      )
      parser.add_argument(
          "--split_size",
          default=50,
          type=int,
          help="How large to make each split, in MB",
      )
      parser.add_argument(
          "--no_make_test_file",
          default=True,
          dest="make_test_file",
          action="store_false",
          help="Don't save a test file.  Honestly, we never even use it.  Best for low resource languages where every bit helps",
      )
      args = parser.parse_args()
  
      print("Processing files:")
      print(f"source root: {args.src_root}")
      print(f"target root: {args.tgt_root}")
      print("")
  
      langs = []
      if len(args.langs) > 0:
          langs = args.langs.split(",")
          print(
              "Only processing the following languages: " + str(langs)
          )
  
      packages = []
      if len(args.packages) > 0:
          packages = args.packages.split(",")
          print(
              "Only processing the following packages: " + str(packages)
          )
  
      src_root = Path(args.src_root)
      tgt_root = Path(args.tgt_root)
  
      lang_dirs = os.listdir(src_root)
      lang_dirs = [
          l for l in lang_dirs if l not in EXCLUDED_FOLDERS
      ]  # skip excluded
      lang_dirs = [
          l for l in lang_dirs if os.path.isdir(src_root / l)
      ]  # skip non-directory
      if len(langs) > 0:  # filter languages if specified
          lang_dirs = [l for l in lang_dirs if l in langs]
      print(f"{len(lang_dirs)} total languages found:")
      print(lang_dirs)
      print("")
  
      split_size = int(args.split_size * 1024 * 1024)
  
      for lang in lang_dirs:
          lang_root = src_root / lang
          data_dirs = os.listdir(lang_root)
          if len(packages) > 0:
              data_dirs = [d for d in data_dirs if d in packages]
          data_dirs = [
              d for d in data_dirs if os.path.isdir(lang_root / d)
          ]
          print(
              f"{len(data_dirs)} total corpus found for language {lang}."
          )
          print(data_dirs)
          print("")
  
          for dataset_name in data_dirs:
              src_dir = lang_root / dataset_name
              tgt_dir = tgt_root / lang / dataset_name
  
              if not os.path.exists(tgt_dir):
                  os.makedirs(tgt_dir)
              print(f"-> Processing {lang}-{dataset_name}")
              prepare_lm_data(
                  src_dir,
                  tgt_dir,
                  lang,
                  dataset_name,
                  args.xz_output,
                  split_size,
                  args.make_test_file,
              )
  
          print("")
  
file_path: stanza/stanza/utils/charlm/make_lm_data.py
imports: import argparse
  import glob
  import os
  from pathlib import Path
  import shutil
  import subprocess
  import tempfile
  from tqdm import tqdm
label_desc: Use image node_id 1 for calling a global function with example usage: main() without return types