code: def prepare_lm_data(
      src_dir,
      tgt_dir,
      lang,
      dataset_name,
      compress,
      split_size,
      make_test_file,
  ):
      """
      Combine, shuffle and split data into smaller files, following a naming convention.
      """
      assert isinstance(src_dir, Path)
      assert isinstance(tgt_dir, Path)
      with tempfile.TemporaryDirectory(dir=tgt_dir) as tempdir:
          tgt_tmp = os.path.join(tempdir, f"{lang}-{dataset_name}.tmp")
          print(f"--> Copying files into {tgt_tmp}...")
          # TODO: we can do this without the shell commands
          input_files = (
              glob.glob(str(src_dir) + "/*.txt")
              + glob.glob(str(src_dir) + "/*.txt.xz")
              + glob.glob(str(src_dir) + "/*.txt.gz")
          )
          for src_fn in tqdm(input_files):
              if src_fn.endswith(".txt"):
                  cmd = f"cat {src_fn} >> {tgt_tmp}"
                  subprocess.run(cmd, shell=True)
              elif src_fn.endswith(".txt.xz"):
                  cmd = f"xzcat {src_fn} >> {tgt_tmp}"
                  subprocess.run(cmd, shell=True)
              elif src_fn.endswith(".txt.gz"):
                  cmd = f"zcat {src_fn} >> {tgt_tmp}"
                  subprocess.run(cmd, shell=True)
              else:
                  raise AssertionError(
                      "should not have found %s" % src_fn
                  )
          tgt_tmp_shuffled = os.path.join(
              tempdir, f"{lang}-{dataset_name}.tmp.shuffled"
          )
  
          print(f"--> Shuffling files into {tgt_tmp_shuffled}...")
          cmd = f"cat {tgt_tmp} | shuf > {tgt_tmp_shuffled}"
          result = subprocess.run(cmd, shell=True)
          if result.returncode != 0:
              raise RuntimeError("Failed to shuffle files!")
          size = os.path.getsize(tgt_tmp_shuffled) / 1024 / 1024 / 1024
          print(f"--> Shuffled file size: {size:.4f} GB")
          if size < 0.1:
              raise RuntimeError(
                  "Not enough data found to build a charlm.  At least 100MB data expected"
              )
  
          print(
              f"--> Splitting into smaller files of size {split_size} ..."
          )
          train_dir = tgt_dir / "train"
          if not os.path.exists(train_dir):  # make training dir
              os.makedirs(train_dir)
          cmd = f"split -C {split_size} -a 4 -d --additional-suffix .txt {tgt_tmp_shuffled} {train_dir}/{lang}-{dataset_name}-"
          result = subprocess.run(cmd, shell=True)
          if result.returncode != 0:
              raise RuntimeError("Failed to split files!")
          total = len(glob.glob(f"{train_dir}/*.txt"))
          print(f"--> {total} total files generated.")
          if total < 3:
              raise RuntimeError(
                  "Something went wrong!  %d file(s) produced by shuffle and split, expected at least 3"
                  % total
              )
  
          dev_file = f"{tgt_dir}/dev.txt"
          test_file = f"{tgt_dir}/test.txt"
          if make_test_file:
              print("--> Creating dev and test files...")
              shutil.move(
                  f"{train_dir}/{lang}-{dataset_name}-000.txt", dev_file
              )
              shutil.move(
                  f"{train_dir}/{lang}-{dataset_name}-001.txt",
                  test_file,
              )
              txt_files = [dev_file, test_file] + glob.glob(
                  f"{train_dir}/*.txt"
              )
          else:
              print("--> Creating dev file...")
              shutil.move(
                  f"{train_dir}/{lang}-{dataset_name}-000.txt", dev_file
              )
              txt_files = [dev_file] + glob.glob(f"{train_dir}/*.txt")
  
          if compress:
              print("--> Compressing files...")
              for txt_file in tqdm(txt_files):
                  subprocess.run(["xz", txt_file])
  
          print("--> Cleaning up...")
      print(f"--> All done for {lang}-{dataset_name}.\n")
  
file_path: stanza/stanza/utils/charlm/make_lm_data.py
imports: import argparse
  import glob
  import os
  from pathlib import Path
  import shutil
  import subprocess
  import tempfile
  from tqdm import tqdm
label_desc: Use image node_id 2 for calling a global function with example usage: prepare_lm_data(src_dir, tgt_dir, lang, dataset_name, compress, split_size, make_test_file) without return types