code: def repack_tar_files_with_not_enough_batches(
      output_dir: Path, num_batches_per_tarfile: int
  ) -> None:
      f"""
      It is possible that number of batches in a fragment is not evenly divisible by ``num_batches_per_tarfile``.
      In such a case excess batches are put in a tar file which matches a pattern
      ``fragment(0|[1-9][0-9]*).num_batches(0|[1-9][0-9]*).(0|[1-9][0-9]*).tar.to_repack``. Such files are repacked by
      ``repack_tar_files_with_not_enough_batches`` function into tar files with correct ``num_batches_per_tarfile``
      batches each. If there is no enough batches in repacked files, then up to ``num_batches_per_tarfile - 1``
      remaining batches may be discarded.
  
      Args:
          output_dir: a path to the output directory which contains files to repack and where new files are saved
          num_batches_per_tarfile: a number of batches in 1 tar file. If number of batches in files matching a pattern
              ``fragment(0|[1-9][0-9]*).num_batches(0|[1-9][0-9]*).(0|[1-9][0-9]*).tar.to_repack`` is not evenly
              divisible by ``num_batches_per_tarfile`` excess batches are discarded.
      """
      files_to_repack_with_matches = [
          (path, TAR_FRAGMENT_PATTERN_TO_REPACK.match(path.name))
          for path in output_dir.iterdir()
          if TAR_FRAGMENT_PATTERN_TO_REPACK.match(path.name) is not None
      ]
      files_to_repack_with_matches = sorted(
          files_to_repack_with_matches, key=lambda x: int(x[1].group(3))
      )
      logging.info(
          f"Found {len(files_to_repack_with_matches)} files for repacking."
      )
      files_to_repack_with_matches = deque(files_to_repack_with_matches)
      total_batches_in_repacked_files = 0
      initial_number_of_files_to_repack = len(
          files_to_repack_with_matches
      )
      pop_file_ds = None
      new_file_sink = None
      new_file_num_batches = 0
      while files_to_repack_with_matches:
          assert pop_file_ds is None or new_file_sink is None
          if new_file_sink is None:
              # `append_file` is a file which content will serve as a start for new tar file. `append_file` content is
              # copied into a `new_file` and then content of other files needing repacking is appended to content of
              # `new_file`.
              (
                  append_file,
                  match,
              ) = files_to_repack_with_matches.popleft()
              new_file = (
                  append_file.parent
                  / TAR_FRAGMENT_TMPL_FINISHED.format(
                      fragment_idx=match.group(1),
                      num_batches=num_batches_per_tarfile,
                      file_idx=match.group(3),
                  )
              )
              new_file_sink = wds.TarWriter(str(new_file))
              append_ds_to_rewrite = (
                  wds.WebDataset(
                      urls=[str(append_file)], nodesplitter=None
                  )
                  .decode(wds.handle_extension(".pyd", decode_pyd))
                  .to_tuple("__key__", "batch.pyd")
              )
              for key, batch in iter(append_ds_to_rewrite):
                  new_file_sink.write(
                      {"__key__": key, "batch.pyd": batch}
                  )
                  new_file_num_batches += 1
                  total_batches_in_repacked_files += 1
              assert (
                  total_batches_in_repacked_files
                  < initial_number_of_files_to_repack
                  * num_batches_per_tarfile
              )
              assert new_file_num_batches == int(match.group(2)), (
                  f"Number of batches {new_file_num_batches} in {append_file} is different from number of batches "
                  f"{match.group(2)} in repacked tar file with name {append_file}."
              )
              append_file.unlink()
          if files_to_repack_with_matches and pop_file_ds is None:
              pop_file, _ = files_to_repack_with_matches.pop()
              pop_file_ds = (
                  wds.WebDataset(
                      urls=[str(pop_file)], nodesplitter=None
                  )
                  .decode(wds.handle_extension(".pyd", decode_pyd))
                  .to_tuple("__key__", "batch.pyd")
              )
              pop_file_ds = iter(pop_file_ds)
          if pop_file_ds is not None and new_file_sink is not None:
              while new_file_num_batches < num_batches_per_tarfile:
                  try:
                      key, batch = next(pop_file_ds)
                  except StopIteration:
                      pop_file_ds = None
                      pop_file.unlink()
                      break
                  new_file_sink.write(
                      {"__key__": key, "batch.pyd": batch}
                  )
                  total_batches_in_repacked_files += 1
                  assert (
                      total_batches_in_repacked_files
                      < initial_number_of_files_to_repack
                      * num_batches_per_tarfile
                  )
                  new_file_num_batches += 1
              if new_file_num_batches >= num_batches_per_tarfile:
                  assert new_file_num_batches == num_batches_per_tarfile
                  new_file_sink.close()
                  new_file_sink = None
                  new_file_num_batches = 0
      if new_file_sink is not None:
          new_file_sink.close()
          new_file.unlink()
          logging.info(f"Discarded {new_file_num_batches} batches.")
      if pop_file_ds is not None:
          pop_file.unlink()
      logging.info(
          f"Repacked {total_batches_in_repacked_files} batches from short tar files"
      )
  
imports: import inspect
  import json
  import multiprocessing
  import os
  import pickle
  import re
  import shutil
  import tempfile
  from collections import deque
  from pathlib import Path
  from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, Type, Union
  import numpy
  import torch
  import webdataset
  from joblib import Parallel, delayed
  from omegaconf import DictConfig
  from torch.utils.data import IterableDataset
  from nemo.collections.common.tokenizers import TokenizerSpec
  from nemo.collections.nlp.data.token_classification.punctuation_capitalization_dataset import LABEL_ID_DIR_FOR_NEMO_CHECKPOINT, BertPunctuationCapitalizationDataset, Progress, create_label_ids, create_masks_and_segment_ids, load_label_ids, raise_not_equal_labels_error
  from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer
  from nemo.core.neural_types import AudioSignal, ChannelType, LabelsType, LengthsType, MaskType, NeuralType
  from nemo.utils import logging
label_desc: Use image node_id 13 for calling a global function with example usage: repack_tar_files_with_not_enough_batches(output_dir, num_batches_per_tarfile) without return types