code: # Copyright (c) Facebook, Inc. and its affiliates.
  from collections import defaultdict
  import itertools
  import logging
  import numpy as np
  import operator
  import pickle
  import torch.utils.data
  from tabulate import tabulate
  from termcolor import colored
  
  from herbarium.config import configurable
  from herbarium.utils.comm import get_world_size
  from herbarium.utils.env import seed_all_rng
  from herbarium.utils.file_io import PathManager
  from herbarium.utils.logger import _log_api_usage, log_first_n
  
  from .catalog import DatasetCatalog, MetadataCatalog
  from .common import AspectRatioGroupedDataset, DatasetFromList, MapDataset
  from .dataset_mapper import DatasetMapper
  from .dataset_utils import check_metadata_consistency
  from .samplers import InferenceSampler, RepeatFactorTrainingSampler, TrainingSampler
  from tqdm import tqdm
  
  """
  This file contains the default logic to build a dataloader for training or testing.
  """
  
  __all__ = [
      "build_batch_data_loader",
      "build_general_train_loader",
      "build_general_test_loader",
      "get_dataset_dicts",
      "print_instances_class_histogram",
  ]
  
  def print_instances_class_histogram(dataset_dicts, class_names):
      """
      Args:
          dataset_dicts (list[dict]): list of dataset dicts.
          class_names (list[str]): list of class names (zero-indexed).
      """
  
      logger = logging.getLogger(__name__)
      logger.info("Build instances class histogram")
  
      num_classes = len(class_names)
      hist_bins = np.arange(num_classes + 1)
      histogram = np.zeros((num_classes,), dtype=np.int)
      for entry in tqdm(dataset_dicts):
          annos = entry["annotations"]
          classes = np.asarray(
              [x["category_id"] for x in annos], dtype=np.int
          )
          if len(classes):
              assert classes.min() >= 0, f"Got an invalid category_id={classes.min()}"
              assert (
                  classes.max() < num_classes
              ), f"Got an invalid category_id={classes.max()} for a dataset of {num_classes} classes"
          histogram += np.histogram(classes, bins=hist_bins)[0]
  
      N_COLS = min(6, len(class_names) * 2)
  
      def short_name(x):
          # make long class names shorter. useful for lvis
          if len(x) > 13:
              return x[:11] + ".."
          return x
  
      data = list(
          itertools.chain(*[[short_name(class_names[i]), int(v)] for i, v in enumerate(histogram)])
      )
      total_num_instances = sum(data[1::2])
      data.extend([None] * (N_COLS - (len(data) % N_COLS)))
      if num_classes > 1:
          data.extend(["total", total_num_instances])
      data = itertools.zip_longest(*[data[i::N_COLS] for i in range(N_COLS)])
  
      # TODO: Too many classes expected. Need to summarize histogram
  
      table = tabulate(
          data,
          headers=["category", "#instances"] * (N_COLS // 2),
          tablefmt="pipe",
          numalign="left",
          stralign="center",
      )
      log_first_n(
          logging.INFO,
          "Distribution of instances among all {} categories:\n".format(num_classes)
          + colored(table, "cyan"),
          key="message",
      )
  
  def get_dataset_dicts(names):
      """
      Load and prepare dataset dicts for instance detection/segmentation and semantic segmentation.
  
      Args:
          names (str or list[str]): a dataset name or a list of dataset names
          filter_empty (bool): whether to filter out images without instance annotations
  
      Returns:
          list[dict]: a list of dicts following the standard dataset dict format.
      """
      if isinstance(names, str):
          names = [names]
      assert len(names), names
      dataset_dicts = [DatasetCatalog.get(dataset_name) for dataset_name in names]
      for dataset_name, dicts in zip(names, dataset_dicts):
          assert len(dicts), "Dataset '{}' is empty!".format(dataset_name)
  
      dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts))
  
      has_instances = "annotations" in dataset_dicts[0]
  
      if has_instances:
          try:
              check_metadata_consistency("thing_classes", names)
              # TODO: printing class histogram for 250M image is amazingly slow.
              #class_names = MetadataCatalog.get(names[0]).thing_classes
              #print_instances_class_histogram(dataset_dicts, class_names)
          except AttributeError:  # class names are not available for this dataset
              pass
  
      assert len(dataset_dicts), "No valid data found in {}.".format(",".join(names))
  
      # TODO: change here to only able in evaluating stage with random index
  
      if "val" in names[0]:
          per_cat_data = defaultdict(list)
          for data in dataset_dicts:
              per_cat_data[data["annotations"][0]["category_id"]].append(data)
          
          dataset_dicts = []
          for cid, val in per_cat_data.items():
              dataset_dicts.extend(val[:5])
  
      return dataset_dicts
  
  
  def build_batch_data_loader(
      dataset, sampler, total_batch_size, *, aspect_ratio_grouping=False, num_workers=0
  ):
      """
      Build a batched dataloader for training.
  
      Args:
          dataset (torch.utils.data.Dataset): map-style PyTorch dataset. Can be indexed.
          sampler (torch.utils.data.sampler.Sampler): a sampler that produces indices
          total_batch_size, aspect_ratio_grouping, num_workers): see
              :func:`build_detection_train_loader`.
  
      Returns:
          iterable[list]. Length of each list is the batch size of the current
              GPU. Each element in the list comes from the dataset.
      """
      world_size = get_world_size()
      assert (
          total_batch_size > 0 and total_batch_size % world_size == 0
      ), "Total batch size ({}) must be divisible by the number of gpus ({}).".format(
          total_batch_size, world_size
      )
  
      batch_size = total_batch_size // world_size
      if aspect_ratio_grouping:
          data_loader = torch.utils.data.DataLoader(
              dataset,
              sampler=sampler,
              num_workers=num_workers,
              batch_sampler=None,
              collate_fn=operator.itemgetter(0),  # don't batch, but yield individual elements
              worker_init_fn=worker_init_reset_seed,
          )  # yield individual mapped dict
          return AspectRatioGroupedDataset(data_loader, batch_size)
      else:
          batch_sampler = torch.utils.data.sampler.BatchSampler(
              sampler, batch_size, drop_last=True
          )  # drop_last so the batch always have the same size
          return torch.utils.data.DataLoader(
              dataset,
              num_workers=num_workers,
              batch_sampler=batch_sampler,
              collate_fn=trivial_batch_collator,
              worker_init_fn=worker_init_reset_seed,
          )
  
  
  def _train_loader_from_config(cfg, mapper=None, *, dataset=None, sampler=None):
      if dataset is None:
          dataset = get_dataset_dicts(
              cfg.DATASETS.TRAIN,
          )
          _log_api_usage("dataset." + cfg.DATASETS.TRAIN[0])
  
      if mapper is None:
          mapper = DatasetMapper(cfg, True)
  
      if sampler is None:
          sampler_name = cfg.DATALOADER.SAMPLER_TRAIN
          logger = logging.getLogger(__name__)
          logger.info("Using training sampler {}".format(sampler_name))
          if sampler_name == "TrainingSampler":
              sampler = TrainingSampler(len(dataset))
          elif sampler_name == "RepeatFactorTrainingSampler":
              repeat_factors = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency(
                  dataset, cfg.DATALOADER.REPEAT_THRESHOLD
              )
              sampler = RepeatFactorTrainingSampler(repeat_factors)
          else:
              raise ValueError("Unknown training sampler: {}".format(sampler_name))
  
      return {
          "dataset": dataset,
          "sampler": sampler,
          "mapper": mapper,
          "total_batch_size": cfg.SOLVER.IMS_PER_BATCH,
          "aspect_ratio_grouping": cfg.DATALOADER.ASPECT_RATIO_GROUPING,
          "num_workers": cfg.DATALOADER.NUM_WORKERS,
      }
  
  
  # TODO can allow dataset as an iterable or IterableDataset to make this function more general
  @configurable(from_config=_train_loader_from_config)
  def build_general_train_loader(
      dataset, *, mapper, sampler=None, total_batch_size, aspect_ratio_grouping=True, num_workers=0
  ):
      """
      Build a dataloader for object detection with some default features.
      This interface is experimental.
  
      Args:
          dataset (list or torch.utils.data.Dataset): a list of dataset dicts,
              or a map-style pytorch dataset. They can be obtained by using
              :func:`DatasetCatalog.get` or :func:`get_detection_dataset_dicts`.
          mapper (callable): a callable which takes a sample (dict) from dataset and
              returns the format to be consumed by the model.
              When using cfg, the default choice is ``DatasetMapper(cfg, is_train=True)``.
          sampler (torch.utils.data.sampler.Sampler or None): a sampler that produces
              indices to be applied on ``dataset``. Default to :class:`TrainingSampler`,
              which coordinates an infinite random shuffle sequence across all workers.
          total_batch_size (int): total batch size across all workers. Batching
              simply puts data into a list.
          aspect_ratio_grouping (bool): whether to group images with similar
              aspect ratio for efficiency. When enabled, it requires each
              element in dataset be a dict with keys "width" and "height".
          num_workers (int): number of parallel data loading workers
  
      Returns:
          torch.utils.data.DataLoader:
              a dataloader. Each output from it is a ``list[mapped_element]`` of length
              ``total_batch_size / num_workers``, where ``mapped_element`` is produced
              by the ``mapper``.
      """
      if isinstance(dataset, list):
          dataset = DatasetFromList(dataset, copy=False)
      if mapper is not None:
          dataset = MapDataset(dataset, mapper)
      if sampler is None:
          sampler = TrainingSampler(len(dataset))
      assert isinstance(sampler, torch.utils.data.sampler.Sampler)
      return build_batch_data_loader(
          dataset,
          sampler,
          total_batch_size,
          aspect_ratio_grouping=aspect_ratio_grouping,
          num_workers=num_workers,
      )
  
  
  def _test_loader_from_config(cfg, dataset_name, mapper=None):
      """
      Uses the given `dataset_name` argument (instead of the names in cfg), because the
      standard practice is to evaluate each test set individually (not combining them).
      """
      dataset = get_dataset_dicts(
          [dataset_name],
      )
      if mapper is None:
          mapper = DatasetMapper(cfg, False)
      return {"dataset": dataset, "mapper": mapper, "num_workers": cfg.DATALOADER.NUM_WORKERS, "total_batch_size": cfg.SOLVER.IMS_PER_BATCH}
  
  
  @configurable(from_config=_test_loader_from_config)
  def build_general_test_loader(dataset, *, mapper, total_batch_size=1, sampler=None, num_workers=0):
      """
      Similar to `build_detection_train_loader`, but uses a batch size of 1,
      and :class:`InferenceSampler`. This sampler coordinates all workers to
      produce the exact set of all samples.
      This interface is experimental.
  
      Args:
          dataset (list or torch.utils.data.Dataset): a list of dataset dicts,
              or a map-style pytorch dataset. They can be obtained by using
              :func:`DatasetCatalog.get` or :func:`get_detection_dataset_dicts`.
          mapper (callable): a callable which takes a sample (dict) from dataset
             and returns the format to be consumed by the model.
             When using cfg, the default choice is ``DatasetMapper(cfg, is_train=False)``.
          sampler (torch.utils.data.sampler.Sampler or None): a sampler that produces
              indices to be applied on ``dataset``. Default to :class:`InferenceSampler`,
              which splits the dataset across all workers.
          num_workers (int): number of parallel data loading workers
  
      Returns:
          DataLoader: a torch DataLoader, that loads the given detection
          dataset, with test-time transformation and batching.
  
      Examples:
      ::
          data_loader = build_detection_test_loader(
              DatasetRegistry.get("my_test"),
              mapper=DatasetMapper(...))
  
          # or, instantiate with a CfgNode:
          data_loader = build_detection_test_loader(cfg, "my_test")
      """
      if isinstance(dataset, list):
          dataset = DatasetFromList(dataset, copy=False)
      if mapper is not None:
          dataset = MapDataset(dataset, mapper)
      if sampler is None:
          sampler = InferenceSampler(len(dataset))
  
      world_size = get_world_size()
      assert (
          total_batch_size > 0 and total_batch_size % world_size == 0
      ), "Total batch size ({}) must be divisible by the number of gpus ({}).".format(
          total_batch_size, world_size
      )
  
      batch_size = total_batch_size // world_size
  
      # Always use 1 image per worker during inference since this is the
      # standard when reporting inference time in papers.
      batch_sampler = torch.utils.data.sampler.BatchSampler(sampler, batch_size, drop_last=False)
      data_loader = torch.utils.data.DataLoader(
          dataset,
          num_workers=num_workers,
          batch_sampler=batch_sampler,
          collate_fn=trivial_batch_collator,
      )
      return data_loader
  
  
  def trivial_batch_collator(batch):
      """
      A batch collator that does nothing.
      """
      return batch
  
  
  def worker_init_reset_seed(worker_id):
      initial_seed = torch.initial_seed() % 2 ** 31
      seed_all_rng(initial_seed + worker_id)
  
apis: numpy.arange numpy.histogram numpy.asarray numpy.zeros