code: def load_dataset(
      self,
      path: str,
      data_keys: Optional[Dict[str, str]] = None,
      ref_keys: Optional[List[str]] = None,
      only_keep_necessary_columns: bool = False,
      load_smallest_split: bool = False,
      num_samples: Optional[int] = None,
      shuffle: bool = False,
      **load_dataset_kwargs,
  ) -> Union[DatasetDict, Dataset]:
      dataset = datasets_load_dataset(path, **load_dataset_kwargs)
  
      if isinstance(dataset, DatasetDict) and load_smallest_split:
          split = load_dataset_kwargs.get("split", None)
          if split is not None:
              raise ValueError(
                  f"A split name was provided ({split}) but load_smallest_split is True, use either one or the other."
              )
          smallest_split = min(
              dataset.items(), key=lambda item: item[1].num_rows
          )[0]
          logger.info(
              "Since no split was explicitely provided and load_smallest_split=True, using the smallest split of the "
              f'dataset called "{smallest_split}".'
          )
          dataset = dataset[smallest_split]
  
      if shuffle:
          dataset = dataset.shuffle()
  
      if isinstance(dataset, Dataset) and num_samples is not None:
          if num_samples > dataset.num_rows:
              raise ValueError(
                  f"There are only {dataset.num_rows} examples in the dataset, but it was requested to select "
                  f"{num_samples} examples."
              )
          dataset = dataset.select(range(num_samples))
  
      column_names = dataset.column_names
      if isinstance(column_names, dict):
          column_names = list(
              set(itertools.chain.from_iterable(column_names.values()))
          )
  
      if data_keys is None:
          logger.warning(
              "As no data keys were provided, trying to guess them."
          )
          data_keys = self.try_to_guess_data_keys(column_names)
          if data_keys is None:
              raise ValueError(
                  "Data keys need to be specified manually since they could not be guessed from "
                  f"{', '.join(column_names)}"
              )
          else:
              logger.warning(
                  f"Guessed the following data keys: {data_keys}"
              )
      elif not set(data_keys.keys()) <= self.ALLOWED_DATA_KEY_NAMES:
          raise ValueError(
              f"data_keys contains unallowed keys {set(data_keys.keys())}, allowed_keys: {self.ALLOWED_DATA_KEY_NAMES}."
          )
  
      if ref_keys is None:
          ref_keys = self.try_to_guess_ref_keys(column_names)
          logger.warning(
              f"As no ref keys were provided, tried to guess them: {ref_keys}."
          )
  
      dataset = self.prepare_dataset(
          dataset, data_keys=data_keys, ref_keys=ref_keys
      )
  
      if only_keep_necessary_columns:
          ref_keys = ref_keys if ref_keys is not None else []
          necessary_columns = (
              self.preprocessor.model_input_names + ref_keys
          )
          if isinstance(dataset, DatasetDict):
              for split_name, split in dataset.items():
                  columns_to_remove = [
                      name
                      for name in split.column_names
                      if name not in necessary_columns
                  ]
                  dataset[split_name] = split.remove_columns(
                      columns_to_remove
                  )
          else:
              columns_to_remove = [
                  name
                  for name in dataset.column_names
                  if name not in necessary_columns
              ]
              dataset = dataset.remove_columns(columns_to_remove)
  
      return dataset
  
imports: import copy
  import functools
  import itertools
  from abc import ABC, abstractmethod
  from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
  from datasets import Dataset, DatasetDict
  from datasets import load_dataset
  from transformers import PreTrainedTokenizerBase
  from transformers.image_processing_utils import BaseImageProcessor
  from ..None import logging
label_desc: Use image node_id 8 for calling the TaskProcessor obj's underlying member method code with example usage: obj.load_dataset(path, data_keys, ref_keys, only_keep_necessary_columns, load_smallest_split, num_samples, shuffle) and returns: dataset