code: def get_new_cluster(self, previous_difficulties):
      cluster_fname = CURRICULUM_LEARNING_CLUSTER_PREFIX
      for metric in self.curriculum_schedulers:
          cluster_fname = f"{cluster_fname}_{metric}{self.current_difficulties[metric]}"
      cluster_path = self.data_efficiency_config[DATA_SAMPLING][
          CURRICULUM_LEARNING
      ][CURRICULUM_LEARNING_CLUSTER_PATH]
      cluster_path = f"{cluster_path}/{cluster_fname}"
      if self.global_rank == 0:
          new_cluster = None
          need_clustering = 0
          for metric in self.clustering_type:
              if (
                  self.clustering_type[metric]
                  != CURRICULUM_LEARNING_SINGLE_CLUSTER
              ):
                  need_clustering += 1
          if need_clustering > 1:
              for metric in self.curriculum_schedulers:
                  if (
                      self.clustering_type[metric]
                      == CURRICULUM_LEARNING_SINGLE_CLUSTER
                  ):
                      metric_cluster = np.arange(
                          start=0,
                          stop=self.one_epoch_total_samples,
                          step=1,
                          dtype=self.index_dtype,
                      )
                  else:
                      if (
                          self.difficulty_type[metric]
                          == CURRICULUM_LEARNING_VALUE_BASED
                      ):
                          metric_cluster = (
                              self.get_sample_based_on_metric_value(
                                  metric,
                                  float("-inf"),
                                  self.current_difficulties[metric],
                              )
                          )
                      elif (
                          self.difficulty_type[metric]
                          == CURRICULUM_LEARNING_PERCENTILE_BASED
                      ):
                          metric_cluster = self.get_sample_based_on_metric_percentile(
                              metric,
                              0,
                              self.current_difficulties[metric],
                          )
                  new_cluster = (
                      metric_cluster
                      if new_cluster is None
                      else np.intersect1d(
                          new_cluster,
                          metric_cluster,
                          assume_unique=True,
                      )
                  )
              for cluster in self.data_clusters:
                  new_cluster = np.setdiff1d(
                      new_cluster, cluster[0], assume_unique=True
                  )
          else:
              if len(self.data_clusters) == 0:
                  new_cluster = np.arange(
                      start=0,
                      stop=self.one_epoch_total_samples,
                      step=1,
                      dtype=self.index_dtype,
                  )
              for metric in self.curriculum_schedulers:
                  if (
                      self.clustering_type[metric]
                      != CURRICULUM_LEARNING_SINGLE_CLUSTER
                  ):
                      if (
                          self.difficulty_type[metric]
                          == CURRICULUM_LEARNING_VALUE_BASED
                      ):
                          new_cluster = (
                              self.get_sample_based_on_metric_value(
                                  metric,
                                  previous_difficulties[metric],
                                  self.current_difficulties[metric],
                              )
                          )
                      elif (
                          self.difficulty_type[metric]
                          == CURRICULUM_LEARNING_PERCENTILE_BASED
                      ):
                          new_cluster = self.get_sample_based_on_metric_percentile(
                              metric,
                              previous_difficulties[metric],
                              self.current_difficulties[metric],
                          )
          if new_cluster is not None and len(new_cluster) > 0:
              logger.info(
                  f"new data cluster (previous_difficulties {previous_difficulties}, current_difficulties {self.current_difficulties}) with size {len(new_cluster)} generated."
              )
              self.np_rng.shuffle(new_cluster)
              cluster_builder = create_mmap_dataset_builder(
                  cluster_path, self.index_dtype
              )
              cluster_builder.add_item_numpy(new_cluster)
              close_mmap_dataset_builder(cluster_builder, cluster_path)
              self.data_clusters.append(
                  MMapIndexedDataset(cluster_path, skip_warmup=True)
              )
              self.data_cluster_sizes.append(
                  len(self.data_clusters[-1][0])
              )
          else:
              logger.info(
                  f"new data cluster (previous_difficulties {previous_difficulties}, current_difficulties {self.current_difficulties}) has no matched data thus skipped."
              )
      dist.barrier(group=self.data_parallel_group)
      if os.path.isfile(f"{cluster_path}.bin"):
          self.data_cluster_paths.append(cluster_fname)
          self.data_cluster_current_position.append(0)
  
imports: import torch
  import os
  import numpy
  import deepspeed.comm
  from deepspeed.utils import logger
  from deepspeed.accelerator import get_accelerator
  from ..constants import *
  from ..curriculum_scheduler import CurriculumScheduler
  from .indexed_dataset import MMapIndexedDataset
  from .utils import create_mmap_dataset_builder, close_mmap_dataset_builder, find_fit_int_dtype
label_desc: Use image node_id 7 for calling the DeepSpeedDataSampler obj's underlying member method code with example usage: obj.get_new_cluster(previous_difficulties) without return types