code: def compute_correlation_distribution(
      dataset_url,
      id_column,
      shuffle_row_drop_partitions,
      num_corr_samples=100,
  ):
      """
      Compute the correlation distribution of a given shuffle_options on an existing dataset.
      Use this to compare 2 different shuffling options compare.
      It is encouraged to use a dataset generated by generate_shuffle_analysis_dataset for this analysis.
  
      :param dataset_url: Dataset url to compute correlation distribution of
      :param id_column: Column where an integer or string id can be found
      :param shuffle_row_drop_partitions: shuffle_row_drop_partitions to test correlation against
      :param num_corr_samples: How many samples of the correlation to take to compute distribution
      :return: (mean, standard deviation) of computed distribution
      """
  
      # Read the dataset without any shuffling in order (need to use a dummy pool for this).
      with make_reader(
          dataset_url,
          shuffle_row_groups=False,
          reader_pool_type="dummy",
      ) as reader:
          unshuffled = [row[id_column] for row in reader]
  
      correlations = []
      for _ in range(num_corr_samples):
          with make_reader(
              dataset_url,
              shuffle_row_groups=True,
              shuffle_row_drop_partitions=shuffle_row_drop_partitions,
          ) as reader:
              shuffled = [row[id_column] for row in reader]
              correlations.append(
                  abs(np.corrcoef(unshuffled, shuffled)[0, 1])
              )
  
      mean = np.mean(correlations)
      std_dev = np.std(correlations)
  
      return mean, std_dev
  
file_path: petastorm/petastorm/test_util/shuffling_analysis.py
imports: from __future__ import division
  from functools import partial
  import numpy
  from pyspark.sql.types import LongType
  from petastorm import make_reader
  from petastorm.codecs import ScalarCodec
  from petastorm.etl.dataset_metadata import materialize_dataset
  from petastorm.unischema import Unischema, UnischemaField, dict_to_spark_row
label_desc: Use image node_id 2 for calling a global function with example usage: compute_correlation_distribution(dataset_url, id_column, shuffle_row_drop_partitions, num_corr_samples) and returns: mean,  std_dev