code: def get_datasets_weights_and_num_samples(
      data_prefix, train_valid_test_num_samples
  ):
      # The data prefix should be in the format of:
      #   weight-1, data-prefix-1, weight-2, data-prefix-2, ..
      assert len(data_prefix) % 2 == 0
      num_datasets = len(data_prefix) // 2
      weights = [0] * num_datasets
      prefixes = [0] * num_datasets
      for i in range(num_datasets):
          weights[i] = float(data_prefix[2 * i])
          prefixes[i] = (data_prefix[2 * i + 1]).strip()
      # Normalize weights
      weight_sum = 0.0
      for weight in weights:
          weight_sum += weight
      assert weight_sum > 0.0
      weights = [weight / weight_sum for weight in weights]
  
      # Add 0.5% (the 1.005 factor) so in case the bleding dataset does
      # not uniformly distribute the number of samples, we still have
      # samples left to feed to the network.
      datasets_train_valid_test_num_samples = []
      for weight in weights:
          datasets_train_valid_test_num_samples.append(
              [
                  int(math.ceil(val * weight * 1.005))
                  for val in train_valid_test_num_samples
              ]
          )
  
      return prefixes, weights, datasets_train_valid_test_num_samples
  
file_path: ColossalAI/examples/tutorial/sequence_parallel/data/datasets/dataset_utils.py
imports: import collections
  import math
  import time
  import numpy
  from colossalai.logging import get_dist_logger
  from .blendable_dataset import BlendableDataset
  from .indexed_dataset import make_dataset
label_desc: Use image node_id 1 for calling a global function with example usage: get_datasets_weights_and_num_samples(data_prefix, train_valid_test_num_samples) and returns: prefixes,  weights,  datasets_train_valid_test_num_samples