code: def build_train_valid_test_datasets(
      cfg,
      trainer,
      data_prefix,
      data_impl,
      splits_string,
      train_valid_test_num_samples,
      max_seq_length,
      masked_lm_prob,
      short_seq_prob,
      seed,
      skip_warmup,
      binary_head=False,
      max_seq_length_dec=None,
      dataset_type="standard_bert",
      tokenizer=None,
      max_ngram_size=3,
      mean_ngram_size=None,
      geometric_dist=True,
      permutation=False,
      whole_word_masking=True,
      favor_long_ngrams=False,
      delete_mask_prob=0,
      respect_document_boundaries=True,
      data_impl_kwargs={},
  ):
      # for VSC and text memmap we need to provide a tokenizer, if not given
      if data_impl in ["text_mmap", "csv_mmap"]:
          if "tokenizer" not in data_impl_kwargs:
              if isinstance(data_impl_kwargs, DictConfig):
                  data_impl_kwargs = OmegaConf.to_object(
                      data_impl_kwargs
                  )
              else:
                  # prevent updating the default
                  data_impl_kwargs = data_impl_kwargs.copy()
  
              data_impl_kwargs["tokenizer"] = tokenizer
  
      if not respect_document_boundaries and data_impl_kwargs != {}:
          raise ValueError(
              "respect_document_boundaries=False is not compatible with text_memmap and csv_memmap (data_impl_kwargs != {})"
          )
  
      if data_impl in ["mock"]:
          logging.info(
              f"Initializing mock dataset, type {dataset_type}, for train, validate, and test"
          )
          if len(data_prefix) != 0:
              # Files from this location will not be read; mock data will be generated instead.
              logging.warning(
                  f"Requested data_impl={data_impl}, so ignoring data_prefix setting: {data_prefix}"
              )
          if dataset_type == DSET_TYPE_T5:
              from nemo.collections.nlp.data.language_modeling.megatron.t5_dataset import (
                  MockT5Dataset,
              )
  
              if tokenizer is None:
                  # Tokenizer is used to infer vocabulary size for mock data.
                  raise ValueError(
                      "Tokenizer is required for a mock T5 dataset"
                  )
              train_ds = MockT5Dataset(
                  cfg,
                  tokenizer,
                  "train",
                  int(train_valid_test_num_samples[0]),
                  max_seq_length,
                  max_seq_length_dec,
                  seed,
              )
              valid_ds = MockT5Dataset(
                  cfg,
                  tokenizer,
                  "valid",
                  int(train_valid_test_num_samples[1]),
                  max_seq_length,
                  max_seq_length_dec,
                  seed,
              )
              test_ds = MockT5Dataset(
                  cfg,
                  tokenizer,
                  "test",
                  int(train_valid_test_num_samples[2]),
                  max_seq_length,
                  max_seq_length_dec,
                  seed,
              )
              return train_ds, valid_ds, test_ds
          else:
              raise NotImplementedError(
                  f"Mock dataset is not implemented for requested type: {dataset_type}"
              )
  
      if isinstance(data_prefix, DictConfig):
          assert (
              data_prefix.get("train") is not None
              and data_prefix.get("test") is not None
              and data_prefix.get("validation") is not None
          ), f"Data prefix dictionary should have train, test and validation keys.  data_prefix currently has only {data_prefix.keys()}"
          if cfg.data.splits_string is not None:
              logging.warning(
                  cfg.data.splits_string
                  + " ignored since data prefix is of type dictionary."
              )
          train_ds = build_dataset(
              cfg,
              trainer,
              data_prefix["train"],
              data_impl,
              int(train_valid_test_num_samples[0]),
              max_seq_length,
              masked_lm_prob,
              short_seq_prob,
              seed,
              skip_warmup,
              binary_head,
              max_seq_length_dec,
              "train",
              dataset_type=dataset_type,
              tokenizer=tokenizer,
              max_ngram_size=max_ngram_size,
              mean_ngram_size=mean_ngram_size,
              geometric_dist=geometric_dist,
              permutation=permutation,
              whole_word_masking=whole_word_masking,
              favor_long_ngrams=favor_long_ngrams,
              delete_mask_prob=delete_mask_prob,
              respect_document_boundaries=respect_document_boundaries,
              data_impl_kwargs=data_impl_kwargs,
          )
          validation_ds = build_dataset(
              cfg,
              trainer,
              data_prefix["validation"],
              data_impl,
              int(train_valid_test_num_samples[1]),
              max_seq_length,
              masked_lm_prob,
              short_seq_prob,
              seed,
              skip_warmup,
              binary_head,
              max_seq_length_dec,
              "valid",
              dataset_type=dataset_type,
              tokenizer=tokenizer,
              max_ngram_size=max_ngram_size,
              mean_ngram_size=mean_ngram_size,
              geometric_dist=geometric_dist,
              permutation=permutation,
              whole_word_masking=whole_word_masking,
              favor_long_ngrams=favor_long_ngrams,
              delete_mask_prob=delete_mask_prob,
              respect_document_boundaries=respect_document_boundaries,
              data_impl_kwargs=data_impl_kwargs,
          )
          test_ds = build_dataset(
              cfg,
              trainer,
              data_prefix["test"],
              data_impl,
              int(train_valid_test_num_samples[2]),
              max_seq_length,
              masked_lm_prob,
              short_seq_prob,
              seed,
              skip_warmup,
              binary_head,
              max_seq_length_dec,
              "test",
              dataset_type=dataset_type,
              tokenizer=tokenizer,
              max_ngram_size=max_ngram_size,
              mean_ngram_size=mean_ngram_size,
              geometric_dist=geometric_dist,
              permutation=permutation,
              whole_word_masking=whole_word_masking,
              favor_long_ngrams=favor_long_ngrams,
              delete_mask_prob=delete_mask_prob,
              respect_document_boundaries=respect_document_boundaries,
              data_impl_kwargs=data_impl_kwargs,
          )
          return train_ds, validation_ds, test_ds
  
      else:
          if len(data_prefix) == 1:
              return _build_train_valid_test_datasets(
                  cfg,
                  trainer,
                  data_prefix[0],
                  data_impl,
                  splits_string,
                  train_valid_test_num_samples,
                  max_seq_length,
                  masked_lm_prob,
                  short_seq_prob,
                  seed,
                  skip_warmup,
                  binary_head,
                  max_seq_length_dec,
                  dataset_type=dataset_type,
                  tokenizer=tokenizer,
                  max_ngram_size=max_ngram_size,
                  mean_ngram_size=mean_ngram_size,
                  geometric_dist=geometric_dist,
                  permutation=permutation,
                  whole_word_masking=whole_word_masking,
                  favor_long_ngrams=favor_long_ngrams,
                  delete_mask_prob=delete_mask_prob,
                  respect_document_boundaries=respect_document_boundaries,
                  data_impl_kwargs=data_impl_kwargs,
              )
          # Blending dataset.
          # Parse the values.
          output = get_datasets_weights_and_num_samples(
              data_prefix, train_valid_test_num_samples
          )
          (
              prefixes,
              weights,
              datasets_train_valid_test_num_samples,
          ) = output
          train_n, valid_n, test_n = map(
              sum, zip(*datasets_train_valid_test_num_samples)
          )
  
          # Build individual datasets.
          train_datasets = []
          valid_datasets = []
          test_datasets = []
          for i in range(len(prefixes)):
              (
                  train_ds,
                  valid_ds,
                  test_ds,
              ) = _build_train_valid_test_datasets(
                  cfg,
                  trainer,
                  prefixes[i],
                  data_impl,
                  splits_string,
                  datasets_train_valid_test_num_samples[i],
                  max_seq_length,
                  masked_lm_prob,
                  short_seq_prob,
                  seed,
                  skip_warmup,
                  binary_head,
                  max_seq_length_dec,
                  dataset_type=dataset_type,
                  tokenizer=tokenizer,
                  max_ngram_size=max_ngram_size,
                  mean_ngram_size=mean_ngram_size,
                  geometric_dist=geometric_dist,
                  permutation=permutation,
                  whole_word_masking=whole_word_masking,
                  favor_long_ngrams=favor_long_ngrams,
                  delete_mask_prob=delete_mask_prob,
                  respect_document_boundaries=respect_document_boundaries,
                  data_impl_kwargs=data_impl_kwargs,
              )
              if train_ds:
                  train_datasets.append(train_ds)
              if valid_ds:
                  valid_datasets.append(valid_ds)
              if test_ds:
                  test_datasets.append(test_ds)
  
              # Blend.
          blending_train_dataset = None
          if train_datasets:
              blending_train_dataset = BlendableDataset(
                  train_datasets, weights, train_n
              )
          blending_valid_dataset = None
          if valid_datasets:
              blending_valid_dataset = BlendableDataset(
                  valid_datasets, weights, valid_n
              )
          blending_test_dataset = None
          if test_datasets:
              blending_test_dataset = BlendableDataset(
                  test_datasets, weights, test_n
              )
  
          return (
              blending_train_dataset,
              blending_valid_dataset,
              blending_test_dataset,
          )
  
imports: import collections
  import os
  import subprocess
  import time
  from typing import Any
  import numpy
  import torch
  from omegaconf import OmegaConf
  from omegaconf.dictconfig import DictConfig
  from nemo.collections.nlp.data.language_modeling.megatron.base_dataset_utils import get_datasets_weights_and_num_samples, get_train_valid_test_split_
  from nemo.collections.nlp.data.language_modeling.megatron.blendable_dataset import BlendableDataset
  from nemo.collections.nlp.data.language_modeling.megatron.indexed_dataset import deallocate_indexed_dataset_memory
  from nemo.collections.nlp.data.language_modeling.megatron.indexed_dataset import make_dataset
  from nemo.collections.nlp.data.language_modeling.megatron.indexed_dataset import make_indexed_dataset_compatibility
  from nemo.collections.nlp.data.language_modeling.megatron.length_distribution_type import LengthDistribution
  from nemo.collections.nlp.data.language_modeling.megatron.lm_adapted_t5_dataset import T5LMAdaptedDataset
  from nemo.utils import logging
  from nemo.utils.get_rank import is_global_rank_zero
label_desc: Use image node_id 12 for calling a global function with example usage: build_train_valid_test_datasets(cfg, trainer, data_prefix, data_impl, splits_string, train_valid_test_num_samples, max_seq_length, masked_lm_prob, short_seq_prob, seed, skip_warmup, binary_head, max_seq_length_dec, dataset_type, tokenizer, max_ngram_size, mean_ngram_size, geometric_dist, permutation, whole_word_masking, favor_long_ngrams, delete_mask_prob, respect_document_boundaries, data_impl_kwargs) and returns: train_ds,  validation_ds,  test_ds, blending_train_dataset,  blending_valid_dataset,  blending_test_dataset, train_ds,  valid_ds,  test_ds, _build_train_valid_test_datasets