class_docstr: Megatron GPT Supervised Fine-Tuning
class_docstr_tok: ["Megatron","GPT","Supervised","Fine-Tuning"]
code: def _build_dataset(self, data_cfg, is_train=True):
      packed_sequence = data_cfg.get("packed_sequence", False)
      datasets = []
      # Determine if we are using a single dataset or a list of datasets.
      is_list_config = isinstance(data_cfg.file_names, ListConfig)
      if not is_list_config:
          raise ValueError(
              f"SFT train/validation datasets must be provided as a list of individual JSONL files."
          )
  
      if is_train:
          # Construct the data prefix list for `get_datasets_weights_and_num_samples()`
          # that is of the format [weight1,file_name1,weight2,file_name2,...]
          if (
              data_cfg.concat_sampling_probabilities is None
              or not isinstance(
                  data_cfg.concat_sampling_probabilities, ListConfig
              )
          ):
              raise ValueError(
                  (
                      f"concat_sampling_probabilities must be a ListConfig with the same number of files in file_names."
                      f"Found: {data_cfg.concat_sampling_probabilities}"
                  )
              )
  
          if len(
              data_cfg.get("concat_sampling_probabilities", None)
          ) != len(data_cfg.file_names):
              raise ValueError(
                  (
                      f"concat_sampling_probabilities must be of the same size as file_names.",
                      f"Provided size {len(data_cfg.concat_sampling_probabilities)}, number of datasets {len(data_cfg.file_names)}",
                  )
              )
  
          data_prefix = []
          for weight, prefix in zip(
              data_cfg.concat_sampling_probabilities,
              data_cfg.file_names,
          ):
              data_prefix.append(weight)
              data_prefix.append(prefix)
  
          if (
              self.trainer.max_steps is None
              or self.trainer.max_steps <= 0
          ):
              raise ValueError(
                  f"Trainer max_steps must be set to a positive integer. Found {self.trainer.max_steps}"
              )
          num_train_samples = [
              self.trainer.max_steps * data_cfg.global_batch_size
          ]
          (
              _,
              _,
              num_train_samples_per_dataset,
          ) = get_datasets_weights_and_num_samples(
              data_prefix, num_train_samples
          )
          num_train_samples_after_blend = sum(
              [x[0] for x in num_train_samples_per_dataset]
          )
      else:
          num_train_samples_per_dataset = [[None]] * len(
              data_cfg.file_names
          )
  
      # Check dataset max_seq_legnth and max_position_embeddings size
      if (
          self.cfg.get("position_embedding_type", None)
          in [None, "learned_absolute"]
          and data_cfg.max_seq_length > self.cfg.max_position_embeddings
      ):
          logging.warning(
              f"Set dataset max_seq_length to max_position_embeddings {self.cfg.max_position_embeddings} if using learned_absolute position embedding"
          )
          data_cfg.max_seq_length = self.cfg.max_position_embeddings
  
      for file_path, num_samples in zip(
          data_cfg.file_names, num_train_samples_per_dataset
      ):
          if self.cfg.data.get("chat", False):
              dataset_cls = GPTSFTChatDataset
          elif packed_sequence:
              dataset_cls = GPTSFTPackedDataset
              assert (
                  data_cfg.micro_batch_size == 1
              ), "Micro batch size must be 1 if using packed sequence"
          else:
              dataset_cls = GPTSFTDataset
          dataset = dataset_cls(
              file_path=file_path,
              tokenizer=self.tokenizer,
              max_seq_length=data_cfg.max_seq_length,
              min_seq_length=data_cfg.min_seq_length,
              add_bos=data_cfg.get("add_bos", False),
              add_eos=data_cfg.get("add_eos", True),
              add_sep=data_cfg.get("add_sep", False),
              sep_id=self.sep_id,
              max_num_samples=num_samples[0]
              if not packed_sequence
              else None,
              seed=data_cfg.get("seed", 1234),
              label_key=data_cfg.get("label_key", "answer"),
              answer_only_loss=self.cfg.get("answer_only_loss", True),
              truncation_field=data_cfg.get("truncation_field", "text"),
              pad_to_max_length=data_cfg.get(
                  "pad_to_max_length", False
              ),
              index_mapping_dir=data_cfg.get("index_mapping_dir", None),
              prompt_template=data_cfg.get("prompt_template", None),
              virtual_tokens=self.virtual_tokens,
              tokens_to_generate=data_cfg.get(
                  "tokens_to_generate", 0
              ),  # used at inference time to allocate tensor positions for tokens that will be generated by inf procedure.
              memmap_workers=data_cfg.get(
                  "memmap_workers", None
              ),  # used to set num. of workers to create the memmap index files
              hf_dataset=data_cfg.get(
                  "hf_dataset", False
              ),  # Whether to load the json file with the HuggingFace dataset. otherwise, will load the jsonl file with the JSONLMemMapDataset.
              truncation_method=data_cfg.get(
                  "truncation_method", "right"
              ),  # used to choose truncation method. Options: ['random', 'left', 'right']
              special_tokens=self.cfg.data.get(
                  "chat_prompt_tokens", None
              ),  # special tokens for the chat prompts, a dictionary of {token_type: token}. Default: {'system_turn_start': '<extra_id_0>', 'turn_start': '<extra_id_1>', 'label_start': '<extra_id_2>', 'end_of_turn': '\n', "end_of_name": "\n"}
              is_test=not is_train,
          )
          datasets.append(dataset)
      if is_train:
          if packed_sequence:
              num_train_samples_after_blend = sum(
                  len(dataset) for dataset in datasets
              )
          dataset = BlendableDataset(
              datasets=datasets,
              weights=data_cfg.concat_sampling_probabilities,
              size=num_train_samples_after_blend,
          )
          return dataset
      else:
          return datasets
  
imports: import itertools
  import json
  from functools import partial
  from typing import Any, Optional
  import torch
  from omegaconf import DictConfig, ListConfig
  from pytorch_lightning.trainer.trainer import Trainer
  from nemo.collections.common.metrics import MetricStringToTorchMetric
  from nemo.collections.nlp.data.language_modeling.megatron.base_dataset_utils import get_datasets_weights_and_num_samples
  from nemo.collections.nlp.data.language_modeling.megatron.blendable_dataset import BlendableDataset
  from nemo.collections.nlp.data.language_modeling.megatron.gpt_sft_chat_dataset import GPTSFTChatDataset
  from nemo.collections.nlp.data.language_modeling.megatron.gpt_sft_dataset import GPTSFTDataset, GPTSFTPackedDataset
  from nemo.collections.nlp.data.language_modeling.megatron.megatron_batch_samplers import MegatronPretrainingBatchSampler
  from nemo.collections.nlp.models.language_modeling.megatron_gpt_model import MegatronGPTModel
  from nemo.collections.nlp.modules.common.megatron.utils import get_iterator_k_split
  from nemo.collections.nlp.modules.common.text_generation_utils import generate, get_computeprob_response
  from nemo.collections.nlp.parts.mixins.nlp_adapter_mixins import NLPAdapterModelMixin
  from nemo.collections.nlp.parts.utils_funcs import get_last_rank
  from nemo.utils import AppState, logging
label_desc: Use image node_id 5 for calling the MegatronGPTSFTModel obj's underlying member method code with example usage: obj._build_dataset(data_cfg, is_train) and returns: dataset, datasets