class_docstr: The base GPT dataset
  
  Args:
      indexed_dataset (MMapIndexedDataset): The MMapIndexedDataset around which to build the
      MegatronDataset
  
      indexed_indices (numpy.ndarray): The set of the documents indices to expose
  
      num_samples (int): The number of samples to draw from the indexed dataset
  
      index_split (Split): The indexed_indices Split
  
      config (GPTDatasetConfig): The GPT-specific container for all config sourced parameters
class_docstr_tok: ["The","base","GPT","dataset","Args",":","indexed_dataset","(","MMapIndexedDataset",")",":","The","MMapIndexedDataset","around","which","to","build","the","MegatronDataset","indexed_indices","(","numpy.ndarray",")",":","The","set","of","the","documents","indices","to","expose","num_samples","(","int",")",":","The","number","of","samples","to","draw","from","the","indexed","dataset","index_split","(","Split",")",":","The","indexed_indices","Split","config","(","GPTDatasetConfig",")",":","The","GPT-specific","container","for","all","config","sourced","parameters"]
docstr: Get the text (token ids) and document ids for a given index
  
  Args:
      idx (int): The index into the dataset
  
  Returns:
      Tuple[numpy.ndarray, numpy.ndarray]: The text ids and document ids
docstr_tok: ["Get","the","text","(","token","ids",")","and","document","ids","for","a","given","index","Args",":","idx","(","int",")",":","The","index","into","the","dataset","Returns",":","Tuple","[","numpy.ndarray",",","numpy.ndarray","]",":","The","text","ids","and","document","ids"]
code: def _query_document_sample_shuffle_indices(
      self, idx: int
  ) -> Tuple[numpy.ndarray, numpy.ndarray]:
      """Get the text (token ids) and document ids for a given index
  
      Args:
          idx (int): The index into the dataset
  
      Returns:
          Tuple[numpy.ndarray, numpy.ndarray]: The text ids and document ids
      """
      # Do the shuffle mapping
      idx = self.shuffle_index[idx]
  
      # Get the beginning and end documents and offsets
      doc_index_beg, doc_index_beg_offset = self.sample_index[idx]
      doc_index_end, doc_index_end_offset = self.sample_index[idx + 1]
  
      document_ids = []
      sample_parts = []
  
      # Sample spans a single document
      if doc_index_beg == doc_index_end:
          # Add the document id
          document_ids.append(self.document_index[doc_index_beg])
  
          # Add the entire sample
          sample_parts.append(
              self.indexed_dataset.get(
                  self.document_index[doc_index_beg],
                  offset=doc_index_beg_offset,
                  length=doc_index_end_offset
                  - doc_index_beg_offset
                  + 1,
              )
          )
  
      # Sample spans multiple documents
      else:
          for i in range(doc_index_beg, doc_index_end + 1):
              # Add the document id
              document_ids.append(self.document_index[i])
  
              # Add the sample part
              offset = 0 if i > doc_index_beg else doc_index_beg_offset
              length = (
                  None
                  if i < doc_index_end
                  else doc_index_end_offset + 1
              )
              sample_parts.append(
                  self.indexed_dataset.get(
                      self.document_index[i],
                      offset=offset,
                      length=length,
                  )
              )
  
      return (
          numpy.array(
              numpy.concatenate(sample_parts), dtype=numpy.int64
          ),
          numpy.array(document_ids, dtype=numpy.int64),
      )
  
imports: import logging
  import os
  import time
  from dataclasses import dataclass
  from typing import Dict, Tuple, Union
  import numpy
  import torch
  from megatron.core.datasets.blended_megatron_dataset_config import BlendedMegatronDatasetConfig
  from megatron.core.datasets.indexed_dataset import MMapIndexedDataset
  from megatron.core.datasets.megatron_dataset import MegatronDataset
  from megatron.core.datasets.utils import Split, log_single_rank
label_desc: Use image node_id 7 for calling the GPTDataset obj's underlying member method code with example usage: obj._query_document_sample_shuffle_indices(idx) and returns: numpy,  numpy