class_docstr: Memory Map Index and Binary file for RETRO DATA.
  It provides `chunks` to the original MMap data so data can be fetched at both document and chunk level.
  It can be used both for training data and Retrieval Data.
  Retrieval Dataset adds an extra `chunk_size` padded tokens at the end of each document. '
  `self._index.retrieval_db` is indicating whether it is retrieval dataset or not.
  
  It is built by `preprocess_data_for_megatron.py` script.
class_docstr_tok: ["Memory","Map","Index","and","Binary","file","for","RETRO","DATA",".","It","provides","`","chunks","`","to","the","original","MMap","data","so","data","can","be","fetched","at","both","document","and","chunk","level",".","It","can","be","used","both","for","training","data","and","Retrieval","Data",".","Retrieval","Dataset","adds","an","extra","`","chunk_size","`","padded","tokens","at","the","end","of","each","document",".","'","`","self._index.retrieval_db","`","is","indicating","whether","it","is","retrieval","dataset","or","not",".","It","is","built","by","`","preprocess_data_for_megatron.py","`","script","."]
docstr: return a single document or a slice of documents, excluding the paddings for the retrieval db
docstr_tok: ["return","a","single","document","or","a","slice","of","documents",",","excluding","the","paddings","for","the","retrieval","db"]
code: def __getitem__(self, idx):
      """
      return a single document or a slice of documents, excluding the paddings for the retrieval db
      """
      if isinstance(idx, int):
          # no need to handle retrieval_db since size exclude the paddings
          ptr, size = self._index[idx]
          np_array = np.frombuffer(
              self._bin_buffer,
              dtype=self._index.dtype,
              count=size,
              offset=ptr,
          )
          return np_array
      elif isinstance(idx, slice):
          start, stop, step = idx.indices(len(self))
          if step != 1:
              raise ValueError(
                  "Slices into indexed_dataset must be contiguous"
              )
          ptr = self._index._pointers[start]
          if self._index.retrieval_db:
              # for retrieval db, need to add the padding of chunk_size at the end of each document
              sizes = self._index._sizes[idx] + self._index.chunk_size
          else:
              sizes = self._index._sizes[idx]
          # offsets get the number of tokens for each document including the paddings
          offsets = list(accumulate(sizes))
          total_size = sum(sizes)
          np_array = np.frombuffer(
              self._bin_buffer,
              dtype=self._index.dtype,
              count=total_size,
              offset=ptr,
          )
          sents = np.split(np_array, offsets[:-1])
          if self._index.retrieval_db:
              # remove the paddings
              sents = [
                  sent[: -self._index.chunk_size] for sent in sents
              ]
          return sents
  
imports: import os
  import shutil
  import struct
  from functools import lru_cache
  from itertools import accumulate
  from typing import List
  import numpy
  import torch
  from nemo.utils import logging
label_desc: Use image node_id 6 for calling the MMapRetrievalIndexedDataset obj's underlying member method code with example usage: obj.__getitem__(idx) and returns: np_array, sents