class_docstr: Memory Map Index and Binary file for RETRO DATA.
  It provides `chunks` to the original MMap data so data can be fetched at both document and chunk level.
  It can be used both for training data and Retrieval Data.
  Retrieval Dataset adds an extra `chunk_size` padded tokens at the end of each document. '
  `self._index.retrieval_db` is indicating whether it is retrieval dataset or not.
  
  It is built by `preprocess_data_for_megatron.py` script.
class_docstr_tok: ["Memory","Map","Index","and","Binary","file","for","RETRO","DATA",".","It","provides","`","chunks","`","to","the","original","MMap","data","so","data","can","be","fetched","at","both","document","and","chunk","level",".","It","can","be","used","both","for","training","data","and","Retrieval","Data",".","Retrieval","Dataset","adds","an","extra","`","chunk_size","`","padded","tokens","at","the","end","of","each","document",".","'","`","self._index.retrieval_db","`","is","indicating","whether","it","is","retrieval","dataset","or","not",".","It","is","built","by","`","preprocess_data_for_megatron.py","`","script","."]
docstr: Retrieves a single chunk item from the dataset.
  It will get chunk_size tokens for training data
  or 2*chunk_size tokens for retrieval data.
  If force_no_cont_ids=True, it will always get chunk_size tokens
docstr_tok: ["Retrieves","a","single","chunk","item","from","the","dataset",".","It","will","get","chunk_size","tokens","for","training","data","or","2","*","chunk_size","tokens","for","retrieval","data",".","If","force_no_cont_ids=True",",","it","will","always","get","chunk_size","tokens"]
code: def get_chunk(self, chunk_id, force_no_cont_ids=False):
      """Retrieves a single chunk item from the dataset.
      It will get chunk_size tokens for training data
      or 2*chunk_size tokens for retrieval data.
      If force_no_cont_ids=True, it will always get chunk_size tokens
      """
      if isinstance(chunk_id, (int, np.int64, np.int32)):
          ptr = self._index.get_chunk_address(chunk_id)
          if self._index.retrieval_db and (not force_no_cont_ids):
              size = self._index.chunk_size * 2
          else:
              size = self._index.chunk_size
          np_array = np.frombuffer(
              self._bin_buffer,
              dtype=self._index.dtype,
              count=size,
              offset=ptr,
          )
          return np_array
      elif isinstance(chunk_id, slice):
          start, stop, step = chunk_id.indices(self.chunks)
          if step != 1:
              raise ValueError(
                  "Slices into indexed_dataset must be contiguous"
              )
          if self._index.retrieval_db and (not force_no_cont_ids):
              chunk_size = self._index.chunk_size * 2
          else:
              chunk_size = self._index.chunk_size
          ptr = self._index.get_chunk_address(start)
          end_address = (
              self._index.get_chunk_address(stop - 1)
              + chunk_size * self._index._dtype_size
          )
          address = self._index._chunk_address[chunk_id]
          starting_pos = address // self._index._dtype_size
          total_size = (end_address - ptr) // self._index._dtype_size
          np_array = np.frombuffer(
              self._bin_buffer,
              dtype=self._index.dtype,
              count=total_size,
              offset=ptr,
          )
          sents = [
              np_array[pos : pos + chunk_size]
              for pos in starting_pos - starting_pos[0]
          ]
          return sents
  
imports: import os
  import shutil
  import struct
  from functools import lru_cache
  from itertools import accumulate
  from typing import List
  import numpy
  import torch
  from nemo.utils import logging
label_desc: Use image node_id 10 for calling the MMapRetrievalIndexedDataset obj's underlying member method code with example usage: obj.get_chunk(chunk_id, force_no_cont_ids) and returns: np_array, sents