code: def _build_index_from_memdata(fn, newline_int):
      """
      Build index of delimiter positions between samples in memmap.
      Can be provided externally.
  
      Returns a 1D array of ints.
      """
      # use memmap to read file
      mdata = np.memmap(fn, dtype=np.uint8, mode="r")
      # find newline positions
      midx = np.where(mdata == newline_int)[0]
      midx_dtype = midx.dtype
      # make sure to account for all data
      midx = midx.tolist()
      # add last item in case there is no new-line at the end of the file
      if (len(midx) == 0) or (midx[-1] + 1 != len(mdata)):
          midx = midx + [len(mdata) + 1]
  
      # remove empty lines from end of file
      while len(midx) > 1 and (midx[-1] - midx[-2]) < 2:
          midx.pop(-1)
      midx = np.asarray(midx, dtype=midx_dtype)
  
      # free memmap
      mdata._mmap.close()
      del mdata
  
      return midx
  
imports: import datetime
  import json
  import multiprocessing
  import os
  import pickle
  import time
  from functools import partial
  from typing import Callable, List, Optional, Type
  import numpy
  import torch
  from nemo.core import Dataset
  from nemo.utils import AppState, logging
label_desc: Use image node_id 1 for calling a global function with example usage: _build_index_from_memdata(fn, newline_int) and returns: midx