code: def load_data(paths):
      """Load multiple hdf5 files to single numpy array."""
  
      # Read data shapes.
      shape_map = defaultdict(lambda: (0, None))
      for p in paths:
          f = h5py.File(p, "r")
          for k in f.keys():
              shape = tuple(f[k].shape)
              shape_map[k] = (shape_map[k][0] + shape[0], shape[1])
          f.close()
  
      # Allocate output array.
      data_map = {
          k: np.empty(s, dtype="f4") for k, s in shape_map.items()
      }
      start_map = {k: 0 for k in shape_map}
  
      # Load files.
      for pi, p in enumerate(tqdm(paths, "load data")):
          f = h5py.File(p, "r")
          for k in f.keys():
              i0 = start_map[k]
              i1 = i0 + len(f[k])
              data_map[k][i0:i1] = f[k]
              start_map[k] += len(f[k])
          f.close()
  
      return data_map
  
file_path: megatron-lm/tools/bert_embedding/utils.py
imports: from collections import defaultdict
  import glob
  import numpy
  import os
  import torch
  from tqdm import tqdm
  from megatron import print_rank_0
  from megatron.core import parallel_state
  from .external_libs import h5py
label_desc: Use image node_id 2 for calling a global function with example usage: load_data(paths) and returns: data_map