text: # encoding: utf-8
  """
  #lookup.py
  
  This file manages all the mappings between hickle/HDF5 metadata and python
  types.
  There are three dictionaries that are populated by the LoaderManager associated
  with each file created or loaded:
  
  1) types_dict
  Mapping between python types and dataset and group creation functions, e.g.
      types_dict = {
          list: (create_listlike_dataset, 'list'),
          int: (create_python_dtype_dataset, 'int'),
          np.ndarray: (create_np_array_dataset, 'ndarray'),
          }
  
  2) hkl_types_dict
  Mapping between hickle metadata and dataset loading functions, e.g.
      hkl_types_dict = {
          'list': load_list_dataset,
          'tuple': load_tuple_dataset
          }
  
  3) hkl_container_dict
  Mapping between hickle metadata and group container classes, e.g.
      hkl_contianer_dict = {
          'list': ListLikeContainer,
          'tuple': TupleLikeContainer,
          'dict': DictLikeContainer
      }
  
  ## Extending hickle to add support for other classes and types
  
  The process to add new load/dump capabilities is as follows:
  
  1) Create a file called load_[newmodule/newpackage].py
  2) In the load_[newmodule/newpackage].py file, define your create_dataset,
     load_dataset functions and PyContainer objects, along with the 
     'class_register' and 'exclude_register' tables. See the other loaders
     in <Python path>/dist_packages/hickle/loaders directory for examples.
     The columns in register_class table correspond to argument list of 
     hickle.lookup.LoaderManager.register_class method. 
     
  
  3) store the load_[newmodule/newpackage].py file in one of the following locations
     <Python path>/dist_packages/hickle/loaders/
         the loaders supported by hickle globally
  
     <Python path>/dist_packages/hickle_loaders/
         loaders installed during installation of additional single file python modules
         through pip or legacy os installer
  
     <Python path>/dist_packages/[newpackage]/hickle_loaders/
         loaders provided by [newpackage] installed through pip or legacy os installer
  
     [MyPackage]/hickle_loaders/
         loaders specific for objects and classes defined by the modules of [MyPackage] package
  
     basedir([MyModule|MyApplication])/hickle_loaders
         loaders specific for object and classes defined by [MyModule|MyApplication].py file
         not part of a python package
  
  Loader for a single object or class can also be created by calling 
  hickle.lookup.register_class prior to calling hickle.dump and hickle.load
  
  1) create a `create_[MyClass]_dataset` function, a `load_[MyClass]_fcn` function and/or
     `[MyClass]Container` class for [MyClass] class to be dumped and loaded by hickle.
     Examples can be found in the loader modules in <Python path>/dist_packages/hickle/loaders
     directory
  2) call `hickle.lookup.LoaderManager.register_class` method as follows from your code
  
     ```
     from hickle.lookup import LoaderManager
  
     LoaderManager.register_class(
         [MyClass],'<myclass_type>',
         create_[MyClass]_dataset,
         load_[MyClass]_fcnt, # or None if [MyClass] is mapped to h5py.Group only
         [MyClass]Container , # or None if [MyClass] is mapped to h5py.Dataset only
         True, # if False [MyClass] object will be stored explicitly on any occurrence
         'custom', # set to None to enforce unconditional use of loader
     )
  
     hickle.dump('my_[MyClass]_object,'myfile.hkl','w',options={'custom':True})
     new_[MyClass]_object = hickle.load('myfile.hkl')
     ```
  
  """
  
  
  # %% IMPORTS
  # Built-in imports
  import sys
  import warnings
  import types
  import re
  import weakref
  import os.path
  from importlib.util import find_spec, module_from_spec,spec_from_file_location,spec_from_loader
  from importlib import invalidate_caches
  
  # Package imports
  import collections
  import pickle
  import numpy as np
  import h5py as h5
  
  # hickle imports
  from .helpers import PyContainer,not_dumpable,nobody_is_my_name,no_compression,NotHicklable
  from .loaders import optional_loaders, attribute_prefix
  
  if sys.version_info[:2] <= (3,5): # pragma: no cover
      # define ModuleNotFoundError on __builtins__ to ensure below code is working
      setattr(sys.modules['builtins'],'ModuleNotFoundError',getattr(sys.modules['builtins'],'ModuleNotFoundError',ImportError))
  
  # %% GLOBALS
  
  # %% FUNCTION DEFINITIONS
  
  
  def load_nothing(h_node, base_type , py_obj_type): # pragma: no cover
      """
      loads nothing
      """
      return nobody_is_my_name
  
  def dump_nothing(py_obj, h_group, name, **kwargs): # pragma: no cover
      """
      dumps nothing
      """
      return nobody_is_my_name
  
  
  # %% CLASS DEFINITIONS
  
  class _DictItem(): # pragma: no cover
      """
      dummy py_obj for dict_item loader
      """
  
  class NodeReference(): # pragma: no cover
      """
      dummy py_obj_type returned by ReferenceManager.get_type
      when encountering dataset of h5py.ref_dtype  which expose 
      no explicit 'type' attribute.
      """
  
  class ReferenceError(Exception): # pragma: no cover
      """
      exception thrown by ReferenceManager
      """
  
  class LookupError(Exception): # pragma: no cover
      """
      exception thrown if type lookup fails
      """
  
  class SerializedWarning(UserWarning): # pragma: no cover
      """ An object type was not understood
  
      The data will be serialized using pickle.
      """
  
  class PackageImportDropped(UserWarning): # pragma: no cover
      """
      Package or module defining type/class was removed from 
      sys.modules
      """
  
  class MockedLambdaWarning(UserWarning): # pragma: no cover
      """
      In Python >= 3.8 lambda function fails pickle.loads
  
      faking restoring lambda to keep legacy hickle 4.x files
      loading properly
      """
  
  class DataRecoveredWarning(UserWarning): # pragma: no cover
      """
      Raised when hickle does not find an appropriate loader for
      a specific type and thus has to fall back to '!recover!' loader
      recover_custom_dataset function or RecoverGroupContainer PyContainer
      object
      """
  
  class AttemptRecoverCustom():
      """
      Dummy type indicating that the data of specific py_obj_type listed in the
      hickle_types_table could not be restored. Most likely pickle.loads encountered
      an ImportError/ModuleNotFoundError or AttributeError indicating that the package
      and/or module defining the py_obj_type is not installed or does not anymore 
      provide the definition of py_object_type.
  
      In this case hickle tries to at least recover the data and corresponding meta
      data stored in h5py.Group or h5py.Dataset attributes and the base_type string
      indicating the loader used to dump the data.  Only in case this attempt fails
      an exception is thrown.
      """
  
  class RecoveredGroup(dict,AttemptRecoverCustom):
      """
      dict type object representing the content and hickle meta data of a h5py.group
      """
  
      __slots__ = ('attrs',)
  
      def __init__(self,*args,attrs={},**kwargs):
          super().__init__(*args,**kwargs)
          self.attrs={name:value for name,value in attrs.items() if name not in {'type'}}
  
  class RecoveredDataset(np.ndarray,AttemptRecoverCustom):
      """
      numpy.ndarray type object representing the content and hickle meta data  of a
      h5py.dataset
      """
  
      __slots__ = ('attrs',)
  
      def __new__(cls,input_array,dtype=None,attrs={}):
          array_copy = np.array(input_array,dtype=dtype)
          obj = super().__new__(
              cls,
              shape = array_copy.shape,
              dtype = array_copy.dtype,
              buffer=array_copy,
              offset=0,
              strides=array_copy.strides,
              order = 'C' if array_copy.flags.c_contiguous else 'F'
          )
          obj.attrs = {name:value for name,value in attrs.items() if name not in {'type'}}
          return obj
      
      def __array_finalize__(self,obj):
          if obj is not None:
              self.attrs = getattr(obj,'attrs',{})
  
  class ManagerMeta(type):
      """
      Metaclass for all manager classes derived from the BaseManager class.
      Ensures that the __managers__ class attribute of each immediate child
      class of BaseManager is initialized to a dictionary and ensures that
      it can not be overwritten by any grandchild class and down. The
      __managers__ attribute declared by any grandchild shadowing the
      __managers__ attribute of any of its ancestors is dropped without any
      further notice.
      """
  
      def __new__(cls, name, bases, namespace, **kwords):
          for _ in ( True for base in bases if isinstance(getattr(base,'__managers__',None),dict) ):
              namespace.pop('__managers__',None)
              break
          else:
              if not isinstance(namespace.get('__managers__',None),dict):
                  namespace['__managers__'] = dict() if bases and not object in bases else None
          return super().__new__(cls,name,bases,namespace,**kwords)
                  
  class BaseManager(metaclass = ManagerMeta):
      """
      Base class providing basic management of simultaneously open managers.
      Must be subclassed.
      """
      __slots__ = ('__weakref__',)
  
      __managers__ = None
  
      @classmethod
      def _drop_manager(cls, fileid):
          """
          finalizer callback to properly remove a <manager> object from
          the <manager>.__managers__ structure when corresponding file is
          closed or <manager> object is garbage collected
  
          Parameters
          ----------
          cls (BaseManager):
              the child <manager> class for which to drop the hdf5 file instance
              referenced to by the specified file id
  
          fileid (h5py.FileId):
              Id identifying the hdf5 file the ReferenceManager was created for
  
          """
          try:
              manager = cls.__managers__.get(fileid,None)
              if manager is None:
                  return
              cls.__managers__.pop(fileid,None)
          except: # pragma: no cover
              # only triggered in case of race
              pass
  
      @classmethod
      def create_manager(cls, h_node, create_entry):
          """
          Check whether an instance already exists for the file the h_node belongs to and
          call create_entry if no entry yet exists.
  
          Parameters
          ----------
          cls:
              the manager class to create a new instance for
  
          h_node (h5py.File, h5py.Group, h5py.Dataset):
              the h5py node or its h_root_group or file to create a new ReferenceManager
              object for.
  
          create_entry (callable):
              function or method which returns a new table entry. A table entry
              is a tuple or list with contains as its first item the newly
              created <manager> object. It may include further items specific to
              the actual BaseManager subclass.
                  
          Raises
          ------
          LookupError:
              if manager has already been created for h_node or its h_root_group 
                  
          """
          manager = cls.__managers__.get(h_node.file.id,None)
          if manager is not None:
              raise LookupError(
                  "'{}' type manager already created for file '{}'".format(
                  cls.__name__,h_node.file.filename
              )
          )
          table = cls.__managers__[h_node.file.id] = create_entry()
          weakref.finalize(table[0],cls._drop_manager,h_node.file.id)
          return table[0]
  
      @classmethod
      def get_manager(cls, h_node):
          """
          return manager responsible for the file containing h_node 
      
          Parameters
          ----------
          h_node (h5py.File, h5py.Group, h5py.Dataset):
              the h5py node to obtain the responsible manager for.
              
          Raises
          ------
          LookupError:
              if no manager has been created yet for h_node or its h_root_group 
          """
          try:
              return cls.__managers__[h_node.file.id][0]
          except KeyError:
              raise ReferenceError("no managers exist for file '{}'".format(h_node.file.filename))
  
      def __init__(self):
          if type.mro(self.__class__)[0] is BaseManager:
              raise TypeError("'BaseManager' class must be subclassed")
  
      def __enter__(self):
          raise NotImplementedError("'{}' type object must implement Python ContextManager protocol")
  
      def __exit__(self, exc_type, exc_value, exc_traceback, h_node=None):
          # remove this ReferenceManager object from the table of active ReferenceManager objects
          # and cleanly unlink from any h5py object instance and id references managed. Finalize
          # 'hickle_types_table' overlay if it was created by __init__ for hickle 4.0.X file
          self.__class__._drop_manager(h_node.file.id)
  
  class ReferenceManager(BaseManager, dict):
      """
      Manages all object and type references created for basic
      and type special memoisation.
  
      To create a ReferenceManager call ReferenceManager.create_manager
      function. The value returned can be and shall be used within a
      with statement to ensure it is garbage collected before file is closed.
      For example:
  
          with ReferenceManager.create_manager(h_root_group) as memo:
              _dump(data,h_root_group,'data',memo,loader,**kwargs)
  
          with ReferenceManager.create_manager(h_root_group) as memo:
              _load(py_container,'data',h_root_group['data'],memo,loader)
  
          with ReferenceManager.create_manager(h_root_group,fix_lambda_obj_type) as memo:
              _load(py_container,'data',h_root_group['data'],memo,loader)
  
      NOTE: for creating appropriate loader object see LoaderManager
      """
  
      __slots__ = (
          '_py_obj_type_table', # hickle_types_table h5py.Group storing type information
          '_py_obj_type_link', # dictionary linking py_obj_type and representation in hickle_types_table
          '_base_type_link', # dictionary linking base_type string and representation in hickle_types_table
          '_overlay', # in memory hdf5 dummy file hosting dummy hickle_types_table for hickle 4.x files
          'pickle_loads' # reference to pickle.loads method
      )
  
      
      @staticmethod
      def get_root(h_node):
          """
          returns the h_root_group the passed h_node belongs to.
          """
  
          # try to resolve the 'type' attribute of the h_node
          entry_ref = h_node.attrs.get('type',None)
          if isinstance(entry_ref,h5.Reference):
  
              # return the grandparent of the referenced py_obj_type dataset as it
              # also the h_root_group of h_node
              try:
                  entry = h_node.file.get(entry_ref,None)
              except ValueError: # pragma: no cover
                  entry = None
              if entry is not None:
                  return entry.parent.parent
          if h_node.parent == h_node.file:
  
              # h_node is either the h_root_group it self or the file node representing
              # the open hickle file. 
              return h_node if isinstance(h_node,h5.Group) else h_node.file
  
          # either h_node has not yet a 'type' assigned or contains pickle string
          # which has implicit b'pickle' type. try to resolve h_root_group from its
          # parent 'type' entry if any
          entry_ref = h_node.parent.attrs.get('type',None)
          if not isinstance(entry_ref,h5.Reference):
              if entry_ref is None:
  
                  # parent has neither a 'type' assigned
                  return h_node if isinstance(h_node,h5.Group) else h_node.file
  
              # 'type' seems to be a byte string or string fallback to h_node.file
              return h_node.file
          try:
              entry = h_node.file.get(entry_ref,None)
          except ValueError: # pragma: no cover
              entry = None
          if entry is None:
  
              # 'type' reference seems to be stale
              return h_node if isinstance(h_node,h5.Group) else h_node.file
  
          # return the grand parent of the referenced py_obj_type dataset as it
          # is also the h_root_group of h_node
          return  entry.parent.parent
  
      @staticmethod
      def _drop_overlay(h5file):
          """
          closes in memory overlay file providing dummy 'hickle_types_table' structure
          for hdf5 files which were created by hickle 4.x 
          """
          h5file.close()
  
      @classmethod
      def create_manager(cls,h_node, pickle_loads = pickle.loads):
          """
          creates a new ReferenceManager object for the h_root_group the h_node
          belongs to. 
  
  
          Parameters
          ----------
          h_node (h5py.Group, h5py.Dataset):
              the h5py node or its h_root_group to create a new ReferenceManager
              object for.
  
          pickle_loads (FunctionType,MethodType):
              method to be used to expand py_obj_type pickle strings.
              defaults to pickle.loads. Must be set to fix_lambda_obj_type 
              for hickle file created by hickle 4.x.
  
          Raises
          ------
          LookupError:
              if ReferenceManager has already been created for h_node or its h_root_group 
          """
  
          def create_manager():
              return ( 
                  ReferenceManager(h_node,pickle_loads = pickle_loads),
                  ReferenceManager.get_root(h_node)
              )
          return super().create_manager(h_node,create_manager)
  
      def __init__(self, h_root_group, *args,pickle_loads = pickle.loads, **kwargs):
          """
          constructs ReferenceManager object
  
          Parameters
          ----------
          h_root_group (h5py.Group):
              see ReferenceManager.create_manager
  
          args (tuple,list):
              passed to dict.__init__
  
          pickle_loads (FunctionType,MethodType):
              see ReferenceManager.create_manager
  
          kwargs (dict):
              passed to dict.__init__
  
          Raises
          ------
          ReferenceError:
              In case an error occurs while loading 'hickle_types_table' from an
              existing file opened for reading and writing
          """
          super().__init__(*args,**kwargs)
          self._py_obj_type_link = dict()
          self._base_type_link = dict()
          self._overlay = None
          self.pickle_loads = pickle_loads
  
          # get the 'hickle_types_table' member of h_root_group or create it anew
          # in case none found. In case hdf5 file is opened for reading only
          # create an in memory hdf5 file (managed by hdf5 'core' driver) providing
          # an empty dummy hickle_types_table. This is necessary to ensue that
          # ReferenceManager.resolve_type works properly on hickle 4.x files which
          # store type information directly in h5py.Group and h5py.Datasets attrs
          # structure.
          self._py_obj_type_table = h_root_group.get('hickle_types_table',None)
          if self._py_obj_type_table is None:
              if h_root_group.file.mode == 'r+':
                  self._py_obj_type_table = h_root_group.create_group("hickle_types_table",track_order = True)
              else:
                  h5_overlay = h5.File(
                      '{}.hover'.format(h_root_group.file.filename.rsplit('.',1)[0]),
                      mode='w', driver='core',backing_store=False
                  )
                  self._py_obj_type_table = h5_overlay.create_group("hickle_types_table",track_order = True)
                  self._overlay = weakref.finalize(self,ReferenceManager._drop_overlay,h5_overlay)
              return
  
          # verify that '_py_obj_type_table' is a valid h5py.Group object
          if not isinstance(self._py_obj_type_table,h5.Group):
              raise ReferenceError("'hickle_types_table' invalid: Must be HDF5 Group entry")
  
          # if h_root_group.file was opened for writing restore '_py_obj_type_link' and
          # '_base_type_link' table entries from '_py_obj_type_table' to ensure when
          # h5py.Group and h5py.Dataset are added anew to h_root_group tree structure
          # their 'type' attribute is set to the correct py_obj_type reference by the
          # ReferenceManager.store_type method. Each of '_py_obj_type_link' and
          # '_base_type_link' tables can be used to properly restore  the 'py_obj_type'
          # and 'base_type' when loading the file as well as assigning to the 'type'
          # attribute the appropriate 'py_obj_type' dataset reference from the
          # '_py_obj_type_table' when dumping data to the file.
          if h_root_group.file.mode != 'r+':
              return
          for _, entry in self._py_obj_type_table.items():
              if entry.shape is None and entry.dtype == 'S1':
                  base_type = entry.name.rsplit('/',1)[-1].encode('ascii')
                  self._base_type_link[base_type] = entry
                  self._base_type_link[entry.id] = base_type
                  continue
              base_type_ref = entry.attrs.get('base_type',None)
              if not isinstance(base_type_ref,h5.Reference):
                  raise ReferenceError(
                      "inconsistent 'hickle_types_table' entries for py_obj_type '{}': "
                      "no base_type".format(py_obj_type)
                  )
              try:
                  base_type_entry = entry.file.get(base_type_ref,None)
              except ValueError: # pragma: no cover
                  base_type_entry = None
              if base_type_entry is None:
                  raise ReferenceError(
                      "inconsistent 'hickle_types_table' entries for py_obj_type '{}': "
                      "stale base_type".format(py_obj_type)
                  )
              base_type = self._base_type_link.get(base_type_entry.id,None)
              if base_type is None:
                  base_type = base_type_entry.name.rsplit('/',1)[-1].encode('ascii')
              try:
                  py_obj_type = pickle.loads(entry[()])
              except (ImportError,AttributeError):
                  py_obj_type = AttemptRecoverCustom
                  entry_link = py_obj_type,'!recover!',base_type
              else:
                  entry_link = py_obj_type,base_type
                  self._py_obj_type_link[id(py_obj_type)] = entry
              self._py_obj_type_link[entry.id] = entry_link
  
      def store_type(self, h_node, py_obj_type, base_type = None, attr_name = 'type', **kwargs):
          """
          assigns a 'py_obj_type' entry reference to the attribute specified by attr_name
          of h_node and creates if not present the appropriate 'hickle_types_table'
          entries for py_obj_type and base_type.
  
          Note
          ----
              Storing and restoring the content of nodes containing pickle byte strings
              is fully managed by pickle.dumps and pickle.loads functions including
              selection of appropriate py_obj_type. Therefore no explicit entry for 
              object and b'pickle' py_obj_type and base_type pairs indicating pickled
              content of pickled dataset are created.
  
          Parameters
          ----------
          h_node (h5py.Group, h5py.Dataset):
              node the 'type' attribute a 'hickle_types_table' entry corresponding to 
              the provided py_obj_type, base_type entry pair shall be assigned to.
  
          py_obj_type (any type or class):
              the type or class of the object instance represented by h_node
  
          base_type (bytes):
              the base-type bytes string of the loader used to create the h_node and
              restore an object instance form on load. If None no 'hickle_types_table'
              will be crated for py_obj_type if not already present and a LookupError
              exception is raised instead.
  
          attr_name (str):
              the name of the attribute the type reference shall be stored to. Defaults
              to 'type'
  
          kwargs (dict):
              keyword arguments to be passed to h5py.Group.create_dataset function
              when creating the entries for py_obj_type and base_type anew
  
          Raises
          ------
          ValueError:
              if base_type is not a valid bytes string
  
          LookupError:
              if base_type is None and no 'hickle_types_table' entry exists for
              py_obj_type yet
  
          """
  
          # return immediately if py_obj_type is object as h_node contains pickled byte
          # string of the actual object dumped
          if py_obj_type is object:
              return
  
          # if no entry within the 'hickle_types_table' exists yet
          # for py_obj_type create the corresponding pickle string dataset
          # and store appropriate entries in the '_py_obj_type_link' table for
          # further use by ReferenceManager.store_type and ReferenceManager.resolve_type
          # methods
          py_obj_type_id = id(py_obj_type)
          entry = self._py_obj_type_link.get(py_obj_type_id,None)
          if entry is None:
              if base_type is None:
                  raise LookupError(
                      "no entry found for py_obj_type '{}'".format(py_obj_type.__name__)
                  ) 
              if not isinstance(base_type,(str,bytes)) or not base_type:
                  raise ValueError("base_type must be non empty bytes string")
              type_entry = memoryview(pickle.dumps(py_obj_type))
              type_entry = np.array(type_entry,copy = False)
              type_entry.dtype = 'S1'
              entry = self._py_obj_type_table.create_dataset(
                  str(len(self._py_obj_type_table)),
                  data=type_entry,
                  shape=(1,type_entry.size),
                  **kwargs
              )
              # assign a reference to base_type entry within 'hickle_types_table' to
              # the 'base_type' attribute of the newly created py_obj_type entry.
              # if 'hickle_types_table' does not yet contain empty dataset entry for
              # base_type create it and store appropriate entries in the '_base_type_link' table
              # for further use by ReferenceManager.store_type and ReferenceManager,resolve_type
              # methods
              base_entry = self._base_type_link.get(base_type,None)
              if base_entry is None:
                  base_entry = self._base_type_link[base_type] = self._py_obj_type_table.create_dataset(
                      base_type.decode('ascii'),
                      shape=None,dtype = 'S1',
                      **no_compression(kwargs)
                  )
                  self._base_type_link[base_entry.id] = base_type
              entry.attrs['base_type'] = base_entry.ref
              self._py_obj_type_link[py_obj_type_id] = entry
              self._py_obj_type_link[entry.id] = (py_obj_type,base_type)
          h_node.attrs[attr_name] = entry.ref
  
      def resolve_type(self,h_node,attr_name = 'type',base_type_type = 1):
          """
          resolves the py_obj_type and base_type pair referenced to by the 'type' attribute and
          if present the 'base_type' attribute.
  
          Note: If the 'base_type' attribute is present it is assumed that the dataset was
              created by hickle 4.x version. Consequently it is assumed that the 'type' attribute
              contains a pickle bytes string to load the py_obj_type from instead of a reference
              to a 'hickle_types_table' entry representing the py_obj_type base_type pair of h_node.
  
          Note: If 'type' attribute is not present h_node represents either a h5py.Reference to
              the actual node of the object to be restored or contains a pickle bytes string.
              In either case the corresponding implicit py_obj_type base_type pair 
              (NodeReference, b'!node-reference!') or (object,b'pickle') respective is assumed and
              returned.
  
          Note: If restoring 'py_object_type' from pickle string stored in type attribute or 
              'hickle_types_table' fails than implicit py_obj_type base_type pair 
              (AttemptRecoverCustom,'!recover!') is returned instead of the actual 'py_obj_type' 
              base_type pair is returned. The latter can be retrieved by setting 'base_type_type'
              to 2 in this case
  
          Parameters
          ----------
          h_node (h5py.Group,h5py.Dataset):
              the node to resolve py_obj_type and base_type for using reference stored in
              attribute specified by attr_name
  
          attr_name (str):
              the name of the attribute the type reference shall be restored from. Defaults
              to and must be 'type' in case not a h5py.Reference.
  
          base_type_type (int):
              1 (default) base_type used to select loader
             -1 original base_type corresponding to not understood py_obj_type of
                recovered h5py.Group or h5py.Dataset
  
          Returns
          -------
          tuple containing (py_obj_type,base_type,is_container)
  
          py_obj_type:
              the python type of the restored object
  
          base_type:
              the base_type string indicating the loader to be used for properly
              restoring the py_obj_type instance or the base_type string 
  
          is_container:
              boolean flag indicating whether h_node represents a h5py.Group or
              h5py.Reference both of which have to be handled by corresponding
              PyContainer type loaders or a h5py.Dataset for which the appropriate
              load_fn is to be called.
          """
  
          # load the type attribute indicated by attr_name. If not present check if h_node
          # is h5py.Reference dataset or a dataset containing a pickle bytes
          # string. In either case assume (NodeReference,b'!node-reference!') or (object,b'pickle')
          # respective as (py_obj_type, base_type) pair and set is_container flag to True for
          # h5py.Reference and False otherwise.
          # 
          # NOTE: hickle 4.x legacy file does not store 'type' attribute for h5py.Group nodes with 
          #       b'dict_item' base_type. As h5py.Groups do not have a dtype attribute the check
          #       whether h_node.dtype equals h5py.ref_dtype will raise AttributeError.
          #       If h_node represents a b'dict_item' than self.pickle_loads will point to
          #       fix_lambda_obj_type below which will properly handle None value of type_ref
          #       in any other case file is not a hickle 4.x legacy file and thus has to be
          #       considered broken
          type_ref = h_node.attrs.get(attr_name,None)
          if not isinstance(type_ref,h5.Reference):
              if type_ref is None:
                  try:
                      metadata = h_node.dtype.metadata
                  except (AttributeError,):
                      pass
                  else:
                      if metadata is not None and issubclass(metadata.get('ref',object),h5.Reference):
                          return NodeReference,b'!node-reference!',True
                      return object,b'pickle',False
  
              # check if 'type' attribute of h_node contains a reference to a 'hickle_types_table'
              # entry. If not use pickle to restore py_object_type from the 'type' attribute value
              # directly if possible
              try:
  
                  # set is_container_flag to True if h_node is h5py.Group type object and false
                  # otherwise
                  return self.pickle_loads(type_ref), h_node.attrs.get('base_type', b'pickle'), isinstance(h_node, h5.Group)
              except (ModuleNotFoundError,AttributeError):
                  # module missing or py_object_type not provided by module
                  return AttemptRecoverCustom,( h_node.attrs.get('base_type',b'pickle') if base_type_type == 2 else b'!recover!' ),isinstance(h_node,h5.Group)
              except (TypeError, pickle.UnpicklingError, EOFError):
                  raise ReferenceError(
                      "node '{}': '{}' attribute ('{}')invalid: not a pickle byte string".format(
                          h_node.name,attr_name,type_ref
                      )
                  )
          try:
              entry = self._py_obj_type_table[type_ref]
          except (ValueError, KeyError):
              raise ReferenceError(
                  "node '{}': '{}' attribute invalid: stale reference".format(
                      h_node.name,attr_name
                  )
              )
  
          # load (py_obj_type,base_type) pair from _py_obj_type_link for 'hickle_types_table' entry
          # referenced by 'type' entry. Create appropriate _py_obj_type_link and _base_type_link
          # entries if if not present for (py_obj_type,base_type) pair for further use by
          # ReferenceManager.store_type and ReferenceManager.resolve_type methods.
          type_info = self._py_obj_type_link.get(entry.id, None)
          if type_info is None:
              base_type_ref = entry.attrs.get('base_type', None)
              if base_type_ref is None:
                  base_type = b'pickle'
              else:
                  try:
                      base_type_entry = self._py_obj_type_table[base_type_ref]
                  except ( ValueError,KeyError ):
                      # TODO should be recovered here instead?
                      raise ReferenceError(
                          "stale base_type reference encountered for '{}' type table entry".format(
                              entry.name
                          )
                      )
                  base_type = self._base_type_link.get(base_type_entry.id,None)
                  if base_type is None:
  
                      # get the relative table entry name form full path name of entry node
                      base_type = base_type_entry.name.rsplit('/',1)[-1].encode('ASCII')
                      self._base_type_link[base_type] = base_type_entry
                      self._base_type_link[base_type_entry.id] = base_type
              try:
                  py_obj_type = self.pickle_loads(entry[()])
              except (ModuleNotFoundError,AttributeError):
                  py_obj_type = AttemptRecoverCustom
                  entry_link = (py_obj_type,b'!recover!',base_type)
              else:
                  entry_link = (py_obj_type,base_type)
                  self._py_obj_type_link[id(py_obj_type)] = entry
              type_info = self._py_obj_type_link[entry.id] = entry_link
          # return (py_obj_type,base_type). set is_container flag to true if 
          # h_node is h5py.Group object and false otherwise
          return (type_info[0],type_info[base_type_type],isinstance(h_node,h5.Group))
  
      def __enter__(self):
          if not isinstance(self._py_obj_type_table, h5.Group) or not self._py_obj_type_table:
              raise RuntimeError(
                  "Stale ReferenceManager, call ReferenceManager.create_manager to create a new one"
              )
          return self
  
      def __exit__(self, exc_type, exc_value, exc_traceback):
          if not isinstance(self._py_obj_type_table, h5.Group) or not self._py_obj_type_table:
              return
  
          # remove this ReferenceManager object from the table of active ReferenceManager objects
          # and cleanly unlink from any h5py object instance and id references managed. Finalize
          # 'hickle_types_table' overlay if it was created by __init__ for hickle 4.x file
          super().__exit__(exc_type, exc_value, exc_traceback, self._py_obj_type_table)
          self._py_obj_type_table = None
          self._py_obj_type_link = None
          self._base_type_link = None
          self.pickle_loads = None
          if self._overlay is not None:
              self._overlay()
              self._overlay = None
  
  #####################
  # loading optional  #
  #####################
  
  _managed_by_hickle = {'hickle', ''}
  
  _custom_loader_enabled_builtins = {'__main__':('','')}
  
  class LoaderManager(BaseManager):
      """
      Handles the file specific lookup of loader to be used to dump or load
      a python object of a specific type 
  
      To create a LoaderManager call LoaderManager.create_manager
      function. The value returned can be and shall be used within a
      with statement for example as follows:
  
          with LoaderManager.create_manager(h_root_group) as loader:
              _dump(data,h_root_group,'data',memo,loader,**kwargs)
  
          with LoaderManager.create_manager(h_root_group,False,{'custom':true}) as loader:
              _load(py_container,'data',h_root_group['data'],memo,loader)
  
          with LoaderManager.create_manager(h_root_group,True) as memo:
              _load(py_container,'data',h_root_group['data'],memo,loader)
  
      NOTE: for creating appropriate memo object see ReferenceManager
  
      """
  
      # Define dict of all acceptable types dependent upon loader option
      __py_types__ = { 
          None: {},
          'hickle-4.x': {},
          **{ option:{} for option in optional_loaders }
      }
  
      # Define dict of all acceptable load function dependent upon loader option
      __hkl_functions__ = {
          None: {},
          'hickle-4.x': {},
          **{ option:{} for option in optional_loaders }
      }
  
      # Define dict of all acceptable hickle container types dependent upon loader option
      __hkl_container__ = {
          None: {},
          'hickle-4.x': {},
          **{ option:{} for option in optional_loaders }
      }
  
      # Empty list (hashtable) of loaded loader names
      __loaded_loaders__ = set()
  
  
      @classmethod
      def register_class(
          cls, myclass_type, hkl_str, dump_function=None, load_function=None,
          container_class=None, memoise = True, option=None
      ):
          """
          Register a new class to be recognized and dumped or restored by hickle.
      
          Parameters
          ----------
          myclass_type (type.class):
              the class to register dump_fcn, load_fcn, PyContainer for
  
          hkl_str (str):
              String to write to HDF5 file identifying class and loader suitable
              for restoring the py_object described by the data stored in hickle
              file.
  
              NOTE: dict_item, pickle, !node-refrence!, !recover! and any other
                  string enclosed within a pair of !! can not be modified if once
                  registered. Strings quoted by !! must be added as global loader
                  with option = None.
                  
          dump_function (callable):
              callable to write data to HDF5
  
          load_function (callable):
              function to load data from HDF5
  
          container_class (PyContainer):
              PyContainer type proxy class to load data from HDF5
  
          memoise (bool): 
              True: references to the object instances of class shall be remembered
                  during dump and load for properly resolving multiple references
                  to the same object instance.
  
              False: every occurrence of an instance of the object has to be dumped
                  and restored on load disregarding instances already present.
  
          option (str, None):
              String identifying set of loaders which shall only be used when
              specific feature or category is requested on top of global loaders.
              If None than loader is globally to be used if there is no other loader
              registered for myclass_type. 
  
              NOTE: only strings listed in 'optional_loaders' exported by 
                    hickle.loaders.__init__ and 'hickle-4.x' are accepted. 
                  
          Raises
          ------
          TypeError:
              loader for myclass_type may only be registered by hickle core modules not
              loaded from hickle/loaders/ directory, <packagepath>/hickle_loaders/,
              <modulepath>/hickle_loaders/ or <__main__path>/hickle_loaders/ directory
              by explicitly calling LoaderManager.register_class method.
  
          ValueError:
              if optional loader modules tries to shadow 'dict_item', 'pickle' and any
              loader marked as essential to proper function of hickle.dump and hickle.load 
              by ! prefix and postfix ('!node-reference!', '!recover!').
  
          LookupError:
              if optional loader denoted by option is unknown. Any new option must be
              listed in 'optional_loaders' exported by 'hickle.loaders.__init__.py'
              file to be recognized as valid option
          """
      
          if (
              myclass_type is object or
              isinstance(
                  myclass_type,
                  (types.FunctionType, types.BuiltinFunctionType, types.MethodType, types.BuiltinMethodType)
              ) or
              issubclass(myclass_type,(type,_DictItem))
          ):
              # object, all functions, methods, class objects and the special _DictItem class
              # type objects are to be handled by hickle core only. 
              dump_module = getattr(dump_function, '__module__', '').split('.', 2)
              load_module = getattr(load_function, '__module__', '').split('.', 2)
              container_module = getattr(container_class, '__module__', '').split('.', 2)
              if {dump_module[0], load_module[0], container_module[0]} - _managed_by_hickle:
                  raise TypeError(
                      "loader for '{}' type managed by hickle only".format(
                          myclass_type.__name__
                      )
                  )
              if "loaders" in {*dump_module[1:2], *load_module[1:2], *container_module[1:2]}:
                  raise TypeError(
                      "loader for '{}' type managed by hickle core only".format(
                          myclass_type.__name__
                      )
                  )
          if ( 
              ( cls.__hkl_functions__[None].get(hkl_str) or cls.__hkl_container__[None].get(hkl_str) ) and
              ( hkl_str[:1] == hkl_str[-1:] == b'!' or hkl_str in disallow_in_option )
          ):
              raise ValueError(
                  "'{}' base_type may not be shadowed by loader".format(hkl_str)
              )
          # add loader
          try:
              if dump_function is not None:
                  cls.__py_types__[option][myclass_type] = ( dump_function, hkl_str,memoise)
              if load_function is not None:
                  cls.__hkl_functions__[option][hkl_str] = load_function
                  cls.__hkl_functions__[option][hkl_str.decode('ascii')] = load_function
              if container_class is not None:
                  cls.__hkl_container__[option][hkl_str] = container_class
                  cls.__hkl_container__[option][hkl_str.decode('ascii')] = container_class
          except KeyError:
              raise LookupError("Invalid option '{}' encountered".format(option))
          
              
  
  
      @classmethod
      def register_class_exclude(cls, hkl_str_to_ignore, option = None):
          """
          Tell loading function to ignore any HDF5 dataset with attribute 'type=XYZ'
      
          Parameters
          ----------
          hkl_str_to_ignore (str): attribute type=string to ignore and exclude
              from loading.
  
          option (str, None):
              String identifying set of optional loaders from which class shall
              be excluded
                  
          Raises
          ------
          ValueError:
              class is managed by hickle core machinery and thus may not be ignored
  
          LookupError:
              option loader shall belong to is unknown. Any new option must be listed
              in 'optional_loaders' exported by 'hickle.loaders.__init__.py' file to be
              recognized as valid option
          """
      
          if hkl_str_to_ignore[0] == hkl_str_to_ignore[-1] == b'!' or hkl_str_to_ignore in disallowed_to_ignore:
              raise ValueError(
                  "excluding '{}' base_type managed by hickle core not possible".format(
                      hkl_str_to_ignore
                  )
              )
          try:
              cls.__hkl_functions__[option][hkl_str_to_ignore] = load_nothing
              cls.__hkl_container__[option][hkl_str_to_ignore] = NoContainer
              cls.__hkl_functions__[option][hkl_str_to_ignore.decode('ascii')] = load_nothing
              cls.__hkl_container__[option][hkl_str_to_ignore.decode('ascii')] = NoContainer
          except KeyError:
              raise LookupError("'{}' option unknown".format(option))
  
      __slots__ = ( 'types_dict', 'hkl_types_dict', 'hkl_container_dict', '_mro', '_file')
  
  
      _option_formatter = '{}{{}}'.format(attribute_prefix)
      _option_parser = re.compile(r'^{}(.*)$'.format(attribute_prefix),re.I)
  
      def __init__(self, h_root_group, legacy = False, options = None):
          """
          constructs LoaderManager object
  
          Parameters
          ----------
          h_root_group (h5py.Group):
              see LoaderManager.create_manager
  
          legacy (bool):
              If true the file h_node belongs to is in legacy hickle 4.x format.
              Ensure lambda py_obj_type strings are loaded properly and
              'hickle-4.x' type loaders are included within types_dict, 
              'hkl_types_dict' and 'hkl_container_dict'
  
          options (dict):
              optional loaders to be loaded. Each key names one loader and
              its value indicates whether to be used (True) or excluded (False) 
  
          Raises
          ------
          LookupError:
              option loader unknown
          """
  
          # initialize lookup dictionaries with set of common loaders
          self.types_dict = collections.ChainMap(self.__class__.__py_types__[None])
          self.hkl_types_dict = collections.ChainMap(self.__class__.__hkl_functions__[None])
          self.hkl_container_dict = collections.ChainMap(self.__class__.__hkl_container__[None])
  
          # Select source of optional loader flags. If option is None try to read options
          # from h_root_group.attrs structure. Otherwise use content of options dict store
          # each entry to be used within h_root_group.attrs structure or update entry there
          if options is None:
              option_items = ( 
                  match.group(1).lower() 
                  for match,on in ( 
                      ( LoaderManager._option_parser.match(name), value )
                      for name, value in h_root_group.attrs.items() 
                  ) 
                  if match and on
              )
          else:
              def set_option_items():
                  for option_key,on in options.items():
                      if not on:
                          continue
                      h_root_group.attrs[LoaderManager._option_formatter.format(option_key.upper())] = on 
                      yield option_key
              option_items = set_option_items()
          # try to include loader set indicated by option_name
          try:
              for option_name in option_items:
                  self.types_dict.maps.insert(0,self.__class__.__py_types__[option_name])
                  self.hkl_types_dict.maps.insert(0,self.__class__.__hkl_functions__[option_name])
                  self.hkl_container_dict.maps.insert(0,self.__class__.__hkl_container__[option_name])
          except KeyError:
              raise LookupError("Option '{}' invalid".format(option_name))
              
          # add loaders required to properly load legacy files created by hickle 4.x and
          # ensure that non class types are properly reported by load_loader
          if legacy:
              self._mro = type_legacy_mro
              self.types_dict.maps.insert(0,self.__class__.__py_types__['hickle-4.x'])
              self.hkl_types_dict.maps.insert(0,self.__class__.__hkl_functions__['hickle-4.x'])
              self.hkl_container_dict.maps.insert(0,self.__class__.__hkl_container__['hickle-4.x'])
          else:
              self._mro = type.mro
          self._file = h_root_group.file
          
      def load_loader(self, py_obj_type,*,base_type=None):
          """
          Checks if given `py_obj` requires an additional loader to be handled
          properly and loads it if so.
      
          Parameters
          ----------
          py_obj:
              the Python object to find an appropriate loader for
      
          Returns
          -------
          tuple containing (py_obj, (create_dataset, base_type, memoise))
  
          py_obj:
               the Python object the loader was requested for
      
          (create_dataset,base_type,memoise):
               tuple providing create_dataset function, name of base_type used
               to represent py_obj and the boolean memoise flag indicating
               whether loaded object shall be remembered for restoring further
               references to it or must be loaded every time encountered.
      
          Raises
          ------
          RuntimeError:
              in case py object is defined by hickle core machinery.
          """
      
          types_dict = self.types_dict
          loaded_loaders = self.__class__.__loaded_loaders__
          # loop over the entire mro_list of py_obj_type
          for mro_item in self._mro(py_obj_type):
  
              # Check if mro_item is already listed in types_dict and return if found 
              loader_item = types_dict.get(mro_item,None)
              if loader_item is not None:
                  return py_obj_type,loader_item
      
              # Obtain the package name of mro_item
              package_list = mro_item.__module__.split('.',2)
      
              package_file = None 
              if package_list[0] == 'hickle':
                  if package_list[1] != 'loaders':
                      if base_type is not None and ( base_type in self.hkl_types_dict or base_type in self.hkl_container_dict):
                          return py_obj_type,(not_dumpable,base_type,True)
                      print(mro_item,package_list)
                      raise RuntimeError(
                          "objects defined by hickle core must be registered"
                          " before first dump or load"
                      )
                  if (
                      len(package_list) < 3 or
                      not package_list[2].startswith("load_") or
                      '.' in package_list[2][5:]
                  ):
                      warnings.warn(
                          "ignoring '{!r}' dummy type not defined by loader module".format(py_obj_type),
                          RuntimeWarning
                      )
                      continue
  
                  # dummy objects are not dumpable ensure that future lookups return that result
                  loader_item = types_dict.get(mro_item,None)
                  if loader_item is None:
                      loader_item = types_dict[mro_item] = ( not_dumpable, b'NotHicklable',False )
  
                  # ensure module of mro_item is loaded as loader as it will contain
                  # loader which knows how to handle group or dataset with dummy as 
                  # py_obj_type 
                  loader_name = mro_item.__module__
                  if loader_name in loaded_loaders:
  
                      # loader already loaded as triggered by dummy abort search and return
                      # what found so far as fallback to further bases does not make sense
                      return py_obj_type,loader_item
              else:
                  loader_name,package_file = _custom_loader_enabled_builtins.get(package_list[0],(None,''))
                  if loader_name is None:
                      # construct the name of the associated loader
                      loader_name = 'hickle.loaders.load_{:s}'.format(package_list[0])
                  elif not loader_name:
                      # try to resolve module name for __main__ script and other generic modules
                      package_module = sys.modules.get(package_list[0],None)
                      if package_module is None:
                          warnings.warn(
                              "package/module '{}' defining '{}' type dropped".format(
                                  package_list[0],mro_item.__name__
                              ),
                              PackageImportDropped
                          )
                          continue
                      package_file = getattr(package_module,'__file__',None)
                      if package_file is None:
                          package_loader = getattr(package_module,'__loader__',None)
                          if package_loader is None: # pragma: no cover
                              # just to secure against "very smart" tinkering
                              # with python import machinery, no serious testable use-case known and expected
                              continue
                          package_spec = spec_from_loader(package_list[0],package_loader)
                          if not getattr(package_spec,'has_location',False):
                              continue
                          package_file = package_spec.origin
                      if not os.path.isabs(package_file): # pragma: no cover
                          # not sure if this case wouldn't just be result of "very smart" tinkering
                          # with python import machinery, no serious testable use-case known yet
                          package_spec = find_spec(os.path.basename(package_file.rsplit('.')[0]))
                          if not getattr(package_spec,'has_location',False): # pargma: no cover
                              # not sure if this case wouldn't just be result of "very smart" tinkering
                              # with python import machinery, no serious testable use-case known yet
                              continue
                          package_file = package_spec.origin
                      package_list[0],allow_custom_loader = os.path.basename(package_file).rsplit('.')[0],package_list[0]
                      loader_name = 'hickle.loaders.load_{:s}'.format(package_list[0])
                      _custom_loader_enabled_builtins[allow_custom_loader] = loader_name, package_file
      
                  # Check if this module is already loaded
                  if loader_name in loaded_loaders:
                      # loader is loaded but does not define loader for mro_item
                      # check next base class
                      continue
      
              # check if loader module has already been loaded. If use that instead
              # of importing it anew
              loader = sys.modules.get(loader_name,None)
              if loader is None:
                  # Try to load a loader with this name
                  loader_spec = find_spec(loader_name)
                  if loader_spec is None:
                      assert isinstance(package_file,str), "package_file name for _custom_loader_enabled_builtins must be string"
                      if not package_file:
                          package_spec = getattr(sys.modules.get(package_list[0],None),'__spec__',None)
                          if package_spec is None:
                              package_spec = find_spec(package_list[0])
                          if not getattr(package_spec,'has_location',False):
                              # can't resolve package or base module hosting mro_item
                              continue
                          package_file = package_spec.origin
                      package_path = os.path.dirname(package_file)
                      package_loader_path = os.path.join(
                          package_path, "hickle_loaders", "load_{:s}.py".format(package_list[0])
                      )
                      try:
                          fid = open(package_loader_path,'rb')
                      except FileNotFoundError:
                          try:
                              package_loader_path += 'c'
                              fid = open(package_loader_path,'rb')
                          except FileNotFoundError:
                              # no file for loader module found
                              continue
                          else:
                              fid.close()
                      else:
                          fid.close()
                      loader_spec = spec_from_file_location(loader_name,package_loader_path)
                  # import the the loader module described by module_spec
                  # any import errors and exceptions result at this stage from
                  # errors inside module and not cause loader module does not
                  # exist
                  loader = module_from_spec(loader_spec)
                  loader_spec.loader.exec_module(loader)
                  sys.modules[loader_name] = loader
      
              # load all loaders defined by loader module
              for next_loader in loader.class_register:
                  self.register_class(*next_loader)
              for drop_loader in ( loader if isinstance(loader,(list,tuple)) else (loader,None) for loader in loader.exclude_register) :
                  self.register_class_exclude(*drop_loader)
              loaded_loaders.add(loader_name)
      
              # check if loader module defines a loader for base_class mro_item
              loader_item = types_dict.get(mro_item,None)
              if loader_item is None:
                  # the new loader does not define loader for mro_item
                  # check next base class
                  continue
  
              # return loader for base_class mro_item
              return py_obj_type,loader_item
      
          # no appropriate loader found. Lower py_object_type to object and
          # return fallback to pickle
          return object,(create_pickled_dataset,b'pickle',True)
  
      @classmethod
      def create_manager(cls, h_node, legacy = False, options = None):
          """
          creates an new LoaderManager object for the h_root_group the h_node
          belongs to. 
  
  
          Parameters
          ----------
          h_node (h5py.Group, h5py.Dataset):
              the h5py node or its h_root_group to create a new LoaderManager
              object for.
  
          legacy (bool):
              if true file h_node belongs to is in legacy hickle 4.x format
              ensure lambda py_obj_type strings are loaded properly and
              'hickle-4.x' type loaders are included within types_dict, 
              'hkl_types_dict' and 'hkl_container_dict'
  
          options (dict):
              optional loaders to be loaded. Each key names one loader and
              its value indicates whether to be used (True) or excluded (False) 
  
          Raises
          ------
          LookupError:
              if ReferenceManager has already been created for h_node or its h_root_group 
          """
  
          def create_manager():
              return (LoaderManager(h_node,legacy,options),)
          return super().create_manager(h_node,create_manager)
  
      def __enter__(self):
          if not isinstance(self._file,h5.File) or not self._file:
              raise RuntimeError(
                  "Stale LoaderManager, call LoaderManager.create_manager to create a new one"
              )
          return self
  
      def __exit__(self, exc_type, exc_value, exc_traceback):
          if not isinstance(self._file,h5.File) or not self._file:
              return
          super().__exit__(exc_type, exc_value, exc_traceback, self._file)
          self._file = None
          self._mro = None
          self.types_dict = None
          self.hkl_types_dict = None
          self.hkl_container_dict = None
          
  def type_legacy_mro(cls):
      """
      drop in replacement of type.mro for loading legacy hickle 4.x files which were
      created without generalized PyContainer objects available. Consequently some
      h5py.Datasets and h5py.Group objects expose function objects as their py_obj_type
      type.mro expects classes only.
  
      Parameters
      ----------
      cls (type):
          the py_obj_type/class of the object to load or dump
  
      Returns
      -------
          mro list for cls as returned by type.mro  or in case cls is a function or method
          a single element tuple is returned
      """
      if isinstance(
          cls,
          (types.FunctionType,types.BuiltinFunctionType,types.MethodType,types.BuiltinMethodType)
      ):
          return (cls,)
      return type.mro(cls) 
  
  # %% BUILTIN LOADERS (not maskable)
  
  # list of below hkl_types which may not be ignored
  # NOTE: types which are enclosed in !! pair are disallowed in any case
  disallowed_to_ignore = {b'dict_item', b'pickle' }
  
  # list of below hkl_types which may not be redefined by optional loader
  # NOTE: types which are enclosed in !! pair are disallowed in any case
  disallow_in_option = {b'pickle'}
  
  class NoContainer(PyContainer): # pragma: no cover
      """
      load nothing container
      """
  
      def convert(self):
          pass
  
  
  class _DictItemContainer(PyContainer):
      """
      PyContainer reducing hickle version 4.x dict_item type h5py.Group to 
      its content for inclusion within dict h5py.Group
      """
  
      def convert(self):
          return self._content[0]
  
  LoaderManager.register_class(
      _DictItem, b'dict_item', dump_nothing, load_nothing, _DictItemContainer, False, 'hickle-4.x'
  )
  
          
  class ExpandReferenceContainer(PyContainer):
      """
      PyContainer for properly restoring additional references to an object
      instance shared multiple times within the dumped object structure
      """
  
      def filter(self,h_parent):
          """
          resolves the h5py.Reference link and yields the the node
          it refers to as sub item of h_parent so that it can be
          properly loaded by recursively calling hickle._load method
          independent whether it can be directly loaded from the memo
          dictionary or has to be restored from file. 
          """
  
          try:
              referred_node = h_parent.file.get(h_parent[()],None)
          except ( ValueError, KeyError ): # pragma no cover
              referred_node = None
          if referred_node is None:
              raise ReferenceError("node '{}' stale node reference".format(h_parent.name))
          yield referred_node.name.rsplit('/',1)[-1], referred_node
  
      def convert(self):
          """
          returns the object the reference was pointing to
          """
  
          return self._content[0]
  
  # objects created by resolving h5py.Reference datasets are already stored inside
  # memo dictionary so no need to memoise them.
  LoaderManager.register_class(
      NodeReference, b'!node-reference!', dump_nothing, load_nothing, ExpandReferenceContainer, False
  )
  
  
  def create_pickled_dataset(py_obj, h_group, name, reason = None, **kwargs):
      """
      Create pickle string as object can not be mapped to any other h5py
      structure.
  
      Parameters
      ----------
      py_obj:
          python object to dump; default if item is not matched.
  
      h_group (h5.File.group):
          group to dump data into.
  
      name (str):
          the name of the resulting dataset
  
      reason (str,None):
          reason why py_object has to be pickled eg. string
          provided by NotHicklable exception
  
      Warnings
      -------
      SerializedWarning:
          issued before pickle string is created
  
      """
  
      # for what ever reason py_obj could not be successfully reduced
      # ask pickle for help and report to user.
      reason_str = " (Reason: %s)" % (reason) if reason is not None else ""
      warnings.warn(
          "{!r} type not understood, data is serialized:{:s}".format(
              py_obj.__class__.__name__, reason_str
          ),
          SerializedWarning
      )
  
      # store object as pickle string
      pickled_obj = pickle.dumps(py_obj)
      d = h_group.create_dataset(name, data = memoryview(pickled_obj), **kwargs)
      return d,() 
  
  def load_pickled_data(h_node, base_type, py_obj_type):
      """
      loade pickle string and return resulting py_obj
      """
      try:
          return pickle.loads(h_node[()])
      except (ImportError,AttributeError):
          return RecoveredDataset(h_node[()],dtype = h_node.dtype,attrs = dict(h_node.attrs))
  
          
  # no dump method is registered for object as this is the default for
  # any unknown object and for classes, functions and methods
  LoaderManager.register_class(object,b'pickle',None,load_pickled_data)
  
  def recover_custom_dataset(h_node,base_type,py_obj_type):
      """
      drop in load_fcn for any base_type no appropriate loader could be found
      """
      manager = ReferenceManager.get_manager(h_node)
      _,base_type,_ = manager.resolve_type(h_node,base_type_type = -1)
      warnings.warn(
          "loader '{}' missing for '{}' type object. Data recovered ({})".format(
              base_type, 
              py_obj_type.__name__ if not isinstance(py_obj_type, AttemptRecoverCustom) else None,
              h_node.name.rsplit('/')[-1]
          ),
          DataRecoveredWarning
      )
      attrs = dict(h_node.attrs)
      attrs['base_type'] = base_type
      return RecoveredDataset(h_node[()],dtype=h_node.dtype,attrs=attrs)
  
  class RecoverGroupContainer(PyContainer):
      """
      drop in PyContainer for any base_type not appropriate loader could be found
      """
      def __init__(self,h5_attrs, base_type, object_type):
          super().__init__(h5_attrs, base_type, object_type,_content = {})
      
      def filter(self,h_parent):
          """
          switch base_type to the one loader is missing for
          """
  
          warnings.warn(
              "loader '{}' missing for '{}' type object. Data recovered ({})".format(
                  self.base_type,
                  self.object_type.__name__ if not isinstance(self.object_type,AttemptRecoverCustom) else None,
                  h_parent.name.rsplit('/')[-1]
              ),
              DataRecoveredWarning
          )
          manager = ReferenceManager.get_manager(h_parent)
          _,self.base_type,_ = manager.resolve_type(h_parent,base_type_type = -1)
          yield from h_parent.items()
  
      def append(self,name,item,h5_attrs):
          if isinstance(item,AttemptRecoverCustom):
              self._content[name] = item
          else:
              self._content[name] = (item,{ key:value for key,value in h5_attrs.items() if key not in {'type'}})
  
      def convert(self):
          attrs = {key:value for key,value in self._h5_attrs.items() if key not in {'type'}}
          attrs['base_type'] = self.base_type
          return RecoveredGroup(self._content,attrs=attrs)
  
  
  LoaderManager.register_class(AttemptRecoverCustom,b'!recover!',None,recover_custom_dataset,RecoverGroupContainer,True)
      
  
  def _moc_numpy_array_object_lambda(x):
      """
      drop in replacement for lambda object types which seem not
      any more be accepted by pickle for Python 3.8 and onward.
      see fix_lambda_obj_type function below
  
      Parameters
      ----------
      x (list):
          itemlist from which to return first element
  
      Returns
      -------
      first element of provided list
      """
      return x[0]
  
  LoaderManager.register_class(
      _moc_numpy_array_object_lambda, b'!moc_lambda!', dump_nothing, load_nothing, None, True, 'hickle-4.x'
  )
  
  def fix_lambda_obj_type(bytes_object, *, fix_imports=True, encoding="ASCII", errors="strict"):
      """
      drop in replacement for pickle.loads method when loading files created by hickle 4.x 
      It captures any TypeError thrown by pickle.loads when encountering a pickle string 
      representing a lambda function used as py_obj_type for a h5py.Dataset or h5py.Group.
      While in Python <3.8 pickle loads creates the lambda Python >= 3.8 throws an 
      error when encountering such a pickle string. This is captured and
      _moc_numpy_array_object_lambda returned instead. Further some h5py.Group and h5py.Datasets
      do not provide any py_obj_type for them object is returned assuming that proper loader has
      been identified by other objects already
      """
      if bytes_object is None:
          return object
      try:
          return pickle.loads(bytes_object, fix_imports=fix_imports, encoding=encoding, errors=errors)
      except TypeError:
          warnings.warn(
              "presenting '{!r}' instead of stored lambda 'type'".format(
                  _moc_numpy_array_object_lambda
              ),
              MockedLambdaWarning
          )
          return _moc_numpy_array_object_lambda
  