"""Fiber

A class used to implement the fibers (or the entire **fibertree**) of
a tensor.

"""

import logging

from functools import partialmethod
from copy import deepcopy
import yaml
import random

from .coord_payload import CoordPayload
from .payload import Payload

#
# Set up logging
#
module_logger = logging.getLogger('fibertree.core.fiber')


#
# Define an error class
#
class CoordinateError(Exception):
    """CoordinateError"""
    pass


class PayloadError(Exception):
    """PayloadError"""
    pass


#
# Define the fiber class
#
class Fiber:
    """A base class for representing a tensor as a tree of fibers

    The Fiber class implements the concept of a fiber of a tensor.  A
    Fiber is comprised of a list of **coordinates** and associated
    **payloads**.

    Fiber coordinates are unique tags for each element of the Fiber,
    they are often integers in the open range [0 to **shape**), where
    **shape** is an attribute of the Fiber.  However, coordinates may
    be other types, such as `tuples`. For example, `tuple` coordinates
    can be generated by `Fiber.flattenRanks()`.

    Fiber payloads are a value associated with each
    coordinate. Payloads may themselves be Fibers, which permits the
    creation of **fibertrees**. The payloads of the leaf level of the
    fibertree will be terminal values. Since we want to enable code to
    hold and update a payload (outside the context of the Fiber) we
    usually **box** the leaf level payloads using the `Payload` class
    (see `fibertree.core.payload`).

    A Fiber may have not have values for every possible coordinate
    (i.e., the coordinate list may have gaps). Such missing
    coordinates are referred to as being **empty**.

    Fibers can be iterated and return a series of `CoordPayload`
    elements (see `fibertree.core.coord_payload`).

    Attributes
    ----------

    Fibers have a set of attributes that can be set and
    accessed. These include:

    - A **default** value, which is the value associated with an
      **empty** coordinate and is accessed with `Fiber.getDefault()`
      and may be returned by methods that access a point in the Fiber
      such as `Fiber.getPayload()` and `Fiber.getPayloadRef()`.

    - A **shape**, which specifies the **size** of the fiber and
      controls the range of legal (integer) coordinates. The shape of
      an individual fiber is an integer, while the shape of a
      fibertree will be a list of integers corresponding to the
      shapes of the fibers at each level of the tree. A fiber's (or
      fibertree's) shape can be accessed with `Fiber.getShape()`

    - An **ordered** attribute which indicates and maintains the
      invariant that the coordinates of the fiber are monitonically
      increasing.

    - A **unique** attribute which indicates and maintains the
      invariant that the coordinates of the fiber are unique.

    - An **owner** rank. Since fibers are often part of a `Tensor`,
      the fiber may have an owning rank (see `Fiber.setOwner()`). When
      owned by a rank, some fiber atributes are actually obtained from
      that rank rather than being stored locally.

    - A **savedPos**, which is a value stored by certain methods that
      records a postion in a fiber that can be used in the future as a
      **shortcut** to accelerate a search in the fiber by another
      method. See `Fiber.getRange()` for an example of a method that
      both saves and can use these **shortcuts**.

    Constructor
    -----------

    The `Fiber` constructor creates an fiber with a given set of
    coordinates and payloads (in struct of lists form).

    Parameters
    ----------

    coords: list, default=[]
        List of coordinate values

    payloads: list, default=[]
        List of corresponding payloads for the coordinates

    default: value, default=0
        Default value of a payload in this fiber

    shape: integer, default=None
        Declared shape for the fiber

    initial: value, default=None
        A value to initialize all payloads.

    max_coord: value, default=no maximum coordinate
            The maximum legal coordinate value (this is really "shape-1")

    ordered: Boolean, default=True
        Attribute specifing that the coordinates are monotonically increasing

    unique: Boolean, default=True
        Attribute specifing that the coordinates are unique


    Notes
    -----

    Elements of the fiber have a `position` that can be used to
    reference the element. Generally one can think of the `position` as
    the relative address of the element in the concrete represention of
    the fiber. A variety of methods use a `postion` to reference an
    element, such as `Fiber.__getitem__()` and `Fiber.__setitem__()`

    Currently, the internal implementation of a Fiber is to hold a list
    of coordinates and a list of payloads (struct of lists) and the
    instance variables holding those lists (coords and payloads) are
    currently left public...

    """


    def __init__(self,
                 coords=None,
                 payloads=None,
                 default=0,
                 shape=None,
                 initial=None,
                 max_coord=None,
                 ordered=True,
                 unique=True):

        #
        # Set up logging
        #
        self.logger = logging.getLogger('fibertree.core.fiber')

        #
        # Collect attributes
        #
        self._ordered = ordered
        self._unique = unique

        #
        # Handle cases with missing inputs
        #
        if coords is None:
            if payloads is None:
                #
                # If neither coords or payloads are given create an empty fiber
                #
                coords = []
                payloads = []
            else:
                #
                # If only payloads are given create a "dense" fiber
                #
                # TBD: Reconcile coords with shape
                #
                coords = list(range(len(payloads)))
        else:
            if payloads is None:
                #
                # If only coords are given create a set of payloads
                #
                # TBD: Creating a set of zeros is odd...
                #
                payloads = len(coords)*[initial]


        assert (len(coords) == len(payloads)), \
            "Coordinates and payloads must be same length"

        #
        # Note:
        #    If a payload is already boxed, Payload.maybe_box() will
        #    NOT double box it.
        #
        #    We do not eliminate explicit zeros in the payloads
        #    so zeros will be preserved.
        #
        self.coords = coords
        """The list of coordinates of the fiber"""

        self.payloads = [Payload.maybe_box(p) for p in payloads]
        """The list of payloads of the fiber"""

        #
        # Check that fiber attributes are satisfied
        #
        self._checkOrdered()
        self._checkUnique()

        #
        # Set a specific contstant value for the shape of this fiber
        #
        # Note1: this value is overridden by the owning rank's shape
        # Note2: no checks are done to see if this value is violated
        #
        self._shape = shape
        #
        # Set a specific constant value to the maximum legal coordinate
        #
        # TBD: If not None there are lots of places this should be checked
        #
        self._max_coord = max_coord
        if max_coord is not None:
            Fiber._deprecated("Explicitly setting a fiber's max_coord is deprecated")
        #
        # Owner rank... set later when fiber is appended to a rank
        #
        self.setOwner(None)

        #
        # Create default value
        #
        self._default = Payload.maybe_box(default)

        #
        # Initialize "saved position"
        #
        self._saved_pos = 0

        #
        # Clear all stats
        #
        self.clearStats()


    @classmethod
    def fromCoordPayloadList(cls, *cp, **kwargs):
        """Construct a Fiber from a coordinate/payload list.

        The base Fiber contructor creates a fiber from a list of a
        coordinates and a list of payloads (i.e., struct of lists
        form) this constructor takes a list of coordinate/payload
        tuples (i.e., list of structs form).

        Parameters
        ----------

        cp: list of tuples
            A sequence of fiber elements as (coord, payload) tuples

        kwargs: keyword arguments
            Keyword arguments accepted by `Fiber.__init__()`

        Todo
        ----

        Support a list of `CoordPayload` elements

        """

        (coords, payloads) = zip(*cp)

        return cls(coords, payloads, **kwargs)


    @classmethod
    def fromYAMLfile(cls, yamlfile, default=0, **kwargs):
        """Construct a Fiber from a YAML file

        Parameters
        ----------

        yamlfile: str
            The name of a YAML file holding a description of a fiber

        kwargs: keyword arguments
            Keyword arguments accepted by `Fiber.__init__()`

        """

        (coords, payloads) = Fiber.parse(yamlfile, default)

        return cls(coords, payloads, default=default, **kwargs)


    @classmethod
    def fromUncompressed(cls, payload_list):
        """Construct a Fiber from an uncompressed nest of lists.

        The coordinates of the Fiber will be infered from the postions
        of the values in the lists. Zeros in the lists will be
        interpreted as the empty value of the Fiber.


        Parameters
        ----------

        payload_list: list
            A nest of lists holding the values of the Fiber.

        kwargs: keyword arguments
            Keyword arguments accepted by `Fiber.__init__()`


        Notes
        -----

        Zero values and sub-fibers that are all zeros are squeezed
        out, i.e., they will have no coordinates.  Unless the entire
        input is zeros.

        """

        f = Fiber._makeFiber(payload_list)

        #
        # Check if the list was all zeros, so return an empty fiber
        #
        if f is None:
            # Return something for an entirely empty input
            return Fiber([], [])

        return f


    @staticmethod
    def _makeFiber(payload_list):
        """Recursively make a fiber out of an uncompressed nest of lists"""

        assert(isinstance(payload_list, list))

        if isinstance(payload_list[0], list):
            size_check = len(payload_list[0])

            for p in payload_list:
                assert size_check == len(p), \
                       "All lists must be the same length"

        # Create zipped list of (non-empty) coordinates/payloads
        zipped = [(c, p) for c, p in enumerate(payload_list) if p != 0]

        #
        # Recursively unzip the lists into a Fiber
        #
        if len(zipped) == 0:
            # Got an empty subtree
            return None

        if isinstance(payload_list[0], list):
            coords = []
            payloads = []

            for c, p in zipped:
                real_p = Fiber._makeFiber(p)
                if real_p is not None:
                    coords.append(c)
                    payloads.append(real_p)
        else:
            coords = [c for c, _ in zipped]
            payloads = [p for _, p in zipped]

        if len(coords) == 0:
            return None

        #
        # Create fiber
        #
        # Note: max_coord dervived from input argument list and
        #       assuming coordinates start at 0
        #
        return Fiber(coords, payloads, shape=len(payload_list))


    @classmethod
    def fromRandom(cls, shape, density, interval=10, seed=None):
        """Create a fiber populated with random values.

        Multi-level fibers are supported by recursively creating
        fibers.

        Parameters
        -----------

        shape: list
            The `shape` (i.e., size) of the fibers at each level of
            the tree.

        density: list or scalar
            The probability that an element of the fiber will not be
            empty for each level of the tree. A scalar is density for
            leaf level of tree and other levels have density 1.0.

        interval: number
            The range (from 0 to `interval`) of each value at the leaf
            level of the tree.

        seed: a valid argument for `random.seed`
            A seed to pass to `random.seed`.

        """

        if not isinstance(density, list):
            #
            # Convert scalar density to per rank density list
            #
            density = (len(shape)-1)*[1.0] + [density]


        assert len(shape) == len(density), \
               "Density and shape arrays must be same length"

        if seed is not None:
            random.seed(seed)

        coords = []
        payloads = []

        for c in range(shape[0]):
            if random.random() < density[0]:
                if len(shape) == 1:
                    payload = random.randint(1, interval)
                    if payload == 0:
                        break
                else:
                    payload = Fiber.fromRandom(shape[1:],
                                               density[1:],
                                               interval)
                    if payload.isEmpty():
                        continue

                coords.append(c)
                payloads.append(payload)

        f = Fiber(coords, payloads)

        return f


#
# Stats-related methods
#

    def clearStats(self):
        """Clear savedPos-related statistics

        See `Fiber.getSavedPos()` for more information.

        """

        self._clearSavedPosStats()


#
# Accessor methods
#
    def getCoords(self):
        """Return the list of coordinates in fiber

        Returns
        -------
        coord_list: list
            List of coordinates

        Notes
        -----

        This method should be used in preference to accessing the
        `Fiber.coords` class instance variable directly.

        """

        return self.coords

    def getPayloads(self):
        """Return the list of payloads in fiber

        Returns
        -------
        payload_list: list
            List of payloads

        Notes
        -----

        This method should be used in preference to accessing the
        `Fiber.payloads` class instance variable directly.

        """

        return self.payloads


    def isOrdered(self):
        """Return the status of the "ordered" attribute

        Returns
        -------
        is_ordered: Boolean
            Set to True if the coordinates are ordered


        Note: this attribute cannot be changed after fiber creation.

        """

        return self._ordered


    def isUnique(self):
        """Return the status of the "unique" attribute

        Returns
        -------
        is_unique: Boolean
            Set to True if the coordinates are ordered

        Note: this attribute cannot be changed after fiber creation.

        """

        return self._unique


#
# Coordinate-based methods
#
# The following set of methods all reference an element in the fiber
# by coordinate. Some just obtain information about the element (non-mutating)
# others change the content of the fiber (mutating)
#

    def getPayload(self, *coords, default=None, allocate=True, start_pos=None):
        """Get the payload at a **point** in a fibertree

        Return the final payload after recursively traversing the
        levels of the fiber tree for at each coordinate in coords. If
        the list of coordinates reaches a leaf of the tree, it returns
        a value otherwise it will return a fiber. [Non-mutating]

        This method operates in two modes: allocate=True and False.

        For "allocate=True" mode, if any coordinate refers to a
        non-existent element, a payload is created (a fiber for at a
        non-leaf level or a zero at the leaf) recursively, but not
        inserted into the fiber tree. The final such payload is
        returned to the caller.

        For "allocate=False" mode, if any coordinate refers to a
        non-existent element, nothing is created and the `default`
        value is returned.

        If `start_pos` is specified it is used as a shortcut to start the
        search for the coordinate. And a new position is saved for use in
        a later search. Only works for a one-deep search.

        Parameters
        ----------

        coords: list
            list of coordinates to traverse

        allocate: Boolean, default=True
            Automatically generate the default value if needed at each
            level of the tree, but don't insert into the tree.

        default: value, default=None
            A constant default value to return if coordinate is empty on
            no-allocate

        start_pos: scalar or Payload() containing a scalar, default=None
            An optional shortcut value to optimize search

        Returns
        -------

        payload: a scalar or Fiber
             The payload of the element at the specified coordinate(s)

        Raises
        ------

        None

        """

        assert default is None or not allocate
        assert start_pos is None or len(coords) == 1

        start_pos = Payload.get(start_pos)

        # TBD: Actually optimize the search

        try:
            index = self.coords.index(coords[0])
            payload = self.payloads[index]
        except Exception:
            #
            # The requested coordinate did not exist
            #
            # If we are allocating missing elements or
            # are not at the last coordinate in the given coord list
            # create a default value to return (or recurse into),
            # but do not change anything in the actual fiber.
            #
            # Otherwise return the provided default.
            #
            # TBD: We (arbitarily) record we found it at the final
            #      index, this may not work for the next
            #      shortcut-based lookup
            #
            index = len(self.coords)

            if allocate or len(coords) > 1:
                payload = self._createDefault(addtorank=False)
            else:
                payload = Payload.maybe_box(default)

        if len(coords) > 1:
            assert Payload.contains(payload, Fiber), \
                "getPayload too many coordinates"

            # Recurse to the next level's fiber
            return payload.getPayload(*coords[1:],
                                      default=default,
                                      allocate=allocate)

        if start_pos is not None:
            self.setSavedPos(index, distance=index - start_pos)

        return payload


    def getPayloadRef(self, *coords, start_pos=None):
        """Get a (mutable) reference to a payload in a fibertree

        Return the final payload after recursively traversing the
        levels of the fiber tree for at each coordinate in coords,
        which are essential the coordinates of a `point`.  If the
        payload is empty, then recursively return the `default`
        payload

        If `start_pos` is specified it is used as a shortcut to start the
        search for the coordinate. And a new position is saved for use in
        a later search. Only works for a one-deep search.

        Parameters
        ----------
        coords: coordinates
            List of coordinates to traverse, i.e., a "point"

        start_pos: scalar or Payload() containing a scalar
            Optional shortcut value to optimize search

        Returns
        -------
        payload: a (boxed) scalar or Fiber
           The payload of the element at the specified coordinate(s)

        Raises
        ------

        None

        """

        assert start_pos is None or len(coords) == 1

        # TBD: Actually optimize the search

        start_pos = Payload.get(start_pos)

        try:
            #
            # Try to find the index for the first coordinate in the "point"
            # and get the payload at that index
            #
            index = self.coords.index(coords[0])
            payload = self.payloads[index]
        except Exception:
            #
            # Coordinate didn't exist so create a payload
            # at that coordinate then the index will exist
            #
            payload = self._create_payload(coords[0])
            index = self.coords.index(coords[0])

        if len(coords) > 1:
            # Recurse to the next level's fiber
            assert Payload.contains(payload, Fiber), "Too many coordinates"

            return payload.getPayloadRef(*coords[1:])

        if start_pos is not None:
            self.setSavedPos(index, distance=index - start_pos)

        assert Payload.is_payload(payload)

        return payload


    def _create_payload(self, coord):
        """Create a payload in the fiber at coord

        Optionally insert into the owners rank.

        """

        # Create a payload at coord
        # Iemporary value (should be None)

        payload = self._createDefault()

        assert Payload.is_payload(payload)

        try:
            index = next(x for x, val in enumerate(self.coords) if val > coord)
            self.coords.insert(index, coord)
            self.payloads.insert(index, payload)
        except StopIteration:
            index = len(self.coords)
            self.coords.append(coord)
            self.payloads.append(payload)


        #
        # Get the payload out of the payloads array
        # TBD: Not sure why I felt this was needed
        #
        payload = self.payloads[index]

        assert Payload.is_payload(payload)

        return payload


    def getRange(self,
                 start_coord,
                 size=None,
                 end_coord=None,
                 trans_fn=None,
                 start_pos=None):
        """Extract a range of coordinates from a Fiber

        Return a fiber in the range starting at `start_coord` and ending
        either when the `size` is exceeded or the fiber reaches the end
        of the open interval ending at `end_coord`.

        Parameters
        ----------
        start_coord: coordinate
            A coordinate indicating where to start the new fiber

        size: integer, default=None
            The size of the range in coordinate space

        end_coord: coordinate, default=None
            A coordinate indicating the end of the open interval

        trans_fn: function: coord -> coord, default=None
            A function that converts a coordinate in the orginal fiber
            into a cordinate in the new fiber

        start_pos: scalar or Payload() containing a scalar, default=None
            Optional **shortcut** value to optimize search for `start_coord`

        Returns
        -------

        Fiber
            Fiber containing the requested range


        Notes
        -----

        1) Either `size` or `end_coord` must be specified, but not both.

        2) The resulting fiber will NOT include `end_coord`


        """

        assert not (size is None and end_coord is None)
        assert size is not None or end_coord is not None
        assert self._ordered

        if trans_fn is None:
            # Default trans_fn is identify function (inefficient but easy)
            trans_fn = lambda x: x

        start_pos = Payload.get(start_pos)

        if start_pos is not None:
            assert start_pos < len(self.coords)
            assert start_pos == 0 or self.coords[start_pos - 1] < start_coord

            range_start = start_pos
        else:
            range_start = 0

        # Invariant: trans_fn is order preserving, but we check for reversals

        min = start_coord

        if size is not None:
            max = start_coord + size
        else:
            max = end_coord

        coords = []
        payloads = []

        # Start at start_pos (if any)

        first_pos = None

        for pos in range(range_start, len(self.coords)):
            c = self.coords[pos]
            p = self.payloads[pos]
            new_c = trans_fn(c)
            if new_c >= max:
                break
            if new_c >= min:

                # For statistics
                if first_pos is None:
                    first_pos = pos

                coords.append(new_c)
                payloads.append(p)


        # Note: This reversal implies a complex read order

        if len(coords) > 1 and coords[1] < coords[0]:
            coords.reverse()
            payloads.reverse()

        if start_pos is not None:
            if first_pos is None:
                self.setSavedPos(pos)
            else:
                self.setSavedPos(pos, distance=first_pos - start_pos)


        return self._newFiber(coords, payloads)


    def prune(self, trans_fn=None, start_pos=None):
        """Create a new fiber by pruning the elements of an existing fiber

        Return a fiber containing a subset of the coordinates of the
        input fiber. The input fiber is traversed calling `trans_fn`
        for each element. Based on the return value the element is
        included [True] (or not [False]) in the new fiber or
        traversal is stopped [None].

        Note: It is the responsibility of the `trans_fn` to cope with
        the fact that the coordinates of the fiber are "ordered"
        and/or "unique".

        Parameters
        ----------
        trans_fn: function: postion, coord, payload -> {True, False, None}, default=None

            A function that specifies what to do with each element of
            the original fiber.


        start_pos: scalar or Payload() containing a scalar, default=None
            Optional **shortcut** value to optimize search for `start_coord`

        Returns
        -------

        Fiber
            Fiber containing the pruned element of the input Fiber.

        """

        start_pos = Payload.get(start_pos)

        if start_pos is not None:
            assert start_pos < len(self.coords)
            range_start = start_pos
        else:
            range_start = 0

        coords = []
        payloads = []

        # Start at start_pos (if any)

        first_pos = None
        end_pos_offset = 1

        #
        # Traverse positions in fiber starting at range_start
        #
        for pos in range(range_start, len(self.coords)):
            c = self.coords[pos]
            p = self.payloads[pos]

            #
            # Call pruning function
            #
            status = trans_fn(pos, c, p)

            #
            # End processing if status is None
            #
            if status is None:
                end_pos_offset = 0
                break

            #
            # Include element if status is True
            #
            if status:
                if first_pos is None:
                    first_pos = pos

                coords.append(c)
                payloads.append(p)

        #
        # Record start_pos information
        #
        if start_pos is not None:
            if first_pos is None:
                self.setSavedPos(pos + end_pos_offset)
            else:
                self.setSavedPos(pos + end_pos_offset,
                                 distance=first_pos - start_pos)


        return self._newFiber(coords, payloads)


    def getPosition(self, coord, start_pos=None):
        """Find position of element associated with a coordinate [non-mutating]

        Return the position of the elemnt at `coord`, if any. If the
        the element at `coord` is **empty** return None. This method
        is non-mutating, i.e., the fiber will not change as a side
        effect of a call to this method.

        If `start_pos` is specified it is used as a shortcut to start the
        search for the coordinate. And a new position is saved for use in
        a later search. Only works for a one-deep search.

        Parameters
        ----------
        coord: coordinate
            Coordinate to look up

        start_pos: scalar or Payload() containing a scalar, default=None
            Optional shortcut value to optimize search

        Returns
        -------

        position: integer or None
            An index that can be used to _getitem_()

        Raises
        ------

        None

        """

        # TBD: Actually optimize the search

        start_pos = Payload.get(start_pos)

        try:
            index = self.coords.index(coord)
        except Exception:
            index = None

        if start_pos is not None and index is not None:
            self.setSavedPos(index, distance=index - start_pos)

        return index


    def getPositionRef(self, coord, start_pos=None):
        """Find position of element associated with a coordinate [mutating]

        Return the position of the elemnt at `coord`. If the the
        element at `coord` is **empty** then create it and assign the
        appropriate default payload, e.g., an empty Fiber or a zero.

        If `start_pos` is specified it is used as a shortcut to start the
        search for the coordinate. And a new position is saved for use in
        a later search. Only works for a one-deep search.


        Parameters
        ----------
        coord: coordinate
            Coordinate to look up

        start_pos: scalar or Payload() containing a scalar
            Optional shortcut value to optimize search

        Returns
        -------

        position: integer
            An index that can be used to _getitem_()


        Raises
        ------

        None

        """

        start_pos = Payload.get(start_pos)

        # TBD: Actually optimize the search

        try:
            index = self.coords.index(coord)
        except Exception:
            self._create_payload(coord)
            index = len(self.payloads) - 1  # TODO: This is wrong...

        if start_pos is not None and index is not None:
            self.setSavedPos(index, distance=index - start_pos)

        return index


    def project(self, trans_fn=None, interval=None):
        """Create a new fiber with coordinates projected according to `trans_fn`

        This method creates a new fiber with the same payloads as the
        original fiber, but with the coordinates transformed by `trans_fn`.


        Parameters
        ----------

        trans_fn: function with signature: lambda coord -> coord
            Function to convert a original fiber coordinate into a new
            fiber coordinate.

        interval: tuple, default=None (all coordinates)
            Restict projection to this range of original coordinates

        Returns
        -------

        fiber
            Fiber with coordinates projected according to `trans_fn`

        Raises
        ------

        None


        Notes
        -----

        This method returns a fiber that carries forward the "ordered"
        and "unique" attributes of the original fiber.  However, it
        largely does not check that the `trans_fn` maintains those
        attributes. Although it does a crude check to see if the
        coordinates seem to have been reversed.

        TBD
        ----

        Add support for **shortcuts**.

        """

        if trans_fn is None:
            # Default trans_fn is identify function (inefficient but easy)
            trans_fn = lambda x: x

        # Invariant: trans_fn is order preserving, but we check for reversals

        if interval is None:
            # All coordinates are legal

            coords = [trans_fn(c) for c in self.coords]
            payloads = self.payloads
        else:
            # Only pass coordinates in [ interval[0], interval[1] )

            min = interval[0]
            max = interval[1]

            coords = []
            payloads = []

            for c, p in zip(self.coords, self.payloads):
                new_c = trans_fn(c)
                if new_c >= min and new_c < max:
                    coords.append(new_c)
                    payloads.append(p)

            # Note: This reversal implies a complex read order

            if len(coords) > 1 and coords[1] < coords[0]:
                coords.reverse()
                payloads.reverse()

        return self._newFiber(coords, payloads)

#
# Deprecated coordinate-based methods
#

    def insertOrLookup(self, coord, value=None):
        """.. deprecated::"""

        Fiber._deprecated("Fiber.insertOrLookup() is deprecated use getPayloadRef()")

        if value is None:
            value = self._createDefault()

        payload = Payload.maybe_box(value)

        index = 0
        try:
            index = next(x for x, val in enumerate(self.coords) if val >= coord)
            if self.coords[index] == coord:
                return self.payloads[index]
            self.coords.insert(index, coord)
            self.payloads.insert(index, payload)
            return self.payloads[index]
        except StopIteration:
            self.coords.append(coord)
            self.payloads.append(payload)
            return self.payloads[-1]


    def insert(self, coord, value):
        """.. deprecated::"""

        Fiber._deprecated("Fiber.insert() is deprecated use getPayloadRef()")

        payload = Payload.maybe_box(value)

        try:
            index = next(x for x, val in enumerate(self.coords) if val > coord)
            self.coords.insert(index, coord)
            self.payloads.insert(index, payload)
        except StopIteration:
            self.coords.append(coord)
            self.payloads.append(payload)

        return None


    #
    # Owner rank related methods
    #
    def setOwner(self, owner):
        """Set rank that owns this fiber

        This method allows one to set the **owning rank** of a
        Fiber. This allows support for certain attributes of a Fiber
        that are common to all the fibers in a rank to be accessed
        from the rank. This includes `Fiber.getDefault()`,
        `Fiber.getShape() and `Fiber.getRankId()`.

        Parameters
        ----------

        owner: Rank
            The rank that owns this fiber


        """

        self._owner = owner

    def getOwner(self):
        """Get rank that owns this fiber.

        This method allows one to get the **owning rank** of a
        Fiber.

        Parameters
        ----------
        None

        Returns
        -------
        owner_rank: Rank
            The rank that owns this fiber

        """

        return self._owner


    #
    # Default payload methods
    #
    def setDefault(self, default):
        """.. deprecated::"""

        Fiber._deprecated("Fiber.setDefault() default values should be set by the owning rank")

        self._setDefault(default)


    def _setDefault(self, default):
        """_setDefault - internal use version"""

        owner = self.getOwner()

        #
        # Try to set default at owning rank, otherwise hold value locally
        #
        if owner is not None:
            owner.setDefault(default)
        else:
            self._default = Payload.maybe_box(default)


    def getDefault(self):
        """Get the default payload for this fiber.

        Ideally the **default** value from a fiber is obtained from the owner rank.

        Parameters
        ----------
        None

        Returns
        -------
        value: value
            A copy of the default payload of fibers in this rank

        Raises
        ------
        None

        """

        #
        # Try to get default from owning rank
        #
        owner = self.getOwner()

        if owner is not None:
            return owner.getDefault()

        #
        # For unowned fibers, try to guess a default value
        #
        # Note: the payload being a Fiber overwrides self._default
        #
        if len(self.payloads) > 0 and Payload.contains(self.payloads[0], Fiber):
            return Fiber

        if self._default != 0:
            return deepcopy(self._default)

        return Payload(0)



    def _createDefault(self, addtorank=True):
        """_createDefault

        Obtain the default payload for a fiber. This method goes one
        step further than getDefault() because if the default payload
        is itself a fiber it creates a Fiber().

        Finally, if the current fiber is part of a a non-leaf rank
        it (optionally) adds the new fiber into the **next** rank.

        TBD: Fold this into an option to getDefault()

        """

        owner = self.getOwner()

        default = self.getDefault()

        a_payload = (self.payloads or [None])[0]
        if isinstance(a_payload, Fiber):
            next_default = a_payload.getDefault()
        else:
            next_default = None

        return Fiber._instantiateDefault(owner, default, next_default, addtorank)


    @staticmethod
    def _instantiateDefault(owner, default, next_default=None, addtorank=False):
        """_instantiateDefault

        Create (recursively for default values that are tuples) an
        instance of a default payload for a fiber. This method goes
        one step further than getDefault() because if the default
        payload is itself (or contains) a fiber it creates a Fiber()
        object.

        Finally, if the newly created fiber is part of a a non-leaf
        rank it (optionally) adds the new fiber into the **next**
        rank.

        Parameters
        ----------

        owner: rank
            The rank that owns the fiber we are creating a payload for

        default: a payload (boxed or unboxed)
            A default value from a fiber

        next_default: a payload (boxed or unboxed)
            If `default` is a Fiber then a default payload for that fiber

        addtorank: Boolean
            If the newly created value is a fiber, then should that fiber
            be added to the its owning rank (owner.next_rank)

        Returns
        -------

        A (boxed) payload

        """
        #
        # Selectively unbox the default
        #
        default = Payload.get(default)

        if isinstance(default, tuple):
            #
            # Recursively create defaults. Note each of the elements of the tuple
            # will be **boxed** as will the final result...
            #
            return Payload(tuple([Fiber._instantiateDefault(owner, e) for e in default]))

        if callable(default):
            #
            # Call the method to create the value
            #
            # Note, currently, this must be a fiber..
            #
            value = default()

            #
            # Conditionaly set the owning rank of the
            # newly created fiber by appending it to the
            # next rank of the tensor.
            #
            # Adding it to the owner.next_rank sets the
            # "default" for the rank, otherwise we set a
            # "default" explcitly.
            #
            # TBD: This is a messy interaction with rank
            #      See Rank.append()
            #
            if Payload.contains(value, Fiber):
                if owner and owner.next_rank is not None:
                    #
                    # The new fiber is nominally part of
                    # "owner.next_rank"
                    #
                    # TBD: Rank.append() sets owner, maybe that should be done here.
                    #
                    if addtorank:
                        #
                        # Actually add it to the rank
                        #
                        owner.next_rank.append(value)
                    else:
                        #
                        # Set the owner, but do not add to rank
                        #
                        value.setOwner(owner.next_rank)
                else:
                    value._setDefault(next_default)
            else:
                assert False, "Unsupported Payload type"
        else:
            assert not isinstance(default, Payload)

            value = default

        return Payload.maybe_box(value)


    #
    # Saved position shortcut related methods
    #

    def setSavedPos(self, position, distance=None):
        """Set the postion for a **shortcut**

        Save the postion in a fiber for use as a future **shortcut**.
        Typically to shorten the duration of some search. The
        (optional) distance is used to maintain statistics on the
        number of elements traversed before arriving at this new
        postion.

        Parameters
        ----------
        postion: integer
            Postion (index) in the fiber to remember

        distance, integer, default=None
            Distance searched to arrive at this postion

        Returns
        -------
        None

        See also
        --------

        `Fiber.getSavedPos()`
        `Fiber.getSavedPosStats()`

        """

        position = Payload.get(position)

        self._saved_pos = position

        #
        # Optionally save distanced moved statistics
        #
        if distance is not None:
            self._saved_count += 1
            self._saved_dist += abs(distance)


    def getSavedPos(self):
        """Set the postion for a **shortcut**

        Get the postion in a fiber for use as a **shortcut**.
        Typically to shorten the duration of some search.

        Parameters
        ----------
        postion: integer
            Postion (index) in the fiber last remembered


        Returns
        -------
        postion: integer


        See also
        --------

        `Fiber.setSavedPos()`
        `Fiber.getSavedPosStats()`

        """

        return self._saved_pos


    def getSavedPosStats(self, clear=True):
        """Get the statistcs assocaited with **shortcuts**

        Get the number of shortcuts used and the distance searched
        using those shortcuts, and optionally clear the statistics


        Parameters
        ----------
        clear: Bool
            Clear the statistics


        Returns
        -------
        stats: tuple
            Tuple of number of **shortcuts** set and total search distance


        See also
        --------

        `Fiber.getSavedPos()`
        `Fiber.setSavedPos()`

        """

        stats = (self._saved_count, self._saved_dist)

        if clear:
            self._clearSavedPosStats()

        return stats


    def _clearSavedPosStats(self):
        """_clearSavedPosStats"""

        self._saved_count = 0
        self._saved_dist = 0

    #
    # Computed attribute acccessors
    #
    def minCoord(self):
        """Return the minimum coordinate that exists in the fiber

        Parameters
        ----------
        None

        Returns
        -------
        min_coordinate: coordinate


        Notes
        -----

        This is only meaningful for coordinates that have an lexographical order.

        """

        # TBD: Should check that the candidate is not an explicit zero

        if len(self.coords) == 0:
            return None

        return min(self.coords)

    def maxCoord(self):
        """Return the maximum coordinate that exists in the fiber

        Parameters
        ----------
        None

        Returns
        -------
        max_coordinate: coordinate


        Notes
        -----

        This is only meaningful for coordinates that have an lexographical order.

        """

        #
        # If _max_coord is set we assume it is correct
        #
        # TBD: _max_coord is not always maintained properly for
        #      some fiber mutations
        #
        if self._max_coord is not None:
            return self._max_coord

        if len(self.coords) == 0:
            return None

        #
        # TBD: Check is there is a lexographical order...
        #
        if not isinstance(self.coords[0], int):
            #
            # Coordinates aren't integers, so maxCoord doesn't make sense
            #
            return None

        #
        # TBD: Maybe should actually look for largest non-empty coordinate
        #
        return max(self.coords)


    def countValues(self):
        """Count values in the fiber tree

        Count the number of leaf elements in the fibertree that are
        not **empty** and have do not have the a payload with the
        fiber's **default** value.

        Parameters
        ----------
        None

        Returns
        -------
        value_count: integer
            Number of non-empty, non-default values in the fibertee.

        Notes
        -----

        An explcit zero scalar value will NOT count as a value in a

        """

        count = 0
        for p in self.payloads:
            if Payload.contains(p, Fiber):
                count += Payload.get(p).countValues()
            else:
                count += 1 if not Payload.isEmpty(p) else 0

        return count

#
# Position based methods
#
    def __getitem__(self, keys):
        """__getitem__

        For an integer key return a (coordinate, payload) tuple
        containing the contents of a fiber at `position`, i.e., an
        offset in the coordinate and payload arrays. For a slice key
        return a new fiber for the slice

        Parameters
        ----------
        keys: single integer/slicr or tuple of integers/slices
            The positions or slices in an n-D fiber

        Returns
        -------
        tuple or Fiber
            A tuple of a coordinate and payload or a Fiber of the slice

        Raises
        ------

        IndexError
            Index out of range

        TypeError
            Invalid key type

        """

        if not isinstance(keys, tuple):
            # Keys is a single value for 1-D access
            key = keys
            key_cdr = ()
        else:
            # Keys is a tuple for for n-D access
            key = keys[0]
            key_cdr = keys[1:]

        if isinstance(key, int):
            # Handle key as single index

            if key < 0:
                # Handle negative indices
                key += len(self)

            if key < 0 or key >= len(self):
                raise(IndexError, f"The index ({key}) is out of range")

            new_payload = self.payloads[key]

            if len(key_cdr):
                # Recurse down the fiber tree
                new_payload = new_payload[key_cdr]

            return CoordPayload(self.coords[key], new_payload)

        if isinstance(key, slice):
            # Key is a slice

            # Get the start, stop, and step from the slice
            slice_range = range(*key.indices(len(self)))

            coords = [self.coords[ii] for ii in slice_range]

            if len(key_cdr):
                # Recurse down the fiber tree for each payload in slice
                payloads = [self.payloads[ii][key_cdr] for ii in slice_range]
            else:
                # Just use each payload in slice
                payloads = [self.payloads[ii] for ii in slice_range]

            return Fiber(coords, payloads)

        raise(TypeError, "Invalid key type.")


    def __setitem__(self, key, newvalue):
        """__setitem__

        The `newvalue` parameter is either a CoordPayload or an
        arbitrary value to assign to the position "key" in the fiber.
        If `newvalue` is not a CoordPayload or the Coord in the
        CoordPayload is None the current coordinate will be left
        unchanged. The payload will be boxed if appropriate. If the
        payload is None, then the payload will be left unchanged.

        Parameters
        ----------
        key: single integer
            The position in the fiber to be set

        newvalue: a CoordPayload or a payload value
            The coordinate/payload or just payload to assign

        Returns
        -------
        Nothing

        Raises
        ------

        IndexError
            Index out of range

        TypeError
            Invalid key type

        CoordinateError
            Invalid coordinate


        Notes
        ------

        If this fiber does has the "unique" attribute but not the
        "ordered" attribute this method does not check that the new
        coordinate is unique.

        """

        position = key

        #
        # TBD: Get isinstance of CoordPayload to work...
        #
        try:
            coord = newvalue.coord
            payload = newvalue.payload
        except Exception:
            coord = None
            payload = newvalue

        if coord is not None:
            #
            # Check that coordinate order is maintained
            #
            if self._ordered:
                if position > 0 and coord <= self.coords[position - 1]:
                    raise CoordinateError

                if position + 1 < len(self.coords) and coord >= self.coords[position + 1]:
                    raise CoordinateError

            self.coords[position] = coord

        #
        # A payload of None just updates the coordinate
        #
        if payload is not None:
            self.payloads[position] = Payload.maybe_box(payload)


    def __len__(self):
        """__len__"""

        return len(self.coords)


    def isEmpty(self):
        """Check if Fiber is empty

        Empty is defined as of zero length, only containing
        **default** values or only containing subfibers that are
        empty.

        Returns
        -------

        empty_p: Bool
            Boolean indicating the fiber was empty


        Notes
        -----

        Need to check for **default** values that are not zero

        """

        return all(map(Payload.isEmpty, self.payloads))


    def nonEmpty(self):
        """Create Fiber with only non-empty elements

        Because our fiber representation might have explicit zeros in
        it this method (recursively) creates a new fiber with those
        elements pruned out.

        Returns
        -------

        pruned_fiber: Fiber
            Copy of original fiber with only non-empty elements

        """
        coords = []
        payloads = []

        for c, p in zip(self.coords, self.payloads):
            if not Payload.isEmpty(p):
                coords.append(c)
                if Payload.contains(p, Fiber):
                    payloads.append(p.nonEmpty())
                else:
                    payloads.append(p)

        return self._newFiber(coords, payloads)

#
# Iterator methods
#

    def __iter__(self):
        """__iter__"""

        for i in range(len(self.coords)):
            yield CoordPayload(self.coords[i], self.payloads[i])

    def __reversed__(self):
        """Return reversed fiber"""

        for coord, payload in zip(reversed(self.coords),
                                  reversed(self.payloads)):
            yield CoordPayload(coord, payload)


    def iterShape(self):
        """Iterate over fiber shape

        Iterate over every coordinate in the shape, returning a
        CoordPayload for each one, with a **default** value for
        empty payloads.

        Parameters
        ----------
        None

        """

        for c in range(self.getShape(all_ranks=False)):
            p = self.getPayload(c)
            yield CoordPayload(c, p)


    def iterShapeRef(self):
        """Iterate over fiber shape

        Iterate over every coordinate in the shape, returning a
        CoordPayload for each one, and creating elements for empty
        payloads.

        Parameters
        ----------
        None

        """

        for c in range(self.getShape(all_ranks=False)):
            p = self.getPayloadRef(c)
            yield CoordPayload(c, p)


#
# Core methods
#

    def clear(self):
        """Clear all coordinates/payloads in a fiber

        Returns
        -------
        Nothing

        """

        self.coords.clear()
        self.payloads.clear()


    def payload(self, coord):
        """.. deprecated::"""

        Fiber._deprecated("Fiber.payload() is deprecated use getPayload()")

        return self.getPayload(coord)


    def append(self, coord, value):
        """Append an element at the end of fiber

        Parameters
        ----------

        coord: scalar
            The coordinate of the element to add to the fiber

        value: payload
            The payload of the elemnt to add to the fiber

        Note
        ----

        For "ordered" fibers, the coordinates in the Fiber must be
        monotonically increasing.

        The "unique" property is not checked for "unordered" fibers.

        The payload will be optionally be **boxed**.

        """

        if self._ordered:
            assert self.maxCoord() is None or self.maxCoord() < coord, \
                   "Fiber coordinates in 'ordered' fibers must be monotonically increasing"

        payload = Payload.maybe_box(value)

        self.coords.append(coord)
        self.payloads.append(payload)


    def extend(self, other):
        """Extend a fiber with another fiber

        Extends the fiber with the contents of another fiber

        Parameters
        ----------
        other: Fiber
            A fiber to extend the original fiber with


        Returns
        -------
        Nothing


        Notes
        ------

        The `other` fiber is not copied, so beware of multiple
        references to the same objects.

        The "unique" property is not checked for "unordered" fibers.

        """

        assert Payload.contains(other, Fiber), \
            "Fibers can only be extended with another fiber"

        if other.isEmpty():
            # Extending with an empty fiber is a nop
            return None

        if self._ordered:
            assert self.maxCoord() is None or self.maxCoord() < other.coords[0], \
                "Fiber coordinates in 'ordered' fibers must be monotonically increasing"

        self.coords.extend(other.coords)
        self.payloads.extend(other.payloads)

        return None


    def updateCoords(self, func, depth=0, rankid=None):
        """Update (rewrite) the values of the coordinates of a fiber

        Update each coordinate in the the fibers at a depth of `depth`
        below `self` by invoking `func` on it.  Therefore, a depth of
        zero will update the coordinates in the current fiber. Higher
        depths with result in a depth first search down to `depth`
        before traversing the coordinates.

        Parameters
        ----------

        func: function: position, coordinate, payload -> coordinate
            A function that is invoked with each coordinate as its argument

        depth: integer
            The depth in the fiber tree to dive before traversing

        rankid: string, default=None
            The name of a rank, i.e., a rankid, at which to perform
            the split, overrides the `depth` argument.

        Returns
        --------
        Nothing

        Raises
        ------

        Nothing


        Notes
        -----

        This method checks and that coordinates remain monotonically
        increasing and re-orders to make sure
        self.coords/self.payloads preserve monotonacity.

        The "unique" property is not checked.

        """
        if rankid is not None:
            depth = self._rankid2depth(rankid)

        if depth > 0:
            # Recurse down to depth...
            for p in self.payloads:
                p.updateCoords(func, depth=depth - 1)
        else:
            # Update my coordinates

            no_sort_needed = True

            last_coord = None

            for i in range(len(self.coords)):
                new_coord = func(i, self.coords[i], self.payloads[i])
                self.coords[i] = new_coord

                no_sort_needed = no_sort_needed and ((last_coord is None) or (last_coord <= new_coord))
                last_coord = new_coord

            if self._ordered and not no_sort_needed:
                #
                # Resort the coords/payloads
                #
                self.logger.debug("Fiber.updateCoords() - sort needed")

                zipped_cp = zip(self.coords, self.payloads)
                sorted_cp = sorted(zipped_cp)
                self.coords, self.payloads = [ list(tuple) for tuple in zip(*sorted_cp)]

        return None


    def updatePayloads(self, func, depth=0, rankid=None):
        """Update the values of the payloads of a fiber

        Update each payload in the the fibers at a depth of `depth`
        below "self" by invoking "func" on it.  Therefore, a depth of
        zero will update the payloads in the current fiber. Higher
        depths with result in a depth first search down to `depth`
        before traversing the payloads.

        Parameters
        ----------

        func: function: postion, coordinate, payload -> payload
             A function that is invoked with each payload as its argument

        depth: integer
            The depth in the fibertree to dive before traversing

        rankid: string, default=None
            The name of a rank, i.e., a rankid, at which to perform
            the split, overrides the `depth` argument.

        Returns
        --------

        None

        Raises
        ------

        TBD: currently nothing

        """
        if rankid is not None:
            depth = self._rankid2depth(rankid)

        if depth > 0:
            # Recurse down to depth...
            for p in self.payloads:
                p.updatePayloads(func, depth=depth - 1)
        else:
            # Update my payloads
            for i in range(len(self.payloads)):
                self.payloads[i] = func(self.payloads[i])

        return None


    def unzip(self):
        """Unzip the payloads of a fiber

        Unzip a fiber whose payloads are a tuple into two fibers each
        with the same coordinates

        Parameters
        ----------

        None

        Returns
        -------

        unziped_fibers: tuple
            A tuple of fibers

        """

        coords_a = list(self.coords)
        coords_b = list(self.coords)

        (payloads_a, payloads_b) = zip(*self.payloads)

        return (self._newFiber(coords_a, payloads_a),
                self._newFiber(coords_b, payloads_b))

#
# Shape-related methods
#

    def getShape(self, all_ranks=True):
        """Return the shape of a fibertree

        Find the **shape** of the current fiber (`all_ranks`=False) or
        the entire fibertree rooted at the current fiber (`all_ranks`=True)

        Parameters
        ----------

        all_ranks: Bool, default=True

        Returns
        -------

        shape: integer or list of integers
            The shape of the current fiber or the entire tree

        """

        owner = self.getOwner()

        shape = None

        if owner is not None:
            shape = owner.getShape(all_ranks=all_ranks)

        if shape is not None:
            return shape

        if self._shape is not None:
            return self._shape

        #
        # Backup for cases where there is no  owner
        # or owner didn't know shape
        #
        shape = self.estimateShape(all_ranks=all_ranks)

        return shape


    def estimateShape(self, all_ranks=True):
        """estimateShape

        Traverse a fiber tree to estimate its shape

        """
        shape = self._calcShape(all_ranks=all_ranks)

        #
        # Since _calcShape() always returns a list we may
        # need to get out first value
        #
        if not all_ranks:
            shape = shape[0]

        return shape


    def _calcShape(self, shape=None, level=0, all_ranks=True):
        """ _calcShape()

        Find the maximum coordinate at each level of the tree

        TBD: Using maximum coordinate isn't really right because
             the original array may have a empty value at its
             maximum coordinate location

        """

        #
        # Start recursion
        #
        if shape is None:
            shape = []

        #
        # Conditionaly append a new level to the shape array
        #
        if len(shape) < level + 1:
            shape.append(0)

        #
        # If fiber is empty then shape doesn't change
        #
        if not len(self.coords):
            return shape

        #
        # Try to determine the maximum coordinate
        #
        max_coord = self.maxCoord()

        #
        # The fiber is not empty, but max_coord isn't meaningful,
        # so assume fiber is dense and coodinates start at zero,
        # and return count of elements minus one.
        #
        if max_coord is None:
            max_coord = len(self.coords)-1

        #
        # Update shape for this Fiber at this level
        #
        shape[level] = max(shape[level], max_coord + 1)

        #
        # Recursively process payloads that are Fibers
        #
        if all_ranks and Payload.contains(self.payloads[0], Fiber):
            for p in self.payloads:
                shape = Payload.get(p)._calcShape(shape, level + 1)

        return shape

#
# Rankid methods
#
    def getRankIds(self, all_ranks=True):
        """Return rankids of a fibertree

        Find the **rank ids** of the current fiber (`all_ranks`=False) or
        the entire fibertree rooted at the current fiber (`all_ranks`=True)

        Parameters
        ----------

        all_ranks: Bool, default=True

        Returns
        -------

        rank_ids: str or list of str
            The rank ids of the current fiber or the entire tree

        """

        owner = self.getOwner()

        if owner is not None:
            return owner.getRankIds(all_ranks=True)

        #
        # Approximate rankids for fiber not in a tensor
        #

        rankids = [f"X.{d}" for d in reversed(range(self.getDepth()))]

        return rankids

#
# Dimensionality method
#
    def getDepth(self):
        """Get the depth of the fiber

        Get the depth, i.e., number of dimensions, of the fiber

        Parameters
        ----------
        None

        Returns
        -------
        depth: integer
            The depth of the fibertree

        Raises
        ------
        None

        """

        owner = self.getOwner()

        if owner is not None:
            #
            # In a tensor, so get the number of ranks starting at this fiber
            #
            depth = 0

            while owner is not None:
                depth += 1
                owner = owner.next_rank

            return depth

        #
        # Just have a raw fiber, so count levels
        #
        fiber = self

        depth = 1

        while len(fiber.payloads) > 0 and isinstance(fiber.payloads[0], Fiber):
            depth += 1
            fiber = fiber.payloads[0]

        return depth


#
# Miscelaneous methods
#
    def uncompress(self, shape=None, level=0):
        """Return an uncompressed fibertree (i.e., a nest of lists)

        Recursively create a nest of lists that corresponding to the
        **uncompressed** represention of the current
        fibertree. **Empty** coordinates at a fiber will be
        converted into the **default** value for that fiber.

        Parameters
        ----------

        shape: list of integers, default=None
            Impose a fixed shape on the result


        Returns
        -------
        uncompressed: list of lists

        Notes
        ------

        All elements of the lists are **unboxed**, i.e., never of type
        `Payload`. However, nested elements, e.g., as part of a
        `tuple`, of type `Payload` are not **unboxed**.

        This method only works for "ordered", "unique" fibers.

        """

        assert self._ordered and self._unique

        if shape is None:
            shape = self.getShape(all_ranks=True)

        f = []

        shape_fiber = Fiber(coords=list(range(shape[level])), initial=1)
        for c, (mask, p, _) in self | shape_fiber:

            if (mask == "AB"):
                if Payload.contains(p, Fiber):
                    f.append(Payload.get(p).uncompress(shape, level + 1))
                else:
                    f.append(Payload.get(p))

            if (mask == "B"):
                f.append(self._fillempty(shape, level + 1))

        return f


    def _fillempty(self, shape, level):
        """Recursive fill empty"""

        if level + 1 > len(shape):
            #
            # Find a fiber at the leaf level
            #
            f = self
            while isinstance(f.payloads[0], Fiber):
                f = f.payloads[0]

            #
            # Use the **unboxed** default from the leaf level fiber
            #
            return Payload.get(f.getDefault())

        f = []

        for i in range(shape[level]):
            f.append(self._fillempty(shape, level + 1))

        return f

#
# Arithmetic operations
#

    def __ilshift__(self, other):
        """Fiber assignment

        This operator will make a recursive assignment of all the
        elements of one fiber (`other`) into another fiber (`self`)
        using getPayloadRef(), so subfibers in new fiber are
        properly inserted into their owning rank/tensor

        Note: we use <<= in place of base '=' since we don't want a
        pointer to the existing fiber but an copy of `other` in the
        new fiber.

        Parameters
        ----------

        other: Fiber
            A fiber whose elements will be inserted into `self`

        Notes
        -----

        There is an analogous assignment operator for the `Payload`
        and `CoordPayload` classes, so one can "assign" a new value to
        a "payload" irrespective of whether the "payload" is a
        `Payload`, a `CoordPayload` or a `Fiber`.

        This method is not supported for fibers without the "unique"
        attribute.

        """

        assert Payload.contains(other, Fiber)
        assert self._unique

        if len(self.coords) != 0:
            #
            # Clear out any existing data
            #
            self.coords = []
            self.payloads = []


        for c, p in other:
            #
            # For each non-empty element of other, insert it into the
            # target, note that this works regardless of whether p is
            # a Fiber or a Payload
            #
            if Payload.isEmpty(p):
                continue

            ref = self.getPayloadRef(c)
            ref <<= p

        return self


    def __add__(self, other):
        """Scalar/fiber or fiber/fiber elementwise addition

        This operation does one of two things based on the type of
        `other`. If `other` is a fiber then `other` is added
        element-wise with `self`. Otherwise `other` is treated as a
        scalar and added to each element of `self`. In either case a
        new fiber is created with those sums.

        Parameters
        ----------

        other: scalar | fiber
            The scalar to add to each element of `self`
            or the fiber to add elementwise to `self`

        Returns
        -------

        result_fiber: Fiber
            The fiber after the addition of `other` to `self`


        Examples
        --------

        >>> f = Fiber.fromUncompressed([1, 2, 3, 0, 0, 6])
        >>> f + 2
        Fiber([0, 1, 2, 5], [3, 4, 5, 2, 2, 8])


        Note
        -----

        From the persepctive of modeling activity, this operation has
        implict loops that get exectuted atomically. Therefore, it
        should be used selectively when one is trying to show all the
        activity in a program's flow.

        When doing scalar/fiber addition, empty elements of `self`
        will be treated as zero and the value of `other` will appear
        in the output for those coordinates. When doing fiber/fiber
        addition the result will have the union of the coordinates of
        `self` and `other`.

        """

        coords = []
        payloads = []

        #
        # If `other` is a fiber element-wise add the two fibers
        # Otherwise add `other` to each element of `self`
        #
        if isinstance(other, Fiber):
            for c, (_, self_val, other_val) in self | other:
                coords.append(c)
                payloads.append(self_val + other_val)
        else:
            for c, p in self.iterShape():
                coords.append(c)
                payloads.append(other + p.value)

        return self._newFiber(coords, payloads)


    def __radd__(self, other):
        """ Scalar/fiber addition

        See __add__ for more information.

        Examples
        --------

        >>> f = Fiber.fromUncompressed([1, 2, 3, 0, 0, 6])
        >>> 2 + f
        Fiber([0, 1, 2, 5], [3, 4, 5, 2, 2, 8])

        """

        return self.__add__(other)


    def __iadd__(self, other):
        """Add a scalar or a fiber to a fiber

        This operation does one of two things based on the type of
        `other`. If `other` is a fiber then `other` is added
        elementwise to `self`. Otherwise it is treated as a scalar and
        added to each element of `self`. In either case `self` is
        updated with the sum.

        Parameters
        ----------

        other: scalar | fiber
            The scalar to add to each element of `self`
            or the fiber to add elementwise to `self`

        Returns
        -------

        None


        Examples
        --------

        >>> f = Fiber.fromUncompressed([1, 2, 3, 0, 0, 6])
        >>> f += 2
        >>> f
        Fiber([0, 1, 2, 5], [3, 4, 5, 2, 2, 8])


        Note
        -----

        From the persepctive of modeling activity, this operation has
        implict loops that get exectuted atomically. Therefore, it
        should be used selectively when one is trying to show all the
        activity in a program's flow.

        When doing the addtions, empty elements of `self` will be
        treated as zero and and will be included in the sum (only with
        non-empty coordiantes of `other` if it is a fiber), creating
        new non-empty elements in `self`.

        """

        #
        # If `other` is a fiber add each element of `other` to `self`
        #
        if isinstance(other, Fiber):
            for _, (self_ref, other_val) in self << other:
                self_ref += other_val
            return self

        #
        # Othewise add `other` to each element of `self`
        #
        for c, p in self.iterShapeRef():
            p += other

        return self


    def __mul__(self, other):
        """Scalar/fiber and fiber/fiber elementwise multiplication

        This operation does one of two things based on the type of
        `other`. If `other` is a fiber then `other` is multiplied
        element-wise with `self`. Otherwise `other` is treated as a
        scalar and used to scale each element of `self`. In either
        case a new fiber is created with those products.

        Parameters
        ----------

        other: scalar | Fiber
            The scalar to scale each element of `self` by
            or the fiber to multiply elementwise with `self`.

        Returns
        -------

        result_fiber: Fiber
            A fiber scaled or elementwise multiplied by `other`


        Examples
        --------

        >>> f = Fiber.fromUncompressed([1, 2, 3, 0, 0, 6])
        >>> f * 2
        Fiber([0, 1, 2, 5], [2, 4, 6, 12])

        Notes
        -----

        From the persepctive of modeling activity, this operation has
        implict loops that get exectuted atomically. Therefore, it
        should be used selectively when one is trying to show all the
        activity in a program's flow.

        """

        coords = []
        payloads = []

        if isinstance(other, Fiber):
            for c, (a_val, b_val) in self & other:
                coords.append(c)
                payloads.append(a_val * b_val)
        else:
            for c, p in self:
                coords.append(c)
                payloads.append(other * p.value)

        result = self._newFiber(coords, payloads)
        return result


    def __rmul__(self, other):
        """ Scalar/fiber multiplication

        See __mul__ for more information.

        Examples
        --------

        >>> f = Fiber.fromUncompressed([1, 2, 3, 0, 0, 6])
        >>> 2 * f
        Fiber([0, 1, 2, 5], [2, 4, 6, 12])

        """

        return self.__mul__(other)


    def __imul__(self, other):
        """__imul__

        This operation does one of two things based on the type of
        `other`. If `other` is a fiber then `other` is multiplied
        elementwise by `self`. Otherwise it is treated as a scalar and
        scales each element of `self`. In either case `self` is
        updated with the products.

        Parameters
        ----------

        other: scalar | Fiber
            The scalar to scale each element of `self`
            or the fiber to multiply elementwise to `self`

        Returns
        -------

        None


        Examples
        --------

        >>> f = Fiber.fromUncompressed([1, 2, 3, 0, 0, 6])
        >>> f *= 2
        >>> f
        Fiber([0, 1, 2, 5], [2, 4, 6, 12])

        Notes
        -----

        From the persepctive of modeling activity, this operation has
        implict loops that get exectuted atomically. Therefore, it
        should be used selectively when one is trying to show all the
        activity in a program's flow.

        """

        #
        # If `other` is a fiber, elementwise multiply `other` by `self`
        #
        if isinstance(other, Fiber):
            for c, (self_val, other_val) in self & other:
                #
                # Get a reference to the c coordinate in `self`.  Note
                # that is may be a zero and therefore a hard zero will
                # be included in the final fiber.
                #
                self_ref = self.getPayloadRef(c)
                self_ref <<= self_val * other_val

            return self

        #
        # Othewise multiply `other` to each element of `self`
        #
        for _, p in self:
            p *= other

        return self

#
# Split methods
#
# Note: all these methods return a new fiber
#
    def __truediv__(self, partitions):
        """Split a fiber uniformly in coordinate space into `partitions` partitions

        Parameters
        ----------

        partitions: integer
            The number of partions to split the fiber into


        Returns
        -------

        split_fiber: Fiber
            A fiber split uniformly in coorindate space


        Examples
        --------

        >>> f = Fiber.fromUncompressed([1, 2, 3, 0, 0, 6])
        >>> f / 2
        F[ 0 -> F [ 0 -> 1, 1 -> 2, 2-> 3],
           3 -> F [ 5 -> 6]]


        Notes
        -----

        This method depends on maxCoord() being meaningful

        TBD
        ---

        Is there a reasonable semantic if `partitions` is a fiber
        """

        shape = self.getShape(all_ranks=False)

        assert shape is not None, \
               "Cannot partition a fiber without a maximum coordinate"

        return self.splitUniform((shape+partitions-1)//partitions)


    def __floordiv__(self, partitions):
        """Split a fiber evenly in position space into `partitions` partitions

        Parameters
        ----------

        partitions: integer
            The number of partions to split the fiber into


        Returns
        -------

        split_fiber: Fiber
            A fiber split equally in postion space


        Examples
        --------

        >>> f = Fiber.fromUncompressed([1, 2, 3, 0, 0, 6])
        >>> f / 2
        F[ 0 -> F [ 0 -> 1, 1 -> 2 ],
           2 -> F [ 2 -> 3, 5 -> 6 ]]


        Notes
        -----

        None

        TBD
        ---

        Is there a reasonable semantic if `partitions` is a fiber
        """


        occupancy = len(self.coords)

        return self.splitEqual ((occupancy+partitions-1)//partitions)


    def splitUniform(self, step, partitions=1, relativeCoords=False, depth=0, rankid=None):
        """Split a fiber uniformly in coordinate space

        Parameters
        ----------
        step: integer
            The `step` between initial coordinates in each split

        relative_coords: Bool
            Should the coordinate in the split fibers match the
            original coordinates (`relativeCoords`=False) or always
            start at zero (`relativeCoords`=True)

        depth: integer, default=0
            The depth in the fibertree to perform the split

        rankid: string, default=None
            The name of a rank, i.e., a rankid, at which to perform
            the split, overrides the `depth` argument.

        Returns
        -------
        split_fiber: Fiber
            A fibertree with one more level corresonding to the
            splits of the original fiber.

        Notes:
        -------
        References to pieces of the original tensor may be returned as
        pieces of the returned tensor.

        """

        class _SplitterUniform():

            def __init__(self, step):
                self.step = step
                self.cur_group = 0

            def nextGroup(self, i, c):
                count = 0
                last_group = self.cur_group

                while c >= self.cur_group:
                    count += 1
                    last_group = self.cur_group
                    self.cur_group += self.step

                return count, last_group

        if rankid is not None:
            depth = self._rankid2depth(rankid)

        if depth > 0:
            split_fiber = deepcopy(self)
            update_lambda = lambda p: p.splitUniform(step, partitions, relativeCoords)
            split_fiber.updatePayloads(update_lambda, depth=depth-1)
            return split_fiber

        splitter = _SplitterUniform(step)

        return self._splitGeneric(splitter,
                                  partitions,
                                  relativeCoords=relativeCoords)


    def splitNonUniform(self, splits, partitions=1, relativeCoords=False, depth=0, rankid=None):
        """Split a fiber non-uniformly in coordinate space

        Parameters
        ----------
        splits: list of integers
            A list of the starting coordinates for each split

        relative_coords: Bool
            Should the coordinate in the split fibers match the
            original coordinates (`relativeCoords`=False) or always
            start at zero (`relativeCoords`=True)

        depth: integer, default=0
            The depth in the fibertree to perform the split

        rankid: string, default=None
            The name of a rank, i.e., a rankid, at which to perform
            the split, overrides the `depth` argument.

        Returns
        -------
        split_fiber: Fiber
            A fibertree with one more level corresonding to the
            splits of the original fiber


        Notes:
        -------
        One does not needs to include a split starting at coordinate zero.

        References to pieces of the original tensor may be returned as
        pieces of the returned tensor.

        """

        class _SplitterNonUniform():

            def __init__(self, splits):
                if Payload.contains(splits, Fiber):
                    self.splits = splits.coords.copy()
                else:
                    self.splits = splits.copy()

                self.cur_split = self.splits.pop(0)

            def nextGroup(self, i, c):
                count = 0
                last_group = self.cur_split

                while c >= self.cur_split:
                    count += 1
                    last_group = self.cur_split
                    if self.splits:
                        self.cur_split = self.splits.pop(0)
                    else:
                        self.cur_split = float("inf")

                return count, last_group

        if rankid is not None:
            depth = self._rankid2depth(rankid)

        if depth > 0:
            split_fiber = deepcopy(self)
            update_lambda = lambda p: p.splitNonUniform(splits, partitions, relativeCoords)
            split_fiber.updatePayloads(update_lambda, depth=depth-1)
            return split_fiber

        splitter = _SplitterNonUniform(splits)

        return self._splitGeneric(splitter,
                                  partitions,
                                  relativeCoords=relativeCoords)


    def splitEqual(self, step, partitions=1, relativeCoords=False, depth=0, rankid=None):
        """Split a fiber equally in postion space

        Parameters
        ----------
        step: integer
            The `step` in number of elements in each split

        relative_coords: Bool
            Should the coordinate in the split fibers match the
            original coordinates (`relativeCoords`=False) or always
            start at zero (`relativeCoords`=True)

        depth: integer, default=0
            The depth in the fibertree to perform the split

        rankid: string, default=None
            The name of a rank, i.e., a rankid, at which to perform
            the split, overrides the `depth` argument.

        Returns
        -------
        split_fiber: Fiber
            A fibertree with one more level corresonding to the
            splits of the original fiber

        Notes:
        -------
        References to pieces of the original tensor may be returned as
        pieces of the returned tensor.

        """

        class _SplitterEqual():

            def __init__(self, step):
                self.step = step
                self.cur_count = 0

            def nextGroup(self, i, c):
                count = 0

                while i >= self.cur_count:
                    count += 1
                    self.cur_count += self.step

                return count, c

        if rankid is not None:
            depth = self._rankid2depth(rankid)

        if depth > 0:
            split_fiber = deepcopy(self)
            update_lambda = lambda p: p.splitEqual(step, partitions, relativeCoords)
            split_fiber.updatePayloads(update_lambda, depth=depth-1)
            return split_fiber

        splitter = _SplitterEqual(step)

        return self._splitGeneric(splitter,
                                  partitions,
                                  relativeCoords=relativeCoords)


    def splitUnEqual(self, sizes, partitions=1, relativeCoords=False, depth=0, rankid=None):
        """Split a fiber unequally in postion space

        Split a fiber by the sizes in `sizes`.

        Parameters
        ----------
        sizes: list of integers
            The `sizes` of the splits in number of elements in each split

        relative_coords: Bool
            Should the coordinate in the split fibers match the
            original coordinates (`relativeCoords`=False) or always
            start at zero (`relativeCoords`=True)

        depth: integer, default=0
            The depth in the fibertree to perform the split

        rankid: string, default=None
            The name of a rank, i.e., a rankid, at which to perform
            the split, overrides the `depth` argument.

        Returns
        -------
        split_fiber: Fiber
            A fibertree with one more level corresonding to the
            splits of the original fiber

        Notes
        ------
        If there are more coordinates than the sum of the `sizes` all
        remaining coordinates are put into the final split.

        References to pieces of the original tensor may be returned as
        pieces of the returned tensor.

        """

        class _SplitterUnEqual():

            def __init__(self, sizes):
                self.sizes = sizes.copy()
                self.cur_count = -1

            def nextGroup(self, i, c):
                count = 0

                while i > self.cur_count:
                    count += 1
                    if self.sizes:
                        self.cur_count += self.sizes.pop(0)
                    else:
                        self.cur_count = float("inf")

                return count, c

        if rankid is not None:
            depth = self._rankid2depth(rankid)

        if depth > 0:
            split_fiber = deepcopy(self)
            update_lambda = lambda p: p.splitUnEqual(sizes, partitions, relativeCoords)
            split_fiber.updatePayloads(update_lambda, depth=depth-1)
            return split_fiber

        splitter = _SplitterUnEqual(sizes)

        return self._splitGeneric(splitter,
                                  partitions,
                                  relativeCoords=relativeCoords)


    def _rankid2depth(self, rankid):
        """_rankid2depth

        Finds the depth of a given rankid
        """

        owner = self.getOwner()

        assert owner is not None, "Rankids exist only for fibers in a tensor"

        rankids = owner.getRankIds()

        return rankids.index(rankid)



    def _splitGeneric(self, splitter, partitions, relativeCoords):
        """_splitGeneric

        Takes the current fiber and splits it according to the
        boundaries defined by splitter().  The result is a new rank
        (for paritions = 1) or two new ranks (for partitions > 1).

        rank2 - uppermost rank with one coordinate per partition
                only exists for partitions > 1
        rank1 - middle rank with one coordinate per split
        rank0 - lowest rank with fibers split out from the original fiber

        """

        rank0_fiber_group = []
        rank0_fiber_coords = []
        rank0_fiber_payloads = []

        rank1_fiber_coords = []
        rank1_fiber_payloads = []

        # Create arrays for rank1 fibers per partition

        for i in range(partitions):
            rank1_fiber_coords.append([])
            rank1_fiber_payloads.append([])

        cur_coords = None
        rank1_count = -1

        # Split apart the fiber into groups according to "splitter"

        for i0, (c0, p0) in enumerate(zip(self.coords, self.payloads)):
            # Check if we need to start a new rank0 fiber
            count, next_rank1_coord = splitter.nextGroup(i0, c0)
            if (count > 0):
                rank1_count += count

                # Old style: upper rank's coordinates were a dense range
                # rank1_coord = rank1_count

                # New style: upper rank's coordinates are first coordinate of group
                rank1_coord = next_rank1_coord
                rank0_offset = rank1_coord

                rank0_fiber_group.append(rank1_coord)

                cur_coords = []
                rank0_fiber_coords.append(cur_coords)

                cur_payloads = []
                rank0_fiber_payloads.append(cur_payloads)

            # May not be in a group yet
            if cur_coords is not None:
                if relativeCoords:
                    cur_coords.append(c0 - rank0_offset)
                else:
                    cur_coords.append(c0)

                cur_payloads.append(p0)


        # Deal the split fibers out to the partitions

        partition = 0

        for c1, c0, p0 in zip(rank0_fiber_group,
                              rank0_fiber_coords,
                              rank0_fiber_payloads):

            rank1_fiber_coords[partition].append(c1)
            rank1_fiber_payloads[partition].append(Fiber(c0, p0))
            partition = (partition + 1) % partitions

        # For 1 partition don't return a extra level of Fiber

        if partitions == 1:
            fiber = self._newFiber(rank1_fiber_coords[0], rank1_fiber_payloads[0])
            fiber._setDefault(Fiber())
            return fiber

        # For >1 partitions return a Fiber with a payload for each partition

        payloads = []

        for c1, p1 in zip(rank1_fiber_coords, rank1_fiber_payloads):
            payload = self._newFiber(c1, p1)
            payloads.append(payload)

        return self._newFiber(payloads=payloads)

#
# Operation methods
#

    def concat(self, other):
        """ concat

        Concatenate two fibers

        TBD: Make sure coordinates are monitonically increasing

        """

        assert Payload.contains(other, Fiber), \
            "Fiber concatenation must involve two fibers"

        #
        # TBD: Set default for Fiber
        #
        return self._newFiber(coords=self.coords + other.coords,
                              payloads=self.payloads + other.payloads)



#
# Aggretated intersection/union methods
#

    @staticmethod
    def intersection(*args):
        """Intersect a set of fibers.

        Create a new fiber containing all the coordinates that are
        common to **all** the fibers in `args` and for each of those
        coordinates create a payload that is the combination of the
        payloads of all the input fibers. Note, however, unlike a
        sequence of two-operand intersections (see Fiber.__and__()`)
        the payloads are combined together in one long `tuple`.

        Parameters
        ----------

        args: list of Fibers
            The set of fibers to intersect

        Returns
        -------

        result: Fiber
            A fiber containing the intersection of all the input fibers.


        Note
        ----

        Currently only supported for "ordered", "unique" fibers.

        """

        nested_result = args[0] & args[1]

        for arg in args[2:]:
            nested_result = nested_result & arg

        c_result = []
        p_result = []

        for c_nested, p_nested in nested_result:
            #
            # Add coordinate
            #
            c_result.append(c_nested)

            #
            # Get out payload value
            #
            p_nested = Payload.get(p_nested)

            #
            # Create flattened payload
            #
            p = ()
            while isinstance(p_nested, tuple):
                p = (p_nested[1],) + p
                p_nested = p_nested[0]
                p_nested = Payload.get(p_nested)


            p = (Payload.maybe_box(p_nested),) + p

            p_result.append(Payload(p))

        result = Fiber(coords=c_result, payloads=p_result)
        result._setDefault(tuple([arg.getDefault() for arg in args]))

        return result

    @staticmethod
    def union(*args):
        """Union a set of fibers.

        Create a new fiber containing the coordinates that exist in
        **any** of the fibers in `args` and for each of those
        coordinates create a payload that is the combination of the
        payloads of all the input fibers. Note, however, unlike a
        sequence of two-operand unions (see `Fiber.__or__()`) the
        payloads are combined together in one long `tuple` with a mask
        at the begining indicating all the fibers that had a non-empty
        payload at that coordinate.

        Parameters
        ----------

        args: list of Fibers
            The set of fibers to union

        Returns
        -------

        result: Fiber
            A fiber containing the union of all the input fibers.

        Note
        ----

        Currently only supported for "ordered", "unique" fibers.

        """

        nested_result = args[0] | args[1]

        for arg in args[2:]:
            nested_result = nested_result | arg

        c_result = []
        p_result = []

        for c_nested, p_nested in nested_result:
            #
            # Add coordinate
            #
            c_result.append(c_nested)

            #
            # Get out payload value
            #
            p_nested = p_nested.value

            #
            # Create flattened payload
            #
            p = ()
            mask=""
            mask_num = ord("A")+len(args)

            while isinstance(p_nested, tuple):
                mask_num -= 1
                ab_mask = p_nested[0]

                if "B" in ab_mask:
                    mask = chr(mask_num) + mask

                p = (p_nested[2],) + p
                p_nested = Payload.get(p_nested[1])

            if "A" in ab_mask:
                mask = "A" + mask

            p = (mask, Payload.maybe_box(p_nested),) + p

            p_result.append(Payload(p))

        result = Fiber(coords=c_result, payloads=p_result)
        result._setDefault(tuple([""]+[arg.getDefault() for arg in args]))

        return result



#
# Merge methods
#
    def __and__(self, other):
        """Two-operand intersection

        Return the intersection of `self` and `other` by considering
        all possible coordinates and returning a fiber consisting of
        payloads containing a tuple of the payloads of the inputs for
        coordinates where the following truth table returns True:

        ```
                         coordinate not     |      coordinate
                        present in `other`  |    present in `other`
                    +-----------------------+-----------------------+
                    |                       |                       |
        coordinate  |                       |                       |
        not present |         False         |        False          |
        in `self`   |                       |                       |
                    |                       |                       |
        ------------+-----------------------+-----------------------+
                    |                       |                       |
        coordinate  |                       |                       |
        present in  |         False         |        True           |
        `self`      |                       |                       |
                    |                       |                       |
        ------------+-----------------------+-----------------------+
        ```

        Parameters
        ----------
        other: Fiber
            A fiber to intersect with the current fiber


        Returns
        --------
        result: Fiber
            A fiber created according to the intersection rules

        Note
        ----

        Currently only supported for "ordered", "unique" fibers.

        """

        def get_next(iter):
            """get_next"""

            try:
                coord, payload = next(iter)
            except StopIteration:
                return (None, None)

            return CoordPayload(coord, payload)


        def get_next_nonempty(iter):
            """get_next_nonempty"""

            (coord, payload) = get_next(iter)

            while Payload.isEmpty(payload):
                (coord, payload) = get_next(iter)

            return CoordPayload(coord, payload)

        assert self._ordered and self._unique

        a_fiber = self
        b_fiber = other

        a = self.__iter__()
        b = other.__iter__()

        z_coords = []
        z_payloads = []

        a_coord, a_payload = get_next_nonempty(a)
        b_coord, b_payload = get_next_nonempty(b)

        while not (a_coord is None or b_coord is None):
            if a_coord == b_coord:
                z_coords.append(a_coord)
                z_payloads.append((a_payload, b_payload))

                a_coord, a_payload = get_next_nonempty(a)
                b_coord, b_payload = get_next_nonempty(b)
                continue

            if a_coord < b_coord:
                a_coord, a_payload = get_next_nonempty(a)
                continue

            if a_coord > b_coord:
                b_coord, b_payload = get_next_nonempty(b)
                continue

        result = Fiber(z_coords, z_payloads)
        result._setDefault((a_fiber.getDefault(), b_fiber.getDefault()))

        return result


    def __or__(self, other):
        """__or__

        Return the union of `self` and `other` by considering all possible
        coordinates and returning a fiber consisting of payloads containing
        a tuple of the payloads of the inputs for coordinates where the
        following truth table returns True:


        ```
                         coordinate not     |      coordinate
                        present in `other`  |    present in `other`
                    +-----------------------+-----------------------+
                    |                       |                       |
        coordinate  |                       |                       |
        not present |         False         |        True           |
        in `self`   |                       |                       |
                    |                       |                       |
        ------------+-----------------------+-----------------------+
                    |                       |                       |
        coordinate  |                       |                       |
        present in  |         True          |        True           |
        `self`      |                       |                       |
                    |                       |                       |
        ------------+-----------------------+-----------------------+
        ```

        Parameters
        ----------
        other: Fiber
            A fiber to union with the current fiber


        Returns
        --------
        result: Fiber
            A fiber created according to the union rules

        Note
        ----

        Currently only supported for "ordered", "unique" fibers.

        """


        def get_next(iter):
            """get_next"""

            try:
                coord, payload = next(iter)
            except StopIteration:
                return (None, None)

            return CoordPayload(coord, payload)

        def get_next_nonempty(iter):
            """get_next_nonempty"""

            (coord, payload) = get_next(iter)

            while Payload.isEmpty(payload):
                (coord, payload) = get_next(iter)

            return CoordPayload(coord, payload)

        assert self._ordered and self._unique

        a_fiber = self
        b_fiber = other

        a = self.__iter__()
        b = other.__iter__()

        z_coords = []
        z_payloads = []

        a_coord, a_payload = get_next_nonempty(a)
        b_coord, b_payload = get_next_nonempty(b)

        while not (a_coord is None or b_coord is None):
            if a_coord == b_coord:
                z_coords.append(a_coord)

                z_payloads.append(("AB", a_payload, b_payload))

                a_coord, a_payload = get_next_nonempty(a)
                b_coord, b_payload = get_next_nonempty(b)
                continue

            if a_coord < b_coord:
                z_coords.append(a_coord)

                b_default = b_fiber._createDefault()
                z_payloads.append(("A", a_payload, b_default))

                a_coord, a_payload = get_next_nonempty(a)
                continue

            if a_coord > b_coord:
                z_coords.append(b_coord)

                a_default = a_fiber._createDefault()
                z_payloads.append(("B", a_default, b_payload))

                b_coord, b_payload = get_next_nonempty(b)
                continue

        while a_coord is not None:
            z_coords.append(a_coord)

            b_default = b_fiber._createDefault()
            z_payloads.append(("A", a_payload, b_default))

            a_coord, a_payload = get_next_nonempty(a)

        while b_coord is not None:
            z_coords.append(b_coord)

            a_default = a_fiber._createDefault()
            z_payloads.append(("B", a_default, b_payload))

            b_coord, b_payload = get_next_nonempty(b)

        result = Fiber(z_coords, z_payloads)
        result._setDefault(("", a_fiber.getDefault(), b_fiber.getDefault()))

        return result


    def __xor__(self, other):
        """__xor__

        Return the xor of `self` and `other` by considering all possible
        coordinates and returning a fiber consisting of payloads containing
        a tuple of the payloads of the inputs for coordinates where the
        following truth table returns True:


        ```
                         coordinate not     |      coordinate
                        present in `other`  |    present in `other`
                    +-----------------------+-----------------------+
                    |                       |                       |
        coordinate  |                       |                       |
        not present |         False         |        True           |
        in `self`   |                       |                       |
                    |                       |                       |
        ------------+-----------------------+-----------------------+
                    |                       |                       |
        coordinate  |                       |                       |
        present in  |         True          |        False          |
        `self`      |                       |                       |
                    |                       |                       |
        ------------+-----------------------+-----------------------+
        ```

        Parameters
        ----------
        other: Fiber
            A fiber to xor with the current fiber


        Returns
        --------
        result: Fiber
            A fiber created according to the xor rules

        Note
        ----

        Currently only supported for "ordered", "unique" fibers.

        """


        def get_next(iter):
            """get_next"""

            try:
                coord, payload = next(iter)
            except StopIteration:
                return (None, None)
            return CoordPayload(coord, payload)

        def get_next_nonempty(iter):
            """get_next_nonempty"""

            (coord, payload) = get_next(iter)

            while Payload.isEmpty(payload):
                (coord, payload) = get_next(iter)

            return CoordPayload(coord, payload)

        assert self._ordered and self._unique

        a_fiber = self
        b_fiber = other

        a = self.__iter__()
        b = other.__iter__()

        z_coords = []
        z_payloads = []

        a_coord, a_payload = get_next_nonempty(a)
        b_coord, b_payload = get_next_nonempty(b)

        while not (a_coord is None or b_coord is None):
            if a_coord == b_coord:
                a_coord, a_payload = get_next_nonempty(a)
                b_coord, b_payload = get_next_nonempty(b)
                continue

            if a_coord < b_coord:
                z_coords.append(a_coord)

                b_default = b_fiber._createDefault()
                z_payloads.append(("A", a_payload, b_default))

                a_coord, a_payload = get_next_nonempty(a)
                continue

            if a_coord > b_coord:
                z_coords.append(b_coord)

                a_default = a_fiber._createDefault()
                z_payloads.append(("B", a_default, b_payload))

                b_coord, b_payload = get_next_nonempty(b)
                continue

        while a_coord is not None:
            z_coords.append(a_coord)

            b_default = b_fiber._createDefault()
            z_payloads.append(("A", a_payload, b_default))

            a_coord, a_payload = get_next_nonempty(a)

        while b_coord is not None:
            z_coords.append(b_coord)

            a_default = a_fiber._createDefault()
            z_payloads.append(("B", a_default, b_payload))

            b_coord, b_payload = get_next_nonempty(b)

        result = Fiber(z_coords, z_payloads)
        result._setDefault(("", a_fiber.getDefault(), b_fiber.getDefault()))

        return result



    def __lshift__(self, other):
        """Fiber assignment

        Return the "assignment" of `other` to `self` by considering
        all possible coordinates and returning a fiber consisting of
        payloads containing a tuple of the payloads of the inputs for
        coordinates where the following truth table returns True:


        ```
                         coordinate not     |      coordinate
                        present in `other`  |    present in `other`
                    +-----------------------+-----------------------+
                    |                       |                       |
        coordinate  |                       |                       |
        not present |         False         |        True           |
        in `self`   |                       |                       |
                    |                       |                       |
        ------------+-----------------------+-----------------------+
                    |                       |                       |
        coordinate  |                       |                       |
        present in  |         False         |        True           |
        `self`      |                       |                       |
                    |                       |                       |
        ------------+-----------------------+-----------------------+
        ```

        Parameters
        ----------
        other: Fiber
            A fiber to assign into the current fiber


        Returns
        --------
        result: Fiber
            A fiber created according to the assignment rules


        Notes
        ------

        An explcit zero in the input will NOT generate a corresponding
        coordinate in the output!

        """


        def get_next(iter):
            """get_next"""

            try:
                coord, payload = next(iter)
            except StopIteration:
                return (None, None)

            return CoordPayload(coord, payload)

        def get_next_nonempty(iter):
            """get_next_nonempty"""

            (coord, payload) = get_next(iter)

            while Payload.isEmpty(payload):
                (coord, payload) = get_next(iter)

            return CoordPayload(coord, payload)

        a_fiber = self
        b_fiber = other

        # "a" is self!
        b = other.__iter__()

        z_coords = []
        z_a_payloads = []
        z_b_payloads = []
        z_payloads = []

        b_coord, b_payload = get_next_nonempty(b)

        while b_coord is not None:
            z_coords.append(b_coord)

            # TBD: Optimize with co-iteration...

            a_payload = self.getPayload(b_coord, allocate=False)

            z_a_payloads.append(a_payload)
            z_b_payloads.append(b_payload)

            b_coord, b_payload = get_next_nonempty(b)

        #
        # Collect z_payloads allowing for repeated coordinates
        #
        for b_coord, a_payload, b_payload in zip(z_coords, z_a_payloads, z_b_payloads):

            if a_payload is None:
                a_payload = self._create_payload(b_coord)

            z_payloads.append((a_payload, b_payload))

        result = self._newFiber(z_coords, z_payloads)

        return result


    def __sub__(self, other):
        """__sub__

        Return the "difference" of `other` from `self` by considering
        all possible coordinates and returning a fiber consisting of
        payloads containing a tuple of the payloads of the inputs for
        coordinates where the following truth table returns True:


        ```
                         coordinate not     |      coordinate
                        present in `other`  |    present in `other`
                    +-----------------------+-----------------------+
                    |                       |                       |
        coordinate  |                       |                       |
        not present |         False         |        False          |
        in `self`   |                       |                       |
                    |                       |                       |
        ------------+-----------------------+-----------------------+
                    |                       |                       |
        coordinate  |                       |                       |
        present in  |          True         |        False          |
        `self`      |                       |                       |
                    |                       |                       |
        ------------+-----------------------+-----------------------+
        ```

        Parameters
        ----------
        other: Fiber
            A fiber to subtract from the current fiber


        Returns
        --------
        result: Fiber
            A fiber created according to the subtraction rules


        Note
        ----

        Currently only supported for "ordered", "unique" fibers.

        """


        def get_next(iter):
            """get_next"""

            try:
                coord, payload = next(iter)
            except StopIteration:
                return (None, None)
            return CoordPayload(coord, payload)

        def get_next_nonempty(iter):
            """get_next_nonempty"""

            (coord, payload) = get_next(iter)

            while Payload.isEmpty(payload):
                (coord, payload) = get_next(iter)

            return CoordPayload(coord, payload)

        assert self._ordered and self._unique

        a_fiber = self
        b_fiber = other

        a = self.__iter__()
        b = other.__iter__()

        z_coords = []
        z_payloads = []

        a_coord, a_payload = get_next(a)
        b_coord, b_payload = get_next_nonempty(b)

        while not (a_coord is None or b_coord is None):
            if a_coord == b_coord:
                a_coord, a_payload = get_next(a)
                b_coord, b_payload = get_next_nonempty(b)
                continue

            if a_coord < b_coord:
                z_coords.append(a_coord)
                z_payloads.append(a_payload)

                a_coord, a_payload = get_next(a)
                continue

            if a_coord > b_coord:
                b_coord, b_payload = get_next(b)
                continue

        while a_coord is not None:
            z_coords.append(a_coord)
            z_payloads.append(a_payload)

            a_coord, a_payload = get_next(a)

        result = Fiber(z_coords, z_payloads)
        result._setDefault(a_fiber.getDefault())

        return result


#
# Multilayer methods
#
# Note: all these methods return a new fiber
#
    def swapRanks(self):
        """Swap the (highest) two ranks of the fiber.

        By swapping two ranks this method effects the equivalent of
        merging those two ranks.

        Returns
        -------

        result: Fiber
            The result of swapping the top two ranks of the fiber

        Notes
        -----

        This function relies on flattenRanks() and unflattenRanks().
        FIXME: flattenRanks() could be more general to support all p1 types,
        including tuples.

        Note
        ----

        Currently only supported for "ordered", "unique" fibers.

        """

        assert self._ordered and self._unique

        #
        # Flatten the (highest) two ranks
        #
        flattened = self.flattenRanks(style="pair")

        # Make sure that the flattened fiber has at least one coordinate
        assert(len(flattened.coords) > 0)

        # Make sure the coord is a >=2-element tuple
        assert(len(flattened.coords[0]) >= 2)

        #
        # Sort on secord coordinate of flattened fiber
        # and create new sorted fiber
        #
        sorted_cp = sorted([(c[::-1], p) for c, p in flattened])

        coords = [c for c, _ in sorted_cp]
        payloads = [p for _, p in sorted_cp]

        flattened_sorted = Fiber(coords, payloads)

        #
        # Unflatten to get original coordinates in swapped ranks
        #
        swapped = flattened_sorted.unflattenRanks()

        #
        # TBD: set default
        #
        return swapped


    def flattenRanks(self, levels=1, style="tuple"):
        """Flatten two ranks into one - COO-style

        Takes `levels` ranks and **flattens** them into a single
        rank.The coordinates of the combined rank can be created with
        a specified `style`:

        - tuple - flattened tuple of coordinates from all flattend ranks
        - pair - nested tuples of coordinates from all flattend ranks
        - absolute - the coordinate of the lowest rank
        - relative -  the sum of the corrdinate of the flattened ranks

        Parameters
        ----------

        levels: integer
            Number of levels to flatten into the top level

        style: One of ['tuple', 'pair', 'absolute', 'relative'], default='tuple'


        Returns
        -------

        result: Fiber
            Fiber with `level` ranks flattened into the current rank

        Note
        ----

        Currently only supported for "ordered", "unique" fibers.

        """

        assert self._ordered and self._unique

        #
        # Flatten deeper levels first, if requested
        #
        if levels == 1:
            cur_payloads = self.payloads
        else:
            assert Payload.contains(self.payloads[0], Fiber), \
                "Insuffient levels to flatten"

            cur_payloads = []

            for p in self.payloads:
                cur_payloads.append(p.flattenRanks(levels=levels - 1, style=style))

        #
        # Flatten this level
        #
        coords = []
        payloads = []

        for c1, p1 in zip(self.coords, cur_payloads):

            if Payload.contains(p1, Fiber):
                #
                # Handle case where payload of next rank is a Fiber
                #
                p1 = Payload.get(p1)

                for c0, p0 in p1:
                    coords.append(self._flattenCoords(c1,
                                                      c0,
                                                      style=style))
                    payloads.append(p0)

            elif Payload.contains(p1, tuple):
                #
                # Handle case where payload is a tuple. In this case,
                # look for the first fiber in the "p1" tuple and use
                # that fiber to flatten the ranks. The payloads of the
                # new flattened rank is a new tuple containing a
                # payload from that fiber and the the other values in
                # the "p1" tuple.
                #
                # Note: unflattening the fiber tree created in this
                # scenario will not recreate the original pre-flattened
                # fiber tree
                #
                p1 = Payload.get(p1)

                #
                # Find the fiber..
                #
                p1_fiber = None

                for n, p1_p in enumerate(p1):
                    if Payload.contains(p1_p, Fiber):
                        p1_fiber = p1_p

                        if len(p1) == 2:
                            # value is other element of tuple
                            p1_value = p1[(n + 1) % 2]
                        else:
                            # value is tuple without fiber
                            p1_value = p1[:n] + p1[n + 1:]

                        break

                if p1_fiber is None:
                    raise PayloadError

                #
                # Create the flattened fiber
                #
                for c0, p0 in p1_fiber:
                    coords.append(self._flattenCoords(c1,
                                                      c0,
                                                      style=style))

                    payloads.append((p0, p1_value))
            else:
                #
                # I don't know how to handle this payload
                #
                raise PayloadError
        #
        # TBD: set default
        #
        return Fiber(coords, payloads)

    @staticmethod
    def _flattenCoords(c1, c0, style="nested"):
        """_flattenCoords"""

        if style == "tuple":
            #
            # Combine coordinates into a single flat tuple, flattening
            # contents of the individual coordinates that are tuples
            #
            # Convert c1 to tuple, if necessary
            #
            if not isinstance(c1, tuple):
                c1 = (c1, )

            #
            # Convert c0 to tuple, if necessary
            #
            if not isinstance(c0, tuple):
                c0 = (c0,)

            #
            # Concatenate the two coordinates
            #
            c1_c0 = c1 + c0
        elif style == "pair":
            #
            # Create a new coordinate as a two element tuple
            #
            c1_c0 = (c1, c0)
        elif style == "absolute":
            c1_c0 = c0
        elif style == "relative":
            c1_c0 = c1 + c0
        else:
            assert False, \
                f"Supported coordinate styles are: tuple, pair, absolute, relative. Got: {style}"

        return c1_c0


    def unflattenRanks(self, levels=1):
        """Unflatten a ranks into two or more ranks

        Takes a single level of a fiber and expands extract out
        `levels` more levels.

        Parameters
        ----------

        levels: integer
           The number of extra levels to create


        Returns
        -------

        result: Fiber
            A fiber with `levels`' levels unflatten from the top level

        Notes
        -----

        This method may not produce the correct result if the original
        fiber was flattened with either the 'relative' or 'absolute'
        styles.

        Note
        ----

        Currently only supported for "ordered", "unique" fibers.

        """

        assert isinstance(self.coords[0], tuple)
        assert self._ordered and self._unique

        #
        # Place to collect cordinates/payloads for new top rank
        #
        coords1 = []
        payloads1 = []

        first = True

        #
        # Traverse the rank being unflattened in order to collect all
        # the coordinates/payloads to put in the final top (c1) rank
        #
        for cx, p0 in zip(self.coords, self.payloads):
            #
            # Little dance to get the coordinates of the two ranks,
            # which also deals with the case where the coordinates
            # were not nested.
            #
            c1 = cx[0]
            if len(cx) > 2:
                c0 = cx[1:]
            else:
                c0 = cx[1]

            #
            # Check if we finished all the elements of the lower (c0)
            # rank and have moved on to a new coordinate in the higher
            # (c1) rank. If so add the collected c0 coordinates and
            # payloads as a fiber to the c1 rank, and start a new
            # collection of coordinates and fibers.
            #
            if first or (c1 > c1_last):
                if not first:
                    #
                    # Except when starting to work on the first c1
                    # coordinate, add a new coordinate/payload pair
                    # (maybe after a recursive unflattening) to the
                    # new top (c1) rank
                    #
                    coords1.append(c1_last)

                    cur_fiber = Fiber(coords0, payloads0)
                    if levels > 1:
                        cur_fiber = cur_fiber.unflattenRanks(levels=levels - 1)

                    payloads1.append(cur_fiber)

                #
                # Start working on a new c1 coordinate (c1_last) and
                # create a place to collect lower rank (c0)
                # coordinates/payloads to form the payload (fiber) of
                # that c1 coordinate.
                #
                first = False
                c1_last = c1
                coords0 = []
                payloads0 = []

            coords0.append(c0)
            payloads0.append(p0)

        #
        # Pick up the last element of the new top rank
        #
        coords1.append(c1_last)

        cur_fiber = Fiber(coords0, payloads0)
        if levels > 1:
            cur_fiber = cur_fiber.unflattenRanks(levels=levels - 1)

        payloads1.append(cur_fiber)

        #
        # Create (and return) the new top (c1) rank
        #
        #
        # TBD: set default
        #
        return self._newFiber(coords1, payloads1)

#
# Closures to operate on all payloads at a specified depth
#
# Note: all these methods mutate the fibers
#
# TBD: Reimpliment with Guowei's cleaner Python closure/wrapper
#

    def updatePayloadsBelow(self, func, *args, depth=0, **kwargs):
        """updatePayloadsBelow

        Utility function used as a closure on updatePayloads() to
        change all the payloads in fibers at `depth` in the tree by
        applying `func` with parameters *args and **kwargs to the
        payloads.

        """

        update_lambda = lambda p: func(p, *args, **kwargs)
        return self.updatePayloads(update_lambda, depth=depth)


    splitUniformBelow = partialmethod(updatePayloadsBelow,
                                      splitUniform)

    splitNonUniformBelow = partialmethod(updatePayloadsBelow,
                                         splitNonUniform)

    splitEqualBelow = partialmethod(updatePayloadsBelow,
                                    splitEqual)

    splitUnEqualBelow = partialmethod(updatePayloadsBelow,
                                      splitUnEqual)

    swapRanksBelow = partialmethod(updatePayloadsBelow,
                                   swapRanks)

    flattenRanksBelow = partialmethod(updatePayloadsBelow,
                                      flattenRanks)

    unflattenRanksBelow = partialmethod(updatePayloadsBelow,
                                        unflattenRanks)


#
#  Comparison operations
#

    def __eq__(self, other):
        """__eq__ - Equality check for Fibers

        Note: explict zeros do not result in inequality
        """

        if not isinstance(other, Fiber):
            return False

        for c, (mask, ps, po) in self | other:
            if mask == "A" and not Payload.isEmpty(ps):
                return False

            if mask == "B" and not Payload.isEmpty(po):
                return False

            if mask == "AB" and not (ps == po):
                return False

        return True

#
#  String methods
#
    def print(self, title=None):
        """Print a fiber"""

        if title is not None:
            print("%s" % title)

        print("%s" % self)
        print("")

    def __format__(self, format):
        """__format__

        Format a fiber

        Spec:

        [(<coord spec>,<scalar spec>)][n][*]

        where:
                "n" means add newlines
                "*" means do not truncate with elipsis

        """
        import re

        kwargs = {}

        regex0 = r'(\(.*,.*\))?(n)?(\*)?'
        match0 = re.search(regex0, format)
        group1 = match0.group(1)

        if group1 is not None:
            regex1 = r'\((.*),(.*)\)'
            match1 = re.search(regex1, group1)
            kwargs['coord_fmt'] = match1.group(1)
            kwargs['payload_fmt'] = match1.group(2)

        if match0.group(2) == 'n':
            kwargs['newline'] = True

        if match0.group(3) == '*':
            kwargs['cutoff'] = 10000

        return self.__str__(**kwargs)


    def __str__(self,
                coord_fmt="d",
                payload_fmt="d",
                newline=False,
                cutoff=2,
                indent=0):
        """__str__"""

        def format_coord(coord):
            """Return "coord" properly formatted with "coord_fmt" """

            if not isinstance(coord, tuple):
                return f"{coord:{coord_fmt}}"

            return '(' + ', '.join(format_coord(c) for c in coord) + ')'


        def format_payload(payload):
            """Return "payload" properly formatted with "payload_fmt" """

            try:
                result = f"{payload:{payload_fmt}}"
            except Exception:
                result = f"{payload}"

            return result


        def cond_string(string):
            """Return "string" if newline is True"""

            if newline:
                return string

            return ''

        str = ''

        if self._owner is None:
            str += "F/["
        else:
            str += f"F({self._owner.getId()})/["

        coord_indent = 0
        next_indent = 0
        items = len(self.coords)

        if self.payloads and Payload.contains(self.payloads[0], Fiber):

            for (c, p) in zip(self.coords[0:cutoff], self.payloads[0:cutoff]):
                if coord_indent == 0:
                    coord_indent = indent + len(str)
                    str += f"( {format_coord(c)} -> "
                    if newline:
                        next_indent = indent + len(str)
                else:
                    str += cond_string('\n' + coord_indent * ' ')
                    str += f"( {format_coord(c)} -> "

                str += p.__str__(coord_fmt=coord_fmt,
                                 payload_fmt=payload_fmt,
                                 newline=newline,
                                 cutoff=cutoff,
                                 indent=next_indent)
                str += ')'

            if items > cutoff:
                str += cond_string('\n')
                str += next_indent * ' ' + "..."
                str += cond_string('\n')
                str += next_indent * ' ' + "..."

            return str

        if newline:
            next_indent = indent + len(str)

        for i in range(min(items, cutoff)):
            if coord_indent != 0:
                str += cond_string('\n')

            str += cond_string(coord_indent * ' ')
            str += f"({format_coord(self.coords[i])} -> "
            str += f"{format_payload(self.payloads[i])}) "
            coord_indent = next_indent

        if items > cutoff:
            str += cond_string('\n' + next_indent * ' ')
            str += " ... "
            str += cond_string('\n' + next_indent * ' ')
            str += " ... "

        str += "]"
        return str

    def __repr__(self):
        """__repr__"""

        # TBD: Owner is not properly reflected in representation

        payloads = [Payload.get(r) for r in self.payloads]
        str = f"Fiber({self.coords!r}, {payloads!r}"

        if self._owner:
            str += f", owner={self._owner.getId()}"

        str += ")"

        return str

#
# Yaml input/output methods
#

    @staticmethod
    def parse(yamlfile, default):
        """Parse a yaml file containing a tensor"""

        with open(yamlfile, 'r') as stream:
            try:
                y_file = yaml.safe_load(stream)
            except yaml.YAMLError as exc:
                print(exc)
                exit(1)

        #
        # Make sure key "fiber" exists
        #
        if not isinstance(y_file, dict) or 'fiber' not in y_file:
            print("Yaml is not a fiber")
            exit(1)

        newfiber = Fiber.dict2fiber(y_file)

        return (newfiber.getCoords(), newfiber.getPayloads())



    def dump(self, yamlfile):
        """Dump a tensor to a file in YAML format"""

        fiber_dict = self.fiber2dict()

        with open(yamlfile, 'w') as stream:
            yaml.dump(fiber_dict, stream)

#
# Conversion methods - to/from dictionaries
#

    @staticmethod
    def dict2fiber(y_payload_dict, level=0):
        """Parse a yaml-based tensor payload, creating Fibers as appropriate"""

        if isinstance(y_payload_dict, dict) and 'fiber' in y_payload_dict:
            # Got a fiber, so need to get into the Fiber class

            y_fiber = y_payload_dict['fiber']

            #
            # Error checking
            #
            if not isinstance(y_fiber, dict):
                print("Malformed payload")
                exit(0)

            if 'coords' not in y_fiber:
                print("Malformed fiber")
                exit(0)

            if 'payloads' not in y_fiber:
                print("Malformed fiber")
                exit(0)

            #
            # Process corrdinates and payloads
            #
            f_coords = y_fiber['coords']
            y_f_payloads = y_fiber['payloads']

            f_payloads = []
            for y_f_payload in y_f_payloads:
                f_payloads.append(Fiber.dict2fiber(y_f_payload, level + 1))

            #
            # Turn into a fiber
            #
            subtree = Fiber(coords=f_coords, payloads=f_payloads)
        else:
            # Got scalars, so format is unchanged
            subtree = y_payload_dict

        return subtree



    def fiber2dict(self):
        """Return dictionary with fiber information"""

        f = {'fiber':
             {'coords': self.coords,
              'payloads': [Payload.payload2dict(p) for p in self.payloads]}}

        return f

#
# Utility functions
#
    def _newFiber(self,
                  coords=None,
                  payloads=None,
                  default=None,
                  shape=None,
                  initial=None,
                  max_coord=None,
                  ordered=None,
                  unique=None):
        """Create a new fiber carrying over attributes from `self`

        Note: Input parameters must be kept in sync with `__init__`,
        and certain input parameters that come in as `None` take their
        values from `self`.

        """

        if ordered is None:
            ordered = self._ordered

        if unique is None:
            unique = self._unique

        if default is None:
            default = self._default

        if shape is None:
            default = self._shape

        return Fiber(coords=coords,
                     payloads=payloads,
                     default=default,
                     shape=shape,
                     initial=initial,
                     max_coord=max_coord,
                     ordered=ordered,
                     unique=unique)


    def _checkOrdered(self):
        """ Check that coordinates satisfy the "ordered" attribute """

        if not self._ordered or len(self.coords) == 0:
            return True

        coords = self.coords

        last = coords[0]

        for c in coords[1:]:
            if c <= last:
                assert False, "Illegal non-monotonic coordinate"

            last = c


    def _checkUnique(self):
        """ Check that coordinates satisfy the "unique" attribute """

        if not self._unique:
            return True

        if self._ordered:
            coords = self.coords
        else:
            coords = sorted(self.coords)

        last = None

        for c in coords:
            if c == last:
                assert False, "Illegal repeated coordinate"

            last = c


    @staticmethod
    def _deprecated(message):
        import warnings

        warnings.warn(message, FutureWarning, stacklevel=3)


#
# Pdoc stuff
#
__pdoc__ = {'Fiber.dict2fiber':    False,
            'Fiber.fiber2dict':    False,
            'Fiber.parse':         False,
            'Fiber.__setitem__':   True,
            'Fiber.__getitem__':   True,
            'Fiber.__ilshift__':   True,
            'Fiber.__truediv__':   True,
            'Fiber.__floordiv__':  True,
            'Fiber.__add__':  True,
            'Fiber.__radd__':  True,
            'Fiber.__iadd__':  True,
            'Fiber.__mul__':  True,
            'Fiber.__rmul__':  True,
            'Fiber.__imul__':  True,
            'Fiber.__and__':  True,
            'Fiber.__or__':  True,
            'Fiber.__xor__':  True,
            'Fiber.__lshift__':  True,
            'Fiber.__sub__':  True,
           }


if __name__ == "__main__":

    a = Fiber([2, 4, 6], [3, 5, 7])

    print("Simple print")
    a.print()
    print("----\n\n")
