text: /*
   * E57FormatImpl.cpp - implementation of private functions of E57 format.
   *
   * Copyright 2009 - 2010 Kevin Ackley (kackley@gwi.net)
   *
   * Permission is hereby granted, free of charge, to any person or organization
   * obtaining a copy of the software and accompanying documentation covered by
   * this license (the "Software") to use, reproduce, display, distribute,
   * execute, and transmit the Software, and to prepare derivative works of the
   * Software, and to permit third-parties to whom the Software is furnished to
   * do so, all subject to the following:
   *
   * The copyright notices in the Software and this entire statement, including
   * the above license grant, this restriction and the following disclaimer,
   * must be included in all copies of the Software, in whole or in part, and
   * all derivative works of the Software, unless such copies or derivative
   * works are solely in the form of machine-executable object code generated by
   * a source language processor.
   *
   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   * FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
   * SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
   * FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
   * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
   * DEALINGS IN THE SOFTWARE.
   */
  
  #include <cmath>
  #include <cstring>
  
  #include "E57FormatImpl.h"
  
  #include "CheckedFile.h"
  #include "Decoder.h"
  #include "Encoder.h"
  #include "ImageFileImpl.h"
  #include "SourceDestBufferImpl.h"
  
  using namespace e57;
  using namespace std;
  
  /// Section types
  enum
  {
     BLOB_SECTION = 0,
     COMPRESSED_VECTOR_SECTION,
  };
  
  struct BlobSectionHeader
  {
     const uint8_t sectionId = BLOB_SECTION;
  
     uint8_t reserved1[7] = {};         // must be zero
     uint64_t sectionLogicalLength = 0; // byte length of whole section
  
  #ifdef E57_DEBUG
     void dump( int indent = 0, std::ostream &os = std::cout );
  #endif
  };
  
  #ifdef E57_DEBUG
  void BlobSectionHeader::dump( int indent, std::ostream &os )
  {
     os << space( indent ) << "sectionId:            " << sectionId << endl;
     os << space( indent ) << "sectionLogicalLength: " << sectionLogicalLength << endl;
  }
  #endif
  
  struct CompressedVectorSectionHeader
  {
     const uint8_t sectionId = COMPRESSED_VECTOR_SECTION;
  
     uint8_t reserved1[7] = {};         // must be zero
     uint64_t sectionLogicalLength = 0; // byte length of whole section
     uint64_t dataPhysicalOffset = 0;   // offset of first data packet
     uint64_t indexPhysicalOffset = 0;  // offset of first index packet
  
     CompressedVectorSectionHeader();
     void verify( uint64_t filePhysicalSize = 0 );
  
  #ifdef E57_DEBUG
     void dump( int indent = 0, std::ostream &os = std::cout ) const;
  #endif
  };
  
  //================================================================================================
  
  VectorNodeImpl::VectorNodeImpl( ImageFileImplWeakPtr destImageFile, bool allowHeteroChildren ) :
     StructureNodeImpl( destImageFile ), allowHeteroChildren_( allowHeteroChildren )
  {
     /// don't checkImageFileOpen, StructNodeImpl() will do it
  }
  
  bool VectorNodeImpl::isTypeEquivalent( NodeImplSharedPtr ni )
  {
     /// don't checkImageFileOpen
  
     /// Same node type?
     if ( ni->type() != E57_VECTOR )
        return ( false );
  
     /// Downcast to shared_ptr<VectorNodeImpl>
     shared_ptr<VectorNodeImpl> ai( dynamic_pointer_cast<VectorNodeImpl>( ni ) );
     if ( !ai ) // check if failed
        throw E57_EXCEPTION2( E57_ERROR_INTERNAL,
                              "this->elementName=" + this->elementName() + " elementName=" + ni->elementName() );
  
     /// allowHeteroChildren must match
     if ( allowHeteroChildren_ != ai->allowHeteroChildren_ )
        return ( false );
  
     /// Same number of children?
     if ( childCount() != ai->childCount() )
        return ( false );
  
     /// Check each child, must be in same order
     for ( unsigned i = 0; i < childCount(); i++ )
     {
        if ( !children_.at( i )->isTypeEquivalent( ai->children_.at( i ) ) )
           return ( false );
     }
  
     /// Types match
     return ( true );
  }
  
  bool VectorNodeImpl::allowHeteroChildren() const
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return allowHeteroChildren_;
  }
  
  void VectorNodeImpl::set( int64_t index64, NodeImplSharedPtr ni )
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     if ( !allowHeteroChildren_ )
     {
        /// New node type must match all existing children
        for ( auto &child : children_ )
        {
           if ( !child->isTypeEquivalent( ni ) )
           {
              throw E57_EXCEPTION2( E57_ERROR_HOMOGENEOUS_VIOLATION, "this->pathName=" + this->pathName() );
           }
        }
     }
  
     ///??? for now, use base implementation
     StructureNodeImpl::set( index64, ni );
  }
  
  void VectorNodeImpl::writeXml( ImageFileImplSharedPtr imf, CheckedFile &cf, int indent, const char *forcedFieldName )
  {
     /// don't checkImageFileOpen
  
     ustring fieldName;
     if ( forcedFieldName != nullptr )
        fieldName = forcedFieldName;
     else
        fieldName = elementName_;
  
     cf << space( indent ) << "<" << fieldName << " type=\"Vector\" allowHeterogeneousChildren=\""
        << static_cast<int64_t>( allowHeteroChildren_ ) << "\">\n";
     for ( auto &child : children_ )
     {
        child->writeXml( imf, cf, indent + 2, "vectorChild" );
     }
     cf << space( indent ) << "</" << fieldName << ">\n";
  }
  
  #ifdef E57_DEBUG
  void VectorNodeImpl::dump( int indent, ostream &os ) const
  {
     /// don't checkImageFileOpen
     os << space( indent ) << "type:        Vector"
        << " (" << type() << ")" << endl;
     NodeImpl::dump( indent, os );
     os << space( indent ) << "allowHeteroChildren: " << allowHeteroChildren() << endl;
     for ( unsigned i = 0; i < children_.size(); i++ )
     {
        os << space( indent ) << "child[" << i << "]:" << endl;
        children_.at( i )->dump( indent + 2, os );
     }
  }
  #endif
  
  //=============================================================================
  CompressedVectorNodeImpl::CompressedVectorNodeImpl( ImageFileImplWeakPtr destImageFile ) : NodeImpl( destImageFile )
  {
     // don't checkImageFileOpen, NodeImpl() will do it
  }
  
  void CompressedVectorNodeImpl::setPrototype( const NodeImplSharedPtr &prototype )
  {
     // don't checkImageFileOpen, ctor did it
  
     //??? check ok for proto, no Blob CompressedVector, empty?
     //??? throw E57_EXCEPTION2(E57_ERROR_BAD_PROTOTYPE)
  
     /// Can't set prototype twice.
     if ( prototype_ )
        throw E57_EXCEPTION2( E57_ERROR_SET_TWICE, "this->pathName=" + this->pathName() );
  
     /// prototype can't have a parent (must be a root node)
     if ( !prototype->isRoot() )
     {
        throw E57_EXCEPTION2( E57_ERROR_ALREADY_HAS_PARENT,
                              "this->pathName=" + this->pathName() + " prototype->pathName=" + prototype->pathName() );
     }
  
     /// Verify that prototype is destined for same ImageFile as this is
     ImageFileImplSharedPtr thisDest( destImageFile() );
     ImageFileImplSharedPtr prototypeDest( prototype->destImageFile() );
     if ( thisDest != prototypeDest )
     {
        throw E57_EXCEPTION2( E57_ERROR_DIFFERENT_DEST_IMAGEFILE, "this->destImageFile" + thisDest->fileName() +
                                                                     " prototype->destImageFile" +
                                                                     prototypeDest->fileName() );
     }
  
     //!!! check for incomplete CompressedVectors when closing file
     prototype_ = prototype;
  
     /// Note that prototype is not attached to CompressedVector in a parent/child
     /// relationship. This means that prototype is a root node (has no parent).
  }
  
  NodeImplSharedPtr CompressedVectorNodeImpl::getPrototype()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( prototype_ ); //??? check defined
  }
  
  void CompressedVectorNodeImpl::setCodecs( const shared_ptr<VectorNodeImpl> &codecs )
  {
     // don't checkImageFileOpen, ctor did it
  
     //??? check ok for codecs, empty vector, or each element has "inputs" vector
     // of strings, codec
     // substruct
  
     /// Can't set codecs twice.
     if ( codecs_ )
        throw E57_EXCEPTION2( E57_ERROR_SET_TWICE, "this->pathName=" + this->pathName() );
  
     /// codecs can't have a parent (must be a root node)
     if ( !codecs->isRoot() )
     {
        throw E57_EXCEPTION2( E57_ERROR_ALREADY_HAS_PARENT,
                              "this->pathName=" + this->pathName() + " codecs->pathName=" + codecs->pathName() );
     }
  
     /// Verify that codecs is destined for same ImageFile as this is
     ImageFileImplSharedPtr thisDest( destImageFile() );
     ImageFileImplSharedPtr codecsDest( codecs->destImageFile() );
     if ( thisDest != codecsDest )
     {
        throw E57_EXCEPTION2( E57_ERROR_DIFFERENT_DEST_IMAGEFILE, "this->destImageFile" + thisDest->fileName() +
                                                                     " codecs->destImageFile" + codecsDest->fileName() );
     }
  
     codecs_ = codecs;
  
     /// Note that codecs is not attached to CompressedVector in a parent/child
     /// relationship. This means that codecs is a root node (has no parent).
  }
  
  shared_ptr<VectorNodeImpl> CompressedVectorNodeImpl::getCodecs()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( codecs_ ); //??? check defined
  }
  
  bool CompressedVectorNodeImpl::isTypeEquivalent( NodeImplSharedPtr ni )
  {
     // don't checkImageFileOpen
  
     //??? is this test a good idea?
  
     /// Same node type?
     if ( ni->type() != E57_COMPRESSED_VECTOR )
        return ( false );
  
     /// Downcast to shared_ptr<CompressedVectorNodeImpl>
     shared_ptr<CompressedVectorNodeImpl> cvi( dynamic_pointer_cast<CompressedVectorNodeImpl>( ni ) );
     if ( !cvi ) // check if failed
        throw E57_EXCEPTION2( E57_ERROR_INTERNAL,
                              "this->elementName=" + this->elementName() + " elementName=" + ni->elementName() );
  
     /// recordCount must match
     if ( recordCount_ != cvi->recordCount_ )
        return ( false );
  
     /// Prototypes and codecs must match ???
     if ( !prototype_->isTypeEquivalent( cvi->prototype_ ) )
        return ( false );
     if ( !codecs_->isTypeEquivalent( cvi->codecs_ ) )
        return ( false );
  
     return ( true );
  }
  
  bool CompressedVectorNodeImpl::isDefined( const ustring &pathName )
  {
     throw E57_EXCEPTION2( E57_ERROR_NOT_IMPLEMENTED, "this->pathName=" + this->pathName() + " pathName=" + pathName );
  }
  
  void CompressedVectorNodeImpl::setAttachedRecursive()
  {
     /// Mark this node as attached to an ImageFile
     isAttached_ = true;
  
     /// Mark nodes in prototype tree, if defined
     if ( prototype_ )
        prototype_->setAttachedRecursive();
  
     /// Mark nodes in codecs tree if defined
     if ( codecs_ )
        codecs_->setAttachedRecursive();
  }
  
  int64_t CompressedVectorNodeImpl::childCount()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( recordCount_ );
  }
  
  void CompressedVectorNodeImpl::checkLeavesInSet( const StringSet & /*pathNames*/, NodeImplSharedPtr /*origin*/ )
  {
     // don't checkImageFileOpen
  
     /// Since only called for prototype nodes, should't be able to get here since
     /// CompressedVectors can't be in prototypes
     throw E57_EXCEPTION2( E57_ERROR_INTERNAL, "this->pathName=" + this->pathName() );
  }
  
  void CompressedVectorNodeImpl::writeXml( ImageFileImplSharedPtr imf, CheckedFile &cf, int indent,
                                           const char *forcedFieldName )
  {
     // don't checkImageFileOpen
  
     ustring fieldName;
     if ( forcedFieldName != nullptr )
        fieldName = forcedFieldName;
     else
        fieldName = elementName_;
  
     uint64_t physicalStart = cf.logicalToPhysical( binarySectionLogicalStart_ );
  
     cf << space( indent ) << "<" << fieldName << " type=\"CompressedVector\"";
     cf << " fileOffset=\"" << physicalStart;
     cf << "\" recordCount=\"" << recordCount_ << "\">\n";
  
     if ( prototype_ )
        prototype_->writeXml( imf, cf, indent + 2, "prototype" );
     if ( codecs_ )
        codecs_->writeXml( imf, cf, indent + 2, "codecs" );
     cf << space( indent ) << "</" << fieldName << ">\n";
  }
  
  #ifdef E57_DEBUG
  void CompressedVectorNodeImpl::dump( int indent, ostream &os ) const
  {
     os << space( indent ) << "type:        CompressedVector"
        << " (" << type() << ")" << endl;
     NodeImpl::dump( indent, os );
     if ( prototype_ )
     {
        os << space( indent ) << "prototype:" << endl;
        prototype_->dump( indent + 2, os );
     }
     else
        os << space( indent ) << "prototype: <empty>" << endl;
     if ( codecs_ )
     {
        os << space( indent ) << "codecs:" << endl;
        codecs_->dump( indent + 2, os );
     }
     else
        os << space( indent ) << "codecs: <empty>" << endl;
     os << space( indent ) << "recordCount:                " << recordCount_ << endl;
     os << space( indent ) << "binarySectionLogicalStart:  " << binarySectionLogicalStart_ << endl;
  }
  #endif
  
  shared_ptr<CompressedVectorWriterImpl> CompressedVectorNodeImpl::writer( vector<SourceDestBuffer> sbufs )
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
  
     ImageFileImplSharedPtr destImageFile( destImageFile_ );
  
     /// Check don't have any writers/readers open for this ImageFile
     if ( destImageFile->writerCount() > 0 )
     {
        throw E57_EXCEPTION2( E57_ERROR_TOO_MANY_WRITERS, "fileName=" + destImageFile->fileName() +
                                                             " writerCount=" + toString( destImageFile->writerCount() ) +
                                                             " readerCount=" + toString( destImageFile->readerCount() ) );
     }
     if ( destImageFile->readerCount() > 0 )
     {
        throw E57_EXCEPTION2( E57_ERROR_TOO_MANY_READERS, "fileName=" + destImageFile->fileName() +
                                                             " writerCount=" + toString( destImageFile->writerCount() ) +
                                                             " readerCount=" + toString( destImageFile->readerCount() ) );
     }
  
     /// sbufs can't be empty
     if ( sbufs.empty() )
        throw E57_EXCEPTION2( E57_ERROR_BAD_API_ARGUMENT, "fileName=" + destImageFile->fileName() );
  
     if ( !destImageFile->isWriter() )
        throw E57_EXCEPTION2( E57_ERROR_FILE_IS_READ_ONLY, "fileName=" + destImageFile->fileName() );
  
     if ( !isAttached() )
        throw E57_EXCEPTION2( E57_ERROR_NODE_UNATTACHED, "fileName=" + destImageFile->fileName() );
  
     /// Get pointer to me (really shared_ptr<CompressedVectorNodeImpl>)
     NodeImplSharedPtr ni( shared_from_this() );
  
     /// Downcast pointer to right type
     shared_ptr<CompressedVectorNodeImpl> cai( dynamic_pointer_cast<CompressedVectorNodeImpl>( ni ) );
     if ( !cai ) // check if failed
        throw E57_EXCEPTION2( E57_ERROR_INTERNAL,
                              "this->elementName=" + this->elementName() + " elementName=" + ni->elementName() );
  
     /// Return a shared_ptr to new object
     shared_ptr<CompressedVectorWriterImpl> cvwi( new CompressedVectorWriterImpl( cai, sbufs ) );
     return ( cvwi );
  }
  
  shared_ptr<CompressedVectorReaderImpl> CompressedVectorNodeImpl::reader( vector<SourceDestBuffer> dbufs )
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
  
     ImageFileImplSharedPtr destImageFile( destImageFile_ );
  
     /// Check don't have any writers/readers open for this ImageFile
     if ( destImageFile->writerCount() > 0 )
     {
        throw E57_EXCEPTION2( E57_ERROR_TOO_MANY_WRITERS, "fileName=" + destImageFile->fileName() +
                                                             " writerCount=" + toString( destImageFile->writerCount() ) +
                                                             " readerCount=" + toString( destImageFile->readerCount() ) );
     }
     if ( destImageFile->readerCount() > 0 )
     {
        throw E57_EXCEPTION2( E57_ERROR_TOO_MANY_READERS, "fileName=" + destImageFile->fileName() +
                                                             " writerCount=" + toString( destImageFile->writerCount() ) +
                                                             " readerCount=" + toString( destImageFile->readerCount() ) );
     }
  
     /// dbufs can't be empty
     if ( dbufs.empty() )
        throw E57_EXCEPTION2( E57_ERROR_BAD_API_ARGUMENT, "fileName=" + destImageFile->fileName() );
  
     /// Can be read or write mode, but must be attached
     if ( !isAttached() )
        throw E57_EXCEPTION2( E57_ERROR_NODE_UNATTACHED, "fileName=" + destImageFile->fileName() );
  
     /// Get pointer to me (really shared_ptr<CompressedVectorNodeImpl>)
     NodeImplSharedPtr ni( shared_from_this() );
  #ifdef E57_MAX_VERBOSE
     // cout << "constructing CAReader, ni:" << endl;
     // ni->dump(4);
  #endif
  
     /// Downcast pointer to right type
     shared_ptr<CompressedVectorNodeImpl> cai( dynamic_pointer_cast<CompressedVectorNodeImpl>( ni ) );
     if ( !cai ) // check if failed
        throw E57_EXCEPTION2( E57_ERROR_INTERNAL,
                              "this->elementName=" + this->elementName() + " elementName=" + ni->elementName() );
  #ifdef E57_MAX_VERBOSE
        // cout<<"constructing CAReader, cai:"<<endl;
        // cai->dump(4);
  #endif
     /// Return a shared_ptr to new object
     shared_ptr<CompressedVectorReaderImpl> cvri( new CompressedVectorReaderImpl( cai, dbufs ) );
     return ( cvri );
  }
  
  //=====================================================================
  IntegerNodeImpl::IntegerNodeImpl( ImageFileImplWeakPtr destImageFile, int64_t value, int64_t minimum,
                                    int64_t maximum ) :
     NodeImpl( destImageFile ),
     value_( value ), minimum_( minimum ), maximum_( maximum )
  {
     // don't checkImageFileOpen, NodeImpl() will do it
  
     /// Enforce the given bounds
     if ( value < minimum || maximum < value )
     {
        throw E57_EXCEPTION2( E57_ERROR_VALUE_OUT_OF_BOUNDS,
                              "this->pathName=" + this->pathName() + " value=" + toString( value ) +
                                 " minimum=" + toString( minimum ) + " maximum=" + toString( maximum ) );
     }
  }
  
  bool IntegerNodeImpl::isTypeEquivalent( NodeImplSharedPtr ni )
  {
     // don't checkImageFileOpen
  
     /// Same node type?
     if ( ni->type() != E57_INTEGER )
        return ( false );
  
     /// Downcast to shared_ptr<IntegerNodeImpl>
     shared_ptr<IntegerNodeImpl> ii( dynamic_pointer_cast<IntegerNodeImpl>( ni ) );
     if ( !ii ) // check if failed
        throw E57_EXCEPTION2( E57_ERROR_INTERNAL,
                              "this->elementName=" + this->elementName() + " elementName=" + ni->elementName() );
  
     /// minimum must match
     if ( minimum_ != ii->minimum_ )
        return ( false );
  
     /// maximum must match
     if ( maximum_ != ii->maximum_ )
        return ( false );
  
     /// ignore value_, doesn't have to match
  
     /// Types match
     return ( true );
  }
  
  bool IntegerNodeImpl::isDefined( const ustring &pathName )
  {
     // don't checkImageFileOpen
  
     /// We have no sub-structure, so if path not empty return false
     return pathName.empty();
  }
  
  int64_t IntegerNodeImpl::value()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( value_ );
  }
  
  int64_t IntegerNodeImpl::minimum()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( minimum_ );
  }
  
  int64_t IntegerNodeImpl::maximum()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( maximum_ );
  }
  
  void IntegerNodeImpl::checkLeavesInSet( const StringSet &pathNames, NodeImplSharedPtr origin )
  {
     // don't checkImageFileOpen
  
     /// We are a leaf node, so verify that we are listed in set.
     if ( pathNames.find( relativePathName( origin ) ) == pathNames.end() )
        throw E57_EXCEPTION2( E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName() );
  }
  
  void IntegerNodeImpl::writeXml( ImageFileImplSharedPtr /*imf???*/, CheckedFile &cf, int indent,
                                  const char *forcedFieldName )
  {
     // don't checkImageFileOpen
  
     ustring fieldName;
     if ( forcedFieldName != nullptr )
        fieldName = forcedFieldName;
     else
        fieldName = elementName_;
  
     cf << space( indent ) << "<" << fieldName << " type=\"Integer\"";
  
     /// Don't need to write if are default values
     if ( minimum_ != E57_INT64_MIN )
        cf << " minimum=\"" << minimum_ << "\"";
     if ( maximum_ != E57_INT64_MAX )
        cf << " maximum=\"" << maximum_ << "\"";
  
     /// Write value as child text, unless it is the default value
     if ( value_ != 0 )
        cf << ">" << value_ << "</" << fieldName << ">\n";
     else
        cf << "/>\n";
  }
  
  #ifdef E57_DEBUG
  void IntegerNodeImpl::dump( int indent, ostream &os ) const
  {
     // don't checkImageFileOpen
     os << space( indent ) << "type:        Integer"
        << " (" << type() << ")" << endl;
     NodeImpl::dump( indent, os );
     os << space( indent ) << "value:       " << value_ << endl;
     os << space( indent ) << "minimum:     " << minimum_ << endl;
     os << space( indent ) << "maximum:     " << maximum_ << endl;
  }
  #endif
  
  //=============================================================================
  ScaledIntegerNodeImpl::ScaledIntegerNodeImpl( ImageFileImplWeakPtr destImageFile, int64_t rawValue, int64_t minimum,
                                                int64_t maximum, double scale, double offset ) :
     NodeImpl( destImageFile ),
     value_( rawValue ), minimum_( minimum ), maximum_( maximum ), scale_( scale ), offset_( offset )
  {
     // don't checkImageFileOpen, NodeImpl() will do it
  
     /// Enforce the given bounds on raw value
     if ( rawValue < minimum || maximum < rawValue )
     {
        throw E57_EXCEPTION2( E57_ERROR_VALUE_OUT_OF_BOUNDS,
                              "this->pathName=" + this->pathName() + " rawValue=" + toString( rawValue ) +
                                 " minimum=" + toString( minimum ) + " maximum=" + toString( maximum ) );
     }
  }
  //=============================================================================
  ScaledIntegerNodeImpl::ScaledIntegerNodeImpl( ImageFileImplWeakPtr destImageFile, double scaledValue,
                                                double scaledMinimum, double scaledMaximum, double scale,
                                                double offset ) :
     NodeImpl( destImageFile ),
     value_( static_cast<int64_t>( floor( ( scaledValue - offset ) / scale + .5 ) ) ),
     minimum_( static_cast<int64_t>( floor( ( scaledMinimum - offset ) / scale + .5 ) ) ),
     maximum_( static_cast<int64_t>( floor( ( scaledMaximum - offset ) / scale + .5 ) ) ), scale_( scale ),
     offset_( offset )
  {
     // don't checkImageFileOpen, NodeImpl() will do it
  
     /// Enforce the given bounds on raw value
     if ( scaledValue < scaledMinimum || scaledMaximum < scaledValue )
     {
        throw E57_EXCEPTION2( E57_ERROR_VALUE_OUT_OF_BOUNDS, "this->pathName=" + this->pathName() +
                                                                " scaledValue=" + toString( scaledValue ) +
                                                                " scaledMinimum=" + toString( scaledMinimum ) +
                                                                " scaledMaximum=" + toString( scaledMaximum ) );
     }
  }
  
  bool ScaledIntegerNodeImpl::isTypeEquivalent( NodeImplSharedPtr ni )
  {
     // don't checkImageFileOpen
  
     /// Same node type?
     if ( ni->type() != E57_SCALED_INTEGER )
        return ( false );
  
     /// Downcast to shared_ptr<ScaledIntegerNodeImpl>
     shared_ptr<ScaledIntegerNodeImpl> ii( dynamic_pointer_cast<ScaledIntegerNodeImpl>( ni ) );
     if ( !ii ) // check if failed
        throw E57_EXCEPTION2( E57_ERROR_INTERNAL,
                              "this->elementName=" + this->elementName() + " elementName=" + ni->elementName() );
  
     /// minimum must match
     if ( minimum_ != ii->minimum_ )
        return ( false );
  
     /// maximum must match
     if ( maximum_ != ii->maximum_ )
        return ( false );
  
     /// scale must match
     if ( scale_ != ii->scale_ )
        return ( false );
  
     /// offset must match
     if ( offset_ != ii->offset_ )
        return ( false );
  
     /// ignore value_, doesn't have to match
  
     /// Types match
     return ( true );
  }
  
  bool ScaledIntegerNodeImpl::isDefined( const ustring &pathName )
  {
     // don't checkImageFileOpen
  
     /// We have no sub-structure, so if path not empty return false
     return pathName.empty();
  }
  
  int64_t ScaledIntegerNodeImpl::rawValue()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( value_ );
  }
  
  double ScaledIntegerNodeImpl::scaledValue()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( value_ * scale_ + offset_ );
  }
  
  int64_t ScaledIntegerNodeImpl::minimum()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( minimum_ );
  }
  double ScaledIntegerNodeImpl::scaledMinimum()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( minimum_ * scale_ + offset_ );
  }
  
  int64_t ScaledIntegerNodeImpl::maximum()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( maximum_ );
  }
  double ScaledIntegerNodeImpl::scaledMaximum()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( maximum_ * scale_ + offset_ );
  }
  
  double ScaledIntegerNodeImpl::scale()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( scale_ );
  }
  
  double ScaledIntegerNodeImpl::offset()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( offset_ );
  }
  
  void ScaledIntegerNodeImpl::checkLeavesInSet( const StringSet &pathNames, NodeImplSharedPtr origin )
  {
     // don't checkImageFileOpen
  
     /// We are a leaf node, so verify that we are listed in set.
     if ( pathNames.find( relativePathName( origin ) ) == pathNames.end() )
        throw E57_EXCEPTION2( E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName() );
  }
  
  void ScaledIntegerNodeImpl::writeXml( ImageFileImplSharedPtr /*imf*/, CheckedFile &cf, int indent,
                                        const char *forcedFieldName )
  {
     // don't checkImageFileOpen
  
     ustring fieldName;
     if ( forcedFieldName != nullptr )
        fieldName = forcedFieldName;
     else
        fieldName = elementName_;
  
     cf << space( indent ) << "<" << fieldName << " type=\"ScaledInteger\"";
  
     /// Don't need to write if are default values
     if ( minimum_ != E57_INT64_MIN )
        cf << " minimum=\"" << minimum_ << "\"";
     if ( maximum_ != E57_INT64_MAX )
        cf << " maximum=\"" << maximum_ << "\"";
     if ( scale_ != 1.0 )
        cf << " scale=\"" << scale_ << "\"";
     if ( offset_ != 0.0 )
        cf << " offset=\"" << offset_ << "\"";
  
     /// Write value as child text, unless it is the default value
     if ( value_ != 0 )
        cf << ">" << value_ << "</" << fieldName << ">\n";
     else
        cf << "/>\n";
  }
  
  #ifdef E57_DEBUG
  void ScaledIntegerNodeImpl::dump( int indent, ostream &os ) const
  {
     // don't checkImageFileOpen
     os << space( indent ) << "type:        ScaledInteger"
        << " (" << type() << ")" << endl;
     NodeImpl::dump( indent, os );
     os << space( indent ) << "rawValue:    " << value_ << endl;
     os << space( indent ) << "minimum:     " << minimum_ << endl;
     os << space( indent ) << "maximum:     " << maximum_ << endl;
     os << space( indent ) << "scale:       " << scale_ << endl;
     os << space( indent ) << "offset:      " << offset_ << endl;
  }
  #endif
  
  //=============================================================================
  
  FloatNodeImpl::FloatNodeImpl( ImageFileImplWeakPtr destImageFile, double value, FloatPrecision precision,
                                double minimum, double maximum ) :
     NodeImpl( destImageFile ),
     value_( value ), precision_( precision ), minimum_( minimum ), maximum_( maximum )
  {
     // don't checkImageFileOpen, NodeImpl() will do it
  
     /// Since this ctor also used to construct single precision, and defaults for
     /// minimum/maximum are for double precision, adjust bounds smaller if
     /// single.
     if ( precision_ == E57_SINGLE )
     {
        if ( minimum_ < E57_FLOAT_MIN )
           minimum_ = E57_FLOAT_MIN;
        if ( maximum_ > E57_FLOAT_MAX )
           maximum_ = E57_FLOAT_MAX;
     }
  
     /// Enforce the given bounds on raw value
     if ( value < minimum || maximum < value )
     {
        throw E57_EXCEPTION2( E57_ERROR_VALUE_OUT_OF_BOUNDS,
                              "this->pathName=" + this->pathName() + " value=" + toString( value ) +
                                 " minimum=" + toString( minimum ) + " maximum=" + toString( maximum ) );
     }
  }
  
  bool FloatNodeImpl::isTypeEquivalent( NodeImplSharedPtr ni )
  {
     // don't checkImageFileOpen
  
     /// Same node type?
     if ( ni->type() != E57_FLOAT )
        return ( false );
  
     /// Downcast to shared_ptr<FloatNodeImpl>
     shared_ptr<FloatNodeImpl> fi( dynamic_pointer_cast<FloatNodeImpl>( ni ) );
     if ( !fi ) // check if failed
        throw E57_EXCEPTION2( E57_ERROR_INTERNAL,
                              "this->elementName=" + this->elementName() + " elementName=" + ni->elementName() );
  
     /// precision must match
     if ( precision_ != fi->precision_ )
        return ( false );
  
     /// minimum must match
     if ( minimum_ != fi->minimum_ )
        return ( false );
  
     /// maximum must match
     if ( maximum_ != fi->maximum_ )
        return ( false );
  
     /// ignore value_, doesn't have to match
  
     /// Types match
     return ( true );
  }
  
  bool FloatNodeImpl::isDefined( const ustring &pathName )
  {
     // don't checkImageFileOpen
  
     /// We have no sub-structure, so if path not empty return false
     return pathName.empty();
  }
  
  double FloatNodeImpl::value() const
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return value_;
  }
  
  FloatPrecision FloatNodeImpl::precision() const
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return precision_;
  }
  
  double FloatNodeImpl::minimum() const
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return minimum_;
  }
  
  double FloatNodeImpl::maximum() const
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return maximum_;
  }
  
  void FloatNodeImpl::checkLeavesInSet( const StringSet &pathNames, NodeImplSharedPtr origin )
  {
     // don't checkImageFileOpen
  
     /// We are a leaf node, so verify that we are listed in set (either relative
     /// or absolute form)
     if ( pathNames.find( relativePathName( origin ) ) == pathNames.end() &&
          pathNames.find( pathName() ) == pathNames.end() )
        throw E57_EXCEPTION2( E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName() );
  }
  
  void FloatNodeImpl::writeXml( ImageFileImplSharedPtr /*imf*/, CheckedFile &cf, int indent, const char *forcedFieldName )
  {
     // don't checkImageFileOpen
  
     ustring fieldName;
     if ( forcedFieldName != nullptr )
        fieldName = forcedFieldName;
     else
        fieldName = elementName_;
  
     cf << space( indent ) << "<" << fieldName << " type=\"Float\"";
     if ( precision_ == E57_SINGLE )
     {
        cf << " precision=\"single\"";
  
        /// Don't need to write if are default values
        if ( minimum_ > E57_FLOAT_MIN )
           cf << " minimum=\"" << static_cast<float>( minimum_ ) << "\"";
        if ( maximum_ < E57_FLOAT_MAX )
           cf << " maximum=\"" << static_cast<float>( maximum_ ) << "\"";
  
        /// Write value as child text, unless it is the default value
        if ( value_ != 0.0 )
           cf << ">" << static_cast<float>( value_ ) << "</" << fieldName << ">\n";
        else
           cf << "/>\n";
     }
     else
     {
        /// Don't need to write precision="double", because that's the default
  
        /// Don't need to write if are default values
        if ( minimum_ > E57_DOUBLE_MIN )
           cf << " minimum=\"" << minimum_ << "\"";
        if ( maximum_ < E57_DOUBLE_MAX )
           cf << " maximum=\"" << maximum_ << "\"";
  
        /// Write value as child text, unless it is the default value
        if ( value_ != 0.0 )
           cf << ">" << value_ << "</" << fieldName << ">\n";
        else
           cf << "/>\n";
     }
  }
  
  #ifdef E57_DEBUG
  void FloatNodeImpl::dump( int indent, ostream &os ) const
  {
     // don't checkImageFileOpen
     os << space( indent ) << "type:        Float"
        << " (" << type() << ")" << endl;
     NodeImpl::dump( indent, os );
     os << space( indent ) << "precision:   ";
     if ( precision() == E57_SINGLE )
        os << "single" << endl;
     else
        os << "double" << endl;
  
     /// Save old stream config
     const streamsize oldPrecision = os.precision();
     const ios_base::fmtflags oldFlags = os.flags();
  
     os << space( indent ) << scientific << setprecision( 17 ) << "value:       " << value_ << endl;
     os << space( indent ) << "minimum:     " << minimum_ << endl;
     os << space( indent ) << "maximum:     " << maximum_ << endl;
  
     /// Restore old stream config
     os.precision( oldPrecision );
     os.flags( oldFlags );
  }
  #endif
  
  //=============================================================================
  
  StringNodeImpl::StringNodeImpl( ImageFileImplWeakPtr destImageFile, const ustring &value ) :
     NodeImpl( destImageFile ), value_( value )
  {
     // don't checkImageFileOpen, NodeImpl() will do it
  }
  
  bool StringNodeImpl::isTypeEquivalent( NodeImplSharedPtr ni )
  {
     // don't checkImageFileOpen
  
     /// Same node type?
     if ( ni->type() != E57_STRING )
        return ( false );
  
     /// ignore value_, doesn't have to match
  
     /// Types match
     return ( true );
  }
  
  bool StringNodeImpl::isDefined( const ustring &pathName )
  {
     // don't checkImageFileOpen
  
     /// We have no sub-structure, so if path not empty return false
     return pathName.empty();
  }
  
  ustring StringNodeImpl::value()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( value_ );
  }
  
  void StringNodeImpl::checkLeavesInSet( const StringSet &pathNames, NodeImplSharedPtr origin )
  {
     // don't checkImageFileOpen
  
     /// We are a leaf node, so verify that we are listed in set.
     if ( pathNames.find( relativePathName( origin ) ) == pathNames.end() )
        throw E57_EXCEPTION2( E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName() );
  }
  
  void StringNodeImpl::writeXml( ImageFileImplSharedPtr /*imf*/, CheckedFile &cf, int indent,
                                 const char *forcedFieldName )
  {
     // don't checkImageFileOpen
  
     ustring fieldName;
     if ( forcedFieldName != nullptr )
        fieldName = forcedFieldName;
     else
        fieldName = elementName_;
  
     cf << space( indent ) << "<" << fieldName << " type=\"String\"";
  
     /// Write value as child text, unless it is the default value
     if ( value_.empty() )
     {
        cf << "/>\n";
     }
     else
     {
        cf << "><![CDATA[";
  
        size_t currentPosition = 0;
        size_t len = value_.length();
  
        /// Loop, searching for occurences of "]]>", which will be split across
        /// two CDATA directives
        while ( currentPosition < len )
        {
           size_t found = value_.find( "]]>", currentPosition );
  
           if ( found == string::npos )
           {
              /// Didn't find any more "]]>", so can send the rest.
              cf << value_.substr( currentPosition );
              break;
           }
  
           /// Must output in two pieces, first send upto end of "]]"  (don't send
           /// the following ">").
           cf << value_.substr( currentPosition, found - currentPosition + 2 );
  
           /// Then start a new CDATA
           cf << "]]><![CDATA[";
  
           /// Keep looping to send the ">" plus the remaining part of the string
           currentPosition = found + 2;
        }
        cf << "]]></" << fieldName << ">\n";
     }
  }
  
  #ifdef E57_DEBUG
  void StringNodeImpl::dump( int indent, ostream &os ) const
  {
     os << space( indent ) << "type:        String"
        << " (" << type() << ")" << endl;
     NodeImpl::dump( indent, os );
     os << space( indent ) << "value:       '" << value_ << "'" << endl;
  }
  #endif
  
  //=============================================================================
  
  BlobNodeImpl::BlobNodeImpl( ImageFileImplWeakPtr destImageFile, int64_t byteCount ) : NodeImpl( destImageFile )
  {
     // don't checkImageFileOpen, NodeImpl() will do it
  
     ImageFileImplSharedPtr imf( destImageFile );
  
     /// This what caller thinks blob length is
     blobLogicalLength_ = byteCount;
  
     /// Round segment length up to multiple of 4 bytes
     binarySectionLogicalLength_ = sizeof( BlobSectionHeader ) + blobLogicalLength_;
     unsigned remainder = binarySectionLogicalLength_ % 4;
     if ( remainder > 0 )
        binarySectionLogicalLength_ += 4 - remainder;
  
     /// Reserve space for blob in file, extend with zeros since writes will
     /// happen at later time by caller
     binarySectionLogicalStart_ = imf->allocateSpace( binarySectionLogicalLength_, true );
  
     /// Prepare BlobSectionHeader
     BlobSectionHeader header;
     header.sectionLogicalLength = binarySectionLogicalLength_;
  #ifdef E57_MAX_VERBOSE
     header.dump(); //???
  #endif
  
     /// Write header at beginning of section
     imf->file_->seek( binarySectionLogicalStart_ );
     imf->file_->write( reinterpret_cast<char *>( &header ), sizeof( header ) );
  }
  
  BlobNodeImpl::BlobNodeImpl( ImageFileImplWeakPtr destImageFile, int64_t fileOffset, int64_t length ) :
     NodeImpl( destImageFile )
  {
     /// Init blob object that already exists in E57 file currently reading.
  
     // don't checkImageFileOpen, NodeImpl() will do it
  
     ImageFileImplSharedPtr imf( destImageFile );
  
     /// Init state from values read from XML
     blobLogicalLength_ = length;
     binarySectionLogicalStart_ = imf->file_->physicalToLogical( fileOffset );
     binarySectionLogicalLength_ = sizeof( BlobSectionHeader ) + blobLogicalLength_;
  }
  
  bool BlobNodeImpl::isTypeEquivalent( NodeImplSharedPtr ni )
  {
     // don't checkImageFileOpen, NodeImpl() will do it
  
     /// Same node type?
     if ( ni->type() != E57_BLOB )
        return ( false );
  
     /// Downcast to shared_ptr<BlobNodeImpl>
     shared_ptr<BlobNodeImpl> bi( dynamic_pointer_cast<BlobNodeImpl>( ni ) );
     if ( !bi ) // check if failed
        throw E57_EXCEPTION2( E57_ERROR_INTERNAL,
                              "this->elementName=" + this->elementName() + " elementName=" + ni->elementName() );
  
     /// blob lengths must match
     if ( blobLogicalLength_ != bi->blobLogicalLength_ )
        return ( false );
  
     /// ignore blob contents, doesn't have to match
  
     /// Types match
     return ( true );
  }
  
  bool BlobNodeImpl::isDefined( const ustring &pathName )
  {
     // don't checkImageFileOpen, NodeImpl() will do it
  
     /// We have no sub-structure, so if path not empty return false
     return pathName.empty();
  }
  
  int64_t BlobNodeImpl::byteCount()
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     return ( blobLogicalLength_ );
  }
  
  void BlobNodeImpl::read( uint8_t *buf, int64_t start, size_t count )
  {
     //??? check start not negative
  
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     if ( static_cast<uint64_t>( start ) + count > blobLogicalLength_ )
     {
        throw E57_EXCEPTION2( E57_ERROR_BAD_API_ARGUMENT,
                              "this->pathName=" + this->pathName() + " start=" + toString( start ) +
                                 " count=" + toString( count ) + " length=" + toString( blobLogicalLength_ ) );
     }
  
     ImageFileImplSharedPtr imf( destImageFile_ );
     imf->file_->seek( binarySectionLogicalStart_ + sizeof( BlobSectionHeader ) + start );
     imf->file_->read( reinterpret_cast<char *>( buf ),
                       static_cast<size_t>( count ) ); //??? arg1 void* ?
  }
  
  void BlobNodeImpl::write( uint8_t *buf, int64_t start, size_t count )
  {
     //??? check start not negative
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
  
     ImageFileImplSharedPtr destImageFile( destImageFile_ );
  
     if ( !destImageFile->isWriter() )
        throw E57_EXCEPTION2( E57_ERROR_FILE_IS_READ_ONLY, "fileName=" + destImageFile->fileName() );
     if ( !isAttached() )
        throw E57_EXCEPTION2( E57_ERROR_NODE_UNATTACHED, "fileName=" + destImageFile->fileName() );
  
     if ( static_cast<uint64_t>( start ) + count > blobLogicalLength_ )
     {
        throw E57_EXCEPTION2( E57_ERROR_BAD_API_ARGUMENT,
                              "this->pathName=" + this->pathName() + " start=" + toString( start ) +
                                 " count=" + toString( count ) + " length=" + toString( blobLogicalLength_ ) );
     }
  
     ImageFileImplSharedPtr imf( destImageFile_ );
     imf->file_->seek( binarySectionLogicalStart_ + sizeof( BlobSectionHeader ) + start );
     imf->file_->write( reinterpret_cast<char *>( buf ),
                        static_cast<size_t>( count ) ); //??? arg1 void* ?
  }
  
  void BlobNodeImpl::checkLeavesInSet( const StringSet &pathNames, NodeImplSharedPtr origin )
  {
     // don't checkImageFileOpen
  
     /// We are a leaf node, so verify that we are listed in set. ???true for
     /// blobs? what exception get if try blob in compressedvector?
     if ( pathNames.find( relativePathName( origin ) ) == pathNames.end() )
        throw E57_EXCEPTION2( E57_ERROR_NO_BUFFER_FOR_ELEMENT, "this->pathName=" + this->pathName() );
  }
  
  void BlobNodeImpl::writeXml( ImageFileImplSharedPtr /*imf*/, CheckedFile &cf, int indent, const char *forcedFieldName )
  {
     // don't checkImageFileOpen
  
     ustring fieldName;
     if ( forcedFieldName != nullptr )
        fieldName = forcedFieldName;
     else
        fieldName = elementName_;
  
     //??? need to implement
     //??? Type --> type
     //??? need to have length?, check same as in section header?
     uint64_t physicalOffset = cf.logicalToPhysical( binarySectionLogicalStart_ );
     cf << space( indent ) << "<" << fieldName << " type=\"Blob\" fileOffset=\"" << physicalOffset << "\" length=\""
        << blobLogicalLength_ << "\"/>\n";
  }
  
  #ifdef E57_DEBUG
  void BlobNodeImpl::dump( int indent, ostream &os ) const
  {
     // don't checkImageFileOpen
     os << space( indent ) << "type:        Blob"
        << " (" << type() << ")" << endl;
     NodeImpl::dump( indent, os );
     os << space( indent ) << "blobLogicalLength_:           " << blobLogicalLength_ << endl;
     os << space( indent ) << "binarySectionLogicalStart:    " << binarySectionLogicalStart_ << endl;
     os << space( indent ) << "binarySectionLogicalLength:   " << binarySectionLogicalLength_ << endl;
     //    size_t i;
     //    for (i = 0; i < blobLogicalLength_ && i < 10; i++) {
     //        uint8_t b;
     //        read(&b, i, 1);
     //        os << space(indent) << "data[" << i << "]: "<< static_cast<int>(b)
     //        << endl;
     //    }
     //    if (i < blobLogicalLength_)
     //        os << space(indent) << "more data unprinted..." << endl;
  }
  #endif
  
  //================================================================
  
  CompressedVectorSectionHeader::CompressedVectorSectionHeader()
  {
     /// Double check that header is correct length.  Watch out for RTTI
     /// increasing the size.
     static_assert( sizeof( CompressedVectorSectionHeader ) == 32, "Unexpected size of CompressedVectorSectionHeader" );
  }
  
  void CompressedVectorSectionHeader::verify( uint64_t filePhysicalSize )
  {
     /// Verify that section is correct type
     if ( sectionId != COMPRESSED_VECTOR_SECTION )
        throw E57_EXCEPTION2( E57_ERROR_BAD_CV_HEADER, "sectionId=" + toString( sectionId ) );
  
     /// Verify reserved fields are zero. ???  if fileversion==1.0 ???
     for ( unsigned i = 0; i < sizeof( reserved1 ); i++ )
     {
        if ( reserved1[i] != 0 )
        {
           throw E57_EXCEPTION2( E57_ERROR_BAD_CV_HEADER,
                                 "i=" + toString( i ) + " reserved=" + toString( reserved1[i] ) );
        }
     }
  
     /// Check section length is multiple of 4
     if ( sectionLogicalLength % 4 )
        throw E57_EXCEPTION2( E57_ERROR_BAD_CV_HEADER, "sectionLogicalLength=" + toString( sectionLogicalLength ) );
  
     /// Check sectionLogicalLength is in bounds
     if ( filePhysicalSize > 0 && sectionLogicalLength >= filePhysicalSize )
     {
        throw E57_EXCEPTION2( E57_ERROR_BAD_CV_HEADER, "sectionLogicalLength=" + toString( sectionLogicalLength ) +
                                                          " filePhysicalSize=" + toString( filePhysicalSize ) );
     }
  
     /// Check dataPhysicalOffset is in bounds
     if ( filePhysicalSize > 0 && dataPhysicalOffset >= filePhysicalSize )
     {
        throw E57_EXCEPTION2( E57_ERROR_BAD_CV_HEADER, "dataPhysicalOffset=" + toString( dataPhysicalOffset ) +
                                                          " filePhysicalSize=" + toString( filePhysicalSize ) );
     }
  
     /// Check indexPhysicalOffset is in bounds
     if ( filePhysicalSize > 0 && indexPhysicalOffset >= filePhysicalSize )
     {
        throw E57_EXCEPTION2( E57_ERROR_BAD_CV_HEADER, "indexPhysicalOffset=" + toString( indexPhysicalOffset ) +
                                                          " filePhysicalSize=" + toString( filePhysicalSize ) );
     }
  }
  
  #ifdef E57_DEBUG
  void CompressedVectorSectionHeader::dump( int indent, std::ostream &os ) const
  {
     os << space( indent ) << "sectionId:            " << static_cast<unsigned>( sectionId ) << endl;
     os << space( indent ) << "sectionLogicalLength: " << sectionLogicalLength << endl;
     os << space( indent ) << "dataPhysicalOffset:   " << dataPhysicalOffset << endl;
     os << space( indent ) << "indexPhysicalOffset:  " << indexPhysicalOffset << endl;
  }
  #endif
  
  ///================================================================
  
  struct SortByBytestreamNumber
  {
     bool operator()( const shared_ptr<Encoder> &lhs, const shared_ptr<Encoder> &rhs ) const
     {
        return ( lhs->bytestreamNumber() < rhs->bytestreamNumber() );
     }
  };
  
  CompressedVectorWriterImpl::CompressedVectorWriterImpl( shared_ptr<CompressedVectorNodeImpl> ni,
                                                          vector<SourceDestBuffer> &sbufs ) :
     cVector_( ni ),
     isOpen_( false ) // set to true when succeed below
  {
     //???  check if cvector already been written (can't write twice)
  
     /// Empty sbufs is an error
     if ( sbufs.empty() )
     {
        throw E57_EXCEPTION2( E57_ERROR_BAD_API_ARGUMENT,
                              "imageFileName=" + cVector_->imageFileName() + " cvPathName=" + cVector_->pathName() );
     }
  
     /// Get CompressedArray's prototype node (all array elements must match this
     /// type)
     proto_ = cVector_->getPrototype();
  
     /// Check sbufs well formed (matches proto exactly)
     setBuffers( sbufs ); //??? copy code here?
  
     /// For each individual sbuf, create an appropriate Encoder based on the
     /// cVector_ attributes
     for ( unsigned i = 0; i < sbufs_.size(); i++ )
     {
        /// Create vector of single sbuf  ??? for now, may have groups later
        vector<SourceDestBuffer> vTemp;
        vTemp.push_back( sbufs_.at( i ) );
  
        ustring codecPath = sbufs_.at( i ).pathName();
  
        /// Calc which stream the given path belongs to.  This depends on position
        /// of the node in the proto tree.
        NodeImplSharedPtr readNode = proto_->get( sbufs.at( i ).pathName() );
        uint64_t bytestreamNumber = 0;
        if ( !proto_->findTerminalPosition( readNode, bytestreamNumber ) )
           throw E57_EXCEPTION2( E57_ERROR_INTERNAL, "sbufIndex=" + toString( i ) );
  
        /// EncoderFactory picks the appropriate encoder to match type declared in
        /// prototype
        bytestreams_.push_back(
           Encoder::EncoderFactory( static_cast<unsigned>( bytestreamNumber ), cVector_, vTemp, codecPath ) );
     }
  
     /// The bytestreams_ vector must be ordered by bytestreamNumber, not by order
     /// called specified sbufs, so sort it.
     sort( bytestreams_.begin(), bytestreams_.end(), SortByBytestreamNumber() );
  #ifdef E57_MAX_DEBUG
     /// Double check that all bytestreams are specified
     for ( unsigned i = 0; i < bytestreams_.size(); i++ )
     {
        if ( bytestreams_.at( i )->bytestreamNumber() != i )
        {
           throw E57_EXCEPTION2( E57_ERROR_INTERNAL, "bytestreamIndex=" + toString( i ) + " bytestreamNumber=" +
                                                        toString( bytestreams_.at( i )->bytestreamNumber() ) );
        }
     }
  #endif
  
     ImageFileImplSharedPtr imf( ni->destImageFile_ );
  
     /// Reserve space for CompressedVector binary section header, record location
     /// so can save to when writer closes. Request that file be extended with
     /// zeros since we will write to it at a later time (when writer closes).
     sectionHeaderLogicalStart_ = imf->allocateSpace( sizeof( CompressedVectorSectionHeader ), true );
  
     sectionLogicalLength_ = 0;
     dataPhysicalOffset_ = 0;
     topIndexPhysicalOffset_ = 0;
     recordCount_ = 0;
     dataPacketsCount_ = 0;
     indexPacketsCount_ = 0;
  
     /// Just before return (and can't throw) increment writer count  ??? safer
     /// way to assure don't miss close?
     imf->incrWriterCount();
  
     /// If get here, the writer is open
     isOpen_ = true;
  }
  
  CompressedVectorWriterImpl::~CompressedVectorWriterImpl()
  {
  #ifdef E57_MAX_VERBOSE
     cout << "~CompressedVectorWriterImpl() called" << endl; //???
  #endif
  
     try
     {
        if ( isOpen_ )
           close();
     }
     catch ( ... )
     {
        //??? report?
     }
  }
  
  void CompressedVectorWriterImpl::close()
  {
  #ifdef E57_MAX_VERBOSE
     cout << "CompressedVectorWriterImpl::close() called" << endl; //???
  #endif
     ImageFileImplSharedPtr imf( cVector_->destImageFile_ );
  
     /// Before anything that can throw, decrement writer count
     imf->decrWriterCount();
  
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     /// don't call checkWriterOpen();
  
     if ( !isOpen_ )
        return;
  
     /// Set closed before do anything, so if get fault and start unwinding, don't
     /// try to close again.
     isOpen_ = false;
  
     /// If have any data, write packet
     /// Write all remaining ioBuffers and internal encoder register cache into
     /// file. Know we are done when totalOutputAvailable() returns 0 after a
     /// flush().
     flush();
     while ( totalOutputAvailable() > 0 )
     {
        packetWrite();
        flush();
     }
  
     /// Compute length of whole section we just wrote (from section start to
     /// current start of free space).
     sectionLogicalLength_ = imf->unusedLogicalStart_ - sectionHeaderLogicalStart_;
  #ifdef E57_MAX_VERBOSE
     cout << "  sectionLogicalLength_=" << sectionLogicalLength_ << endl; //???
  #endif
  
     /// Prepare CompressedVectorSectionHeader
     CompressedVectorSectionHeader header;
     header.sectionLogicalLength = sectionLogicalLength_;
     header.dataPhysicalOffset = dataPhysicalOffset_;      ///??? can be zero, if no data written ???not set yet
     header.indexPhysicalOffset = topIndexPhysicalOffset_; ///??? can be zero, if no data written ???not set
                                                           /// yet
  #ifdef E57_MAX_VERBOSE
     cout << "  CompressedVectorSectionHeader:" << endl;
     header.dump( 4 ); //???
  #endif
  #ifdef E57_DEBUG
     /// Verify OK before write it.
     header.verify( imf->file_->length( CheckedFile::Physical ) );
  #endif
  
     /// Write header at beginning of section, previously allocated
     imf->file_->seek( sectionHeaderLogicalStart_ );
     imf->file_->write( reinterpret_cast<char *>( &header ), sizeof( header ) );
  
     /// Set address and size of associated CompressedVector
     cVector_->setRecordCount( recordCount_ );
     cVector_->setBinarySectionLogicalStart( sectionHeaderLogicalStart_ );
  
     /// Free channels
     bytestreams_.clear();
  
  #ifdef E57_MAX_VERBOSE
     cout << "  CompressedVectorWriter:" << endl;
     dump( 4 );
  #endif
  }
  
  bool CompressedVectorWriterImpl::isOpen() const
  {
     /// don't checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__), or
     /// checkWriterOpen()
     return isOpen_;
  }
  
  std::shared_ptr<CompressedVectorNodeImpl> CompressedVectorWriterImpl::compressedVectorNode() const
  {
     return cVector_;
  }
  
  void CompressedVectorWriterImpl::setBuffers( vector<SourceDestBuffer> &sbufs )
  {
     /// don't checkImageFileOpen
  
     /// If had previous sbufs_, check to see if new ones have changed in
     /// incompatible way
     if ( !sbufs_.empty() )
     {
        if ( sbufs_.size() != sbufs.size() )
        {
           throw E57_EXCEPTION2( E57_ERROR_BUFFERS_NOT_COMPATIBLE,
                                 "oldSize=" + toString( sbufs_.size() ) + " newSize=" + toString( sbufs.size() ) );
        }
  
        for ( size_t i = 0; i < sbufs_.size(); ++i )
        {
           shared_ptr<SourceDestBufferImpl> oldbuf = sbufs_[i].impl();
           shared_ptr<SourceDestBufferImpl> newBuf = sbufs[i].impl();
  
           /// Throw exception if old and new not compatible
           oldbuf->checkCompatible( newBuf );
        }
     }
  
     /// Check sbufs well formed: no dups, no missing, no extra
     /// For writing, all data fields in prototype must be presented for writing
     /// at same time.
     proto_->checkBuffers( sbufs, false );
  
     sbufs_ = sbufs;
  }
  
  void CompressedVectorWriterImpl::write( vector<SourceDestBuffer> &sbufs, const size_t requestedRecordCount )
  {
     /// don't checkImageFileOpen, write(unsigned) will do it
     /// don't checkWriterOpen(), write(unsigned) will do it
  
     setBuffers( sbufs );
     write( requestedRecordCount );
  }
  
  void CompressedVectorWriterImpl::write( const size_t requestedRecordCount )
  {
  #ifdef E57_MAX_VERBOSE
     cout << "CompressedVectorWriterImpl::write() called" << endl; //???
  #endif
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     checkWriterOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
  
     /// Check that requestedRecordCount is not larger than the sbufs
     if ( requestedRecordCount > sbufs_.at( 0 ).impl()->capacity() )
     {
        throw E57_EXCEPTION2( E57_ERROR_BAD_API_ARGUMENT,
                              "requested=" + toString( requestedRecordCount ) +
                                 " capacity=" + toString( sbufs_.at( 0 ).impl()->capacity() ) +
                                 " imageFileName=" + cVector_->imageFileName() + " cvPathName=" + cVector_->pathName() );
     }
  
     /// Rewind all sbufs so start reading from beginning
     for ( auto &sbuf : sbufs_ )
     {
        sbuf.impl()->rewind();
     }
  
     /// Loop until all channels have completed requestedRecordCount transfers
     uint64_t endRecordIndex = recordCount_ + requestedRecordCount;
     for ( ;; )
     {
        /// Calc remaining record counts for all channels
        uint64_t totalRecordCount = 0;
        for ( auto &bytestream : bytestreams_ )
        {
           totalRecordCount += endRecordIndex - bytestream->currentRecordIndex();
        }
  #ifdef E57_MAX_VERBOSE
        cout << "  totalRecordCount=" << totalRecordCount << endl; //???
  #endif
  
        /// We are done if have no more work, break out of loop
        if ( totalRecordCount == 0 )
           break;
  
           /// Estimate how many records can write before have enough data to fill
           /// data packet to efficient length Efficient packet length is >= 75%
           /// of maximum packet length. It is OK if get too much data (more than
           /// one packet) in an iteration. Reader will be able to handle packets
           /// whose streams are not exactly synchronized to the record
           /// boundaries. But try to do a good job of keeping the stream
           /// synchronization "close enough" (so a reader that can cache only two
           /// packets is efficient).
  
  #ifdef E57_MAX_VERBOSE
        cout << "  currentPacketSize()=" << currentPacketSize() << endl; //???
  #endif
  
  #ifdef E57_WRITE_CRAZY_PACKET_MODE
  ///??? depends on number of streams
  #define E57_TARGET_PACKET_SIZE 500
  #else
  #define E57_TARGET_PACKET_SIZE ( DATA_PACKET_MAX * 3 / 4 )
  #endif
        /// If have more than target fraction of packet, send it now
        if ( currentPacketSize() >= E57_TARGET_PACKET_SIZE )
        { //???
           packetWrite();
           continue; /// restart loop so recalc statistics (packet size may not be
                     /// zero after write, if have too much data)
        }
  
        ///??? useful?
        /// Get approximation of number of bytes per record of CompressedVector
        /// and total of bytes used
        float totalBitsPerRecord = 0; // an estimate of future performance
        for ( auto &bytestream : bytestreams_ )
        {
           totalBitsPerRecord += bytestream->bitsPerRecord();
        }
  
  #ifdef E57_MAX_VERBOSE
        float totalBytesPerRecord = max( totalBitsPerRecord / 8, 0.1F ); //??? trust
  
        cout << "  totalBytesPerRecord=" << totalBytesPerRecord << endl; //???
  #endif
  
        /// Don't allow straggler to get too far behind. ???
        /// Don't allow a single channel to get too far ahead ???
        /// Process channels that are furthest behind first. ???
  
        ///!!!! For now just process one record per loop until packet is full
        /// enough, or completed
        /// request
        for ( auto &bytestream : bytestreams_ )
        {
           if ( bytestream->currentRecordIndex() < endRecordIndex )
           {
              //!!! For now, process up to 50 records at a time
              uint64_t recordCount = endRecordIndex - bytestream->currentRecordIndex();
              recordCount = ( recordCount < 50ULL ) ? recordCount : 50ULL; // min(recordCount, 50ULL);
              bytestream->processRecords( static_cast<unsigned>( recordCount ) );
           }
        }
     }
  
     recordCount_ += requestedRecordCount;
  
     /// When we leave this function, will likely still have data in channel
     /// ioBuffers as well as partial words in Encoder registers.
  }
  
  size_t CompressedVectorWriterImpl::totalOutputAvailable() const
  {
     size_t total = 0;
  
     for ( const auto &bytestream : bytestreams_ )
     {
        total += bytestream->outputAvailable();
     }
  
     return total;
  }
  
  size_t CompressedVectorWriterImpl::currentPacketSize() const
  {
     /// Calc current packet size
     return ( sizeof( DataPacketHeader ) + bytestreams_.size() * sizeof( uint16_t ) + totalOutputAvailable() );
  }
  
  uint64_t CompressedVectorWriterImpl::packetWrite()
  {
  #ifdef E57_MAX_VERBOSE
     cout << "CompressedVectorWriterImpl::packetWrite() called" << endl; //???
  #endif
  
     /// Double check that we have work to do
     size_t totalOutput = totalOutputAvailable();
     if ( totalOutput == 0 )
        return ( 0 );
  #ifdef E57_MAX_VERBOSE
     cout << "  totalOutput=" << totalOutput << endl; //???
  #endif
  
     /// Calc maximum number of bytestream values can put in data packet.
     size_t packetMaxPayloadBytes =
        DATA_PACKET_MAX - sizeof( DataPacketHeader ) - bytestreams_.size() * sizeof( uint16_t );
  #ifdef E57_MAX_VERBOSE
     cout << "  packetMaxPayloadBytes=" << packetMaxPayloadBytes << endl; //???
  #endif
  
     /// Allocate vector for number of bytes that each bytestream will write to
     /// file.
     vector<size_t> count( bytestreams_.size() );
  
     /// See if we can fit into a single data packet
     if ( totalOutput < packetMaxPayloadBytes )
     {
        /// We can fit everything in one packet
        for ( unsigned i = 0; i < bytestreams_.size(); i++ )
           count.at( i ) = bytestreams_.at( i )->outputAvailable();
     }
     else
     {
        /// We have too much data for one packet.  Send proportional amounts from
        /// each bytestream. Adjust packetMaxPayloadBytes down by one so have a
        /// little slack for floating point weirdness.
        float fractionToSend = ( packetMaxPayloadBytes - 1 ) / static_cast<float>( totalOutput );
        for ( unsigned i = 0; i < bytestreams_.size(); i++ )
        {
           /// Round down here so sum <= packetMaxPayloadBytes
           count.at( i ) = static_cast<unsigned>( floor( fractionToSend * bytestreams_.at( i )->outputAvailable() ) );
        }
     }
  #ifdef E57_MAX_VERBOSE
     for ( unsigned i = 0; i < bytestreams_.size(); i++ )
        cout << "  count[" << i << "]=" << count.at( i ) << endl; //???
  #endif
  
  #ifdef E57_DEBUG
     /// Double check sum of count is <= packetMaxPayloadBytes
     size_t totalByteCount = 0;
     for ( size_t i : count )
        totalByteCount += i;
     if ( totalByteCount > packetMaxPayloadBytes )
     {
        throw E57_EXCEPTION2( E57_ERROR_INTERNAL, "totalByteCount=" + toString( totalByteCount ) +
                                                     " packetMaxPayloadBytes=" + toString( packetMaxPayloadBytes ) );
     }
  #endif
  
     /// Get smart pointer to ImageFileImpl from associated CompressedVector
     ImageFileImplSharedPtr imf( cVector_->destImageFile_ );
  
     /// Use temp buf in object (is 64KBytes long) instead of allocating each time
     /// here
     char *packet = reinterpret_cast<char *>( &dataPacket_ );
  #ifdef E57_MAX_VERBOSE
     cout << "  packet=" << packet << endl; //???
  #endif
  
     /// To be safe, clear header part of packet
     dataPacket_.header.reset();
  
     /// Write bytestreamBufferLength[bytestreamCount] after header, in
     /// dataPacket_
     auto bsbLength = reinterpret_cast<uint16_t *>( &packet[sizeof( DataPacketHeader )] );
  #ifdef E57_MAX_VERBOSE
     cout << "  bsbLength=" << bsbLength << endl; //???
  #endif
     for ( unsigned i = 0; i < bytestreams_.size(); i++ )
     {
        bsbLength[i] = static_cast<uint16_t>( count.at( i ) ); // %%% Truncation
  #ifdef E57_MAX_VERBOSE
        cout << "  Writing " << bsbLength[i] << " bytes into bytestream " << i << endl; //???
  #endif
     }
  
     /// Get pointer to end of data so far
     char *p = reinterpret_cast<char *>( &bsbLength[bytestreams_.size()] );
  #ifdef E57_MAX_VERBOSE
     cout << "  after bsbLength, p=" << p << endl; //???
  #endif
  
     /// Write contents of each bytestream in dataPacket_
     for ( size_t i = 0; i < bytestreams_.size(); i++ )
     {
        size_t n = count.at( i );
  
  #ifdef E57_DEBUG
        /// Double check we aren't accidentally going to write off end of
        /// vector<char>
        if ( &p[n] > &packet[DATA_PACKET_MAX] )
           throw E57_EXCEPTION2( E57_ERROR_INTERNAL, "n=" + toString( n ) );
  #endif
  
        /// Read from encoder output into packet
        bytestreams_.at( i )->outputRead( p, n );
  
        /// Move pointer to end of current data
        p += n;
     }
  
     /// Length of packet is difference in beginning pointer and ending pointer
     auto packetLength = static_cast<unsigned>( p - packet ); ///??? pointer diff portable?
  #ifdef E57_MAX_VERBOSE
     cout << "  packetLength=" << packetLength << endl; //???
  #endif
  
  #ifdef E57_DEBUG
     /// Double check that packetLength is what we expect
     if ( packetLength != sizeof( DataPacketHeader ) + bytestreams_.size() * sizeof( uint16_t ) + totalByteCount )
     {
        throw E57_EXCEPTION2( E57_ERROR_INTERNAL, "packetLength=" + toString( packetLength ) + " bytestreamSize=" +
                                                     toString( bytestreams_.size() * sizeof( uint16_t ) ) +
                                                     " totalByteCount=" + toString( totalByteCount ) );
     }
  #endif
  
     /// packetLength must be multiple of 4, if not, add some zero padding
     while ( packetLength % 4 )
     {
        /// Double check we aren't accidentally going to write off end of
        /// vector<char>
        if ( p >= &packet[DATA_PACKET_MAX - 1] )
           throw E57_EXCEPTION1( E57_ERROR_INTERNAL );
        *p++ = 0;
        packetLength++;
  #ifdef E57_MAX_VERBOSE
        cout << "  padding with zero byte, new packetLength=" << packetLength << endl; //???
  #endif
     }
  
     /// Prepare header in dataPacket_, now that we are sure of packetLength
     dataPacket_.header.packetLogicalLengthMinus1 = static_cast<uint16_t>( packetLength - 1 ); // %%% Truncation
     dataPacket_.header.bytestreamCount = static_cast<uint16_t>( bytestreams_.size() );        // %%% Truncation
  
     /// Double check that data packet is well formed
     dataPacket_.verify( packetLength );
  
     /// Write whole data packet at beginning of free space in file
     uint64_t packetLogicalOffset = imf->allocateSpace( packetLength, false );
     uint64_t packetPhysicalOffset = imf->file_->logicalToPhysical( packetLogicalOffset );
     imf->file_->seek( packetLogicalOffset ); //??? have seekLogical and seekPhysical instead?
                                              // more explicit
     imf->file_->write( packet, packetLength );
  
  #ifdef E57_MAX_VERBOSE
  //  cout << "data packet:" << endl;
  //  dataPacket_.dump(4);
  #endif
  
     /// If first data packet written for this CompressedVector binary section,
     /// save address to put in section header
     ///??? what if no data packets?
     ///??? what if have exceptions while write, what is state of file?  will
     /// close report file
     /// good/bad?
     if ( dataPacketsCount_ == 0 )
        dataPhysicalOffset_ = packetPhysicalOffset;
     dataPacketsCount_++;
  
     ///!!! update seekIndex here? if started new chunk?
  
     /// Return physical offset of data packet for potential use in seekIndex
     return ( packetPhysicalOffset ); //??? needed
  }
  
  void CompressedVectorWriterImpl::flush()
  {
     for ( auto &bytestream : bytestreams_ )
     {
        bytestream->registerFlushToOutput();
     }
  }
  
  void CompressedVectorWriterImpl::checkImageFileOpen( const char *srcFileName, int srcLineNumber,
                                                       const char *srcFunctionName ) const
  {
     // unimplemented...
  }
  
  void CompressedVectorWriterImpl::checkWriterOpen( const char *srcFileName, int srcLineNumber,
                                                    const char *srcFunctionName ) const
  {
     if ( !isOpen_ )
     {
        throw E57Exception( E57_ERROR_WRITER_NOT_OPEN,
                            "imageFileName=" + cVector_->imageFileName() + " cvPathName=" + cVector_->pathName(),
                            srcFileName, srcLineNumber, srcFunctionName );
     }
  }
  
  void CompressedVectorWriterImpl::dump( int indent, std::ostream &os )
  {
     os << space( indent ) << "isOpen:" << isOpen_ << endl;
  
     for ( unsigned i = 0; i < sbufs_.size(); i++ )
     {
        os << space( indent ) << "sbufs[" << i << "]:" << endl;
        sbufs_.at( i ).dump( indent + 4, os );
     }
  
     os << space( indent ) << "cVector:" << endl;
     cVector_->dump( indent + 4, os );
  
     os << space( indent ) << "proto:" << endl;
     proto_->dump( indent + 4, os );
  
     for ( unsigned i = 0; i < bytestreams_.size(); i++ )
     {
        os << space( indent ) << "bytestreams[" << i << "]:" << endl;
        bytestreams_.at( i )->dump( indent + 4, os );
     }
  
     /// Don't call dump() for DataPacket, since it may contain junk when
     /// debugging.  Just print a few byte values.
     os << space( indent ) << "dataPacket:" << endl;
     auto p = reinterpret_cast<uint8_t *>( &dataPacket_ );
  
     for ( unsigned i = 0; i < 40; ++i )
     {
        os << space( indent + 4 ) << "dataPacket[" << i << "]: " << static_cast<unsigned>( p[i] ) << endl;
     }
     os << space( indent + 4 ) << "more unprinted..." << endl;
  
     os << space( indent ) << "sectionHeaderLogicalStart: " << sectionHeaderLogicalStart_ << endl;
     os << space( indent ) << "sectionLogicalLength:      " << sectionLogicalLength_ << endl;
     os << space( indent ) << "dataPhysicalOffset:        " << dataPhysicalOffset_ << endl;
     os << space( indent ) << "topIndexPhysicalOffset:    " << topIndexPhysicalOffset_ << endl;
     os << space( indent ) << "recordCount:               " << recordCount_ << endl;
     os << space( indent ) << "dataPacketsCount:          " << dataPacketsCount_ << endl;
     os << space( indent ) << "indexPacketsCount:         " << indexPacketsCount_ << endl;
  }
  
  ///================================================================
  ///================================================================
  ///================================================================
  
  CompressedVectorReaderImpl::CompressedVectorReaderImpl( shared_ptr<CompressedVectorNodeImpl> cvi,
                                                          vector<SourceDestBuffer> &dbufs ) :
     isOpen_( false ), // set to true when succeed below
     cVector_( cvi )
  {
  #ifdef E57_MAX_VERBOSE
     cout << "CompressedVectorReaderImpl() called" << endl; //???
  #endif
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
  
     /// Allow reading of a completed CompressedVector, whether file is being read
     /// or currently being written.
     ///??? what other situations need checking for?
     ///??? check if CV not yet written to?
     ///??? file in error state?
  
     /// Empty dbufs is an error
     if ( dbufs.empty() )
     {
        throw E57_EXCEPTION2( E57_ERROR_BAD_API_ARGUMENT,
                              "imageFileName=" + cVector_->imageFileName() + " cvPathName=" + cVector_->pathName() );
     }
  
     /// Get CompressedArray's prototype node (all array elements must match this
     /// type)
     proto_ = cVector_->getPrototype();
  
     /// Check dbufs well formed (matches proto exactly)
     setBuffers( dbufs );
  
     /// For each dbuf, create an appropriate Decoder based on the cVector_
     /// attributes
     for ( unsigned i = 0; i < dbufs_.size(); i++ )
     {
        vector<SourceDestBuffer> theDbuf;
        theDbuf.push_back( dbufs.at( i ) );
  
        shared_ptr<Decoder> decoder = Decoder::DecoderFactory( i, cVector_, theDbuf, ustring() );
  
        /// Calc which stream the given path belongs to.  This depends on position
        /// of the node in the proto tree.
        NodeImplSharedPtr readNode = proto_->get( dbufs.at( i ).pathName() );
        uint64_t bytestreamNumber = 0;
        if ( !proto_->findTerminalPosition( readNode, bytestreamNumber ) )
           throw E57_EXCEPTION2( E57_ERROR_INTERNAL, "dbufIndex=" + toString( i ) );
  
        channels_.emplace_back( dbufs.at( i ), decoder, static_cast<unsigned>( bytestreamNumber ),
                                cVector_->childCount() );
     }
  
     recordCount_ = 0;
  
     /// Get how many records are actually defined
     maxRecordCount_ = cvi->childCount();
  
     ImageFileImplSharedPtr imf( cVector_->destImageFile_ );
  
     //??? what if fault in this constructor?
     cache_ = new PacketReadCache( imf->file_, 32 );
  
     /// Read CompressedVector section header
     CompressedVectorSectionHeader sectionHeader;
     uint64_t sectionLogicalStart = cVector_->getBinarySectionLogicalStart();
     if ( sectionLogicalStart == 0 )
     {
        //??? should have caught this before got here, in XML read, get this if CV
        // wasn't written to
        // by writer.
        throw E57_EXCEPTION2( E57_ERROR_INTERNAL,
                              "imageFileName=" + cVector_->imageFileName() + " cvPathName=" + cVector_->pathName() );
     }
     imf->file_->seek( sectionLogicalStart, CheckedFile::Logical );
     imf->file_->read( reinterpret_cast<char *>( &sectionHeader ), sizeof( sectionHeader ) );
  
  #ifdef E57_DEBUG
     sectionHeader.verify( imf->file_->length( CheckedFile::Physical ) );
  #endif
  
     /// Pre-calc end of section, so can tell when we are out of packets.
     sectionEndLogicalOffset_ = sectionLogicalStart + sectionHeader.sectionLogicalLength;
  
     /// Convert physical offset to first data packet to logical
     uint64_t dataLogicalOffset = imf->file_->physicalToLogical( sectionHeader.dataPhysicalOffset );
  
     /// Verify that packet given by dataPhysicalOffset is actually a data packet,
     /// init channels
     {
        char *anyPacket = nullptr;
        unique_ptr<PacketLock> packetLock = cache_->lock( dataLogicalOffset, anyPacket );
  
        auto dpkt = reinterpret_cast<DataPacket *>( anyPacket );
  
        /// Double check that have a data packet
        if ( dpkt->header.packetType != DATA_PACKET )
        {
           throw E57_EXCEPTION2( E57_ERROR_BAD_CV_PACKET, "packetType=" + toString( dpkt->header.packetType ) );
        }
  
        /// Have good packet, initialize channels
        for ( auto &channel : channels_ )
        {
           channel.currentPacketLogicalOffset = dataLogicalOffset;
           channel.currentBytestreamBufferIndex = 0;
           channel.currentBytestreamBufferLength = dpkt->getBytestreamBufferLength( channel.bytestreamNumber );
        }
     }
  
     /// Just before return (and can't throw) increment reader count  ??? safer
     /// way to assure don't miss close?
     imf->incrReaderCount();
  
     /// If get here, the reader is open
     isOpen_ = true;
  }
  
  CompressedVectorReaderImpl::~CompressedVectorReaderImpl()
  {
  #ifdef E57_MAX_VERBOSE
     cout << "~CompressedVectorReaderImpl() called" << endl; //???
                                                             // dump(4);
  #endif
  
     if ( isOpen_ )
     {
        try
        {
           close(); ///??? what if already closed?
        }
        catch ( ... )
        {
           //??? report?
        }
     }
  }
  
  void CompressedVectorReaderImpl::setBuffers( vector<SourceDestBuffer> &dbufs )
  {
     /// don't checkImageFileOpen
     /// don't checkReaderOpen
  
     /// Check dbufs well formed: no dups, no extra, missing is ok
     proto_->checkBuffers( dbufs, true );
  
     /// If had previous dbufs_, check to see if new ones have changed in
     /// incompatible way
     if ( !dbufs_.empty() )
     {
        if ( dbufs_.size() != dbufs.size() )
        {
           throw E57_EXCEPTION2( E57_ERROR_BUFFERS_NOT_COMPATIBLE,
                                 "oldSize=" + toString( dbufs_.size() ) + " newSize=" + toString( dbufs.size() ) );
        }
        for ( size_t i = 0; i < dbufs_.size(); i++ )
        {
           shared_ptr<SourceDestBufferImpl> oldBuf = dbufs_[i].impl();
           shared_ptr<SourceDestBufferImpl> newBuf = dbufs[i].impl();
  
           /// Throw exception if old and new not compatible
           oldBuf->checkCompatible( newBuf );
        }
     }
  
     dbufs_ = dbufs;
  }
  
  unsigned CompressedVectorReaderImpl::read( vector<SourceDestBuffer> &dbufs )
  {
     /// don't checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__), read() will
     /// do it
  
     checkReaderOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
  
     /// Check compatible with current dbufs
     setBuffers( dbufs );
  
     return ( read() );
  }
  
  unsigned CompressedVectorReaderImpl::read()
  {
  #ifdef E57_MAX_VERBOSE
     cout << "CompressedVectorReaderImpl::read() called" << endl; //???
  #endif
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
     checkReaderOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
  
     /// Rewind all dbufs so start writing to them at beginning
     for ( auto &dbuf : dbufs_ )
     {
        dbuf.impl()->rewind();
     }
  
     /// Allow decoders to use data they already have in their queue to fill newly
     /// empty dbufs This helps to keep decoder input queues smaller, which
     /// reduces backtracking in the packet cache.
     for ( auto &channel : channels_ )
     {
        channel.decoder->inputProcess( nullptr, 0 );
     }
  
     /// Loop until every dbuf is full or we have reached end of the binary
     /// section.
     while ( true )
     {
        /// Find the earliest packet position for channels that are still hungry
        /// It's important to call inputProcess of the decoders before this call,
        /// so current hungriness level is reflected.
        uint64_t earliestPacketLogicalOffset = earliestPacketNeededForInput();
  
        /// If nobody's hungry, we are done with the read
        if ( earliestPacketLogicalOffset == E57_UINT64_MAX )
           break;
  
        /// Feed packet to the hungry decoders
        feedPacketToDecoders( earliestPacketLogicalOffset );
     }
  
     /// Verify that each channel produced the same number of records
     unsigned outputCount = 0;
     for ( unsigned i = 0; i < channels_.size(); i++ )
     {
        DecodeChannel *chan = &channels_[i];
        if ( i == 0 )
        {
           outputCount = chan->dbuf.impl()->nextIndex();
        }
        else
        {
           if ( outputCount != chan->dbuf.impl()->nextIndex() )
           {
              throw E57_EXCEPTION2( E57_ERROR_INTERNAL, "outputCount=" + toString( outputCount ) +
                                                           " nextIndex=" + toString( chan->dbuf.impl()->nextIndex() ) );
           }
        }
     }
  
     /// Return number of records transferred to each dbuf.
     return outputCount;
  }
  
  uint64_t CompressedVectorReaderImpl::earliestPacketNeededForInput() const
  {
     uint64_t earliestPacketLogicalOffset = E57_UINT64_MAX;
  #ifdef E57_MAX_VERBOSE
     unsigned earliestChannel = 0;
  #endif
  
     for ( unsigned i = 0; i < channels_.size(); i++ )
     {
        const DecodeChannel *chan = &channels_[i];
  
        /// Test if channel needs more input.
        /// Important to call inputProcess just before this, so these tests work.
        if ( !chan->isOutputBlocked() && !chan->inputFinished )
        {
           /// Check if earliest so far
           if ( chan->currentPacketLogicalOffset < earliestPacketLogicalOffset )
           {
              earliestPacketLogicalOffset = chan->currentPacketLogicalOffset;
  #ifdef E57_MAX_VERBOSE
              earliestChannel = i;
  #endif
           }
        }
     }
  #ifdef E57_MAX_VERBOSE
     if ( earliestPacketLogicalOffset == E57_UINT64_MAX )
        cout << "earliestPacketNeededForInput returning none found" << endl;
     else
        cout << "earliestPacketNeededForInput returning " << earliestPacketLogicalOffset << " for channel["
             << earliestChannel << "]" << endl;
  #endif
     return earliestPacketLogicalOffset;
  }
  
  DataPacket *CompressedVectorReaderImpl::dataPacket( uint64_t inLogicalOffset ) const
  {
     char *packet = nullptr;
  
     unique_ptr<PacketLock> packetLock = cache_->lock( inLogicalOffset, packet );
  
     return reinterpret_cast<DataPacket *>( packet );
  }
  
  void CompressedVectorReaderImpl::feedPacketToDecoders( uint64_t currentPacketLogicalOffset )
  {
     /// Read earliest packet into cache and send data to decoders with unblocked
     /// output
     bool channelHasExhaustedPacket = false;
     uint64_t nextPacketLogicalOffset = E57_UINT64_MAX;
  
     /// Get packet at currentPacketLogicalOffset into memory.
     auto dpkt = dataPacket( currentPacketLogicalOffset );
  
     /// Double check that have a data packet.  Should have already determined
     /// this.
     if ( dpkt->header.packetType != DATA_PACKET )
     {
        throw E57_EXCEPTION2( E57_ERROR_INTERNAL, "packetType=" + toString( dpkt->header.packetType ) );
     }
  
     /// Feed bytestreams to channels with unblocked output that are reading from
     /// this packet
     for ( DecodeChannel &channel : channels_ )
     {
        /// Skip channels that have already read this packet.
        if ( channel.currentPacketLogicalOffset != currentPacketLogicalOffset || channel.isOutputBlocked() )
        {
           continue;
        }
  
        /// Get bytestream buffer for this channel from packet
        unsigned int bsbLength = 0;
        const char *bsbStart = dpkt->getBytestream( channel.bytestreamNumber, bsbLength );
  
        /// Calc where we are in the buffer
        const char *uneatenStart = &bsbStart[channel.currentBytestreamBufferIndex];
        const size_t uneatenLength = bsbLength - channel.currentBytestreamBufferIndex;
  
        /// Double check we are not off end of buffer
        if ( channel.currentBytestreamBufferIndex > bsbLength )
        {
           throw E57_EXCEPTION2( E57_ERROR_INTERNAL,
                                 "currentBytestreamBufferIndex =" + toString( channel.currentBytestreamBufferIndex ) +
                                    " bsbLength=" + toString( bsbLength ) );
        }
  
        if ( &uneatenStart[uneatenLength] > &bsbStart[bsbLength] )
        {
           throw E57_EXCEPTION2( E57_ERROR_INTERNAL,
                                 "uneatenLength=" + toString( uneatenLength ) + " bsbLength=" + toString( bsbLength ) );
        }
  
        /// Feed into decoder
        size_t bytesProcessed = channel.decoder->inputProcess( uneatenStart, uneatenLength );
  
  #ifdef E57_MAX_VERBOSE
        cout << "  stream[" << channel.bytestreamNumber << "]: feeding decoder " << uneatenLength << " bytes" << endl;
        if ( uneatenLength == 0 )
        {
           channel.dump( 8 );
        }
  
        cout << "  stream[" << channel.bytestreamNumber << "]: bytesProcessed=" << bytesProcessed << endl;
  #endif
  
        /// Adjust counts of bytestream location
        channel.currentBytestreamBufferIndex += bytesProcessed;
  
        /// Check if this channel has exhausted its bytestream buffer in this
        /// packet
        if ( channel.isInputBlocked() )
        {
  #ifdef E57_MAX_VERBOSE
           cout << "  stream[" << channel.bytestreamNumber << "] has exhausted its input in current packet" << endl;
  #endif
           channelHasExhaustedPacket = true;
           nextPacketLogicalOffset = currentPacketLogicalOffset + dpkt->header.packetLogicalLengthMinus1 + 1;
        }
     }
  
     /// Skip over any index or empty packets to next data packet.
     nextPacketLogicalOffset = findNextDataPacket( nextPacketLogicalOffset );
  
     /// If some channel has exhausted this packet, find next data packet and
     /// update currentPacketLogicalOffset for all interested channels.
     if ( channelHasExhaustedPacket )
     {
        if ( nextPacketLogicalOffset < E57_UINT64_MAX )
        { //??? huh?
           /// Get packet at nextPacketLogicalOffset into memory.
           dpkt = dataPacket( nextPacketLogicalOffset );
  
           /// Got a data packet, update the channels with exhausted input
           for ( DecodeChannel &channel : channels_ )
           {
              if ( ( channel.currentPacketLogicalOffset == currentPacketLogicalOffset ) && channel.isInputBlocked() )
              {
                 channel.currentPacketLogicalOffset = nextPacketLogicalOffset;
                 channel.currentBytestreamBufferIndex = 0;
  
                 /// It is OK if the next packet doesn't contain any data for this
                 /// channel, will skip packet on next iter of loop
                 channel.currentBytestreamBufferLength = dpkt->getBytestreamBufferLength( channel.bytestreamNumber );
  
  #ifdef E57_MAX_VERBOSE
                 cout << "  set new stream buffer for channel[" << channel.bytestreamNumber
                      << "], length=" << channel.currentBytestreamBufferLength << endl;
  #endif
                 /// ??? perform flush if new packet flag set?
              }
           }
        }
        else
        {
           /// Reached end without finding data packet, mark exhausted channels as
           /// finished
  #ifdef E57_MAX_VERBOSE
           cout << "  at end of data packets" << endl;
  #endif
           if ( nextPacketLogicalOffset >= sectionEndLogicalOffset_ )
           {
              for ( DecodeChannel &channel : channels_ )
              {
                 if ( ( channel.currentPacketLogicalOffset == currentPacketLogicalOffset ) && channel.isInputBlocked() )
                 {
  #ifdef E57_MAX_VERBOSE
                    cout << "  Marking channel[" << channel.bytestreamNumber << "] as finished" << endl;
  #endif
                    channel.inputFinished = true;
                 }
              }
           }
        }
     }
  }
  
  uint64_t CompressedVectorReaderImpl::findNextDataPacket( uint64_t nextPacketLogicalOffset )
  {
  #ifdef E57_MAX_VERBOSE
     cout << "  searching for next data packet, nextPacketLogicalOffset=" << nextPacketLogicalOffset
          << " sectionEndLogicalOffset=" << sectionEndLogicalOffset_ << endl;
  #endif
  
     /// Starting at nextPacketLogicalOffset, search for next data packet until
     /// hit end of binary section.
     while ( nextPacketLogicalOffset < sectionEndLogicalOffset_ )
     {
        char *anyPacket = nullptr;
  
        unique_ptr<PacketLock> packetLock = cache_->lock( nextPacketLogicalOffset, anyPacket );
  
        /// Guess it's a data packet, if not continue to next packet
        auto dpkt = reinterpret_cast<const DataPacket *>( anyPacket );
  
        if ( dpkt->header.packetType == DATA_PACKET )
        {
  #ifdef E57_MAX_VERBOSE
           cout << "  Found next data packet at nextPacketLogicalOffset=" << nextPacketLogicalOffset << endl;
  #endif
           return nextPacketLogicalOffset;
        }
  
        /// All packets have length in same place, so can use the field to skip to
        /// next packet.
        nextPacketLogicalOffset += dpkt->header.packetLogicalLengthMinus1 + 1;
     }
  
     /// Ran off end of section, so return failure code.
     return E57_UINT64_MAX;
  }
  
  void CompressedVectorReaderImpl::seek( uint64_t /*recordNumber*/ )
  {
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
  
     ///!!! implement
     throw E57_EXCEPTION1( E57_ERROR_NOT_IMPLEMENTED );
  }
  
  bool CompressedVectorReaderImpl::isOpen() const
  {
     /// don't checkImageFileOpen(__FILE__, __LINE__, __FUNCTION__), or
     /// checkReaderOpen()
     return ( isOpen_ );
  }
  
  std::shared_ptr<CompressedVectorNodeImpl> CompressedVectorReaderImpl::compressedVectorNode() const
  {
     return ( cVector_ );
  }
  
  void CompressedVectorReaderImpl::close()
  {
     /// Before anything that can throw, decrement reader count
     ImageFileImplSharedPtr imf( cVector_->destImageFile_ );
     imf->decrReaderCount();
  
     checkImageFileOpen( __FILE__, __LINE__, static_cast<const char *>( __FUNCTION__ ) );
  
     /// No error if reader not open
     if ( !isOpen_ )
        return;
  
     /// Destroy decoders
     channels_.clear();
  
     delete cache_;
     cache_ = nullptr;
  
     isOpen_ = false;
  }
  
  void CompressedVectorReaderImpl::checkImageFileOpen( const char *srcFileName, int srcLineNumber,
                                                       const char *srcFunctionName ) const
  {
     // unimplemented...
  }
  
  void CompressedVectorReaderImpl::checkReaderOpen( const char *srcFileName, int srcLineNumber,
                                                    const char *srcFunctionName ) const
  {
     if ( !isOpen_ )
     {
        throw E57Exception( E57_ERROR_READER_NOT_OPEN,
                            "imageFileName=" + cVector_->imageFileName() + " cvPathName=" + cVector_->pathName(),
                            srcFileName, srcLineNumber, srcFunctionName );
     }
  }
  
  void CompressedVectorReaderImpl::dump( int indent, std::ostream &os )
  {
     os << space( indent ) << "isOpen:" << isOpen_ << endl;
  
     for ( unsigned i = 0; i < dbufs_.size(); i++ )
     {
        os << space( indent ) << "dbufs[" << i << "]:" << endl;
        dbufs_[i].dump( indent + 4, os );
     }
  
     os << space( indent ) << "cVector:" << endl;
     cVector_->dump( indent + 4, os );
  
     os << space( indent ) << "proto:" << endl;
     proto_->dump( indent + 4, os );
  
     for ( unsigned i = 0; i < channels_.size(); i++ )
     {
        os << space( indent ) << "channels[" << i << "]:" << endl;
        channels_[i].dump( indent + 4, os );
     }
  
     os << space( indent ) << "recordCount:             " << recordCount_ << endl;
     os << space( indent ) << "maxRecordCount:          " << maxRecordCount_ << endl;
     os << space( indent ) << "sectionEndLogicalOffset: " << sectionEndLogicalOffset_ << endl;
  }
  
  //================================================================
  
  DecodeChannel::DecodeChannel( SourceDestBuffer dbuf_arg, shared_ptr<Decoder> decoder_arg, unsigned bytestreamNumber_arg,
                                uint64_t maxRecordCount_arg ) :
     dbuf( dbuf_arg ),
     decoder( decoder_arg ), bytestreamNumber( bytestreamNumber_arg )
  {
     maxRecordCount = maxRecordCount_arg;
     currentPacketLogicalOffset = 0;
     currentBytestreamBufferIndex = 0;
     currentBytestreamBufferLength = 0;
     inputFinished = false;
  }
  
  bool DecodeChannel::isOutputBlocked() const
  {
     /// If we have completed the entire vector, we are done
     if ( decoder->totalRecordsCompleted() >= maxRecordCount )
        return ( true );
  
     /// If we have filled the dest buffer, we are blocked
     return ( dbuf.impl()->nextIndex() == dbuf.impl()->capacity() );
  }
  
  bool DecodeChannel::isInputBlocked() const
  {
     /// If have read until the section end, we are done
     if ( inputFinished )
        return ( true );
  
     /// If have eaten all the input in the current packet, we are blocked.
     return ( currentBytestreamBufferIndex == currentBytestreamBufferLength );
  }
  
  void DecodeChannel::dump( int indent, std::ostream &os )
  {
     os << space( indent ) << "dbuf" << endl;
     dbuf.dump( indent + 4, os );
  
     os << space( indent ) << "decoder:" << endl;
     decoder->dump( indent + 4, os );
  
     os << space( indent ) << "bytestreamNumber:              " << bytestreamNumber << endl;
     os << space( indent ) << "maxRecordCount:                " << maxRecordCount << endl;
     os << space( indent ) << "currentPacketLogicalOffset:    " << currentPacketLogicalOffset << endl;
     os << space( indent ) << "currentBytestreamBufferIndex:  " << currentBytestreamBufferIndex << endl;
     os << space( indent ) << "currentBytestreamBufferLength: " << currentBytestreamBufferLength << endl;
     os << space( indent ) << "inputFinished:                 " << inputFinished << endl;
     os << space( indent ) << "isInputBlocked():              " << isInputBlocked() << endl;
     os << space( indent ) << "isOutputBlocked():             " << isOutputBlocked() << endl;
  }
  