title: How to create a std::vector-like class which can binary read/write huge chunks?
body: <p><strong>PROBLEM</strong></p>
  
  <p>I have this old piece of pre-stl C++ code that I want to translate into std C++11 without losing efficiency.</p>
  
  <pre><code>using T = unsigned;  // but can be any POD
  FILE* fp = fopen( outfile.c_str(), "r" );
  T* x = new T[big_n];
  fread( x, sizeof(T), big_n, fp );
  delete[] x;
  fclose( fp );
  </code></pre>
  
  <p>Note that big_n is really big - like millions of records big, so any inefficiencies are pronounced.</p>
  
  <p><strong>PREVIOUS SOLUTION</strong></p>
  
  <p>In this <a href="https://stackoverflow.com/a/14380356/975129">answer</a> from my previous question, I accepted this solution:</p>
  
  <pre><code>std::vector&lt;T&gt; x(big_n);
  fread(x.data(), sizeof(T), big_n, fp);
  </code></pre>
  
  <p><strong>ISSUE AND ATTEMPTED SOLUTION</strong></p>
  
  <p>That previous solution works, but the constructor actually calls T's default constructor big_n times.  This is very slow when big_n is really big (and totally unnecessary as I am about to fread() the entire chunk from disk).  FWIW, in my test case for one file, it was taking 3 seconds instead of 200ms.</p>
  
  <p>So I tried to use this instead:</p>
  
  <pre><code>std::vector&lt;T&gt; x;
  x.reserve( big_n );
  fread(x.data(), sizeof(T), big_n, fp);
  </code></pre>
  
  <p>This seems to work, but then I run into the issue that size() returns 0 and not big_n.</p>
  
  <p><em>How do I correct this without losing too much efficiency?</em></p>
  
  <p><strong>ADDENDUM</strong></p>
  
  <p>I just noticed that <code>std::vector&lt;&gt;</code> can take a custom allocator.  Could using that form of the constructor solve my problem?  I'm looking into this approach now.</p>
  
  <p><strong>WHAT WORKS FOR ME</strong></p>
  
  <p>I've looked into Ali's custom allocator solution below in addition to jrok's simple array solution.  I have decided to adapt jrock's solution for its ease-of-understanding/lower maintenance.</p>
  
  <p>The working code I came up with is below:</p>
  
  <pre><code>#include &lt;vector&gt;
  #include &lt;set&gt;
  #include &lt;memory&gt;
  #include &lt;fstream&gt;
  #include &lt;iostream&gt;
  #include &lt;cassert&gt;
  
  struct Foo
  {
          int m_i;
          Foo() { }
          Foo( int i ) : m_i( i ) { }
          bool operator==( Foo const&amp; rhs ) const { return m_i==rhs.m_i; }
          bool operator!=( Foo const&amp; rhs ) const { return m_i!=rhs.m_i; }
          friend std::ostream&amp; operator&lt;&lt;( std::ostream&amp; os, Foo const&amp; rhs )
          { os &lt;&lt; rhs.m_i; }
  };
  
  
  // DESIGN NOTES  /*{{{*/
  //
  // LIMITATION  T must be a POD so we can fread/fwrite quickly
  //
  // WHY DO WE NEED THIS CLASS?
  //
  // We want to write a large number of small PODs to disk and read them back without
  //   1. spurious calls to default constructors by std::vector
  //   2. writing to disk a gazillion times
  //
  // SOLUTION
  //   A hybrid class containing a std::vector&lt;&gt; for adding new items and a
  //   std::unique_ptr&lt;T[]&gt; for fast persistence.  From the user's POV, it looks
  //   like a std::vector&lt;&gt;.
  //
  // Algorithm
  //   1. add new items into:
  //      std::vector&lt;T&gt;        m_v;
  //   2. when writing to disk, write out m_v as a chunk
  //   3. when reading from disk, read into m_chunk (m_v will start empty again)
  //   4. m_chunk and m_v combined will represent all the data
  /*}}}*/
  
  template&lt;typename T&gt;
  class vector_chunk
  {
  // STATE  /*{{{*/
          size_t                m_n_in_chunk;
          std::unique_ptr&lt;T[]&gt;  m_chunk;
          std::vector&lt;T&gt;        m_v;
  /*}}}*/
  
  // CONSTRUCTOR, INITIALIZATION  /*{{{*/
  public:
          vector_chunk() : m_n_in_chunk( 0 ) { }
  /*}}}*/
  
  // EQUALITY /*{{{*/
          public:
                  bool operator==( vector_chunk const&amp; rhs ) const
                  {
                          if ( rhs.size()!=size() )
                                  return false;
  
                          for( size_t i=0; i&lt;size(); ++i )
                                  if ( operator[]( i )!=rhs[i] )
                                          return false;
  
                          return true;
                  }
  /*}}}*/
  
  // OSTREAM /*{{{*/
          public:
                  friend std::ostream&amp; operator&lt;&lt;( std::ostream&amp; os, vector_chunk const&amp; rhs )
                  {
                          for( size_t i=0; i&lt;rhs.m_n_in_chunk; ++i )
                                  os &lt;&lt; rhs.m_chunk[i] &lt;&lt; "\n";
                          for( T const&amp; t : rhs.m_v )
                                  os &lt;&lt; rhs.t &lt;&lt; "\n";
                  }
  /*}}}*/
  // BINARY I/O  /*{{{*/
  public:
          void write_as_binary( std::ostream&amp; os ) const
          {
                  // write everything out
                  size_t const  n_total = size();
                  os.write( reinterpret_cast&lt;const char*&gt;( &amp;n_total ), sizeof( n_total ));
                  os.write( reinterpret_cast&lt;const char*&gt;( &amp;m_chunk[0] ), m_n_in_chunk * sizeof( T ));
                  os.write( reinterpret_cast&lt;const char*&gt;( m_v.data() ), m_v.size() * sizeof( T ));
          }
          void read_as_binary(  std::istream&amp; is )
          {
                  // only read into m_chunk, clear m_v
                  is.read( reinterpret_cast&lt;char*&gt;( &amp;m_n_in_chunk ), sizeof( m_n_in_chunk ));
                  m_chunk.reset( new T[ m_n_in_chunk ] );
                  is.read( reinterpret_cast&lt;char*&gt;( &amp;m_chunk[0] ), m_n_in_chunk * sizeof( T ));
                  m_v.clear();
          }
  /*}}}*/
  
  // DELEGATION to std::vector&lt;T&gt;  /*{{{*/
  public:
          size_t size() const                 { return m_n_in_chunk + m_v.size(); }
          void push_back( T const&amp; value )    { m_v.push_back( value ); }
          void push_back( T&amp;&amp;      value )    { m_v.push_back( value ); }
          template&lt; class... Args &gt;
          void emplace_back( Args&amp;&amp;... args ) { m_v.emplace_back( args... ); }
          typename std::vector&lt;T&gt;::const_reference
          operator[]( size_t pos ) const
          { return ((pos &lt; m_n_in_chunk) ? m_chunk[ pos ] : m_v[ pos - m_n_in_chunk]); }
  
          typename std::vector&lt;T&gt;::reference
          operator[]( size_t pos )
          { return ((pos &lt; m_n_in_chunk) ? m_chunk[ pos ] : m_v[ pos - m_n_in_chunk]); }
  /*}}}*/
  };
  
  int main()
  {
          size_t const n = 10;
          vector_chunk&lt;Foo&gt;  v, w;
          for( int i=0; i&lt;n; ++i )
                  v.emplace_back( Foo{ i } );
  
          std::filebuf                   ofb, ifb;
          std::unique_ptr&lt;std::ostream&gt;  osp;
          std::unique_ptr&lt;std::istream&gt;  isp;
  
          ofb.open( "/tmp/junk.bin", (std::ios::out | std::ios::binary));
          osp.reset( new std::ostream( &amp;ofb ));
          v.write_as_binary( *osp );
          ofb.close();
  
          ifb.open( "/tmp/junk.bin", (std::ios::in | std::ios::binary));
          isp.reset( new std::istream( &amp;ifb ));
          w.read_as_binary(  *isp );
          ifb.close();
  
          assert( v==w );
  }
  </code></pre>