title: How to incrementally parse (and act on) a large file with Boost.Spirit.Qi?
body: <p>I've created a Qi parser for a custom text file format. There are tens of thousands of entries to process and each entry usually has between 1-10 subentries. I put a trimmed down working example of my parser
  <a href="http://coliru.stacked-crooked.com/a/220105d357ffbbf0" rel="nofollow noreferrer" title="here">here</a>.</p>
  
  <pre><code>#include &lt;boost/config/warning_disable.hpp&gt;
  #include &lt;boost/spirit/include/qi.hpp&gt;
  #include &lt;boost/spirit/include/phoenix_core.hpp&gt;
  #include &lt;boost/spirit/include/phoenix_operator.hpp&gt;
  #include &lt;boost/spirit/include/phoenix_fusion.hpp&gt;
  #include &lt;boost/spirit/include/phoenix_stl.hpp&gt;
  #include &lt;boost/spirit/include/phoenix_object.hpp&gt;
  #include &lt;boost/spirit/include/support_istream_iterator.hpp&gt;
  #include &lt;boost/fusion/include/adapt_struct.hpp&gt;
  #include &lt;boost/fusion/include/io.hpp&gt;
  
  #include &lt;fstream&gt;
  #include &lt;iostream&gt;
  #include &lt;string&gt;
  
  using std::string;
  using std::vector;
  using std::cout;
  using std::endl;
  
  namespace model
  {
      namespace qi = boost::spirit::qi;
  
      struct spectrum
      {
          string comment;
          string file;
          string nativeId;
          double precursorMz;
          int precursorCharge;
          double precursorIntensity;
      };
  
      struct cluster
      {
          string id;
          vector&lt;spectrum&gt; spectra;
      };
  
      struct clustering
      {
          string name;
          vector&lt;cluster&gt; clusters;
      };
  }
  
  // Tell fusion about the data structures to make them first-class fusion citizens.
  // Must be at global scope.
  
  BOOST_FUSION_ADAPT_STRUCT(
      model::spectrum,
      (string, comment)
      (string, file)
      (string, nativeId)
      (double, precursorMz)
      (int, precursorCharge)
      (double, precursorIntensity)
  )
  
  BOOST_FUSION_ADAPT_STRUCT(
      model::cluster,
      (string, id)
      (std::vector&lt;model::spectrum&gt;, spectra)
  )
  
  BOOST_FUSION_ADAPT_STRUCT(
      model::clustering,
      (string, name)
      (std::vector&lt;model::cluster&gt;, clusters)
  )
  
  namespace {
      struct ReportError
      {
        template&lt;typename, typename, typename, typename&gt; struct result { typedef void type; };
  
        // contract the string to the surrounding new-line characters
        template&lt;typename Iter&gt;
        void operator()(Iter first_iter, Iter last_iter,
                        Iter error_iter, const boost::spirit::qi::info&amp; what) const
        {
          std::string first(first_iter, error_iter);
          std::string last(error_iter, last_iter);
          auto first_pos = first.rfind('\n');
          auto last_pos = last.find('\n');
          auto error_line = ((first_pos == std::string::npos) ? first
                              : std::string(first, first_pos + 1))
                            + std::string(last, 0, last_pos);
          //auto error_pos = (error_iter - first_iter) + 1;
          /*auto error_pos = error
          if (first_pos != std::string::npos)
            error_pos -= (first_pos + 1);*/
  
          std::cerr &lt;&lt; "Error parsing in " &lt;&lt; what &lt;&lt; std::endl
                    &lt;&lt; error_line &lt;&lt; std::endl
                    //&lt;&lt; std::setw(error_pos) &lt;&lt; '^'
                    &lt;&lt; std::endl;
          }
      };
  
      const boost::phoenix::function&lt;ReportError&gt; report_error = ReportError();
  }
  
  namespace model
  {
      template &lt;typename Iterator&gt;
      struct cluster_parser : qi::grammar&lt;Iterator, clustering(), qi::blank_type&gt;
      {
          cluster_parser() : cluster_parser::base_type(clusters)
          {
              using qi::int_;
              using qi::lit;
              using qi::double_;
              using qi::bool_;
              using qi::lexeme;
              using qi::eol;
              using qi::ascii::char_;
              using qi::on_error;
              using qi::fail;
              using namespace qi::labels;
              using boost::phoenix::construct;
              using boost::phoenix::val;
  
              quoted_string %= lexeme['"' &gt; +(char_ - '"') &gt; '"'];
  
              spectrum_start %=
                  lit("SPEC") &gt;
                  "#" &gt; +(char_ - "File:") &gt;
                  "File:" &gt; quoted_string &gt; lit(",") &gt;
                  "NativeID:" &gt; quoted_string &gt;
                  bool_ &gt; double_ &gt; int_ &gt; double_;
  
              cluster_start %= 
                  "=Cluster=" &gt; eol &gt;
                  "id=" &gt; +(char_ - eol) &gt; eol &gt;
                  spectrum_start % eol;
  
              clusters %= 
                  "name=" &gt; +(char_ - eol) &gt; eol &gt;
                  eol &gt;
                  cluster_start % eol;
  
              BOOST_SPIRIT_DEBUG_NODES((clusters)(cluster_start)(quoted_string)(spectrum_start))
  
              //on_error&lt;fail&gt;(clusters, report_error(_1, _2, _3, _4));
              //on_error&lt;fail&gt;(cluster_start, report_error(_1, _2, _3, _4));
              //on_error&lt;fail&gt;(spectrum_start, report_error(_1, _2, _3, _4));
              //on_error&lt;fail&gt;(quoted_string, report_error(_1, _2, _3, _4));
  
              // on_success(cluster_start, quantify_cluster(_1, _2, _3, _4)); ??
          }
  
          qi::rule&lt;Iterator, std::string(), qi::blank_type&gt; quoted_string;
          qi::rule&lt;Iterator, cluster(), qi::blank_type&gt; cluster_start;
          qi::rule&lt;Iterator, spectrum(), qi::blank_type&gt; spectrum_start;
          qi::rule&lt;Iterator, clustering(), qi::blank_type&gt; clusters;
      };
  }
  
  int main()
  {
      using namespace model;
  
      cluster_parser&lt;boost::spirit::istream_iterator&gt; g; // Our grammar
      string str;
      //std::ifstream input("c:/test/Mo_tai.clustering");
      std::istringstream input("name=GreedyClustering_0.99\n"
  "\n"
  "=Cluster=\n"
  "id=9c8c5830-5841-4f77-b819-64180509615b\n"
  "SPEC\t#file=w:\\test\\Mo_Tai_iTRAQ_f4.mgf#id=index=219#title=Mo_Tai_iTRAQ_f4.1254.1254.2 File:\"Mo_Tai_iTRAQ_f4.raw\", NativeID:\"controllerType=0 controllerNumber=1 scan=1254\"\ttrue\t\t300.1374\t2\t\t\t0.0\n"
  "=Cluster=\n"
  "id=f8f384a1-3d5f-4af1-9581-4d03a5aa3342\n"
  "SPEC\t#file=w:\\test\\Mo_Tai_iTRAQ_f9.mgf#id=index=560#title=Mo_Tai_iTRAQ_f9.1666.1666.3 File:\"Mo_Tai_iTRAQ_f9.raw\", NativeID:\"controllerType=0 controllerNumber=1 scan=1666\"\ttrue\t\t300.14413\t3\t\t\t0.0\n"
  "SPEC\t#file=w:\\test\\Mo_Tai_iTRAQ_f9.mgf#id=index=520#title=Mo_Tai_iTRAQ_f9.1621.1621.3 File:\"Mo_Tai_iTRAQ_f9.raw\", NativeID:\"controllerType=0 controllerNumber=1 scan=1621\"\ttrue\t\t300.14197\t3\t\t\t0.0\n"
  "=Cluster=\n"
  "id=b84b79e1-44bc-44c0-a9af-5391ca02582d\n"
  "SPEC\t#file=w:\\test\\Mo_Tai_iTRAQ_f2.mgf#id=index=7171#title=Mo_Tai_iTRAQ_f2.12729.12729.2 File:\"Mo_Tai_iTRAQ_f2.raw\", NativeID:\"controllerType=0 controllerNumber=1 scan=12729\"\ttrue\t\t300.15695\t2\t\t\t0.0");
      input.unsetf(std::ios::skipws);
      boost::spirit::istream_iterator begin(input);
      boost::spirit::istream_iterator end;
  
      clustering clusteringResults;
      bool r = phrase_parse(begin, end, g, qi::blank, clusteringResults);
  
      if (r &amp;&amp; begin == end)
      {
          cout &lt;&lt; "Parsing succeeded (" &lt;&lt; clusteringResults.clusters.size() &lt;&lt; " clusters)\n";
  
          /*for (size_t i = 0; i &lt; std::min((size_t)10, clusteringResults.clusters.size()); ++i)
          {
              cluster&amp; c = clusteringResults.clusters[i];
              cout &lt;&lt; "Cluster " &lt;&lt; c.id &lt;&lt; " - avg. precursor m/z: " &lt;&lt; c.avgPrecursorMz &lt;&lt; ", num. spectra: " &lt;&lt; c.spectra.size() &lt;&lt; endl;
          }*/
          return 1;
      }
      else
      {
          std::cout &lt;&lt; "Parsing failed (" &lt;&lt; clusteringResults.clusters.size() &lt;&lt; " clusters)\n";
  
          if (!clusteringResults.clusters.empty())
          {
              cluster&amp; c = clusteringResults.clusters.back();
              cout &lt;&lt; "Last cluster parsed " &lt;&lt; c.id &lt;&lt; ", num. spectra: " &lt;&lt; c.spectra.size() &lt;&lt; endl;
          }
          return 1;
      }
  }
  </code></pre>
  
  <p>I don't want to parse the entire file into memory before processing it. How can I make it queue up an entry (cluster) for processing after each cluster is finished parsing, delete the cluster after processing, then continue parsing? Even better would be to have another thread handle the processing asynchronously.</p>