title: How to combine boost::spirit::lex & boost::spirit::qi?
body: <p>I have a lexer and based on that lexer I now want to create a grammar that used the tokens generated by this lexer. I tried adapting some examples that I found and now I have something that compiles and works at least a little bit, but one of my tests that should fail does not. Now I want to know why and I also want to know what I'm actually doing there (I want to understand - I just copied some code from some examples but that doesn't really improve the understanding much).</p>
  
  <p>Lexer:</p>
  
  <pre><code>#include &lt;boost/spirit/include/lex_lexertl.hpp&gt;
  
  namespace lex = boost::spirit::lex;
  
  enum LexerIDs { ID_IDENTIFIER, ID_WHITESPACE, ID_INTEGER, ID_FLOAT, ID_PUNCTUATOR };
  
  template &lt;typename Lexer&gt;
  struct custom_lexer : lex::lexer&lt;Lexer&gt;
  {
      custom_lexer()
          : identifier("[a-zA-Z_][a-zA-Z0-9_]*")
          , white_space("[ \\t\\n]+")
          , integer_value("[1-9][0-9]*")
          , hex_value("0[xX][0-9a-fA-F]+")
          , float_value("[0-9]*\\.[0-9]+([eE][+-]?[0-9]+)?")
          , float_value2("[0-9]+\\.([eE][+-]?[0-9]+)?")
          , punctuator("\\[|\\]|\\(|\\)|\\.|&amp;&gt;|\\*\\*|\\*|\\+|-|~|!|\\/|%|&lt;&lt;|&gt;&gt;|&lt;|&gt;|&lt;=|&gt;=|==|!=|\\^|&amp;|\\||\\^\\^|&amp;&amp;|\\|\\||\\?|:|,")// [ ] ( ) . &amp;&gt; ** * + - ~ ! / % &lt;&lt; &gt;&gt; &lt; &gt; &lt;= &gt;= == != ^ &amp; | ^^ &amp;&amp; || ? : ,
      {
          using boost::spirit::lex::_start;
          using boost::spirit::lex::_end;
  
          this-&gt;self.add
              (identifier, ID_IDENTIFIER) 
              /*(white_space, ID_WHITESPACE)*/ 
              (integer_value, ID_INTEGER)
              (hex_value, ID_INTEGER)
              (float_value, ID_FLOAT)
              (float_value2, ID_FLOAT)
              (punctuator, ID_PUNCTUATOR);
  
          this-&gt;self("WS") = white_space;
      }
      lex::token_def&lt;std::string&gt; identifier;
      lex::token_def&lt;lex::omit&gt; white_space;
      lex::token_def&lt;int&gt; integer_value;
      lex::token_def&lt;int&gt; hex_value;
      lex::token_def&lt;double&gt; float_value;
      lex::token_def&lt;double&gt; float_value2;
      lex::token_def&lt;&gt; punctuator;
  };
  </code></pre>
  
  <p>Grammar:</p>
  
  <pre><code>namespace qi  = boost::spirit::qi;
  namespace lex = boost::spirit::lex;
  
  template&lt; typename Iterator, typename Lexer&gt;
  struct custom_grammar : qi::grammar&lt;Iterator, qi::in_state_skipper&lt;Lexer&gt;&gt;
  {
  
      template&lt; typename TokenDef &gt;
      custom_grammar(const TokenDef&amp; tok) : custom_grammar::base_type(ges)
      {
          ges = qi::token(ID_INTEGER) | qi::token(ID_FLOAT);
          BOOST_SPIRIT_DEBUG_NODE(ges);
          debug(ges);
      }
      qi::rule&lt;Iterator, qi::in_state_skipper&lt;Lexer&gt;&gt; ges;
  };
  </code></pre>
  
  <p>And example:</p>
  
  <pre><code>BOOST_AUTO_TEST_CASE(BasicGrammar)
  {
      namespace lex = boost::spirit::lex;
      namespace qi = boost::spirit::qi;
  
      std::string test("1234 56");
  
      typedef lex::lexertl::token&lt;char const*, lex::omit, boost::mpl::true_&gt; token_type;
      typedef lex::lexertl::lexer&lt;token_type&gt; lexer_type;
  
      typedef custom_lexer&lt;lexer_type&gt;::iterator_type iterator_type;
  
      custom_lexer&lt;lexer_type&gt; my_lexer; 
      custom_grammar&lt;iterator_type, custom_lexer&lt;lexer_type&gt;::lexer_def&gt; my_grammar(my_lexer);
  
      char const* first = test.c_str();
      char const* last = &amp;first[test.size()];
  
      lexer_type::iterator_type iter = my_lexer.begin(first, last);
      lexer_type::iterator_type end = my_lexer.end();
  
      bool r = qi::phrase_parse(iter,end,my_grammar, qi::in_state( "WS" )[ my_lexer.self ]);
  
      BOOST_CHECK(r);
  }
  </code></pre>
  
  <p>My assumption is that this returns true because the whitespace is skipped - because auf qi::in_state("WS"). Is that true? Additionally, I know how I can output additional tokens for whitespace - but then I don't know what to put at the location where the qi::in_stat is now - without it it isn't working.</p>
  
  <p>Any ideas what I can improve regarding the structure? Why is the debug output so funny?</p>
  
  <pre><code>&lt;ges&gt;
    &lt;try&gt;[]&lt;/try&gt;
    &lt;success&gt;&lt;/success&gt;
    &lt;attributes&gt;[]&lt;/attributes&gt;
  &lt;/ges&gt;
  </code></pre>
  
  <p>Thank you for your help.</p>
  
  <p>Regards</p>
  
  <p>Tobias</p>