title: Boost Spirit lexer states cross pollinate
body: <p>I am trying to use lexer states to do context specific parsing, but it seems that different lexer states do cross-pollinate. Here is a very basic example</p>
  
  <pre><code>#include &lt;boost/config/warning_disable.hpp&gt;
  #include &lt;boost/spirit/include/qi.hpp&gt;
  #include &lt;boost/spirit/include/lex_lexertl.hpp&gt;
  #include &lt;boost/spirit/include/phoenix_operator.hpp&gt;
  #include &lt;boost/spirit/include/phoenix_container.hpp&gt;
  
  #include &lt;iostream&gt;
  #include &lt;string&gt;
  
  using namespace boost::spirit;
  
  template &lt;typename Lexer&gt;
  struct strip_comments_tokens : lex::lexer&lt;Lexer&gt;
  {
      strip_comments_tokens() 
        : strip_comments_tokens::base_type(lex::match_flags::match_default)
      {
          ccomment = "\\/\\*";
          endcomment = ".*\\*\\/";
          hello = "hello";
  
          this-&gt;self.add
              (ccomment)
              (hello);
  
          this-&gt;self("COMMENT").add
              (endcomment);
      }
  
      lex::token_def&lt;&gt; ccomment, endcomment;
      lex::token_def&lt;std::string&gt; hello;
  };
  
  template &lt;typename Iterator&gt;
  struct strip_comments_grammar : qi::grammar&lt;Iterator&gt;
  {
      template &lt;typename TokenDef&gt;
      strip_comments_grammar(TokenDef const&amp; tok)
        : strip_comments_grammar::base_type(start)
      {
          start =  *(   tok.ccomment 
                        &gt;&gt;  qi::in_state("COMMENT") 
                        [
                            tok.endcomment 
                        ]
                |   tok.hello [ std::cout &lt;&lt; _1 ]
          );
      }
  
      qi::rule&lt;Iterator&gt; start;
  };
  
  
  int main(int argc, char* argv[])
  {
      typedef std::string::iterator base_iterator_type;
  
      typedef 
          lex::lexertl::lexer&lt;lex::lexertl::token&lt;base_iterator_type&gt; &gt; 
      lexer_type;
  
      typedef strip_comments_tokens&lt;lexer_type&gt;::iterator_type iterator_type;
  
      strip_comments_tokens&lt;lexer_type&gt; strip_comments;           // Our lexer
      strip_comments_grammar&lt;iterator_type&gt; g (strip_comments);   // Our parser 
  
      std::string str("hello/*hello*/hello");
      base_iterator_type first = str.begin();
  
      bool r = lex::tokenize_and_parse(first, str.end(), strip_comments, g);
  
      return 0;
  }
  </code></pre>
  
  <p>I would expect the input </p>
  
  <pre><code>"hello/*hello*/hello"
  </code></pre>
  
  <p>to be tokenized as <strong>hello ccomment endcomment hello</strong>. But what happens is the input  gets tokenized as <strong>hello ccomment hello</strong>, so the grammar stops working. If you change the input to </p>
  
  <pre><code>"hello/*anything else*/hello" 
  </code></pre>
  
  <p>everything works as expected. </p>
  
  <p>Any ideas?</p>