original_string: public function tokenize(Reader $reader)
      {
          $stream = new TokenStream();
  
          while (!$reader->isEOF()) {
              foreach ($this->handlers as $handler) {
                  if ($handler->handle($reader, $stream)) {
                      continue 2;
                  }
              }
  
              $stream->push(new Token(Token::TYPE_DELIMITER, $reader->getSubstring(1), $reader->getPosition()));
              $reader->moveForward(1);
          }
  
          return $stream
              ->push(new Token(Token::TYPE_FILE_END, null, $reader->getPosition()))
              ->freeze();
      }
code: public function tokenize(Reader $reader)
      {
          $stream = new TokenStream();
  
          while (!$reader->isEOF()) {
              foreach ($this->handlers as $handler) {
                  if ($handler->handle($reader, $stream)) {
                      continue 2;
                  }
              }
  
              $stream->push(new Token(Token::TYPE_DELIMITER, $reader->getSubstring(1), $reader->getPosition()));
              $reader->moveForward(1);
          }
  
          return $stream
              ->push(new Token(Token::TYPE_FILE_END, null, $reader->getPosition()))
              ->freeze();
      }
code_tokens: public function tokenize ( Reader $ reader ) { $ stream = new TokenStream ( ) ; while ( ! $ reader -> isEOF ( ) ) { foreach ( $ this -> handlers as $ handler ) { if ( $ handler -> handle ( $ reader , $ stream ) ) { continue 2 ; } } $ stream -> push ( new Token ( Token :: TYPE_DELIMITER , $ reader -> getSubstring ( 1 ) , $ reader -> getPosition ( ) ) ) ; $ reader -> moveForward ( 1 ) ; } return $ stream -> push ( new Token ( Token :: TYPE_FILE_END , null , $ reader -> getPosition ( ) ) ) -> freeze ( ) ; }
docstring: Tokenize selector source code.
  
  @return TokenStream
docstring_tokens: Tokenize selector source code .