original_string: public function tokenize($inputString, $encoding)
      {
          $this->reset();
  
          $this->_lexemes     = array();
          $this->_queryString = array();
  
          if (PHP_OS == 'AIX' && $encoding == '') {
              $encoding = 'ISO8859-1';
          }
          $strLength = iconv_strlen($inputString, $encoding);
  
          // Workaround for iconv_substr bug
          $inputString .= ' ';
  
          for ($count = 0; $count < $strLength; $count++) {
              $this->_queryString[$count] = iconv_substr($inputString, $count, 1, $encoding);
          }
  
          for ($this->_queryStringPosition = 0;
               $this->_queryStringPosition < count($this->_queryString);
               $this->_queryStringPosition++) {
              $this->process($this->_translateInput($this->_queryString[$this->_queryStringPosition]));
          }
  
          $this->process(self::IN_WHITE_SPACE);
  
          if ($this->getState() != self::ST_WHITE_SPACE) {
              include_once 'Zend/Search/Lucene/Search/QueryParserException.php';
              throw new Zend_Search_Lucene_Search_QueryParserException('Unexpected end of query');
          }
  
          $this->_queryString = null;
  
          return $this->_lexemes;
      }
code: public function tokenize($inputString, $encoding)
      {
          $this->reset();
  
          $this->_lexemes     = array();
          $this->_queryString = array();
  
          if (PHP_OS == 'AIX' && $encoding == '') {
              $encoding = 'ISO8859-1';
          }
          $strLength = iconv_strlen($inputString, $encoding);
  
          // Workaround for iconv_substr bug
          $inputString .= ' ';
  
          for ($count = 0; $count < $strLength; $count++) {
              $this->_queryString[$count] = iconv_substr($inputString, $count, 1, $encoding);
          }
  
          for ($this->_queryStringPosition = 0;
               $this->_queryStringPosition < count($this->_queryString);
               $this->_queryStringPosition++) {
              $this->process($this->_translateInput($this->_queryString[$this->_queryStringPosition]));
          }
  
          $this->process(self::IN_WHITE_SPACE);
  
          if ($this->getState() != self::ST_WHITE_SPACE) {
              include_once 'Zend/Search/Lucene/Search/QueryParserException.php';
              throw new Zend_Search_Lucene_Search_QueryParserException('Unexpected end of query');
          }
  
          $this->_queryString = null;
  
          return $this->_lexemes;
      }
code_tokens: public function tokenize ( $ inputString , $ encoding ) { $ this -> reset ( ) ; $ this -> _lexemes = array ( ) ; $ this -> _queryString = array ( ) ; if ( PHP_OS == 'AIX' && $ encoding == '' ) { $ encoding = 'ISO8859-1' ; } $ strLength = iconv_strlen ( $ inputString , $ encoding ) ; // Workaround for iconv_substr bug $ inputString .= ' ' ; for ( $ count = 0 ; $ count < $ strLength ; $ count ++ ) { $ this -> _queryString [ $ count ] = iconv_substr ( $ inputString , $ count , 1 , $ encoding ) ; } for ( $ this -> _queryStringPosition = 0 ; $ this -> _queryStringPosition < count ( $ this -> _queryString ) ; $ this -> _queryStringPosition ++ ) { $ this -> process ( $ this -> _translateInput ( $ this -> _queryString [ $ this -> _queryStringPosition ] ) ) ; } $ this -> process ( self :: IN_WHITE_SPACE ) ; if ( $ this -> getState ( ) != self :: ST_WHITE_SPACE ) { include_once 'Zend/Search/Lucene/Search/QueryParserException.php' ; throw new Zend_Search_Lucene_Search_QueryParserException ( 'Unexpected end of query' ) ; } $ this -> _queryString = null ; return $ this -> _lexemes ; }
docstring: This method is used to tokenize query string into lexemes
  
  @param  string $inputString
  @param  string $encoding
  @return array
  @throws Zend_Search_Lucene_Search_QueryParserException
docstring_tokens: This method is used to tokenize query string into lexemes