public function tokenize($inputString, $encoding)
    {
        $this->reset();

        $this->_lexemes     = array();
        $this->_queryString = array();

        if (PHP_OS == 'AIX' && $encoding == '') {
            $encoding = 'ISO8859-1';
        }
        $strLength = iconv_strlen($inputString, $encoding);

        // Workaround for iconv_substr bug
        $inputString .= ' ';

        for ($count = 0; $count < $strLength; $count++) {
            $this->_queryString[$count] = iconv_substr($inputString, $count, 1, $encoding);
        }

        for ($this->_queryStringPosition = 0;
             $this->_queryStringPosition < count($this->_queryString);
             $this->_queryStringPosition++) {
            $this->process($this->_translateInput($this->_queryString[$this->_queryStringPosition]));
        }

        $this->process(self::IN_WHITE_SPACE);

        if ($this->getState() != self::ST_WHITE_SPACE) {
            include_once 'Zend/Search/Lucene/Search/QueryParserException.php';
            throw new Zend_Search_Lucene_Search_QueryParserException('Unexpected end of query');
        }

        $this->_queryString = null;

        return $this->_lexemes;
    }
public function tokenize($inputString, $encoding)
    {
        $this->reset();

        $this->_lexemes     = array();
        $this->_queryString = array();

        if (PHP_OS == 'AIX' && $encoding == '') {
            $encoding = 'ISO8859-1';
        }
        $strLength = iconv_strlen($inputString, $encoding);

        // Workaround for iconv_substr bug
        $inputString .= ' ';

        for ($count = 0; $count < $strLength; $count++) {
            $this->_queryString[$count] = iconv_substr($inputString, $count, 1, $encoding);
        }

        for ($this->_queryStringPosition = 0;
             $this->_queryStringPosition < count($this->_queryString);
             $this->_queryStringPosition++) {
            $this->process($this->_translateInput($this->_queryString[$this->_queryStringPosition]));
        }

        $this->process(self::IN_WHITE_SPACE);

        if ($this->getState() != self::ST_WHITE_SPACE) {
            include_once 'Zend/Search/Lucene/Search/QueryParserException.php';
            throw new Zend_Search_Lucene_Search_QueryParserException('Unexpected end of query');
        }

        $this->_queryString = null;

        return $this->_lexemes;
    }
public function tokenize ( $ inputString , $ encoding ) { $ this -> reset ( ) ; $ this -> _lexemes = array ( ) ; $ this -> _queryString = array ( ) ; if ( PHP_OS == 'AIX' && $ encoding == '' ) { $ encoding = 'ISO8859-1' ; } $ strLength = iconv_strlen ( $ inputString , $ encoding ) ; // Workaround for iconv_substr bug $ inputString .= ' ' ; for ( $ count = 0 ; $ count < $ strLength ; $ count ++ ) { $ this -> _queryString [ $ count ] = iconv_substr ( $ inputString , $ count , 1 , $ encoding ) ; } for ( $ this -> _queryStringPosition = 0 ; $ this -> _queryStringPosition < count ( $ this -> _queryString ) ; $ this -> _queryStringPosition ++ ) { $ this -> process ( $ this -> _translateInput ( $ this -> _queryString [ $ this -> _queryStringPosition ] ) ) ; } $ this -> process ( self :: IN_WHITE_SPACE ) ; if ( $ this -> getState ( ) != self :: ST_WHITE_SPACE ) { include_once 'Zend/Search/Lucene/Search/QueryParserException.php' ; throw new Zend_Search_Lucene_Search_QueryParserException ( 'Unexpected end of query' ) ; } $ this -> _queryString = null ; return $ this -> _lexemes ; }
This method is used to tokenize query string into lexemes

@param  string $inputString
@param  string $encoding
@return array
@throws Zend_Search_Lucene_Search_QueryParserException
This method is used to tokenize query string into lexemes