public function tokenize($data, $encoding = '')
    {
        $this->setInput($data, $encoding);

        $tokenList = array();
        while (($nextToken = $this->nextToken()) !== null) {
            $tokenList[] = $nextToken;
        }

        return $tokenList;
    }
public function tokenize($data, $encoding = '')
    {
        $this->setInput($data, $encoding);

        $tokenList = array();
        while (($nextToken = $this->nextToken()) !== null) {
            $tokenList[] = $nextToken;
        }

        return $tokenList;
    }
public function tokenize ( $ data , $ encoding = '' ) { $ this -> setInput ( $ data , $ encoding ) ; $ tokenList = array ( ) ; while ( ( $ nextToken = $ this -> nextToken ( ) ) !== null ) { $ tokenList [ ] = $ nextToken ; } return $ tokenList ; }
Tokenize text to a terms
Returns array of Zend_Search_Lucene_Analysis_Token objects

Tokens are returned in UTF-8 (internal Zend_Search_Lucene encoding)

@param  string $data
@return array
Tokenize text to a terms Returns array of Zend_Search_Lucene_Analysis_Token objects