void function ( ) { if ( input . hasMoreTokens ( ) ) { String loc0 = input . nextToken ( ) ; if ( loc0 . length ( ) > n ) { for ( int loc1 = 0 ; loc1 < loc0 . length ( ) - n + 1 ; loc1 ++ ) { String loc2 = loc0 . substring ( loc1 , loc1 + n ) ; currentTokens . add ( loc2 ) ; } } else currentTokens . add ( loc0 ) ; } }
read a token from the character stream and store it into currenttoken . if there are no more tokens left store a null value . concode_field_sep TokenEnumeration input concode_elem_sep List currentTokens concode_elem_sep int n concode_elem_sep WVTTokenizer tokenizer concode_field_sep boolean hasMoreTokens concode_elem_sep String nextToken concode_elem_sep TokenEnumeration tokenize