@Test public void parsePhrase() throws Exception { SearchTokenizer tokenizer = new SearchTokenizer(); List<SearchQueryToken> result; assertQuery("abc AND \"x-y_z\" AND olingo"); result = tokenizer.tokenize("\"abc\""); Assert.assertNotNull(result); Assert.assertEquals(PHRASE, result.get(0).getToken()); result = tokenizer.tokenize("\"9988 abs\""); Assert.assertNotNull(result); Assert.assertEquals(PHRASE, result.get(0).getToken()); Assert.assertEquals("\"9988 abs\"", result.get(0).getLiteral()); result = tokenizer.tokenize("\"99_88.\""); Assert.assertNotNull(result); Assert.assertEquals(PHRASE, result.get(0).getToken()); Assert.assertEquals("\"99_88.\"", result.get(0).getLiteral()); assertQuery("abc or \"xyz\"").resultsIn(WORD, WORD, PHRASE); }
public List<SearchQueryToken> tokenize(final String searchQuery) throws SearchTokenizerException { if (searchQuery.contains("%28") || searchQuery.contains("%29") || searchQuery.contains("%22")) { throw new SearchTokenizerException("Invalid Token in Query string '", SearchTokenizerException.MessageKeys.NOT_EXPECTED_TOKEN, searchQuery); } char[] chars = searchQuery.trim().toCharArray(); State state = new SearchExpressionState(); List<SearchQueryToken> states = new ArrayList<>(); for (char aChar : chars) { State next = state.nextChar(aChar); if (state.isFinished()) { states.add(state); } state = next; } if (state.close().isFinished()) { states.add(state); } else { throw new SearchTokenizerException("Last parsed state '" + state.toString() + "' is not finished.", SearchTokenizerException.MessageKeys.NOT_FINISHED_QUERY, state.getTokenName()); } return states; }
SearchTokenizer { public List<SearchQueryToken> tokenize(final String searchQuery) throws SearchTokenizerException { if (searchQuery.contains("%28") || searchQuery.contains("%29") || searchQuery.contains("%22")) { throw new SearchTokenizerException("Invalid Token in Query string '", SearchTokenizerException.MessageKeys.NOT_EXPECTED_TOKEN, searchQuery); } char[] chars = searchQuery.trim().toCharArray(); State state = new SearchExpressionState(); List<SearchQueryToken> states = new ArrayList<>(); for (char aChar : chars) { State next = state.nextChar(aChar); if (state.isFinished()) { states.add(state); } state = next; } if (state.close().isFinished()) { states.add(state); } else { throw new SearchTokenizerException("Last parsed state '" + state.toString() + "' is not finished.", SearchTokenizerException.MessageKeys.NOT_FINISHED_QUERY, state.getTokenName()); } return states; } }
SearchTokenizer { public List<SearchQueryToken> tokenize(final String searchQuery) throws SearchTokenizerException { if (searchQuery.contains("%28") || searchQuery.contains("%29") || searchQuery.contains("%22")) { throw new SearchTokenizerException("Invalid Token in Query string '", SearchTokenizerException.MessageKeys.NOT_EXPECTED_TOKEN, searchQuery); } char[] chars = searchQuery.trim().toCharArray(); State state = new SearchExpressionState(); List<SearchQueryToken> states = new ArrayList<>(); for (char aChar : chars) { State next = state.nextChar(aChar); if (state.isFinished()) { states.add(state); } state = next; } if (state.close().isFinished()) { states.add(state); } else { throw new SearchTokenizerException("Last parsed state '" + state.toString() + "' is not finished.", SearchTokenizerException.MessageKeys.NOT_FINISHED_QUERY, state.getTokenName()); } return states; }  }
SearchTokenizer { public List<SearchQueryToken> tokenize(final String searchQuery) throws SearchTokenizerException { if (searchQuery.contains("%28") || searchQuery.contains("%29") || searchQuery.contains("%22")) { throw new SearchTokenizerException("Invalid Token in Query string '", SearchTokenizerException.MessageKeys.NOT_EXPECTED_TOKEN, searchQuery); } char[] chars = searchQuery.trim().toCharArray(); State state = new SearchExpressionState(); List<SearchQueryToken> states = new ArrayList<>(); for (char aChar : chars) { State next = state.nextChar(aChar); if (state.isFinished()) { states.add(state); } state = next; } if (state.close().isFinished()) { states.add(state); } else { throw new SearchTokenizerException("Last parsed state '" + state.toString() + "' is not finished.", SearchTokenizerException.MessageKeys.NOT_FINISHED_QUERY, state.getTokenName()); } return states; }  List<SearchQueryToken> tokenize(final String searchQuery); }
SearchTokenizer { public List<SearchQueryToken> tokenize(final String searchQuery) throws SearchTokenizerException { if (searchQuery.contains("%28") || searchQuery.contains("%29") || searchQuery.contains("%22")) { throw new SearchTokenizerException("Invalid Token in Query string '", SearchTokenizerException.MessageKeys.NOT_EXPECTED_TOKEN, searchQuery); } char[] chars = searchQuery.trim().toCharArray(); State state = new SearchExpressionState(); List<SearchQueryToken> states = new ArrayList<>(); for (char aChar : chars) { State next = state.nextChar(aChar); if (state.isFinished()) { states.add(state); } state = next; } if (state.close().isFinished()) { states.add(state); } else { throw new SearchTokenizerException("Last parsed state '" + state.toString() + "' is not finished.", SearchTokenizerException.MessageKeys.NOT_FINISHED_QUERY, state.getTokenName()); } return states; }  List<SearchQueryToken> tokenize(final String searchQuery);  }