@Test public void testTokens() { Analyzer analyzer = LuceneIndexConstants.ANALYZER; assertEquals(ImmutableList.of("parent", "child"), LuceneIndex.tokenize("/parent/child", analyzer)); assertEquals(ImmutableList.of("p1234", "p5678"), LuceneIndex.tokenize("/p1234/p5678", analyzer)); assertEquals(ImmutableList.of("first", "second"), LuceneIndex.tokenize("first_second", analyzer)); assertEquals(ImmutableList.of("first1", "second2"), LuceneIndex.tokenize("first1_second2", analyzer)); assertEquals(ImmutableList.of("first", "second"), LuceneIndex.tokenize("first. second", analyzer)); assertEquals(ImmutableList.of("first", "second"), LuceneIndex.tokenize("first.second", analyzer)); assertEquals(ImmutableList.of("hello", "world"), LuceneIndex.tokenize("hello-world", analyzer)); assertEquals(ImmutableList.of("hello", "wor*"), LuceneIndex.tokenize("hello-wor*", analyzer)); assertEquals(ImmutableList.of("*llo", "world"), LuceneIndex.tokenize("*llo-world", analyzer)); assertEquals(ImmutableList.of("*llo", "wor*"), LuceneIndex.tokenize("*llo-wor*", analyzer)); }
static List<String> tokenize(String text, Analyzer analyzer) { List<String> tokens = new ArrayList<String>(); TokenStream stream = null; try { stream = analyzer.tokenStream(FieldNames.FULLTEXT, new StringReader(text)); CharTermAttribute termAtt = stream .addAttribute(CharTermAttribute.class); OffsetAttribute offsetAtt = stream .addAttribute(OffsetAttribute.class); stream.reset(); int poz = 0; boolean hasFulltextToken = false; StringBuilder token = new StringBuilder(); while (stream.incrementToken()) { String term = termAtt.toString(); int start = offsetAtt.startOffset(); int end = offsetAtt.endOffset(); if (start > poz) { for (int i = poz; i < start; i++) { for (char c : fulltextTokens) { if (c == text.charAt(i)) { token.append(c); hasFulltextToken = true; } } } } poz = end; if (hasFulltextToken) { token.append(term); hasFulltextToken = false; } else { if (token.length() > 0) { tokens.add(token.toString()); } token = new StringBuilder(); token.append(term); } } if (poz < text.length()) { for (int i = poz; i < text.length(); i++) { for (char c : fulltextTokens) { if (c == text.charAt(i)) { token.append(c); } } } } if (token.length() > 0) { tokens.add(token.toString()); } stream.end(); } catch (IOException e) { LOG.error("Building fulltext query failed", e.getMessage()); return null; } finally { try { if (stream != null) { stream.close(); } } catch (IOException e) { } } return tokens; }
LuceneIndex implements AdvanceFulltextQueryIndex { static List<String> tokenize(String text, Analyzer analyzer) { List<String> tokens = new ArrayList<String>(); TokenStream stream = null; try { stream = analyzer.tokenStream(FieldNames.FULLTEXT, new StringReader(text)); CharTermAttribute termAtt = stream .addAttribute(CharTermAttribute.class); OffsetAttribute offsetAtt = stream .addAttribute(OffsetAttribute.class); stream.reset(); int poz = 0; boolean hasFulltextToken = false; StringBuilder token = new StringBuilder(); while (stream.incrementToken()) { String term = termAtt.toString(); int start = offsetAtt.startOffset(); int end = offsetAtt.endOffset(); if (start > poz) { for (int i = poz; i < start; i++) { for (char c : fulltextTokens) { if (c == text.charAt(i)) { token.append(c); hasFulltextToken = true; } } } } poz = end; if (hasFulltextToken) { token.append(term); hasFulltextToken = false; } else { if (token.length() > 0) { tokens.add(token.toString()); } token = new StringBuilder(); token.append(term); } } if (poz < text.length()) { for (int i = poz; i < text.length(); i++) { for (char c : fulltextTokens) { if (c == text.charAt(i)) { token.append(c); } } } } if (token.length() > 0) { tokens.add(token.toString()); } stream.end(); } catch (IOException e) { LOG.error("Building fulltext query failed", e.getMessage()); return null; } finally { try { if (stream != null) { stream.close(); } } catch (IOException e) { } } return tokens; } }
LuceneIndex implements AdvanceFulltextQueryIndex { static List<String> tokenize(String text, Analyzer analyzer) { List<String> tokens = new ArrayList<String>(); TokenStream stream = null; try { stream = analyzer.tokenStream(FieldNames.FULLTEXT, new StringReader(text)); CharTermAttribute termAtt = stream .addAttribute(CharTermAttribute.class); OffsetAttribute offsetAtt = stream .addAttribute(OffsetAttribute.class); stream.reset(); int poz = 0; boolean hasFulltextToken = false; StringBuilder token = new StringBuilder(); while (stream.incrementToken()) { String term = termAtt.toString(); int start = offsetAtt.startOffset(); int end = offsetAtt.endOffset(); if (start > poz) { for (int i = poz; i < start; i++) { for (char c : fulltextTokens) { if (c == text.charAt(i)) { token.append(c); hasFulltextToken = true; } } } } poz = end; if (hasFulltextToken) { token.append(term); hasFulltextToken = false; } else { if (token.length() > 0) { tokens.add(token.toString()); } token = new StringBuilder(); token.append(term); } } if (poz < text.length()) { for (int i = poz; i < text.length(); i++) { for (char c : fulltextTokens) { if (c == text.charAt(i)) { token.append(c); } } } } if (token.length() > 0) { tokens.add(token.toString()); } stream.end(); } catch (IOException e) { LOG.error("Building fulltext query failed", e.getMessage()); return null; } finally { try { if (stream != null) { stream.close(); } } catch (IOException e) { } } return tokens; } LuceneIndex(IndexTracker tracker, NodeAggregator aggregator); }
LuceneIndex implements AdvanceFulltextQueryIndex { static List<String> tokenize(String text, Analyzer analyzer) { List<String> tokens = new ArrayList<String>(); TokenStream stream = null; try { stream = analyzer.tokenStream(FieldNames.FULLTEXT, new StringReader(text)); CharTermAttribute termAtt = stream .addAttribute(CharTermAttribute.class); OffsetAttribute offsetAtt = stream .addAttribute(OffsetAttribute.class); stream.reset(); int poz = 0; boolean hasFulltextToken = false; StringBuilder token = new StringBuilder(); while (stream.incrementToken()) { String term = termAtt.toString(); int start = offsetAtt.startOffset(); int end = offsetAtt.endOffset(); if (start > poz) { for (int i = poz; i < start; i++) { for (char c : fulltextTokens) { if (c == text.charAt(i)) { token.append(c); hasFulltextToken = true; } } } } poz = end; if (hasFulltextToken) { token.append(term); hasFulltextToken = false; } else { if (token.length() > 0) { tokens.add(token.toString()); } token = new StringBuilder(); token.append(term); } } if (poz < text.length()) { for (int i = poz; i < text.length(); i++) { for (char c : fulltextTokens) { if (c == text.charAt(i)) { token.append(c); } } } } if (token.length() > 0) { tokens.add(token.toString()); } stream.end(); } catch (IOException e) { LOG.error("Building fulltext query failed", e.getMessage()); return null; } finally { try { if (stream != null) { stream.close(); } } catch (IOException e) { } } return tokens; } LuceneIndex(IndexTracker tracker, NodeAggregator aggregator); @Override double getMinimumCost(); @Override String getIndexName(); @Override List<IndexPlan> getPlans(Filter filter, List<OrderEntry> sortOrder, NodeState rootState); @Override double getCost(Filter filter, NodeState root); @Override String getPlan(Filter filter, NodeState root); @Override String getPlanDescription(IndexPlan plan, NodeState root); @Override Cursor query(final Filter filter, final NodeState root); @Override Cursor query(final IndexPlan plan, NodeState rootState); @Override NodeAggregator getNodeAggregator(); }
LuceneIndex implements AdvanceFulltextQueryIndex { static List<String> tokenize(String text, Analyzer analyzer) { List<String> tokens = new ArrayList<String>(); TokenStream stream = null; try { stream = analyzer.tokenStream(FieldNames.FULLTEXT, new StringReader(text)); CharTermAttribute termAtt = stream .addAttribute(CharTermAttribute.class); OffsetAttribute offsetAtt = stream .addAttribute(OffsetAttribute.class); stream.reset(); int poz = 0; boolean hasFulltextToken = false; StringBuilder token = new StringBuilder(); while (stream.incrementToken()) { String term = termAtt.toString(); int start = offsetAtt.startOffset(); int end = offsetAtt.endOffset(); if (start > poz) { for (int i = poz; i < start; i++) { for (char c : fulltextTokens) { if (c == text.charAt(i)) { token.append(c); hasFulltextToken = true; } } } } poz = end; if (hasFulltextToken) { token.append(term); hasFulltextToken = false; } else { if (token.length() > 0) { tokens.add(token.toString()); } token = new StringBuilder(); token.append(term); } } if (poz < text.length()) { for (int i = poz; i < text.length(); i++) { for (char c : fulltextTokens) { if (c == text.charAt(i)) { token.append(c); } } } } if (token.length() > 0) { tokens.add(token.toString()); } stream.end(); } catch (IOException e) { LOG.error("Building fulltext query failed", e.getMessage()); return null; } finally { try { if (stream != null) { stream.close(); } } catch (IOException e) { } } return tokens; } LuceneIndex(IndexTracker tracker, NodeAggregator aggregator); @Override double getMinimumCost(); @Override String getIndexName(); @Override List<IndexPlan> getPlans(Filter filter, List<OrderEntry> sortOrder, NodeState rootState); @Override double getCost(Filter filter, NodeState root); @Override String getPlan(Filter filter, NodeState root); @Override String getPlanDescription(IndexPlan plan, NodeState root); @Override Cursor query(final Filter filter, final NodeState root); @Override Cursor query(final IndexPlan plan, NodeState rootState); @Override NodeAggregator getNodeAggregator(); static final String NATIVE_QUERY_FUNCTION; }