@Test public void bug15858() throws Exception { Query query = new QueryBuilder().setFreetext("beta").build(); Summarizer instance = new Summarizer(query, new CompatibleAnalyser()); assertNotNull(instance.getSummary("alpha beta gamma delta beta")); }
public Summary getSummary(String text) throws IOException { if (text == null) { return null; } SToken[] tokens = getTokens(text); if (tokens.length == 0) { return new Summary(); } SortedSet<Excerpt> excerptSet = new TreeSet<>(new Comparator<Excerpt>() { @Override public int compare(Excerpt excerpt1, Excerpt excerpt2) { if (excerpt1 == null) { return excerpt2 == null ? 0 : -1; } else if (excerpt2 == null) { return 1; } else { int numToks1 = excerpt1.numUniqueTokens(); int numToks2 = excerpt2.numUniqueTokens(); if (numToks1 < numToks2) { return -1; } else if (numToks1 == numToks2) { return excerpt1.numFragments() - excerpt2.numFragments(); } else { return 1; } } } }); int lastExcerptPos = 0; for (int i = 0; i < tokens.length; i++) { if (highlight.contains(tokens[i].toString())) { int startToken = (i > SUM_CONTEXT) ? i - SUM_CONTEXT : 0; int endToken = Math.min(i + SUM_CONTEXT, tokens.length); int offset = tokens[startToken].startOffset(); int j = startToken; Excerpt excerpt = new Excerpt(); if (i != 0) { excerpt.add(new Summary.Ellipsis()); } while ((j < endToken) && (j - startToken < SUM_LENGTH)) { SToken t = tokens[j]; if (highlight.contains(t.toString())) { excerpt.addToken(t.toString()); excerpt.add(new Summary.Fragment(text.substring(offset, t.startOffset()))); excerpt.add(new Summary.Highlight(text.substring(t.startOffset(), t.endOffset()))); offset = t.endOffset(); endToken = Math.min(j + SUM_CONTEXT, tokens.length); } j++; } lastExcerptPos = endToken; if (j < tokens.length) { excerpt.add(new Summary.Fragment(text.substring(offset, tokens[j].endOffset()))); } excerpt.setNumTerms(j - startToken); excerptSet.add(excerpt); i = j + SUM_CONTEXT; } } if (excerptSet.size() == 0) { Excerpt excerpt = new Excerpt(); int excerptLen = Math.min(SUM_LENGTH, tokens.length); lastExcerptPos = excerptLen; excerpt.add(new Summary.Fragment(text.substring(tokens[0].startOffset(), tokens[excerptLen - 1].startOffset()))); excerpt.setNumTerms(excerptLen); excerptSet.add(excerpt); } double tokenCount = 0; Summary s = new Summary(); while (tokenCount <= SUM_LENGTH && excerptSet.size() > 0) { Excerpt excerpt = excerptSet.last(); excerptSet.remove(excerpt); double tokenFraction = (1.0 * excerpt.getNumTerms()) / excerpt.numFragments(); for (Summary.Fragment f : excerpt.elements()) { if (tokenCount + tokenFraction <= SUM_LENGTH) { s.add(f); } tokenCount += tokenFraction; } } if (tokenCount > 0 && lastExcerptPos < tokens.length) { s.add(new Summary.Ellipsis()); } return s; }
Summarizer { public Summary getSummary(String text) throws IOException { if (text == null) { return null; } SToken[] tokens = getTokens(text); if (tokens.length == 0) { return new Summary(); } SortedSet<Excerpt> excerptSet = new TreeSet<>(new Comparator<Excerpt>() { @Override public int compare(Excerpt excerpt1, Excerpt excerpt2) { if (excerpt1 == null) { return excerpt2 == null ? 0 : -1; } else if (excerpt2 == null) { return 1; } else { int numToks1 = excerpt1.numUniqueTokens(); int numToks2 = excerpt2.numUniqueTokens(); if (numToks1 < numToks2) { return -1; } else if (numToks1 == numToks2) { return excerpt1.numFragments() - excerpt2.numFragments(); } else { return 1; } } } }); int lastExcerptPos = 0; for (int i = 0; i < tokens.length; i++) { if (highlight.contains(tokens[i].toString())) { int startToken = (i > SUM_CONTEXT) ? i - SUM_CONTEXT : 0; int endToken = Math.min(i + SUM_CONTEXT, tokens.length); int offset = tokens[startToken].startOffset(); int j = startToken; Excerpt excerpt = new Excerpt(); if (i != 0) { excerpt.add(new Summary.Ellipsis()); } while ((j < endToken) && (j - startToken < SUM_LENGTH)) { SToken t = tokens[j]; if (highlight.contains(t.toString())) { excerpt.addToken(t.toString()); excerpt.add(new Summary.Fragment(text.substring(offset, t.startOffset()))); excerpt.add(new Summary.Highlight(text.substring(t.startOffset(), t.endOffset()))); offset = t.endOffset(); endToken = Math.min(j + SUM_CONTEXT, tokens.length); } j++; } lastExcerptPos = endToken; if (j < tokens.length) { excerpt.add(new Summary.Fragment(text.substring(offset, tokens[j].endOffset()))); } excerpt.setNumTerms(j - startToken); excerptSet.add(excerpt); i = j + SUM_CONTEXT; } } if (excerptSet.size() == 0) { Excerpt excerpt = new Excerpt(); int excerptLen = Math.min(SUM_LENGTH, tokens.length); lastExcerptPos = excerptLen; excerpt.add(new Summary.Fragment(text.substring(tokens[0].startOffset(), tokens[excerptLen - 1].startOffset()))); excerpt.setNumTerms(excerptLen); excerptSet.add(excerpt); } double tokenCount = 0; Summary s = new Summary(); while (tokenCount <= SUM_LENGTH && excerptSet.size() > 0) { Excerpt excerpt = excerptSet.last(); excerptSet.remove(excerpt); double tokenFraction = (1.0 * excerpt.getNumTerms()) / excerpt.numFragments(); for (Summary.Fragment f : excerpt.elements()) { if (tokenCount + tokenFraction <= SUM_LENGTH) { s.add(f); } tokenCount += tokenFraction; } } if (tokenCount > 0 && lastExcerptPos < tokens.length) { s.add(new Summary.Ellipsis()); } return s; } }
Summarizer { public Summary getSummary(String text) throws IOException { if (text == null) { return null; } SToken[] tokens = getTokens(text); if (tokens.length == 0) { return new Summary(); } SortedSet<Excerpt> excerptSet = new TreeSet<>(new Comparator<Excerpt>() { @Override public int compare(Excerpt excerpt1, Excerpt excerpt2) { if (excerpt1 == null) { return excerpt2 == null ? 0 : -1; } else if (excerpt2 == null) { return 1; } else { int numToks1 = excerpt1.numUniqueTokens(); int numToks2 = excerpt2.numUniqueTokens(); if (numToks1 < numToks2) { return -1; } else if (numToks1 == numToks2) { return excerpt1.numFragments() - excerpt2.numFragments(); } else { return 1; } } } }); int lastExcerptPos = 0; for (int i = 0; i < tokens.length; i++) { if (highlight.contains(tokens[i].toString())) { int startToken = (i > SUM_CONTEXT) ? i - SUM_CONTEXT : 0; int endToken = Math.min(i + SUM_CONTEXT, tokens.length); int offset = tokens[startToken].startOffset(); int j = startToken; Excerpt excerpt = new Excerpt(); if (i != 0) { excerpt.add(new Summary.Ellipsis()); } while ((j < endToken) && (j - startToken < SUM_LENGTH)) { SToken t = tokens[j]; if (highlight.contains(t.toString())) { excerpt.addToken(t.toString()); excerpt.add(new Summary.Fragment(text.substring(offset, t.startOffset()))); excerpt.add(new Summary.Highlight(text.substring(t.startOffset(), t.endOffset()))); offset = t.endOffset(); endToken = Math.min(j + SUM_CONTEXT, tokens.length); } j++; } lastExcerptPos = endToken; if (j < tokens.length) { excerpt.add(new Summary.Fragment(text.substring(offset, tokens[j].endOffset()))); } excerpt.setNumTerms(j - startToken); excerptSet.add(excerpt); i = j + SUM_CONTEXT; } } if (excerptSet.size() == 0) { Excerpt excerpt = new Excerpt(); int excerptLen = Math.min(SUM_LENGTH, tokens.length); lastExcerptPos = excerptLen; excerpt.add(new Summary.Fragment(text.substring(tokens[0].startOffset(), tokens[excerptLen - 1].startOffset()))); excerpt.setNumTerms(excerptLen); excerptSet.add(excerpt); } double tokenCount = 0; Summary s = new Summary(); while (tokenCount <= SUM_LENGTH && excerptSet.size() > 0) { Excerpt excerpt = excerptSet.last(); excerptSet.remove(excerpt); double tokenFraction = (1.0 * excerpt.getNumTerms()) / excerpt.numFragments(); for (Summary.Fragment f : excerpt.elements()) { if (tokenCount + tokenFraction <= SUM_LENGTH) { s.add(f); } tokenCount += tokenFraction; } } if (tokenCount > 0 && lastExcerptPos < tokens.length) { s.add(new Summary.Ellipsis()); } return s; } Summarizer(Query query, Analyzer a); }
Summarizer { public Summary getSummary(String text) throws IOException { if (text == null) { return null; } SToken[] tokens = getTokens(text); if (tokens.length == 0) { return new Summary(); } SortedSet<Excerpt> excerptSet = new TreeSet<>(new Comparator<Excerpt>() { @Override public int compare(Excerpt excerpt1, Excerpt excerpt2) { if (excerpt1 == null) { return excerpt2 == null ? 0 : -1; } else if (excerpt2 == null) { return 1; } else { int numToks1 = excerpt1.numUniqueTokens(); int numToks2 = excerpt2.numUniqueTokens(); if (numToks1 < numToks2) { return -1; } else if (numToks1 == numToks2) { return excerpt1.numFragments() - excerpt2.numFragments(); } else { return 1; } } } }); int lastExcerptPos = 0; for (int i = 0; i < tokens.length; i++) { if (highlight.contains(tokens[i].toString())) { int startToken = (i > SUM_CONTEXT) ? i - SUM_CONTEXT : 0; int endToken = Math.min(i + SUM_CONTEXT, tokens.length); int offset = tokens[startToken].startOffset(); int j = startToken; Excerpt excerpt = new Excerpt(); if (i != 0) { excerpt.add(new Summary.Ellipsis()); } while ((j < endToken) && (j - startToken < SUM_LENGTH)) { SToken t = tokens[j]; if (highlight.contains(t.toString())) { excerpt.addToken(t.toString()); excerpt.add(new Summary.Fragment(text.substring(offset, t.startOffset()))); excerpt.add(new Summary.Highlight(text.substring(t.startOffset(), t.endOffset()))); offset = t.endOffset(); endToken = Math.min(j + SUM_CONTEXT, tokens.length); } j++; } lastExcerptPos = endToken; if (j < tokens.length) { excerpt.add(new Summary.Fragment(text.substring(offset, tokens[j].endOffset()))); } excerpt.setNumTerms(j - startToken); excerptSet.add(excerpt); i = j + SUM_CONTEXT; } } if (excerptSet.size() == 0) { Excerpt excerpt = new Excerpt(); int excerptLen = Math.min(SUM_LENGTH, tokens.length); lastExcerptPos = excerptLen; excerpt.add(new Summary.Fragment(text.substring(tokens[0].startOffset(), tokens[excerptLen - 1].startOffset()))); excerpt.setNumTerms(excerptLen); excerptSet.add(excerpt); } double tokenCount = 0; Summary s = new Summary(); while (tokenCount <= SUM_LENGTH && excerptSet.size() > 0) { Excerpt excerpt = excerptSet.last(); excerptSet.remove(excerpt); double tokenFraction = (1.0 * excerpt.getNumTerms()) / excerpt.numFragments(); for (Summary.Fragment f : excerpt.elements()) { if (tokenCount + tokenFraction <= SUM_LENGTH) { s.add(f); } tokenCount += tokenFraction; } } if (tokenCount > 0 && lastExcerptPos < tokens.length) { s.add(new Summary.Ellipsis()); } return s; } Summarizer(Query query, Analyzer a); Summary getSummary(String text); }
Summarizer { public Summary getSummary(String text) throws IOException { if (text == null) { return null; } SToken[] tokens = getTokens(text); if (tokens.length == 0) { return new Summary(); } SortedSet<Excerpt> excerptSet = new TreeSet<>(new Comparator<Excerpt>() { @Override public int compare(Excerpt excerpt1, Excerpt excerpt2) { if (excerpt1 == null) { return excerpt2 == null ? 0 : -1; } else if (excerpt2 == null) { return 1; } else { int numToks1 = excerpt1.numUniqueTokens(); int numToks2 = excerpt2.numUniqueTokens(); if (numToks1 < numToks2) { return -1; } else if (numToks1 == numToks2) { return excerpt1.numFragments() - excerpt2.numFragments(); } else { return 1; } } } }); int lastExcerptPos = 0; for (int i = 0; i < tokens.length; i++) { if (highlight.contains(tokens[i].toString())) { int startToken = (i > SUM_CONTEXT) ? i - SUM_CONTEXT : 0; int endToken = Math.min(i + SUM_CONTEXT, tokens.length); int offset = tokens[startToken].startOffset(); int j = startToken; Excerpt excerpt = new Excerpt(); if (i != 0) { excerpt.add(new Summary.Ellipsis()); } while ((j < endToken) && (j - startToken < SUM_LENGTH)) { SToken t = tokens[j]; if (highlight.contains(t.toString())) { excerpt.addToken(t.toString()); excerpt.add(new Summary.Fragment(text.substring(offset, t.startOffset()))); excerpt.add(new Summary.Highlight(text.substring(t.startOffset(), t.endOffset()))); offset = t.endOffset(); endToken = Math.min(j + SUM_CONTEXT, tokens.length); } j++; } lastExcerptPos = endToken; if (j < tokens.length) { excerpt.add(new Summary.Fragment(text.substring(offset, tokens[j].endOffset()))); } excerpt.setNumTerms(j - startToken); excerptSet.add(excerpt); i = j + SUM_CONTEXT; } } if (excerptSet.size() == 0) { Excerpt excerpt = new Excerpt(); int excerptLen = Math.min(SUM_LENGTH, tokens.length); lastExcerptPos = excerptLen; excerpt.add(new Summary.Fragment(text.substring(tokens[0].startOffset(), tokens[excerptLen - 1].startOffset()))); excerpt.setNumTerms(excerptLen); excerptSet.add(excerpt); } double tokenCount = 0; Summary s = new Summary(); while (tokenCount <= SUM_LENGTH && excerptSet.size() > 0) { Excerpt excerpt = excerptSet.last(); excerptSet.remove(excerpt); double tokenFraction = (1.0 * excerpt.getNumTerms()) / excerpt.numFragments(); for (Summary.Fragment f : excerpt.elements()) { if (tokenCount + tokenFraction <= SUM_LENGTH) { s.add(f); } tokenCount += tokenFraction; } } if (tokenCount > 0 && lastExcerptPos < tokens.length) { s.add(new Summary.Ellipsis()); } return s; } Summarizer(Query query, Analyzer a); Summary getSummary(String text);  }