@Test public void testMultiSentenceChunks() { for (Token<String> chunk : lexLib.getChunks(joinedText)) { assertEquals(chunk.getToken(), joinedText.substring(chunk.getStart(), chunk.getEnd())); } }
@Override public List<Token<String>> getChunks(String text) { int sentenceStart = 0; List<Token<String>> retChunks = new LinkedList<>(); for (String sentence: extractSentences(text)) { String[] tokens = tokenizer.tokenize(sentence); Span[] spans = tokenizer.tokenizePos(sentence); String[] tags = tagger.tag(tokens); String[] chunks = chunker.chunk(tokens, tags); for (int i = 0; i < chunks.length; i++) { List<String> chunk = new LinkedList<>(); int start = i; if ("B-NP".equals(chunks[i])) { chunk.add(tokens[i]); while (i + 1 < chunks.length && "I-NP".equals(chunks[i + 1])) { chunk.add(tokens[i+1]); i++; } retChunks.add(new NounChunk(on(' ').join(chunk).replace(" ,", ","), sentenceStart + spans[start].getStart(), sentenceStart + spans[i].getEnd())); } else if ("B-VP".equals(chunks[i])) { chunk.add(tokens[i]); while (i + 1 < chunks.length && "I-VP".equals(chunks[i + 1])) { chunk.add(tokens[i+1]); i++; } retChunks.add(new VerbChunk(on(' ').join(chunk).replace(" ,", ","), sentenceStart + spans[start].getStart(), sentenceStart + spans[i].getEnd())); } } sentenceStart += spans[spans.length - 1].getEnd() + 2; } return retChunks; }
LexicalLibOpenNlpImpl implements LexicalLib { @Override public List<Token<String>> getChunks(String text) { int sentenceStart = 0; List<Token<String>> retChunks = new LinkedList<>(); for (String sentence: extractSentences(text)) { String[] tokens = tokenizer.tokenize(sentence); Span[] spans = tokenizer.tokenizePos(sentence); String[] tags = tagger.tag(tokens); String[] chunks = chunker.chunk(tokens, tags); for (int i = 0; i < chunks.length; i++) { List<String> chunk = new LinkedList<>(); int start = i; if ("B-NP".equals(chunks[i])) { chunk.add(tokens[i]); while (i + 1 < chunks.length && "I-NP".equals(chunks[i + 1])) { chunk.add(tokens[i+1]); i++; } retChunks.add(new NounChunk(on(' ').join(chunk).replace(" ,", ","), sentenceStart + spans[start].getStart(), sentenceStart + spans[i].getEnd())); } else if ("B-VP".equals(chunks[i])) { chunk.add(tokens[i]); while (i + 1 < chunks.length && "I-VP".equals(chunks[i + 1])) { chunk.add(tokens[i+1]); i++; } retChunks.add(new VerbChunk(on(' ').join(chunk).replace(" ,", ","), sentenceStart + spans[start].getStart(), sentenceStart + spans[i].getEnd())); } } sentenceStart += spans[spans.length - 1].getEnd() + 2; } return retChunks; } }
LexicalLibOpenNlpImpl implements LexicalLib { @Override public List<Token<String>> getChunks(String text) { int sentenceStart = 0; List<Token<String>> retChunks = new LinkedList<>(); for (String sentence: extractSentences(text)) { String[] tokens = tokenizer.tokenize(sentence); Span[] spans = tokenizer.tokenizePos(sentence); String[] tags = tagger.tag(tokens); String[] chunks = chunker.chunk(tokens, tags); for (int i = 0; i < chunks.length; i++) { List<String> chunk = new LinkedList<>(); int start = i; if ("B-NP".equals(chunks[i])) { chunk.add(tokens[i]); while (i + 1 < chunks.length && "I-NP".equals(chunks[i + 1])) { chunk.add(tokens[i+1]); i++; } retChunks.add(new NounChunk(on(' ').join(chunk).replace(" ,", ","), sentenceStart + spans[start].getStart(), sentenceStart + spans[i].getEnd())); } else if ("B-VP".equals(chunks[i])) { chunk.add(tokens[i]); while (i + 1 < chunks.length && "I-VP".equals(chunks[i + 1])) { chunk.add(tokens[i+1]); i++; } retChunks.add(new VerbChunk(on(' ').join(chunk).replace(" ,", ","), sentenceStart + spans[start].getStart(), sentenceStart + spans[i].getEnd())); } } sentenceStart += spans[spans.length - 1].getEnd() + 2; } return retChunks; } @Inject protected  LexicalLibOpenNlpImpl(
      TokenizerProvider tokenizerProvider,
      SentenceDetectorProvider sentenceProvider,
      PosTaggerProvider posProvider,
      ChunkerProvider chunkerProvider); }
LexicalLibOpenNlpImpl implements LexicalLib { @Override public List<Token<String>> getChunks(String text) { int sentenceStart = 0; List<Token<String>> retChunks = new LinkedList<>(); for (String sentence: extractSentences(text)) { String[] tokens = tokenizer.tokenize(sentence); Span[] spans = tokenizer.tokenizePos(sentence); String[] tags = tagger.tag(tokens); String[] chunks = chunker.chunk(tokens, tags); for (int i = 0; i < chunks.length; i++) { List<String> chunk = new LinkedList<>(); int start = i; if ("B-NP".equals(chunks[i])) { chunk.add(tokens[i]); while (i + 1 < chunks.length && "I-NP".equals(chunks[i + 1])) { chunk.add(tokens[i+1]); i++; } retChunks.add(new NounChunk(on(' ').join(chunk).replace(" ,", ","), sentenceStart + spans[start].getStart(), sentenceStart + spans[i].getEnd())); } else if ("B-VP".equals(chunks[i])) { chunk.add(tokens[i]); while (i + 1 < chunks.length && "I-VP".equals(chunks[i + 1])) { chunk.add(tokens[i+1]); i++; } retChunks.add(new VerbChunk(on(' ').join(chunk).replace(" ,", ","), sentenceStart + spans[start].getStart(), sentenceStart + spans[i].getEnd())); } } sentenceStart += spans[spans.length - 1].getEnd() + 2; } return retChunks; } @Inject protected  LexicalLibOpenNlpImpl(
      TokenizerProvider tokenizerProvider,
      SentenceDetectorProvider sentenceProvider,
      PosTaggerProvider posProvider,
      ChunkerProvider chunkerProvider); @Override List<String> extractSentences(String text); @Override List<PosToken> tagPOS(String sentence); @Override List<Token<String>> getChunks(String text); @Override @Beta List<Token<String>> getEntities(String text); }
LexicalLibOpenNlpImpl implements LexicalLib { @Override public List<Token<String>> getChunks(String text) { int sentenceStart = 0; List<Token<String>> retChunks = new LinkedList<>(); for (String sentence: extractSentences(text)) { String[] tokens = tokenizer.tokenize(sentence); Span[] spans = tokenizer.tokenizePos(sentence); String[] tags = tagger.tag(tokens); String[] chunks = chunker.chunk(tokens, tags); for (int i = 0; i < chunks.length; i++) { List<String> chunk = new LinkedList<>(); int start = i; if ("B-NP".equals(chunks[i])) { chunk.add(tokens[i]); while (i + 1 < chunks.length && "I-NP".equals(chunks[i + 1])) { chunk.add(tokens[i+1]); i++; } retChunks.add(new NounChunk(on(' ').join(chunk).replace(" ,", ","), sentenceStart + spans[start].getStart(), sentenceStart + spans[i].getEnd())); } else if ("B-VP".equals(chunks[i])) { chunk.add(tokens[i]); while (i + 1 < chunks.length && "I-VP".equals(chunks[i + 1])) { chunk.add(tokens[i+1]); i++; } retChunks.add(new VerbChunk(on(' ').join(chunk).replace(" ,", ","), sentenceStart + spans[start].getStart(), sentenceStart + spans[i].getEnd())); } } sentenceStart += spans[spans.length - 1].getEnd() + 2; } return retChunks; } @Inject protected  LexicalLibOpenNlpImpl(
      TokenizerProvider tokenizerProvider,
      SentenceDetectorProvider sentenceProvider,
      PosTaggerProvider posProvider,
      ChunkerProvider chunkerProvider); @Override List<String> extractSentences(String text); @Override List<PosToken> tagPOS(String sentence); @Override List<Token<String>> getChunks(String text); @Override @Beta List<Token<String>> getEntities(String text);  }