@Test public void testGetAnnotationsWithMinLength() throws Exception { when(config.getMinLength()).thenReturn(Integer.MAX_VALUE); List<EntityAnnotation> annotations = processor.getAnnotations(text, config); assertThat(annotations, is(empty())); }
protected List<EntityAnnotation> getAnnotations(String content, EntityFormatConfiguration config) throws InterruptedException { checkNotNull(content); BlockingQueue<List<Token<String>>> queue = startShingleProducer(content); List<EntityAnnotation> annotations = new ArrayList<>(); while (true) { List<Token<String>> tokens = queue.take(); if (tokens.equals(ShingleProducer.END_TOKEN)) { break; } if (LuceneUtils.isStopword(getFirst(tokens, null).getToken()) || LuceneUtils.isStopword(getLast(tokens).getToken())) { continue; } String candidate = combineTokens(tokens); if (candidate.length() < config.getMinLength()) { continue; } int start = tokens.get(0).getStart(); int end = tokens.get(tokens.size() - 1).getEnd(); for (Entity entity : recognizer.getEntities(candidate, config)) { annotations.add(new EntityAnnotation(entity, start, end)); } } List<EntityAnnotation> ret = newArrayList(); for (EntityAnnotationGroup group : getAnnotationGroups(annotations, config.isLongestOnly())) { ret.addAll(group); } Collections.sort(ret); return ret; }
EntityProcessorImpl implements EntityProcessor { protected List<EntityAnnotation> getAnnotations(String content, EntityFormatConfiguration config) throws InterruptedException { checkNotNull(content); BlockingQueue<List<Token<String>>> queue = startShingleProducer(content); List<EntityAnnotation> annotations = new ArrayList<>(); while (true) { List<Token<String>> tokens = queue.take(); if (tokens.equals(ShingleProducer.END_TOKEN)) { break; } if (LuceneUtils.isStopword(getFirst(tokens, null).getToken()) || LuceneUtils.isStopword(getLast(tokens).getToken())) { continue; } String candidate = combineTokens(tokens); if (candidate.length() < config.getMinLength()) { continue; } int start = tokens.get(0).getStart(); int end = tokens.get(tokens.size() - 1).getEnd(); for (Entity entity : recognizer.getEntities(candidate, config)) { annotations.add(new EntityAnnotation(entity, start, end)); } } List<EntityAnnotation> ret = newArrayList(); for (EntityAnnotationGroup group : getAnnotationGroups(annotations, config.isLongestOnly())) { ret.addAll(group); } Collections.sort(ret); return ret; } }
EntityProcessorImpl implements EntityProcessor { protected List<EntityAnnotation> getAnnotations(String content, EntityFormatConfiguration config) throws InterruptedException { checkNotNull(content); BlockingQueue<List<Token<String>>> queue = startShingleProducer(content); List<EntityAnnotation> annotations = new ArrayList<>(); while (true) { List<Token<String>> tokens = queue.take(); if (tokens.equals(ShingleProducer.END_TOKEN)) { break; } if (LuceneUtils.isStopword(getFirst(tokens, null).getToken()) || LuceneUtils.isStopword(getLast(tokens).getToken())) { continue; } String candidate = combineTokens(tokens); if (candidate.length() < config.getMinLength()) { continue; } int start = tokens.get(0).getStart(); int end = tokens.get(tokens.size() - 1).getEnd(); for (Entity entity : recognizer.getEntities(candidate, config)) { annotations.add(new EntityAnnotation(entity, start, end)); } } List<EntityAnnotation> ret = newArrayList(); for (EntityAnnotationGroup group : getAnnotationGroups(annotations, config.isLongestOnly())) { ret.addAll(group); } Collections.sort(ret); return ret; } @Inject protected  EntityProcessorImpl(EntityRecognizer recognizer); }
EntityProcessorImpl implements EntityProcessor { protected List<EntityAnnotation> getAnnotations(String content, EntityFormatConfiguration config) throws InterruptedException { checkNotNull(content); BlockingQueue<List<Token<String>>> queue = startShingleProducer(content); List<EntityAnnotation> annotations = new ArrayList<>(); while (true) { List<Token<String>> tokens = queue.take(); if (tokens.equals(ShingleProducer.END_TOKEN)) { break; } if (LuceneUtils.isStopword(getFirst(tokens, null).getToken()) || LuceneUtils.isStopword(getLast(tokens).getToken())) { continue; } String candidate = combineTokens(tokens); if (candidate.length() < config.getMinLength()) { continue; } int start = tokens.get(0).getStart(); int end = tokens.get(tokens.size() - 1).getEnd(); for (Entity entity : recognizer.getEntities(candidate, config)) { annotations.add(new EntityAnnotation(entity, start, end)); } } List<EntityAnnotation> ret = newArrayList(); for (EntityAnnotationGroup group : getAnnotationGroups(annotations, config.isLongestOnly())) { ret.addAll(group); } Collections.sort(ret); return ret; } @Inject protected  EntityProcessorImpl(EntityRecognizer recognizer); @Override List<EntityAnnotation> annotateEntities(EntityFormatConfiguration config); }
EntityProcessorImpl implements EntityProcessor { protected List<EntityAnnotation> getAnnotations(String content, EntityFormatConfiguration config) throws InterruptedException { checkNotNull(content); BlockingQueue<List<Token<String>>> queue = startShingleProducer(content); List<EntityAnnotation> annotations = new ArrayList<>(); while (true) { List<Token<String>> tokens = queue.take(); if (tokens.equals(ShingleProducer.END_TOKEN)) { break; } if (LuceneUtils.isStopword(getFirst(tokens, null).getToken()) || LuceneUtils.isStopword(getLast(tokens).getToken())) { continue; } String candidate = combineTokens(tokens); if (candidate.length() < config.getMinLength()) { continue; } int start = tokens.get(0).getStart(); int end = tokens.get(tokens.size() - 1).getEnd(); for (Entity entity : recognizer.getEntities(candidate, config)) { annotations.add(new EntityAnnotation(entity, start, end)); } } List<EntityAnnotation> ret = newArrayList(); for (EntityAnnotationGroup group : getAnnotationGroups(annotations, config.isLongestOnly())) { ret.addAll(group); } Collections.sort(ret); return ret; } @Inject protected  EntityProcessorImpl(EntityRecognizer recognizer); @Override List<EntityAnnotation> annotateEntities(EntityFormatConfiguration config);  }