EmailPipeline extends AbstractPipeline { @Override public List<NamedEntity> process(Document doc) { Annotations annotations = new Annotations(doc.getId(), getType(), doc.getLanguage()); Matcher matcher = pattern.matcher(doc.getContent()); while (matcher.find()) { String email = matcher.group(0); int start = matcher.start(); annotations.add(NlpStage.NER, start, start + email.length(), NamedEntity.Category.EMAIL); } List<NamedEntity> neList = allFrom(doc.getContent(), annotations); if ("message/rfc822".equals(doc.getContentType())) { String metadataString = parsedEmailHeaders.stream().map(key -> doc.getMetadata().getOrDefault(key, "").toString()).collect(joining(" ")); Matcher metaMatcher = pattern.matcher(metadataString); while (metaMatcher.find()) { neList.add(NamedEntity.create(NamedEntity.Category.EMAIL, metaMatcher.group(0), -1, doc.getId(), doc.getRootDocument(), EMAIL, doc.getLanguage())); } } return neList; } @Inject  EmailPipeline(final PropertiesProvider propertiesProvider); @Override List<NamedEntity> process(Document doc); static String tikaRawHeader(String s); static String tikaMsgHeader(String s); static String tika(String s); @Override Type getType(); @Override void terminate(Language language); @Override Map<Language, Set<NlpStage>> supportedStages(); @Override boolean supports(NlpStage stage, Language language); @Override List<NamedEntity.Category> getTargetEntities(); @Override List<NlpStage> getStages(); @Override boolean isCaching(); @Override Charset getEncoding(); @Override boolean initialize(Language language); @Override Optional<String> getPosTagSet(Language language);  }
@Test public void test_no_email() { List<NamedEntity> annotations = pipeline.process(createDocument("this is a content without email but with an arobase (@).", "docId", Language.ENGLISH)); assertThat(annotations).isEmpty(); }
@Test public void test_one_email() { String content = "this is a content with email@domain.com"; List<NamedEntity> annotations = pipeline.process(createDocument(content, "docId", Language.ENGLISH)); assertThat(annotations).hasSize(1); assertThat(annotations.get(0).getOffset()).isEqualTo(23); assertThat(annotations.get(0).getCategory()).isEqualTo(NamedEntity.Category.EMAIL); assertThat(annotations.get(0).getMention()).isEqualTo("email@domain.com"); }
@Test public void test_one_email_twice() { String content = "this is a content with email@domain.com\n" + "that is twice in the document\n" + "email@domain.com"; List<NamedEntity> annotations = pipeline.process(createDocument(content, "docId", Language.ENGLISH)); assertThat(annotations).hasSize(2); NamedEntity nlpTag = annotations.get(1); assertThat(nlpTag.getOffset()).isEqualTo(70); assertThat(nlpTag.getMention()).isEqualTo("email@domain.com"); }
@Test public void test_three_emails() { List<NamedEntity> annotations = pipeline.process(createDocument("this is a content with email@domain.com\n" + "and another one : foo@bar.com\n" + "and baz@qux.fr", "docId", Language.ENGLISH)); assertThat(annotations).hasSize(3); }
@Test public void test_acceptance() throws IOException { Path emailFile = Paths.get(getClass().getResource("/email.eml").getPath()); String content = new String(Files.readAllBytes(emailFile)); List<NamedEntity> annotations = pipeline.process(createDocument(content, "docId", Language.ENGLISH)); assertThat(annotations).hasSize(10); }