RegexTokenAnnotator extends AbstractRegexAnnotator { public String getReplacement() { return replacement; } RegexTokenAnnotator(String regex, String replacement, String analysisText, String sessionId);  RegexTokenAnnotator(String regex, String replacement, String analysisText, int groupIndex, boolean caseSensitive, boolean diacricticSensitive,
      boolean autoWordBoundaries, String sessionId);  RegexTokenAnnotator(String descriptor, Map<String, String> defaultParams, String sessionId); String getReplacement(); String getAnalysisText();  }
@Test public void testApply() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; String regex = "\\b[\\w.%-]+@[-.\\w]+\\.[A-Za-z]{2,4}\\b"; String replacement = "Email"; RegexTokenAnnotator filter = new RegexTokenAnnotator(regex, replacement, null, sessionId); Sentence text = new Sentence("My address is joe.schmoe@test.com.", sessionId); filter.annotate(text); LOG.debug(text.getAnnotations().toString()); List<Annotation<TokenPlaceholder>> placeholders = text.getAnnotations(TokenPlaceholder.class); assertEquals(1, placeholders.size()); Annotation<TokenPlaceholder> placeholder = placeholders.get(0); assertEquals(14, placeholder.getStart()); assertEquals(33, placeholder.getEnd()); assertEquals("Email", placeholder.getData().getReplacement()); }
@Test public void testApplyWithDollars() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; String regex = "\\b([\\w.%-]+)(@[-.\\w]+\\.[A-Za-z]{2,4})\\b"; String replacement = "\\$Email$2:$1"; RegexTokenAnnotator filter = new RegexTokenAnnotator(regex, replacement, null, sessionId); Sentence text = new Sentence("My address is joe.schmoe@test.com.", sessionId); filter.annotate(text); List<Annotation<TokenPlaceholder>> placeholders = text.getAnnotations(TokenPlaceholder.class); LOG.debug(placeholders.toString()); assertEquals(1, placeholders.size()); Annotation<TokenPlaceholder> placeholder = placeholders.get(0); assertEquals(14, placeholder.getStart()); assertEquals(33, placeholder.getEnd()); assertEquals("$Email@test.com:joe.schmoe", placeholder.getData().getReplacement()); }
@Test public void testApplyWithConsecutiveDollars() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; String regex = "\\b([\\w.%-]+)(@[-.\\w]+\\.[A-Za-z]{2,4})\\b"; String replacement = "\\$Email$2$1"; RegexTokenAnnotator filter = new RegexTokenAnnotator(regex, replacement, null, sessionId); Sentence text = new Sentence("My address is joe.schmoe@test.com.", sessionId); filter.annotate(text); List<Annotation<TokenPlaceholder>> placeholders = text.getAnnotations(TokenPlaceholder.class); LOG.debug(placeholders.toString()); assertEquals(1, placeholders.size()); Annotation<TokenPlaceholder> placeholder = placeholders.get(0); assertEquals(14, placeholder.getStart()); assertEquals(33, placeholder.getEnd()); assertEquals("$Email@test.comjoe.schmoe", placeholder.getData().getReplacement()); }
@Test public void testApplyWithUnmatchingGroups() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; String regex = "\\b(\\d)(\\d)?\\b"; String replacement = "Number$1$2"; RegexTokenAnnotator filter = new RegexTokenAnnotator(regex, replacement, null, sessionId); Sentence text = new Sentence("Two-digit number: 42. One-digit number: 7.", sessionId); filter.annotate(text); List<Annotation<TokenPlaceholder>> placeholders = text.getAnnotations(TokenPlaceholder.class); LOG.debug(placeholders.toString()); assertEquals(2, placeholders.size()); Annotation<TokenPlaceholder> placeholder = placeholders.get(0); assertEquals("Two-digit number: ".length(), placeholder.getStart()); assertEquals("Two-digit number: 42".length(), placeholder.getEnd()); assertEquals("Number42", placeholder.getData().getReplacement()); placeholder = placeholders.get(1); assertEquals("Two-digit number: 42. One-digit number: ".length(), placeholder.getStart()); assertEquals("Two-digit number: 42. One-digit number: 7".length(), placeholder.getEnd()); assertEquals("Number7", placeholder.getData().getReplacement()); }