TokenPattern { List<Pattern> parsePattern(String regexp) throws TalismaneException { boolean inLiteral = false; boolean inException = false; boolean inGrouping = false; boolean groupingHasLetters = false; int groupingStart = 0; List<Pattern> parsedPattern = new ArrayList<Pattern>(); int currentStart = 0; int currentEnd = 0; for (int i = 0; i < regexp.length(); i++) { char c = regexp.charAt(i); if (!inLiteral && c == '\\') { inLiteral = true; } else if (inLiteral) { if (c == 'd' || c == 'D' || c == 'z') { currentEnd = i + 1; } else if (inGrouping) { currentEnd = i + 1; } else { this.addPattern(regexp, currentStart, currentEnd, parsedPattern, inException); this.addPattern(regexp, i - 1, i + 1, parsedPattern, inException); currentStart = i + 1; currentEnd = i + 1; } inLiteral = false; } else if (c == '[') { inGrouping = true; groupingHasLetters = false; groupingStart = i; currentEnd = i + 1; } else if (c == ']') { if (!groupingHasLetters) { if (groupingStart > 0) { this.addPattern(regexp, currentStart, groupingStart, parsedPattern, inException); } this.addPattern(regexp, groupingStart, i + 1, parsedPattern, inException); currentStart = i + 1; currentEnd = i + 1; } else { currentEnd = i + 1; } inGrouping = false; } else if (c == '{') { this.addPattern(regexp, currentStart, currentEnd, parsedPattern, inException); inException = true; currentStart = i + 1; currentEnd = i + 1; } else if (c == '}') { this.addPattern(regexp, currentStart, currentEnd, parsedPattern, inException); inException = false; currentStart = i + 1; currentEnd = i + 1; } else if (c == '.' || c == '+' || c == '(' || c == '|' || c == ')' || c == '^' || c == '?' || c == '!') { currentEnd = i + 1; } else if (c == '-') { if (inGrouping) { } else { this.addPattern(regexp, currentStart, currentEnd, parsedPattern, inException); this.addPattern(regexp, i, i + 1, parsedPattern, inException); currentStart = i + 1; currentEnd = i + 1; } } else if (separatorPattern.matcher("" + c).find()) { if (inGrouping) { if (groupingHasLetters) { throw new TalismaneException("Cannot mix separators and non-separators in same grouping"); } } else { this.addPattern(regexp, currentStart, currentEnd, parsedPattern, inException); this.addPattern(regexp, i, i + 1, parsedPattern, inException); currentStart = i + 1; currentEnd = i + 1; } } else { if (inGrouping) { groupingHasLetters = true; } currentEnd = i + 1; } } this.addPattern(regexp, currentStart, currentEnd, parsedPattern, inException); if (LOG.isTraceEnabled()) { int i = 0; LOG.trace("Parsed " + regexp); for (Pattern pattern : parsedPattern) { boolean test = indexesToTest.contains(i); LOG.trace("Added " + pattern.pattern() + " Test? " + test); i++; } } if (indexesToTest.size() == 0) { throw new InvalidTokenPatternException("No indexes to test in pattern: " + this.getName()); } return parsedPattern; } TokenPattern(String regexp, Pattern separatorPattern); String getRegExp(); List<Pattern> getParsedPattern(); int getTokenCount(); List<Integer> getIndexesToTest(); List<TokenPatternMatchSequence> match(TokenSequence tokenSequence); String getName(); void setName(String name); String getGroupName(); void setGroupName(String groupName); @Override String toString(); @Override int hashCode(); @Override boolean equals(Object obj); boolean isSeparatorClass(int index);  }
@Test public void testParsePattern() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; String regexp = "(?![cdCD]\\z|qu\\z|jusqu\\z).+'.+"; TokenPattern tokenPattern = new TokenPattern(regexp, Tokeniser.getTokenSeparators(sessionId)); List<Pattern> patterns = tokenPattern.getParsedPattern(); Assert.assertEquals(3, patterns.size()); int i = 0; for (Pattern pattern : patterns) { if (i == 0) { Assert.assertEquals("(?![cdCD]\\z|[qQ]u\\z|[jJ]usqu\\z).+", pattern.pattern()); } else if (i == 1) { Assert.assertEquals("'", pattern.pattern()); } else if (i == 2) { Assert.assertEquals(".+", pattern.pattern()); } i++; } regexp = "être (de|d)"; tokenPattern = new TokenPattern(regexp, Tokeniser.getTokenSeparators(sessionId)); patterns = tokenPattern.getParsedPattern(); Assert.assertEquals(3, patterns.size()); i = 0; for (Pattern pattern : patterns) { if (i == 0) { Assert.assertEquals("[êÊE]tre", pattern.pattern()); } else if (i == 1) { Assert.assertEquals(" ", pattern.pattern()); } else if (i == 2) { Assert.assertEquals("(de|d)", pattern.pattern()); } i++; } regexp = ".+\\.\\p"; tokenPattern = new TokenPattern(regexp, Tokeniser.getTokenSeparators(sessionId)); patterns = tokenPattern.getParsedPattern(); Assert.assertEquals(3, patterns.size()); i = 0; for (Pattern pattern : patterns) { if (i == 0) { Assert.assertEquals(".+", pattern.pattern()); } else if (i == 1) { Assert.assertEquals("\\.", pattern.pattern()); } else if (i == 2) { Assert.assertEquals(Tokeniser.getTokenSeparators(sessionId).pattern(), pattern.pattern()); } i++; } regexp = ".+qu'"; tokenPattern = new TokenPattern(regexp, Tokeniser.getTokenSeparators(sessionId)); patterns = tokenPattern.getParsedPattern(); Assert.assertEquals(2, patterns.size()); i = 0; for (Pattern pattern : patterns) { if (i == 0) { Assert.assertEquals(".+qu", pattern.pattern()); } else if (i == 1) { Assert.assertEquals("'", pattern.pattern()); } i++; } regexp = "\\D+\\.a[ \\)]c[abc]"; tokenPattern = new TokenPattern(regexp, Tokeniser.getTokenSeparators(sessionId)); patterns = tokenPattern.getParsedPattern(); LOG.debug(patterns.toString()); Assert.assertEquals(5, patterns.size()); i = 0; for (Pattern pattern : patterns) { if (i == 0) { Assert.assertEquals("\\D+", pattern.pattern()); } else if (i == 1) { Assert.assertEquals("\\.", pattern.pattern()); } else if (i == 2) { Assert.assertEquals("a", pattern.pattern()); } else if (i == 3) { Assert.assertEquals("[ \\)]", pattern.pattern()); } else if (i == 4) { Assert.assertEquals("c[abc]", pattern.pattern()); } i++; } }