TokenPattern { public List<TokenPatternMatchSequence> match(TokenSequence tokenSequence) { List<TokenPatternMatchSequence> matchingSequences = new ArrayList<TokenPatternMatchSequence>(); boolean matchSentenceStart = false; if (this.getParsedPattern().get(0).pattern().equals("\\b")) { matchSentenceStart = true; } boolean matchSentenceEnd = false; if (this.getParsedPattern().get(this.getParsedPattern().size() - 1).pattern().equals("\\b")) { matchSentenceEnd = true; } for (int t0 = -1; t0 < tokenSequence.listWithWhiteSpace().size(); t0++) { boolean haveMatch = false; List<Token> matchingSequence = new ArrayList<Token>(); if (t0 >= 0) { Token token = tokenSequence.listWithWhiteSpace().get(t0); if (checkTokenForMatch(this.getParsedPattern().get(0), token)) { haveMatch = true; matchingSequence.add(token); } } else if (matchSentenceStart) { haveMatch = true; matchingSequence.add(null); } if (haveMatch) { int p = 1; int t1 = t0 + 1; while (p < this.getParsedPattern().size() && t1 < tokenSequence.listWithWhiteSpace().size()) { Token aToken = tokenSequence.listWithWhiteSpace().get(t1); Pattern pattern = this.getParsedPattern().get(p); if (checkTokenForMatch(pattern, aToken)) { matchingSequence.add(aToken); } else { haveMatch = false; break; } p++; t1++; } if (t1 == tokenSequence.listWithWhiteSpace().size() && p == this.getParsedPattern().size() - 1 && matchSentenceEnd) { matchingSequence.add(null); } if (matchingSequence.size() != this.getParsedPattern().size()) { haveMatch = false; } } if (haveMatch) { TokenPatternMatchSequence tokenPatternMatchSequence = new TokenPatternMatchSequence(this, matchingSequence); matchingSequences.add(tokenPatternMatchSequence); for (Token aToken : matchingSequence) { tokenPatternMatchSequence.addMatch(aToken); } } } if (LOG.isTraceEnabled()) { if (matchingSequences.size() > 0) LOG.trace(this.getName() + ": matchingSequences = " + matchingSequences); } return matchingSequences; } TokenPattern(String regexp, Pattern separatorPattern); String getRegExp(); List<Pattern> getParsedPattern(); int getTokenCount(); List<Integer> getIndexesToTest(); List<TokenPatternMatchSequence> match(TokenSequence tokenSequence); String getName(); void setName(String name); String getGroupName(); void setGroupName(String groupName); @Override String toString(); @Override int hashCode(); @Override boolean equals(Object obj); boolean isSeparatorClass(int index);  }
@Test public void testMatch() throws TalismaneException { final String separators = "[\\s\\p{Punct}]"; final List<TokenPatternMatch> matches3 = new ArrayList<TokenPatternMatch>(); final List<TokenPatternMatch> matches4 = new ArrayList<TokenPatternMatch>(); final List<TokenPatternMatch> matches5 = new ArrayList<TokenPatternMatch>(); final List<TokenPatternMatch> matches6 = new ArrayList<TokenPatternMatch>(); final List<TokenPatternMatch> matches7 = new ArrayList<TokenPatternMatch>(); final TokenSequence tokenSequence = mock(TokenSequence.class); final Token token0 = mock(Token.class); final Token token1 = mock(Token.class); final Token token2 = mock(Token.class); final Token token3 = mock(Token.class); final Token token4 = mock(Token.class); final Token token5 = mock(Token.class); final Token token6 = mock(Token.class); final Token token7 = mock(Token.class); final List<Token> listWithWhiteSpaces = Arrays.asList(token0, token1, token2, token3, token4, token5, token6, token7); when (tokenSequence.listWithWhiteSpace()).thenReturn(listWithWhiteSpaces); when (token0.getAnalyisText()).thenReturn("Moi"); when (token0.isSeparator()).thenReturn(false); when (token0.getIndex()).thenReturn(0); when (token1.getAnalyisText()).thenReturn(","); when (token1.isSeparator()).thenReturn(true); when (token1.getIndex()).thenReturn(1); when (token2.getAnalyisText()).thenReturn(" "); when (token2.isSeparator()).thenReturn(true); when (token2.getIndex()).thenReturn(2); when (token3.getAnalyisText()).thenReturn("j"); when (token3.isSeparator()).thenReturn(false); when (token3.getIndex()).thenReturn(3); when (token3.getMatches()).thenReturn(matches3); when (token4.getAnalyisText()).thenReturn("'"); when (token4.isSeparator()).thenReturn(true); when (token4.getIndex()).thenReturn(4); when (token4.getMatches()).thenReturn(matches4); when (token5.getAnalyisText()).thenReturn("aim"); when (token5.isSeparator()).thenReturn(false); when (token5.getIndex()).thenReturn(5); when (token5.getMatches()).thenReturn(matches5); when (token6.getAnalyisText()).thenReturn("'"); when (token6.isSeparator()).thenReturn(true); when (token6.getIndex()).thenReturn(6); when (token6.getMatches()).thenReturn(matches6); when (token7.getAnalyisText()).thenReturn("rais"); when (token7.isSeparator()).thenReturn(false); when (token7.getIndex()).thenReturn(7); when (token7.getMatches()).thenReturn(matches7); Pattern separatorPattern = Pattern.compile(separators, Pattern.UNICODE_CHARACTER_CLASS); TokenPattern tokeniserPatternImpl = new TokenPattern(".+'.+", separatorPattern); List<TokenPatternMatchSequence> patternMatches = tokeniserPatternImpl.match(tokenSequence); assertEquals(2, patternMatches.size()); List<Token> patternMatch = patternMatches.get(0).getTokenSequence(); assertEquals(3, patternMatch.size()); for (int i = 0; i < 3; i++) { Token token = patternMatch.get(i); if (i == 0) { assertEquals(3, token.getIndex()); assertEquals("j", token.getAnalyisText()); assertEquals(1, token.getMatches().size()); assertEquals(0, token.getMatches().get(0).getIndex()); } if (i == 1) { assertEquals(4, token.getIndex()); assertEquals("'", token.getAnalyisText()); assertEquals(1, token.getMatches().size()); assertEquals(1, token.getMatches().get(0).getIndex()); } if (i == 2) { assertEquals(5, token.getIndex()); assertEquals("aim", token.getAnalyisText()); assertEquals(2, token.getMatches().size()); assertEquals(2, token.getMatches().get(0).getIndex()); } } patternMatch = patternMatches.get(1).getTokenSequence(); assertEquals(3, patternMatch.size()); for (int i = 0; i < 3; i++) { Token token = patternMatch.get(i); if (i == 0) { assertEquals(5, token.getIndex()); assertEquals("aim", token.getAnalyisText()); assertEquals(2, token.getMatches().size()); assertEquals(0, token.getMatches().get(1).getIndex()); } if (i == 1) { assertEquals(6, token.getIndex()); assertEquals("'", token.getAnalyisText()); assertEquals(1, token.getMatches().size()); assertEquals(1, token.getMatches().get(0).getIndex()); } if (i == 2) { assertEquals(7, token.getIndex()); assertEquals("rais", token.getAnalyisText()); assertEquals(1, token.getMatches().size()); assertEquals(2, token.getMatches().get(0).getIndex()); } } }
@Test public void testMatch2() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; final Sentence sentence = new Sentence("Qu'ensuite il aille...", sessionId); TokenSequence tokenSequence = new TokenSequence(sentence, sessionId); tokenSequence.findDefaultTokens(); TokenPattern tokenPattern = new TokenPattern("{(?![cdjlmnstCDJLMNST]\\z|qu\\z|jusqu\\z|puisqu\\z|lorsqu\\z|aujourd\\z|prud\\z|quelqu\\z|quoiqu\\z).+'}.+", Tokeniser.getTokenSeparators(sessionId)); List<TokenPatternMatchSequence> patternMatches = tokenPattern.match(tokenSequence); assertEquals(0, patternMatches.size()); }
@Test public void testMatch3() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; final Sentence sentence = new Sentence("Z'ensuite il aille...", sessionId); TokenSequence tokenSequence = new TokenSequence(sentence, sessionId); tokenSequence.findDefaultTokens(); TokenPattern tokenPattern = new TokenPattern("{(?![cdjlmnstCDJLMNST]\\z|qu\\z|jusqu\\z|puisqu\\z|lorsqu\\z|aujourd\\z|prud\\z|quelqu\\z|quoiqu\\z).+'}.+", Tokeniser.getTokenSeparators(sessionId)); List<TokenPatternMatchSequence> patternMatches = tokenPattern.match(tokenSequence); assertEquals(1, patternMatches.size()); TokenPatternMatchSequence matchSequence = patternMatches.get(0); assertEquals(3, matchSequence.getTokenSequence().size()); assertEquals("Z", matchSequence.getTokenSequence().get(0).getOriginalText()); }
@Test public void testMatch4() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; final Sentence sentence = new Sentence("Aix-les-Bains", sessionId); TokenSequence tokenSequence = new TokenSequence(sentence, sessionId); tokenSequence.findDefaultTokens(); TokenPattern tokenPattern = new TokenPattern(".+-{(ce|je|la|le|les|leur|lui|moi|nous|toi|tu)[^-]}", Tokeniser.getTokenSeparators(sessionId)); List<TokenPatternMatchSequence> patternMatches = tokenPattern.match(tokenSequence); assertEquals(0, patternMatches.size()); }