StringSequenceGenerator { public List<String[]> tokenSequences(JCas aJCas) throws FeaturePathException { return psg.tokenSequences(aJCas).stream() .map(this::phrases2String) .collect(Collectors.toList()); } protected  StringSequenceGenerator(PhraseSequenceGenerator.Builder builder); List<String[]> tokenSequences(JCas aJCas);  }
@Test public void testGenerateSequenceFeaturePath() throws FeaturePathException, UIMAException, IOException { String featurePath = Token.class.getName(); int expectedSize = 2; String expectedFirstToken = "Token1"; String expectedLastToken = "Token2"; JCas jCas = jCasWithTokens(); StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .featurePath(featurePath) .buildStringSequenceGenerator(); String[] sequence = sequenceGenerator.tokenSequences(jCas).get(0); assertEquals(expectedSize, sequence.length); assertEquals(expectedFirstToken, sequence[0]); assertEquals(expectedLastToken, sequence[sequence.length - 1]); }
@Test public void testGenerateSequenceFeaturePathLowercase() throws FeaturePathException, UIMAException, IOException { String featurePath = Token.class.getName(); int expectedSize = 2; String expectedFirstToken = "token1"; String expectedLastToken = "token2"; JCas jCas = jCasWithTokens(); StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .featurePath(featurePath) .lowercase(true) .buildStringSequenceGenerator(); String[] sequence = sequenceGenerator.tokenSequences(jCas).get(0); assertEquals(expectedSize, sequence.length); assertEquals(expectedFirstToken, sequence[0]); assertEquals(expectedLastToken, sequence[sequence.length - 1]); }
@Test public void testGenerateSequenceFeaturePathLemmas() throws UIMAException, FeaturePathException, IOException { String featurePath = Token.class.getName() + "/lemma/value"; int expectedSize = 2; String expectedFirstLemma = "lemma1"; String expectedLastLemma = "lemma2"; JCas jCas = jCasWithLemmas(); StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .featurePath(featurePath) .buildStringSequenceGenerator(); String[] sequence = sequenceGenerator.tokenSequences(jCas).get(0); assertEquals(expectedSize, sequence.length); assertEquals(expectedFirstLemma, sequence[0]); assertEquals(expectedLastLemma, sequence[sequence.length - 1]); }
@Test public void testFeaturePathNamedEntities() throws UIMAException, IOException, FeaturePathException { String featurePath = NamedEntity.class.getCanonicalName(); int expectedSize = 1; String expectedNamedEntity = "token1"; JCas jCas = jcasWithNamedEntity(); StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .featurePath(featurePath) .buildStringSequenceGenerator(); String[] sequence = sequenceGenerator.tokenSequences(jCas).get(0); assertEquals(expectedSize, sequence.length); assertEquals(expectedNamedEntity, sequence[0]); }
@Test public void testGenerateSequenceFeaturePathCovering() throws FeaturePathException, UIMAException, IOException { String featurePath = Token.class.getName(); int expectedSize = 2; String expectedFirstToken = "Token1"; String expectedLastToken = "Token2"; String covering = Sentence.class.getTypeName(); JCas jCas = jCasWithSentence(); StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .featurePath(featurePath) .lowercase(false) .coveringType(covering) .buildStringSequenceGenerator(); List<String[]> sequences = sequenceGenerator.tokenSequences(jCas); assertEquals(1, sequences.size()); String[] sequence = sequences.get(0); Assert.assertEquals(expectedSize, sequence.length); Assert.assertEquals(expectedFirstToken, sequence[0]); Assert.assertEquals(expectedLastToken, sequence[sequence.length - 1]); }
@Test public void testFilterRegex() throws UIMAException, IOException, FeaturePathException { JCas jCas = jCasWithTokens(); String filterRegex = ".*1"; int expectedSize = 1; String expectedToken = "Token2"; StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .filterRegex(filterRegex) .buildStringSequenceGenerator(); List<String[]> sequences = sequenceGenerator.tokenSequences(jCas); assertEquals(1, sequences.size()); String[] sequence = sequences.get(0); assertEquals(expectedSize, sequence.length); assertEquals(expectedToken, sequence[0]); }
@Test public void testFilterRegexReplace() throws UIMAException, IOException, FeaturePathException { JCas jCas = jCasWithTokens(); String filterRegex = ".*1"; String replacement = "REPLACED"; int expectedSize = 2; String expectedToken2 = "Token2"; StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .filterRegex(filterRegex) .filterRegexReplacement(replacement) .buildStringSequenceGenerator(); List<String[]> sequences = sequenceGenerator.tokenSequences(jCas); assertEquals(1, sequences.size()); String[] sequence = sequences.get(0); assertEquals(expectedSize, sequence.length); assertEquals(replacement, sequence[0]); assertEquals(expectedToken2, sequence[1]); }
@Test public void testFilterRegexMultiple() throws UIMAException, IOException, FeaturePathException { JCas jCas = jCasWithTokens(); String filterRegex1 = ".*1"; String filterRegex2 = "xyz"; int expectedSize = 1; String expectedToken = "Token2"; StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .filterRegex(filterRegex1) .filterRegex(filterRegex2) .buildStringSequenceGenerator(); List<String[]> sequences = sequenceGenerator.tokenSequences(jCas); assertEquals(1, sequences.size()); String[] sequence = sequences.get(0); assertEquals(expectedSize, sequence.length); assertEquals(expectedToken, sequence[0]); }
@Test public void testFilterStopwordsURL() throws UIMAException, FeaturePathException, IOException { JCas jcas = jCasWithTokens(); URL stopwordsFile = this.getClass().getResource("/stopwords.txt"); int expectedSize = 1; String expectedFirst = "Token2"; StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .stopwordsURL(stopwordsFile) .buildStringSequenceGenerator(); String[] sequence = sequenceGenerator.tokenSequences(jcas).get(0); assertEquals(expectedSize, sequence.length); assertEquals(expectedFirst, sequence[0]); }
@Test public void testFilterStopwordsFileString() throws UIMAException, FeaturePathException, IOException { JCas jcas = jCasWithTokens(); String stopwordsFile = "src/test/resources/stopwords.txt"; int expectedSize = 1; String expectedFirst = "Token2"; StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .stopwordsFile(stopwordsFile) .buildStringSequenceGenerator(); String[] sequence = sequenceGenerator.tokenSequences(jcas).get(0); assertEquals(expectedSize, sequence.length); assertEquals(expectedFirst, sequence[0]); }
@Test public void testFilterStopwordsFile() throws UIMAException, FeaturePathException, IOException { JCas jcas = jCasWithTokens(); File stopwordsFile = new File("src/test/resources/stopwords.txt"); int expectedSize = 1; String expectedFirst = "Token2"; StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .stopwordsFile(stopwordsFile) .buildStringSequenceGenerator(); String[] sequence = sequenceGenerator.tokenSequences(jcas).get(0); assertEquals(expectedSize, sequence.length); assertEquals(expectedFirst, sequence[0]); }
@Test public void testCharacterSequence() throws UIMAException, FeaturePathException, IOException { JCas jcas = jCasWithTokens(); int expectedSize = 13; String expectedFirst = "T"; String expectedLast = "2"; StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .characters(true) .buildStringSequenceGenerator(); String[] sequence = sequenceGenerator.tokenSequences(jcas).get(0); assertEquals(expectedSize, sequence.length); assertEquals(expectedFirst, sequence[0]); assertEquals(expectedLast, sequence[expectedSize - 1]); }
@Test public void testCharacterSequenceLowercase() throws UIMAException, FeaturePathException, IOException { JCas jcas = jCasWithTokens(); int expectedSize = 13; String expectedFirst = "t"; String expectedLast = "2"; StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .lowercase(true) .characters(true) .buildStringSequenceGenerator(); String[] sequence = sequenceGenerator.tokenSequences(jcas).get(0); assertEquals(expectedSize, sequence.length); assertEquals(expectedFirst, sequence[0]); assertEquals(expectedLast, sequence[expectedSize - 1]); }
@Test public void testCharacterSequenceWithCovering() throws UIMAException, FeaturePathException, IOException { String covering = Sentence.class.getTypeName(); JCas jCas = jCasWithSentence(); int expectedSequences = 1; int expectedSize = 13; String expectedFirst = "T"; String expectedLast = "2"; StringSequenceGenerator sequenceGenerator = new PhraseSequenceGenerator.Builder() .coveringType(covering) .characters(true) .buildStringSequenceGenerator(); List<String[]> sequences = sequenceGenerator.tokenSequences(jCas); assertEquals(expectedSequences, sequences.size()); String[] sequence = sequences.get(0); assertEquals(expectedSize, sequence.length); assertEquals(expectedFirst, sequence[0]); assertEquals(expectedLast, sequence[expectedSize - 1]); }