DictionaryBasedContextGenerator { public List<String> extract(Token focusToken, String prevTag) { if (!(focusToken instanceof W)) { return ImmutableList.of(); } String form = focusToken.getCoveredText(); if (!WordUtils.isRussianWord(form)) { return ImmutableList.of("DL=NotRussian"); } form = WordUtils.normalizeToDictionaryForm(form); List<Wordform> dictWfs = morphDict.getEntries(form); if (dictWfs == null || dictWfs.isEmpty()) { return ImmutableList.of("DL=Unknown"); } List<BitSet> dictWfBitSets = Lists.transform(dictWfs, Wordform.allGramBitsFunction(morphDict)); Set<BitSet> tokenPossibleTags = Sets.newHashSetWithExpectedSize(dictWfBitSets.size()); for (BitSet dictWfBits : dictWfBitSets) { BitSet tokenPossibleBits = (BitSet) dictWfBits.clone(); tokenPossibleBits.and(targetCategoriesMask); tokenPossibleTags.add(tokenPossibleBits); } List<String> resultList = Lists.newArrayListWithExpectedSize(tokenPossibleTags.size()); for (BitSet tokenPossibleBits : tokenPossibleTags) { String featValue; if (tokenPossibleBits.isEmpty()) { featValue = "NULL"; } else { featValue = gramJoiner.join(gramModel.toGramSet(tokenPossibleBits)); } resultList.add("DL=" + featValue); } if (prevTag != null && !PunctuationUtils.isPunctuationTag(prevTag)) { BitSet prevTagBits = toGramBits(gramModel, tagMapper.parseTag(prevTag, focusToken.getCoveredText())); for (Map.Entry<String, TwoTagPredicate> predEntry : namedPredicates.entrySet()) { for (BitSet dictTag : tokenPossibleTags) { if (predEntry.getValue().apply(prevTagBits, dictTag)) { resultList.add(predEntry.getKey()); break; } } } } return resultList; } DictionaryBasedContextGenerator(Iterable<String> targetGramCategories,
                                           MorphDictionary morphDict); List<String> extract(Token focusToken, String prevTag);  }
@Test public void testOnPunctuation() { Token token = new PM(jCas, 18, 19); assertEquals(ImmutableList.<String>of(), gen.extract(token, null)); }
@Test public void testOnNonRussian() { Token token = new CW(jCas, 9, 17); assertEquals(ImmutableList.<String>of("DL=NotRussian"), gen.extract(token, null)); }
@Test public void testOnNonDictionary() { Token token = new CW(jCas, 46, 53); assertEquals(ImmutableList.<String>of("DL=Unknown"), gen.extract(token, null)); }
@Test public void testOnKnownWordWithoutPrevTag() { Token token = new CW(jCas, 33, 37); Assert.assertThat("Expected at least one dictionary entry", gen.extract(token, null), hasItem("DL=NOUN_femn_sing_nomn")); }
@SuppressWarnings({"unchecked", "rawtypes"}) @Test public void testOnKnownWordWithPrevTag() { Token token = new CW(jCas, 20, 32); assertThat("Expected at least two dictionary entries and agreement", gen.extract(token, "ADJF&femn&sing&accs"), (Matcher) hasItems( allOf(startsWith("DL="), containsString("accs")), allOf(startsWith("DL="), containsString("nomn")), is("CaseAgr"), is("NumberAgr"), is("NumberCaseAgr"))); assertThat("Expected at least two dictionary entries and agreement", gen.extract(token, "ADJF&masc&plur&nomn"), (Matcher) hasItems(is("CaseAgr"), is("GenderAgr"), is("GenderCaseAgr"))); }