@Test public void testSentence() { String[] testSentence = new String[] {"This is an example sentence"}; final int testTokenIndex = 0; AdaptiveFeatureGenerator generator = new TokenPatternFeatureGenerator(); generator.createFeatures(features, testSentence, testTokenIndex, null); Assert.assertEquals(14, features.size()); Assert.assertEquals("stn=5", features.get(0)); Assert.assertEquals("pt2=iclc", features.get(1)); Assert.assertEquals("pt3=iclclc", features.get(2)); Assert.assertEquals("st=this", features.get(3)); Assert.assertEquals("pt2=lclc", features.get(4)); Assert.assertEquals("pt3=lclclc", features.get(5)); Assert.assertEquals("st=is", features.get(6)); Assert.assertEquals("pt2=lclc", features.get(7)); Assert.assertEquals("pt3=lclclc", features.get(8)); Assert.assertEquals("st=an", features.get(9)); Assert.assertEquals("pt2=lclc", features.get(10)); Assert.assertEquals("st=example", features.get(11)); Assert.assertEquals("st=sentence", features.get(12)); Assert.assertEquals("pta=iclclclclc", features.get(13)); }
public void createFeatures(List<String> feats, String[] toks, int index, String[] preds) { String[] tokenized = tokenizer.tokenize(toks[index]); if (tokenized.length == 1) { feats.add("st=" + StringUtil.toLowerCase(toks[index])); return; } feats.add("stn=" + tokenized.length); StringBuilder pattern = new StringBuilder(); for (int i = 0; i < tokenized.length; i++) { if (i < tokenized.length - 1) { feats.add("pt2=" + FeatureGeneratorUtil.tokenFeature(tokenized[i]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 1])); } if (i < tokenized.length - 2) { feats.add("pt3=" + FeatureGeneratorUtil.tokenFeature(tokenized[i]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 1]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 2])); } pattern.append(FeatureGeneratorUtil.tokenFeature(tokenized[i])); if (!noLetters.matcher(tokenized[i]).find()) { feats.add("st=" + StringUtil.toLowerCase(tokenized[i])); } } feats.add("pta=" + pattern.toString()); }
TokenPatternFeatureGenerator implements AdaptiveFeatureGenerator { public void createFeatures(List<String> feats, String[] toks, int index, String[] preds) { String[] tokenized = tokenizer.tokenize(toks[index]); if (tokenized.length == 1) { feats.add("st=" + StringUtil.toLowerCase(toks[index])); return; } feats.add("stn=" + tokenized.length); StringBuilder pattern = new StringBuilder(); for (int i = 0; i < tokenized.length; i++) { if (i < tokenized.length - 1) { feats.add("pt2=" + FeatureGeneratorUtil.tokenFeature(tokenized[i]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 1])); } if (i < tokenized.length - 2) { feats.add("pt3=" + FeatureGeneratorUtil.tokenFeature(tokenized[i]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 1]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 2])); } pattern.append(FeatureGeneratorUtil.tokenFeature(tokenized[i])); if (!noLetters.matcher(tokenized[i]).find()) { feats.add("st=" + StringUtil.toLowerCase(tokenized[i])); } } feats.add("pta=" + pattern.toString()); } }
TokenPatternFeatureGenerator implements AdaptiveFeatureGenerator { public void createFeatures(List<String> feats, String[] toks, int index, String[] preds) { String[] tokenized = tokenizer.tokenize(toks[index]); if (tokenized.length == 1) { feats.add("st=" + StringUtil.toLowerCase(toks[index])); return; } feats.add("stn=" + tokenized.length); StringBuilder pattern = new StringBuilder(); for (int i = 0; i < tokenized.length; i++) { if (i < tokenized.length - 1) { feats.add("pt2=" + FeatureGeneratorUtil.tokenFeature(tokenized[i]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 1])); } if (i < tokenized.length - 2) { feats.add("pt3=" + FeatureGeneratorUtil.tokenFeature(tokenized[i]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 1]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 2])); } pattern.append(FeatureGeneratorUtil.tokenFeature(tokenized[i])); if (!noLetters.matcher(tokenized[i]).find()) { feats.add("st=" + StringUtil.toLowerCase(tokenized[i])); } } feats.add("pta=" + pattern.toString()); } TokenPatternFeatureGenerator();  TokenPatternFeatureGenerator(Tokenizer supportTokenizer); }
TokenPatternFeatureGenerator implements AdaptiveFeatureGenerator { public void createFeatures(List<String> feats, String[] toks, int index, String[] preds) { String[] tokenized = tokenizer.tokenize(toks[index]); if (tokenized.length == 1) { feats.add("st=" + StringUtil.toLowerCase(toks[index])); return; } feats.add("stn=" + tokenized.length); StringBuilder pattern = new StringBuilder(); for (int i = 0; i < tokenized.length; i++) { if (i < tokenized.length - 1) { feats.add("pt2=" + FeatureGeneratorUtil.tokenFeature(tokenized[i]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 1])); } if (i < tokenized.length - 2) { feats.add("pt3=" + FeatureGeneratorUtil.tokenFeature(tokenized[i]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 1]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 2])); } pattern.append(FeatureGeneratorUtil.tokenFeature(tokenized[i])); if (!noLetters.matcher(tokenized[i]).find()) { feats.add("st=" + StringUtil.toLowerCase(tokenized[i])); } } feats.add("pta=" + pattern.toString()); } TokenPatternFeatureGenerator();  TokenPatternFeatureGenerator(Tokenizer supportTokenizer); void createFeatures(List<String> feats, String[] toks, int index, String[] preds); }
TokenPatternFeatureGenerator implements AdaptiveFeatureGenerator { public void createFeatures(List<String> feats, String[] toks, int index, String[] preds) { String[] tokenized = tokenizer.tokenize(toks[index]); if (tokenized.length == 1) { feats.add("st=" + StringUtil.toLowerCase(toks[index])); return; } feats.add("stn=" + tokenized.length); StringBuilder pattern = new StringBuilder(); for (int i = 0; i < tokenized.length; i++) { if (i < tokenized.length - 1) { feats.add("pt2=" + FeatureGeneratorUtil.tokenFeature(tokenized[i]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 1])); } if (i < tokenized.length - 2) { feats.add("pt3=" + FeatureGeneratorUtil.tokenFeature(tokenized[i]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 1]) + FeatureGeneratorUtil.tokenFeature(tokenized[i + 2])); } pattern.append(FeatureGeneratorUtil.tokenFeature(tokenized[i])); if (!noLetters.matcher(tokenized[i]).find()) { feats.add("st=" + StringUtil.toLowerCase(tokenized[i])); } } feats.add("pta=" + pattern.toString()); } TokenPatternFeatureGenerator();  TokenPatternFeatureGenerator(Tokenizer supportTokenizer); void createFeatures(List<String> feats, String[] toks, int index, String[] preds);  }