@Test public void test() { Assert.assertEquals("2d", FeatureGeneratorUtil.tokenFeature("12")); Assert.assertEquals("4d", FeatureGeneratorUtil.tokenFeature("1234")); Assert.assertEquals("an", FeatureGeneratorUtil.tokenFeature("abcd234")); Assert.assertEquals("dd", FeatureGeneratorUtil.tokenFeature("1234-56")); Assert.assertEquals("ds", FeatureGeneratorUtil.tokenFeature("4/6/2017")); Assert.assertEquals("dc", FeatureGeneratorUtil.tokenFeature("1,234,567")); Assert.assertEquals("dp", FeatureGeneratorUtil.tokenFeature("12.34567")); Assert.assertEquals("num", FeatureGeneratorUtil.tokenFeature("123(456)7890")); Assert.assertEquals("lc", FeatureGeneratorUtil.tokenFeature("opennlp")); Assert.assertEquals("sc", FeatureGeneratorUtil.tokenFeature("O")); Assert.assertEquals("ac", FeatureGeneratorUtil.tokenFeature("OPENNLP")); Assert.assertEquals("cp", FeatureGeneratorUtil.tokenFeature("A.")); Assert.assertEquals("ic", FeatureGeneratorUtil.tokenFeature("Mike")); Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("somethingStupid")); Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature(",")); Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature(".")); Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("?")); Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("!")); Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("#")); Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("%")); Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("&")); }
public static String tokenFeature(String token) { StringPattern pattern = StringPattern.recognize(token); String feat; if (pattern.isAllHiragana()) { feat = "jah"; } else if (pattern.isAllKatakana()) { feat = "jak"; } else if (pattern.isAllLowerCaseLetter()) { feat = "lc"; } else if (pattern.digits() == 2) { feat = "2d"; } else if (pattern.digits() == 4) { feat = "4d"; } else if (pattern.containsDigit()) { if (pattern.containsLetters()) { feat = "an"; } else if (pattern.containsHyphen()) { feat = "dd"; } else if (pattern.containsSlash()) { feat = "ds"; } else if (pattern.containsComma()) { feat = "dc"; } else if (pattern.containsPeriod()) { feat = "dp"; } else { feat = "num"; } } else if (pattern.isAllCapitalLetter()) { if (token.length() == 1) { feat = "sc"; } else { feat = "ac"; } } else if (capPeriod.matcher(token).find()) { feat = "cp"; } else if (pattern.isInitialCapitalLetter()) { feat = "ic"; } else { feat = "other"; } return (feat); }
FeatureGeneratorUtil { public static String tokenFeature(String token) { StringPattern pattern = StringPattern.recognize(token); String feat; if (pattern.isAllHiragana()) { feat = "jah"; } else if (pattern.isAllKatakana()) { feat = "jak"; } else if (pattern.isAllLowerCaseLetter()) { feat = "lc"; } else if (pattern.digits() == 2) { feat = "2d"; } else if (pattern.digits() == 4) { feat = "4d"; } else if (pattern.containsDigit()) { if (pattern.containsLetters()) { feat = "an"; } else if (pattern.containsHyphen()) { feat = "dd"; } else if (pattern.containsSlash()) { feat = "ds"; } else if (pattern.containsComma()) { feat = "dc"; } else if (pattern.containsPeriod()) { feat = "dp"; } else { feat = "num"; } } else if (pattern.isAllCapitalLetter()) { if (token.length() == 1) { feat = "sc"; } else { feat = "ac"; } } else if (capPeriod.matcher(token).find()) { feat = "cp"; } else if (pattern.isInitialCapitalLetter()) { feat = "ic"; } else { feat = "other"; } return (feat); } }
FeatureGeneratorUtil { public static String tokenFeature(String token) { StringPattern pattern = StringPattern.recognize(token); String feat; if (pattern.isAllHiragana()) { feat = "jah"; } else if (pattern.isAllKatakana()) { feat = "jak"; } else if (pattern.isAllLowerCaseLetter()) { feat = "lc"; } else if (pattern.digits() == 2) { feat = "2d"; } else if (pattern.digits() == 4) { feat = "4d"; } else if (pattern.containsDigit()) { if (pattern.containsLetters()) { feat = "an"; } else if (pattern.containsHyphen()) { feat = "dd"; } else if (pattern.containsSlash()) { feat = "ds"; } else if (pattern.containsComma()) { feat = "dc"; } else if (pattern.containsPeriod()) { feat = "dp"; } else { feat = "num"; } } else if (pattern.isAllCapitalLetter()) { if (token.length() == 1) { feat = "sc"; } else { feat = "ac"; } } else if (capPeriod.matcher(token).find()) { feat = "cp"; } else if (pattern.isInitialCapitalLetter()) { feat = "ic"; } else { feat = "other"; } return (feat); }  }
FeatureGeneratorUtil { public static String tokenFeature(String token) { StringPattern pattern = StringPattern.recognize(token); String feat; if (pattern.isAllHiragana()) { feat = "jah"; } else if (pattern.isAllKatakana()) { feat = "jak"; } else if (pattern.isAllLowerCaseLetter()) { feat = "lc"; } else if (pattern.digits() == 2) { feat = "2d"; } else if (pattern.digits() == 4) { feat = "4d"; } else if (pattern.containsDigit()) { if (pattern.containsLetters()) { feat = "an"; } else if (pattern.containsHyphen()) { feat = "dd"; } else if (pattern.containsSlash()) { feat = "ds"; } else if (pattern.containsComma()) { feat = "dc"; } else if (pattern.containsPeriod()) { feat = "dp"; } else { feat = "num"; } } else if (pattern.isAllCapitalLetter()) { if (token.length() == 1) { feat = "sc"; } else { feat = "ac"; } } else if (capPeriod.matcher(token).find()) { feat = "cp"; } else if (pattern.isInitialCapitalLetter()) { feat = "ic"; } else { feat = "other"; } return (feat); }  static String tokenFeature(String token); }
FeatureGeneratorUtil { public static String tokenFeature(String token) { StringPattern pattern = StringPattern.recognize(token); String feat; if (pattern.isAllHiragana()) { feat = "jah"; } else if (pattern.isAllKatakana()) { feat = "jak"; } else if (pattern.isAllLowerCaseLetter()) { feat = "lc"; } else if (pattern.digits() == 2) { feat = "2d"; } else if (pattern.digits() == 4) { feat = "4d"; } else if (pattern.containsDigit()) { if (pattern.containsLetters()) { feat = "an"; } else if (pattern.containsHyphen()) { feat = "dd"; } else if (pattern.containsSlash()) { feat = "ds"; } else if (pattern.containsComma()) { feat = "dc"; } else if (pattern.containsPeriod()) { feat = "dp"; } else { feat = "num"; } } else if (pattern.isAllCapitalLetter()) { if (token.length() == 1) { feat = "sc"; } else { feat = "ac"; } } else if (capPeriod.matcher(token).find()) { feat = "cp"; } else if (pattern.isInitialCapitalLetter()) { feat = "ic"; } else { feat = "other"; } return (feat); }  static String tokenFeature(String token);  }