@Test public void testCaseMasks() { String cap = "Exploring"; List<String> ls = PDFPredicateExtractor.getCaseMasks(cap); Assert.assertEquals(ls.size(), 2); Assert.assertTrue(ls.contains("%Xxx")); Assert.assertTrue(ls.contains("%letters")); String nonSimple = "Dharmaratnå"; ls = PDFPredicateExtractor.getCaseMasks(nonSimple); Assert.assertTrue(ls.contains("%hasNonAscii")); Assert.assertTrue(!ls.contains("%hasAt")); String email = "bob@joe.com"; ls = PDFPredicateExtractor.getCaseMasks(email); Assert.assertTrue(ls.contains("%hasAt")); }
public static List<String> getCaseMasks(String tok) { Pattern Xxx = Pattern.compile("[A-Z][a-z]*"); Pattern xxx = Pattern.compile("[a-z]+"); Pattern letters = Pattern.compile("[a-zA-Z]+"); Pattern dig = Pattern.compile("[0-9]+"); Pattern hasNum = Pattern.compile(".*[0-9]+.*"); Pattern letterDot = Pattern.compile("[A-Z]\\."); Pattern hasNonAscii = Pattern.compile(".*[^\\p{ASCII}]+.*"); Pattern wordColon = Pattern.compile("[a-zA-Z]+:"); Pattern hasAt = Pattern.compile(".*@.*"); List<Pattern> pats = Arrays.asList(Xxx, xxx, letters, dig, hasNum, letterDot, hasNonAscii, wordColon, hasAt); List<String> feats = Arrays.asList("%Xxx", "%xxx", "%letters", "%dig", "%hasNum", "%letDot", "%hasNonAscii", "%capWordColon", "%hasAt"); ArrayList<String> out = new ArrayList<String>(); for (int i = 0; i < pats.size(); i++) { Pattern p = pats.get(i); if (p.matcher(tok).matches()) { out.add(feats.get(i)); } } return out; }
PDFPredicateExtractor implements CRFPredicateExtractor<PaperToken, String> { public static List<String> getCaseMasks(String tok) { Pattern Xxx = Pattern.compile("[A-Z][a-z]*"); Pattern xxx = Pattern.compile("[a-z]+"); Pattern letters = Pattern.compile("[a-zA-Z]+"); Pattern dig = Pattern.compile("[0-9]+"); Pattern hasNum = Pattern.compile(".*[0-9]+.*"); Pattern letterDot = Pattern.compile("[A-Z]\\."); Pattern hasNonAscii = Pattern.compile(".*[^\\p{ASCII}]+.*"); Pattern wordColon = Pattern.compile("[a-zA-Z]+:"); Pattern hasAt = Pattern.compile(".*@.*"); List<Pattern> pats = Arrays.asList(Xxx, xxx, letters, dig, hasNum, letterDot, hasNonAscii, wordColon, hasAt); List<String> feats = Arrays.asList("%Xxx", "%xxx", "%letters", "%dig", "%hasNum", "%letDot", "%hasNonAscii", "%capWordColon", "%hasAt"); ArrayList<String> out = new ArrayList<String>(); for (int i = 0; i < pats.size(); i++) { Pattern p = pats.get(i); if (p.matcher(tok).matches()) { out.add(feats.get(i)); } } return out; } }
PDFPredicateExtractor implements CRFPredicateExtractor<PaperToken, String> { public static List<String> getCaseMasks(String tok) { Pattern Xxx = Pattern.compile("[A-Z][a-z]*"); Pattern xxx = Pattern.compile("[a-z]+"); Pattern letters = Pattern.compile("[a-zA-Z]+"); Pattern dig = Pattern.compile("[0-9]+"); Pattern hasNum = Pattern.compile(".*[0-9]+.*"); Pattern letterDot = Pattern.compile("[A-Z]\\."); Pattern hasNonAscii = Pattern.compile(".*[^\\p{ASCII}]+.*"); Pattern wordColon = Pattern.compile("[a-zA-Z]+:"); Pattern hasAt = Pattern.compile(".*@.*"); List<Pattern> pats = Arrays.asList(Xxx, xxx, letters, dig, hasNum, letterDot, hasNonAscii, wordColon, hasAt); List<String> feats = Arrays.asList("%Xxx", "%xxx", "%letters", "%dig", "%hasNum", "%letDot", "%hasNonAscii", "%capWordColon", "%hasAt"); ArrayList<String> out = new ArrayList<String>(); for (int i = 0; i < pats.size(); i++) { Pattern p = pats.get(i); if (p.matcher(tok).matches()) { out.add(feats.get(i)); } } return out; } PDFPredicateExtractor();  PDFPredicateExtractor(ParserLMFeatures plf); }
PDFPredicateExtractor implements CRFPredicateExtractor<PaperToken, String> { public static List<String> getCaseMasks(String tok) { Pattern Xxx = Pattern.compile("[A-Z][a-z]*"); Pattern xxx = Pattern.compile("[a-z]+"); Pattern letters = Pattern.compile("[a-zA-Z]+"); Pattern dig = Pattern.compile("[0-9]+"); Pattern hasNum = Pattern.compile(".*[0-9]+.*"); Pattern letterDot = Pattern.compile("[A-Z]\\."); Pattern hasNonAscii = Pattern.compile(".*[^\\p{ASCII}]+.*"); Pattern wordColon = Pattern.compile("[a-zA-Z]+:"); Pattern hasAt = Pattern.compile(".*@.*"); List<Pattern> pats = Arrays.asList(Xxx, xxx, letters, dig, hasNum, letterDot, hasNonAscii, wordColon, hasAt); List<String> feats = Arrays.asList("%Xxx", "%xxx", "%letters", "%dig", "%hasNum", "%letDot", "%hasNonAscii", "%capWordColon", "%hasAt"); ArrayList<String> out = new ArrayList<String>(); for (int i = 0; i < pats.size(); i++) { Pattern p = pats.get(i); if (p.matcher(tok).matches()) { out.add(feats.get(i)); } } return out; } PDFPredicateExtractor();  PDFPredicateExtractor(ParserLMFeatures plf); static List<String> getCaseMasks(String tok); static boolean isStopWord(String tok); static float getY(PaperToken t, boolean upper); static double smoothFreq(String tok, ObjectDoubleHashMap<String> hm); static float getExtreme(List<PaperToken> toks, TokenPropertySelector s, boolean max); static float linearNormalize(float f, Pair<Float, Float> rng); static Pair<Float, Float> getExtrema(List<PaperToken> toks, TokenPropertySelector s); static float getFixedFont(PaperToken t); double logYDelt(float y1, float y2); @Override List<ObjectDoubleMap<String>> nodePredicates(List<PaperToken> elems); @Override List<ObjectDoubleMap<String>> edgePredicates(List<PaperToken> elems); }
PDFPredicateExtractor implements CRFPredicateExtractor<PaperToken, String> { public static List<String> getCaseMasks(String tok) { Pattern Xxx = Pattern.compile("[A-Z][a-z]*"); Pattern xxx = Pattern.compile("[a-z]+"); Pattern letters = Pattern.compile("[a-zA-Z]+"); Pattern dig = Pattern.compile("[0-9]+"); Pattern hasNum = Pattern.compile(".*[0-9]+.*"); Pattern letterDot = Pattern.compile("[A-Z]\\."); Pattern hasNonAscii = Pattern.compile(".*[^\\p{ASCII}]+.*"); Pattern wordColon = Pattern.compile("[a-zA-Z]+:"); Pattern hasAt = Pattern.compile(".*@.*"); List<Pattern> pats = Arrays.asList(Xxx, xxx, letters, dig, hasNum, letterDot, hasNonAscii, wordColon, hasAt); List<String> feats = Arrays.asList("%Xxx", "%xxx", "%letters", "%dig", "%hasNum", "%letDot", "%hasNonAscii", "%capWordColon", "%hasAt"); ArrayList<String> out = new ArrayList<String>(); for (int i = 0; i < pats.size(); i++) { Pattern p = pats.get(i); if (p.matcher(tok).matches()) { out.add(feats.get(i)); } } return out; } PDFPredicateExtractor();  PDFPredicateExtractor(ParserLMFeatures plf); static List<String> getCaseMasks(String tok); static boolean isStopWord(String tok); static float getY(PaperToken t, boolean upper); static double smoothFreq(String tok, ObjectDoubleHashMap<String> hm); static float getExtreme(List<PaperToken> toks, TokenPropertySelector s, boolean max); static float linearNormalize(float f, Pair<Float, Float> rng); static Pair<Float, Float> getExtrema(List<PaperToken> toks, TokenPropertySelector s); static float getFixedFont(PaperToken t); double logYDelt(float y1, float y2); @Override List<ObjectDoubleMap<String>> nodePredicates(List<PaperToken> elems); @Override List<ObjectDoubleMap<String>> edgePredicates(List<PaperToken> elems); static final List<String> stopWords; static final HashSet<String> stopHash; static final String[] wordEmbeddingFeatureNames; }