@Test public void testPipeAllCapsToken() { ArrayList<String> list = new ArrayList<String>(1); list.add("This is SPARTA !!!"); Instance inst = pipe.pipe(new Instance(list, "", "", "")); TokenSequence tokens = (TokenSequence) inst.getData(); Token third = (Token) tokens.get(2); assertSimilar(third.getFeatureValue("ALLCAPS"), 1.0); assertSimilar(third.getFeatureValue("BWC=A"), 1.0); assertSimilar(third.getFeatureValue("SIZE2"), 1.0); }
public Instance pipe(Instance carrier) { String abstractFileName = (String) carrier.getSource(); @SuppressWarnings("unchecked") List<String> lines = (List<String>) carrier.getData(); Map<String, Integer> unitFreq = getUnitFrequency(lines); TokenSequence data = new TokenSequence(); LabelSequence target = new LabelSequence((LabelAlphabet) getTargetAlphabet()); List<Unit> unitInfo = new ArrayList<>(); for (int i = 0; i < lines.size(); i++) { String line = lines.get(i); if (line.isEmpty()) { continue; } List<Unit> units = getUnits(line); if (units.isEmpty()) continue; for (int j = 0; j < units.size(); j++) { String currUnitRep = units.get(j).rep; String plainUnitRep = getPlainUnit(currUnitRep); String label = "IS"; Token token = new Token(currUnitRep); if (units.get(j).isTokenInternal) token.setFeatureValue("istokeninternal=", 1); if (containsEOSSymbol(currUnitRep)) token.setFeatureValue("endwithEOSSymb=" + getEOSSymbol(currUnitRep), 1); if ((j + 1) == units.size()) { label = "EOS"; } int count = nrEOSSymbolsContained(plainUnitRep); if (count > 0) token.setFeatureValue("hasinnerEOSSymb=" + count, 1); token.setFeatureValue("TOKEN=" + currUnitRep, 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}].*")) token.setFeatureValue("INITCAPS", 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}]")) token.setFeatureValue("ONECAPS", 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}]+")) token.setFeatureValue("ALLCAPS", 1); if (currUnitRep.matches("(.*[\\p{L}\\p{M}].*[0-9].*|.*[0-9].*[\\p{L}\\p{M}].*)")) token.setFeatureValue("ALPHANUMERIC", 1); if (currUnitRep.matches("[IVXDLCM]+")) token.setFeatureValue("ROMAN", 1); if (currUnitRep.matches(".*\\b[IVXDLCM]+\\b.*")) token.setFeatureValue("HASROMAN", 1); if (currUnitRep.matches("[0-9]+")) token.setFeatureValue("NATURALNUMBER", 1); if (currUnitRep.matches("[-0-9]+[.,]+[0-9.,]+")) token.setFeatureValue("REALNUMBER", 1); if (currUnitRep.matches(".*[0-9]+.*")) token.setFeatureValue("HASDIGITS", 1); if (currUnitRep.matches("(\\(.*|\\[.*)")) token.setFeatureValue("BEGINBRACKETS", 1); if (currUnitRep.matches("(\\(.*\\)|\\[.*\\])")) token.setFeatureValue("INSIDEBRACKETS", 1); if (currUnitRep.matches("(\".*|'.*)")) token.setFeatureValue("BEGINQUOTES", 1); if (currUnitRep.matches("(\".*\"|'.*')")) token.setFeatureValue("INSIDEBQUOTES", 1); if (currUnitRep.length() <= 3) token.setFeatureValue("SIZE1", 1); else if (currUnitRep.length() <= 6) token.setFeatureValue("SIZE2", 1); else token.setFeatureValue("SIZE3", 1); if (currUnitRep.matches("[A-Z]\\.")) token.setFeatureValue("ABBR1", 1); if (currUnitRep.matches("([A-Za-z]\\.)+")) token.setFeatureValue("ABBR2", 1); if (currUnitRep.matches("[abcdfghjklmnpqrstvwxyz]+\\.")) token.setFeatureValue("ABBR3", 1); String bwc = plainUnitRep; bwc = bwc.replaceAll("[\\p{Lu}\\p{M}]+", "A"); bwc = bwc.replaceAll("[\\p{Ll}\\p{M}]+", "a"); bwc = bwc.replaceAll("[0-9]+", "0"); bwc = bwc.replaceAll("[^\\p{L}\\p{M}0-9]+", "x"); token.setFeatureValue("BWC=" + bwc, 1); if (containsEOSSymbol(currUnitRep)) { int freq = ((Integer) unitFreq.get(currUnitRep)).intValue(); if (freq > 1) token.setFeatureValue("FreqTokenEOSSymbol", 1); } if (abbrList.contains(currUnitRep)) token.setFeatureValue("KNOWNABBR", 1); data.add(token); target.add(label); } unitInfo.addAll(units); } carrier.setData(data); carrier.setTarget(target); carrier.setName(unitInfo); carrier.setSource(abstractFileName); return carrier; }
Abstract2UnitPipe extends Pipe { public Instance pipe(Instance carrier) { String abstractFileName = (String) carrier.getSource(); @SuppressWarnings("unchecked") List<String> lines = (List<String>) carrier.getData(); Map<String, Integer> unitFreq = getUnitFrequency(lines); TokenSequence data = new TokenSequence(); LabelSequence target = new LabelSequence((LabelAlphabet) getTargetAlphabet()); List<Unit> unitInfo = new ArrayList<>(); for (int i = 0; i < lines.size(); i++) { String line = lines.get(i); if (line.isEmpty()) { continue; } List<Unit> units = getUnits(line); if (units.isEmpty()) continue; for (int j = 0; j < units.size(); j++) { String currUnitRep = units.get(j).rep; String plainUnitRep = getPlainUnit(currUnitRep); String label = "IS"; Token token = new Token(currUnitRep); if (units.get(j).isTokenInternal) token.setFeatureValue("istokeninternal=", 1); if (containsEOSSymbol(currUnitRep)) token.setFeatureValue("endwithEOSSymb=" + getEOSSymbol(currUnitRep), 1); if ((j + 1) == units.size()) { label = "EOS"; } int count = nrEOSSymbolsContained(plainUnitRep); if (count > 0) token.setFeatureValue("hasinnerEOSSymb=" + count, 1); token.setFeatureValue("TOKEN=" + currUnitRep, 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}].*")) token.setFeatureValue("INITCAPS", 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}]")) token.setFeatureValue("ONECAPS", 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}]+")) token.setFeatureValue("ALLCAPS", 1); if (currUnitRep.matches("(.*[\\p{L}\\p{M}].*[0-9].*|.*[0-9].*[\\p{L}\\p{M}].*)")) token.setFeatureValue("ALPHANUMERIC", 1); if (currUnitRep.matches("[IVXDLCM]+")) token.setFeatureValue("ROMAN", 1); if (currUnitRep.matches(".*\\b[IVXDLCM]+\\b.*")) token.setFeatureValue("HASROMAN", 1); if (currUnitRep.matches("[0-9]+")) token.setFeatureValue("NATURALNUMBER", 1); if (currUnitRep.matches("[-0-9]+[.,]+[0-9.,]+")) token.setFeatureValue("REALNUMBER", 1); if (currUnitRep.matches(".*[0-9]+.*")) token.setFeatureValue("HASDIGITS", 1); if (currUnitRep.matches("(\\(.*|\\[.*)")) token.setFeatureValue("BEGINBRACKETS", 1); if (currUnitRep.matches("(\\(.*\\)|\\[.*\\])")) token.setFeatureValue("INSIDEBRACKETS", 1); if (currUnitRep.matches("(\".*|'.*)")) token.setFeatureValue("BEGINQUOTES", 1); if (currUnitRep.matches("(\".*\"|'.*')")) token.setFeatureValue("INSIDEBQUOTES", 1); if (currUnitRep.length() <= 3) token.setFeatureValue("SIZE1", 1); else if (currUnitRep.length() <= 6) token.setFeatureValue("SIZE2", 1); else token.setFeatureValue("SIZE3", 1); if (currUnitRep.matches("[A-Z]\\.")) token.setFeatureValue("ABBR1", 1); if (currUnitRep.matches("([A-Za-z]\\.)+")) token.setFeatureValue("ABBR2", 1); if (currUnitRep.matches("[abcdfghjklmnpqrstvwxyz]+\\.")) token.setFeatureValue("ABBR3", 1); String bwc = plainUnitRep; bwc = bwc.replaceAll("[\\p{Lu}\\p{M}]+", "A"); bwc = bwc.replaceAll("[\\p{Ll}\\p{M}]+", "a"); bwc = bwc.replaceAll("[0-9]+", "0"); bwc = bwc.replaceAll("[^\\p{L}\\p{M}0-9]+", "x"); token.setFeatureValue("BWC=" + bwc, 1); if (containsEOSSymbol(currUnitRep)) { int freq = ((Integer) unitFreq.get(currUnitRep)).intValue(); if (freq > 1) token.setFeatureValue("FreqTokenEOSSymbol", 1); } if (abbrList.contains(currUnitRep)) token.setFeatureValue("KNOWNABBR", 1); data.add(token); target.add(label); } unitInfo.addAll(units); } carrier.setData(data); carrier.setTarget(target); carrier.setName(unitInfo); carrier.setSource(abstractFileName); return carrier; } }
Abstract2UnitPipe extends Pipe { public Instance pipe(Instance carrier) { String abstractFileName = (String) carrier.getSource(); @SuppressWarnings("unchecked") List<String> lines = (List<String>) carrier.getData(); Map<String, Integer> unitFreq = getUnitFrequency(lines); TokenSequence data = new TokenSequence(); LabelSequence target = new LabelSequence((LabelAlphabet) getTargetAlphabet()); List<Unit> unitInfo = new ArrayList<>(); for (int i = 0; i < lines.size(); i++) { String line = lines.get(i); if (line.isEmpty()) { continue; } List<Unit> units = getUnits(line); if (units.isEmpty()) continue; for (int j = 0; j < units.size(); j++) { String currUnitRep = units.get(j).rep; String plainUnitRep = getPlainUnit(currUnitRep); String label = "IS"; Token token = new Token(currUnitRep); if (units.get(j).isTokenInternal) token.setFeatureValue("istokeninternal=", 1); if (containsEOSSymbol(currUnitRep)) token.setFeatureValue("endwithEOSSymb=" + getEOSSymbol(currUnitRep), 1); if ((j + 1) == units.size()) { label = "EOS"; } int count = nrEOSSymbolsContained(plainUnitRep); if (count > 0) token.setFeatureValue("hasinnerEOSSymb=" + count, 1); token.setFeatureValue("TOKEN=" + currUnitRep, 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}].*")) token.setFeatureValue("INITCAPS", 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}]")) token.setFeatureValue("ONECAPS", 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}]+")) token.setFeatureValue("ALLCAPS", 1); if (currUnitRep.matches("(.*[\\p{L}\\p{M}].*[0-9].*|.*[0-9].*[\\p{L}\\p{M}].*)")) token.setFeatureValue("ALPHANUMERIC", 1); if (currUnitRep.matches("[IVXDLCM]+")) token.setFeatureValue("ROMAN", 1); if (currUnitRep.matches(".*\\b[IVXDLCM]+\\b.*")) token.setFeatureValue("HASROMAN", 1); if (currUnitRep.matches("[0-9]+")) token.setFeatureValue("NATURALNUMBER", 1); if (currUnitRep.matches("[-0-9]+[.,]+[0-9.,]+")) token.setFeatureValue("REALNUMBER", 1); if (currUnitRep.matches(".*[0-9]+.*")) token.setFeatureValue("HASDIGITS", 1); if (currUnitRep.matches("(\\(.*|\\[.*)")) token.setFeatureValue("BEGINBRACKETS", 1); if (currUnitRep.matches("(\\(.*\\)|\\[.*\\])")) token.setFeatureValue("INSIDEBRACKETS", 1); if (currUnitRep.matches("(\".*|'.*)")) token.setFeatureValue("BEGINQUOTES", 1); if (currUnitRep.matches("(\".*\"|'.*')")) token.setFeatureValue("INSIDEBQUOTES", 1); if (currUnitRep.length() <= 3) token.setFeatureValue("SIZE1", 1); else if (currUnitRep.length() <= 6) token.setFeatureValue("SIZE2", 1); else token.setFeatureValue("SIZE3", 1); if (currUnitRep.matches("[A-Z]\\.")) token.setFeatureValue("ABBR1", 1); if (currUnitRep.matches("([A-Za-z]\\.)+")) token.setFeatureValue("ABBR2", 1); if (currUnitRep.matches("[abcdfghjklmnpqrstvwxyz]+\\.")) token.setFeatureValue("ABBR3", 1); String bwc = plainUnitRep; bwc = bwc.replaceAll("[\\p{Lu}\\p{M}]+", "A"); bwc = bwc.replaceAll("[\\p{Ll}\\p{M}]+", "a"); bwc = bwc.replaceAll("[0-9]+", "0"); bwc = bwc.replaceAll("[^\\p{L}\\p{M}0-9]+", "x"); token.setFeatureValue("BWC=" + bwc, 1); if (containsEOSSymbol(currUnitRep)) { int freq = ((Integer) unitFreq.get(currUnitRep)).intValue(); if (freq > 1) token.setFeatureValue("FreqTokenEOSSymbol", 1); } if (abbrList.contains(currUnitRep)) token.setFeatureValue("KNOWNABBR", 1); data.add(token); target.add(label); } unitInfo.addAll(units); } carrier.setData(data); carrier.setTarget(target); carrier.setName(unitInfo); carrier.setSource(abstractFileName); return carrier; } Abstract2UnitPipe(boolean splitAfterPunctuation); }
Abstract2UnitPipe extends Pipe { public Instance pipe(Instance carrier) { String abstractFileName = (String) carrier.getSource(); @SuppressWarnings("unchecked") List<String> lines = (List<String>) carrier.getData(); Map<String, Integer> unitFreq = getUnitFrequency(lines); TokenSequence data = new TokenSequence(); LabelSequence target = new LabelSequence((LabelAlphabet) getTargetAlphabet()); List<Unit> unitInfo = new ArrayList<>(); for (int i = 0; i < lines.size(); i++) { String line = lines.get(i); if (line.isEmpty()) { continue; } List<Unit> units = getUnits(line); if (units.isEmpty()) continue; for (int j = 0; j < units.size(); j++) { String currUnitRep = units.get(j).rep; String plainUnitRep = getPlainUnit(currUnitRep); String label = "IS"; Token token = new Token(currUnitRep); if (units.get(j).isTokenInternal) token.setFeatureValue("istokeninternal=", 1); if (containsEOSSymbol(currUnitRep)) token.setFeatureValue("endwithEOSSymb=" + getEOSSymbol(currUnitRep), 1); if ((j + 1) == units.size()) { label = "EOS"; } int count = nrEOSSymbolsContained(plainUnitRep); if (count > 0) token.setFeatureValue("hasinnerEOSSymb=" + count, 1); token.setFeatureValue("TOKEN=" + currUnitRep, 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}].*")) token.setFeatureValue("INITCAPS", 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}]")) token.setFeatureValue("ONECAPS", 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}]+")) token.setFeatureValue("ALLCAPS", 1); if (currUnitRep.matches("(.*[\\p{L}\\p{M}].*[0-9].*|.*[0-9].*[\\p{L}\\p{M}].*)")) token.setFeatureValue("ALPHANUMERIC", 1); if (currUnitRep.matches("[IVXDLCM]+")) token.setFeatureValue("ROMAN", 1); if (currUnitRep.matches(".*\\b[IVXDLCM]+\\b.*")) token.setFeatureValue("HASROMAN", 1); if (currUnitRep.matches("[0-9]+")) token.setFeatureValue("NATURALNUMBER", 1); if (currUnitRep.matches("[-0-9]+[.,]+[0-9.,]+")) token.setFeatureValue("REALNUMBER", 1); if (currUnitRep.matches(".*[0-9]+.*")) token.setFeatureValue("HASDIGITS", 1); if (currUnitRep.matches("(\\(.*|\\[.*)")) token.setFeatureValue("BEGINBRACKETS", 1); if (currUnitRep.matches("(\\(.*\\)|\\[.*\\])")) token.setFeatureValue("INSIDEBRACKETS", 1); if (currUnitRep.matches("(\".*|'.*)")) token.setFeatureValue("BEGINQUOTES", 1); if (currUnitRep.matches("(\".*\"|'.*')")) token.setFeatureValue("INSIDEBQUOTES", 1); if (currUnitRep.length() <= 3) token.setFeatureValue("SIZE1", 1); else if (currUnitRep.length() <= 6) token.setFeatureValue("SIZE2", 1); else token.setFeatureValue("SIZE3", 1); if (currUnitRep.matches("[A-Z]\\.")) token.setFeatureValue("ABBR1", 1); if (currUnitRep.matches("([A-Za-z]\\.)+")) token.setFeatureValue("ABBR2", 1); if (currUnitRep.matches("[abcdfghjklmnpqrstvwxyz]+\\.")) token.setFeatureValue("ABBR3", 1); String bwc = plainUnitRep; bwc = bwc.replaceAll("[\\p{Lu}\\p{M}]+", "A"); bwc = bwc.replaceAll("[\\p{Ll}\\p{M}]+", "a"); bwc = bwc.replaceAll("[0-9]+", "0"); bwc = bwc.replaceAll("[^\\p{L}\\p{M}0-9]+", "x"); token.setFeatureValue("BWC=" + bwc, 1); if (containsEOSSymbol(currUnitRep)) { int freq = ((Integer) unitFreq.get(currUnitRep)).intValue(); if (freq > 1) token.setFeatureValue("FreqTokenEOSSymbol", 1); } if (abbrList.contains(currUnitRep)) token.setFeatureValue("KNOWNABBR", 1); data.add(token); target.add(label); } unitInfo.addAll(units); } carrier.setData(data); carrier.setTarget(target); carrier.setName(unitInfo); carrier.setSource(abstractFileName); return carrier; } Abstract2UnitPipe(boolean splitAfterPunctuation); Instance pipe(Instance carrier); }
Abstract2UnitPipe extends Pipe { public Instance pipe(Instance carrier) { String abstractFileName = (String) carrier.getSource(); @SuppressWarnings("unchecked") List<String> lines = (List<String>) carrier.getData(); Map<String, Integer> unitFreq = getUnitFrequency(lines); TokenSequence data = new TokenSequence(); LabelSequence target = new LabelSequence((LabelAlphabet) getTargetAlphabet()); List<Unit> unitInfo = new ArrayList<>(); for (int i = 0; i < lines.size(); i++) { String line = lines.get(i); if (line.isEmpty()) { continue; } List<Unit> units = getUnits(line); if (units.isEmpty()) continue; for (int j = 0; j < units.size(); j++) { String currUnitRep = units.get(j).rep; String plainUnitRep = getPlainUnit(currUnitRep); String label = "IS"; Token token = new Token(currUnitRep); if (units.get(j).isTokenInternal) token.setFeatureValue("istokeninternal=", 1); if (containsEOSSymbol(currUnitRep)) token.setFeatureValue("endwithEOSSymb=" + getEOSSymbol(currUnitRep), 1); if ((j + 1) == units.size()) { label = "EOS"; } int count = nrEOSSymbolsContained(plainUnitRep); if (count > 0) token.setFeatureValue("hasinnerEOSSymb=" + count, 1); token.setFeatureValue("TOKEN=" + currUnitRep, 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}].*")) token.setFeatureValue("INITCAPS", 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}]")) token.setFeatureValue("ONECAPS", 1); if (currUnitRep.matches("[\\p{Lu}\\p{M}]+")) token.setFeatureValue("ALLCAPS", 1); if (currUnitRep.matches("(.*[\\p{L}\\p{M}].*[0-9].*|.*[0-9].*[\\p{L}\\p{M}].*)")) token.setFeatureValue("ALPHANUMERIC", 1); if (currUnitRep.matches("[IVXDLCM]+")) token.setFeatureValue("ROMAN", 1); if (currUnitRep.matches(".*\\b[IVXDLCM]+\\b.*")) token.setFeatureValue("HASROMAN", 1); if (currUnitRep.matches("[0-9]+")) token.setFeatureValue("NATURALNUMBER", 1); if (currUnitRep.matches("[-0-9]+[.,]+[0-9.,]+")) token.setFeatureValue("REALNUMBER", 1); if (currUnitRep.matches(".*[0-9]+.*")) token.setFeatureValue("HASDIGITS", 1); if (currUnitRep.matches("(\\(.*|\\[.*)")) token.setFeatureValue("BEGINBRACKETS", 1); if (currUnitRep.matches("(\\(.*\\)|\\[.*\\])")) token.setFeatureValue("INSIDEBRACKETS", 1); if (currUnitRep.matches("(\".*|'.*)")) token.setFeatureValue("BEGINQUOTES", 1); if (currUnitRep.matches("(\".*\"|'.*')")) token.setFeatureValue("INSIDEBQUOTES", 1); if (currUnitRep.length() <= 3) token.setFeatureValue("SIZE1", 1); else if (currUnitRep.length() <= 6) token.setFeatureValue("SIZE2", 1); else token.setFeatureValue("SIZE3", 1); if (currUnitRep.matches("[A-Z]\\.")) token.setFeatureValue("ABBR1", 1); if (currUnitRep.matches("([A-Za-z]\\.)+")) token.setFeatureValue("ABBR2", 1); if (currUnitRep.matches("[abcdfghjklmnpqrstvwxyz]+\\.")) token.setFeatureValue("ABBR3", 1); String bwc = plainUnitRep; bwc = bwc.replaceAll("[\\p{Lu}\\p{M}]+", "A"); bwc = bwc.replaceAll("[\\p{Ll}\\p{M}]+", "a"); bwc = bwc.replaceAll("[0-9]+", "0"); bwc = bwc.replaceAll("[^\\p{L}\\p{M}0-9]+", "x"); token.setFeatureValue("BWC=" + bwc, 1); if (containsEOSSymbol(currUnitRep)) { int freq = ((Integer) unitFreq.get(currUnitRep)).intValue(); if (freq > 1) token.setFeatureValue("FreqTokenEOSSymbol", 1); } if (abbrList.contains(currUnitRep)) token.setFeatureValue("KNOWNABBR", 1); data.add(token); target.add(label); } unitInfo.addAll(units); } carrier.setData(data); carrier.setTarget(target); carrier.setName(unitInfo); carrier.setSource(abstractFileName); return carrier; } Abstract2UnitPipe(boolean splitAfterPunctuation); Instance pipe(Instance carrier);  }