Preprocess extends InternalModule { protected String expand(Document doc) throws ParseException, IOException, MaryConfigurationException { String whichCurrency = ""; boolean URLFirst = false; boolean isURL = false; boolean puncSplit = false; boolean dashSplit = false; String webEmailTemp = ""; String retVal = null; TreeWalker tw = ((DocumentTraversal) doc).createTreeWalker(doc, NodeFilter.SHOW_ELEMENT, new NameNodeFilter(MaryXML.TOKEN), false); Element t; while ((t = (Element) tw.nextNode()) != null) { if (URLFirst) { t = (Element) tw.previousNode(); URLFirst = false; } boolean isYear = true; boolean splitContraction = false; if (MaryDomUtils.hasAncestor(t, MaryXML.SAYAS) || t.hasAttribute("ph") || t.hasAttribute("sounds_like")) { continue; } String tokenText = MaryDomUtils.tokenText(t); if (fourDigitsPattern.matcher(tokenText).matches() && !whichCurrency.equals("")) { isYear = false; } if (currencySymbPattern.matcher(tokenText).matches()) { whichCurrency = tokenText; } if (ordinalPattern.matcher(tokenText).matches()) { String matched = tokenText.split("(?i)st|nd|rd|th")[0]; MaryDomUtils.setTokenText(t, expandOrdinal(matched)); } else if (tokenText.matches("[aA]")) { Element checkNextNode = MaryDomUtils.getNextSiblingElement(t); if (checkNextNode == null || myPunctPattern.matcher(MaryDomUtils.tokenText(checkNextNode)).matches() || MaryDomUtils.tokenText(checkNextNode).length() == 1) { MaryDomUtils.setTokenText(t, "_a"); } } else if (datePattern.matcher(tokenText).matches()) { MaryDomUtils.setTokenText(t, expandDate(tokenText)); } else if (numberSPattern.matcher(tokenText).matches()) { MaryDomUtils.setTokenText(t, expandNumberS(tokenText)); } else if (yearPattern.matcher(tokenText).matches()) { MaryDomUtils.setTokenText(t, expandYearBCAD(tokenText)); } else if (fourDigitsPattern.matcher(tokenText).matches() && isYear) { MaryDomUtils.setTokenText(t, expandYear(Double.parseDouble(tokenText))); } else if (numberWordPattern.matcher(tokenText).matches()) { MaryDomUtils.setTokenText(t, expandWordNumber(tokenText)); } else if (moneyPattern.matcher(tokenText).matches()) { MaryDomUtils.setTokenText(t, expandMoney(tokenText)); } else if (realNumPattern.matcher(tokenText).matches()) { if (!"".equals(whichCurrency)) { MaryDomUtils.setTokenText(t, expandMoney(whichCurrency + tokenText)); } else { MaryDomUtils.setTokenText(t, expandRealNumber(tokenText)); } } else if (contractPattern.matcher(tokenText).matches()) { if (MaryRuntimeUtils.checkLexicon("en_US", tokenText).length == 0) { Matcher contractionMatch = contractPattern.matcher(tokenText); if (contractionMatch.find()) { if (!contractions.containsKey(contractionMatch.group(1))) { MaryDomUtils.setTokenText(t, tokenText.replaceAll("'", "")); } } else { throw new IllegalStateException("No match for find()"); } } } else if (acronymPattern.matcher(tokenText).matches()) { MaryDomUtils.setTokenText(t, expandAcronym(tokenText)); } else if ((abbrevPattern.matcher(tokenText).matches() || abbrevMap.containsKey(tokenText.toLowerCase())) && !isURL) { Element testAbbNode = MaryDomUtils.getNextSiblingElement(t); boolean nextTokenIsCapital = false; if (testAbbNode != null && Character.isUpperCase(MaryDomUtils.tokenText(testAbbNode).charAt(0))) { nextTokenIsCapital = true; } MaryDomUtils.setTokenText(t, expandAbbreviation(tokenText, nextTokenIsCapital)); } else if (timePattern.matcher(tokenText).matches()) { MaryDomUtils.setTokenText(t, expandTime(tokenText)); } else if (durationPattern.matcher(tokenText).matches()) { MaryDomUtils.setTokenText(t, expandDuration(tokenText)); } else if (hashtagPattern.matcher(tokenText).matches()) { MaryDomUtils.setTokenText(t, expandHashtag(tokenText)); } else if (URLPattern.matcher(tokenText).matches()) { Matcher urlMatcher = URLPattern.matcher(tokenText); if (urlMatcher.find()) { webEmailTemp = tokenText; isURL = true; MaryDomUtils.setTokenText(t, expandURL(urlMatcher.group(2))); } else { throw new IllegalStateException("No match for find()"); } } else if (tokenText.equals(".") && isURL) { MaryDomUtils.setTokenText(t, "dot"); webEmailTemp = webEmailTemp.replaceFirst("\\.", "dot"); if (!webEmailTemp.contains(".")) { isURL = false; } } else if (symbols.containsKey(tokenText)) { MaryDomUtils.setTokenText(t, symbols.get(tokenText)); } else if (rangePattern.matcher(tokenText).matches()) { MaryDomUtils.setTokenText(t, expandRange(tokenText)); } else if (tokenText.contains("-") || tokenText.contains("_")) { dashSplit = true; String[] tokens = tokenText.split("[-_]"); int i = 0; for (String tok : tokens) { if (tok.matches("\\d+")) { StringBuilder newTok = new StringBuilder(); for (char c : tok.toCharArray()) { newTok.append(expandNumber(String.valueOf(c))).append(" "); } tokens[i] = newTok.toString(); } i++; } MaryDomUtils.setTokenText(t, Arrays.toString(tokens).replaceAll("[,\\]\\[]", "")); } else if (consonantPattern.matcher(tokenText).matches()) { if (MaryRuntimeUtils.checkLexicon("en_US", tokenText).length == 0) { MaryDomUtils.setTokenText(t, expandConsonants(tokenText)); } } else if (punctuationPattern.matcher(tokenText).find() && tokenText.length() > 1) { puncSplit = true; String[] puncTokens = tokenText.split("((?<=\\p{Punct})|(?=\\p{Punct}))"); MaryDomUtils.setTokenText(t, Arrays.toString(puncTokens).replaceAll("[,\\]\\[]", "")); } else if (tokenText.equals("\"")) { } else if (punctuationPattern.matcher(tokenText).matches()) { t.setAttribute("pos", "."); } if (retVal == null) { retVal = MaryDomUtils.tokenText(t); } if (!tokenText.equals(MaryDomUtils.tokenText(t))) { MaryDomUtils.encloseWithMTU(t, tokenText, null); String[] newTokens = MaryDomUtils.tokenText(t).replaceAll("-", " ").split("\\s+"); MaryDomUtils.setTokenText(t, newTokens[0]); for (int i = 1; i < newTokens.length; i++) { requireNonNull(t); MaryDomUtils.appendToken(t, newTokens[i]); t = MaryDomUtils.getNextSiblingElement(t); if (splitContraction && newTokens.length == 2) { if (newTokens[0].substring(newTokens[0].length() - 1).matches("[cfkpt]") && contractions.get(newTokens[i]).length > 1) { t.setAttribute("ph", contractions.get(newTokens[i])[1]); } else { t.setAttribute("ph", contractions.get(newTokens[i])[0]); } } } if (!isURL && !puncSplit && !dashSplit) { tw.setCurrentNode(t); } else { Node n = tw.previousNode(); if (n == null) { URLFirst = true; } puncSplit = false; dashSplit = false; } } } return retVal; } Preprocess(); static Map<Object, Object> loadAbbrevMap(); MaryData process(MaryData d);  }
@Test public void testOneWord() throws Exception { String lemma = "7"; mary.setOutputType(MaryDataType.WORDS.name()); Document doc = mary.generateXML(lemma); String words = "<maryxml xmlns=\"http: "xmlns:xsi=\"http: + lemma + "</t></s></p></maryxml>"; Document expectedDoc = DomUtils.parseDocument(words); module.expand(expectedDoc); Diff diff = XMLUnit.compareXML(expectedDoc, doc); Assert.assertFalse(diff.identical()); }
@Test(dataProvider = "ExpandTestData") public void testExpand(String token, String word) throws Exception { String words = "<maryxml xmlns=\"http: "xmlns:xsi=\"http: + token + "</t></s></p></maryxml>"; String retVal = module.expand(DomUtils.parseDocument(words)); Assert.assertEquals(retVal, word); }