FormattedText implements TextProcessor { @Override public String getSimpleHtml(String rawText) { rawText = rawText.replaceAll("<\\?in-line-formulae description=\"In-line Formulae\" end=\"lead\"\\?>", "<in-line-formula>"); rawText = rawText.replaceAll("<\\?in-line-formulae description=\"In-line Formulae\" end=\"end\"\\?>", "</in-line-formula>"); rawText = rawText.replace("\u201C", "<q>"); rawText = rawText.replace("\u201d", "</q>"); rawText = rawText.replaceAll("<\\?delete-start [^?>]+\\?>", "<del>"); rawText = rawText.replaceAll("<\\?delete-end [^?>]+\\?>", "</del>"); rawText = rawText.replaceAll("<\\?insert-start [^?>]+\\?>", "<ins>"); rawText = rawText.replaceAll("<\\?insert-end [^?>]+\\?>", "</ins>"); Document document = Jsoup.parse("<body>" + rawText + "</body>", "", Parser.xmlParser()); document.select("bold").tagName("b"); for (Element el : document.select("o")) { if (el.hasAttr("ostyle")) { el.attr("class", el.attr("ostyle")); } else { el.attr("class", el.attr("single")); } } Elements forEls = document.select("in-line-formula"); for (int i = 1; i <= forEls.size(); i++) { Element element = forEls.get(i - 1); element.attr("id", "FOR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.tagName("span"); element.addClass("formula"); } for (Element heading : document.select("heading")) { if (heading.hasAttr("lvl")) { heading.attr("level", heading.attr("lvl")); } heading.tagName("h2"); } document.select("p[id^=h-]").tagName("h4"); for (Element par : document.select("p")) { if (par.hasAttr("lvl")) { par.attr("level", par.attr("lvl")); } } Elements figRefEls = document.select("figref"); for (int i = 1; i <= figRefEls.size(); i++) { Element element = figRefEls.get(i - 1); element.attr("id", "FR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.attr("idref", ReferenceTagger.createFigId(element.text())); element.tagName("a"); element.addClass("figref"); } Elements clmRefEls = document.select("claim-ref"); for (int i = 1; i <= clmRefEls.size(); i++) { Element element = clmRefEls.get(i - 1); element.attr("id", "CR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.tagName("a"); element.addClass("claim"); } for (Element element : document.select("patcit")) { element.tagName("a"); element.addClass("patcite"); } for (Element element : document.select("nplcit")) { element.tagName("a"); element.addClass("nplcite"); } for (Element element : document.select("crossref")) { element.tagName("a"); element.addClass("crossref"); } fixFigrefLists(document); for (Element element : document.select("a")) { figrefMergeTrailing(element); } boolean mathFound = false; Elements mathEls = document.select("math"); if (mathEls.size() > 0) { mathFound = true; } for (int i = 1; i <= mathEls.size(); i++) { Element element = mathEls.get(i - 1); String mathml = ""; try { mathml = Base64.getEncoder().encodeToString(element.outerHtml().getBytes("utf-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } Element newEl = new Element(Tag.valueOf("span"), ""); newEl.attr("id", "MTH-" + Strings.padStart(String.valueOf(i), 4, '0')); newEl.addClass("math"); newEl.attr("format", "mathml"); newEl.appendChild(new TextNode(mathml)); element.replaceWith(newEl); figrefMergeTrailing(newEl); } for (Element el : document.select("sub")) { try { String unicode = UnicodeUtil.toSubscript(el.html()); el.replaceWith(new TextNode(unicode)); } catch (ParseException e) { } } for (Element el : document.select("sup")) { try { String unicode = UnicodeUtil.toSuperscript(el.html()); el.replaceWith(new TextNode(unicode)); } catch (ParseException e) { } } for (Element el : document.select("ul")) { if (el.hasAttr("list-style")) { if (el.attr("list-style").equals("none")) { el.attr("style", "list-style-type:none"); } else if (el.attr("list-style").equals("bullet")) { el.attr("style", "list-style-type:disc"); } else if (el.attr("list-style").equals("dash")) { el.attr("class", "ul-dash"); } } } for (Element el : document.select("ol")) { String type = el.hasAttr("style") ? el.attr("style") : el.attr("ol-style"); el.attr("type", type); } Elements tableEls = document.select("table"); for (int i = 1; i <= tableEls.size(); i++) { Element element = tableEls.get(i - 1); element.attr("id", "TBL-" + Strings.padStart(String.valueOf(i), 4, '0')); Element colGroup = element.prependElement("colgroup"); for (Element spec : element.select("colspec")) { colGroup.appendElement("col").attr("width", spec.attr("colwidth")).attr("align", spec.attr("align")); } for (Element row : element.select("thead row")) { for (Element cell : row.select("entry")) { cell.tagName("th"); } row.tagName("tr"); } for (Element row : element.select("tbody row")) { for (Element cell : row.select("entry")) { String rowSpanSt = cell.attr("morerows"); int rowspan = !rowSpanSt.isEmpty() ? Integer.parseInt(rowSpanSt) + 1 : 1; if (rowspan > 1) { cell.attr("rowspan", String.valueOf(rowspan)); } cell.tagName("td"); } row.tagName("tr"); } } Whitelist whitelist = Whitelist.none(); whitelist.addTags(HTML_WHITELIST_TAGS); whitelist.addAttributes(":all", HTML_WHITELIST_ATTRIB); OutputSettings outSettings = new Document.OutputSettings(); outSettings.charset(Charsets.UTF_16); outSettings.syntax(Syntax.xml); outSettings.outline(false); outSettings.prettyPrint(true); Cleaner cleaner = new Cleaner(whitelist); document = cleaner.clean(document); document.outputSettings(outSettings); String docStr = document.select("body").html(); if (mathFound) { document = Jsoup.parse("<body>" + docStr + "</body>", "", Parser.xmlParser()); document.outputSettings(outSettings); for (Element el : document.select("span.math[format=mathml]")) { String html = new String(Base64.getDecoder().decode(el.html())); el.text(""); el.append(html); } docStr = document.select("body").html(); } return docStr; }  @Override String getPlainText(String rawText, FreetextConfig textConfig); @Override String getSimpleHtml(String rawText); void figrefMergeTrailing(Element element); void fixFigrefLists(Document document); void fixFigrefListItem(Element element); @Override List<String> getParagraphText(String rawText); static final ImmutableSet<String> HEADER_ELEMENTS; static final ImmutableSet<String> TABLE_ELEMENTS; static final ImmutableSet<String> LIST_ELEMENTS; }
@Test public void tailingEntityText() { Map<String, String> variations = new HashMap<String, String>(); variations.put("<figref idref=\"DRAWINGS\">FIG. 1</figref><i>a</i>", "<a idref=\"FIG-1\" id=\"FR-0001\" class=\"figref\">FIG. 1a</a>"); variations.put("(<figref idref=\"DRAWINGS\">FIG. 1</figref>a)", "(\n<a idref=\"FIG-1\" id=\"FR-0001\" class=\"figref\">FIG. 1a</a>)"); variations.put("<figref idref=\"DRAWINGS\">FIG. 1</figref>a;", "<a idref=\"FIG-1\" id=\"FR-0001\" class=\"figref\">FIG. 1a</a>;"); variations.put("<figref idref=\"DRAWINGS\">FIG. 1</figref><i>a;</i>", "<a idref=\"FIG-1\" id=\"FR-0001\" class=\"figref\">FIG. 1a</a>;"); variations.put("<figref idref=\"DRAWINGS\">FIG. 1</figref><i>aa</i>", "<a idref=\"FIG-1\" id=\"FR-0001\" class=\"figref\">FIG. 1</a>aa"); for (Entry<String, String> entry : variations.entrySet()) { String actual = format.getSimpleHtml(entry.getKey()); assertEquals(entry.getValue(), actual); } }
@Test public void ClaimRef() { String input = "<claim-ref idref=\"CLM-00001\">claim 1</claim-ref>"; String expect = "<a idref=\"CLM-00001\" id=\"CR-0001\" class=\"claim\">claim 1</a>"; String actual = format.getSimpleHtml(input); assertEquals(expect, actual); }
@Test public void table() { StringBuilder stb = new StringBuilder(); stb.append("<table frame=\"none\" colsep=\"0\" rowsep=\"0\">\n"); stb.append("<tgroup align=\"left\" colsep=\"0\" rowsep=\"0\" cols=\"2\">\n"); stb.append("<colspec colname=\"offset\" colwidth=\"21pt\" align=\"left\"/>\n"); stb.append("<colspec colname=\"1\" colwidth=\"196pt\" align=\"left\"/>\n"); stb.append( "<thead><row><entry>head1</entry><entry namest=\"offset\" nameend=\"1\" align=\"center\" rowsep=\"1\">head2</entry></row></thead>\n"); stb.append("<tbody valign=\"top\"><row><entry/><entry morerows=\"1\">cell data</entry></row></tbody>\n"); stb.append("</tgroup></table>"); String input = stb.toString(); String expect = "<table id=\"TBL-0001\">\n" + " <colgroup>\n" + " <col width=\"21pt\" align=\"left\" />\n" + " <col width=\"196pt\" align=\"left\" />\n" + " </colgroup> \n" + " <thead>\n" + " <tr>\n" + " <th>head1</th>\n" + " <th align=\"center\">head2</th>\n" + " </tr>\n" + " </thead> \n" + " <tbody valign=\"top\">\n" + " <tr>\n" + " <td></td>\n" + " <td rowspan=\"2\">cell data</td>\n" + " </tr>\n" + " </tbody> \n" + "</table>"; String actual = format.getSimpleHtml(input); assertEquals(expect, actual); }
@Test public void MathML_html() { String input = "<math><mrow><mrow><msup><mi>x</mi><mn>2</mn></msup><mo>+</mo><mrow><mn>4</mn><mo>+</mo><mi>x</mi></mrow><mo>+</mo><mn>4</mn></mrow><mo>=</mo><mn>0</mn></mrow></math>"; String expect = "<span id=\"MTH-0001\" class=\"math\" format=\"mathml\">\n" + " <math> \n" + " <mrow> \n" + " <mrow> \n" + " <msup> \n" + " <mi>\n" + " x \n" + " </mi> \n" + " <mn>\n" + " 2 \n" + " </mn> \n" + " </msup> \n" + " <mo>\n" + " + \n" + " </mo> \n" + " <mrow> \n" + " <mn>\n" + " 4 \n" + " </mn> \n" + " <mo>\n" + " + \n" + " </mo> \n" + " <mi>\n" + " x \n" + " </mi> \n" + " </mrow> \n" + " <mo>\n" + " + \n" + " </mo> \n" + " <mn>\n" + " 4 \n" + " </mn> \n" + " </mrow> \n" + " <mo>\n" + " = \n" + " </mo> \n" + " <mn>\n" + " 0 \n" + " </mn> \n" + " </mrow> \n" + " </math></span>"; String actual = format.getSimpleHtml(input); assertEquals(expect, actual); }
@Test public void subSupUnicode() { String input = "H<sub>2</sub>O <sup>1 + 2</sup>"; String expect = "H\u2082O \u00B9 \u207A \u00B2"; String actual = format.getSimpleHtml(input); assertEquals(expect, actual); }
@Test public void subSupSpaceUnicode() { String input = "R<sub>1 </sub>"; String expect = "R\u2081"; String actual = format.getSimpleHtml(input); assertEquals(expect, actual); }
@Test public void formulae() { String input = "<?in-line-formulae description=\"In-line Formulae\" end=\"lead\"?>CH<sub>4</sub><?in-line-formulae description=\"In-line Formulae\" end=\"tail\"?>"; String expect = "<span id=\"FOR-0001\" class=\"formula\">CH\u2084</span>"; String actual = format.getSimpleHtml(input); assertEquals(expect, actual); }
@Test public void Updates_Insert_Deletions() { String input = "<p id=\"p-0006\" num=\"0005\">In the past" + "<?insert-start id=\"REI-00008\" date=\"20180102\" ?>, " + "<?insert-end id=\"REI-00008\" ?>" + "the freezing and/or extended refrigeration of a formulated fried egg has " + "<?delete-start id=\"REI-00009\" date=\"20180102\" ?>lead<?delete-end id=\"REI-00009\" ?> " + "<?insert-start id=\"REI-00010\" date=\"20180102\" ?>led <?insert-end id=\"REI-00010\" ?>" + "to a loss of a cohesive texture and the degradation of other sensory perceptions such as mouth feel, taste, elasticity, and/or the food product not being tender or appealing to an individual. Another common problem encountered during delayed consumption, extended refrigeration and/or freezing of a formulated fried egg is that the food product exhibits syneresis, or the loss of water when frozen and reheated, or when stored for an extended period of time." + "</p>"; String expect = "<p id=\"p-0006\" num=\"0005\">In the past" + "<ins>, </ins>the freezing and/or extended refrigeration of a formulated fried egg has " + "<del>lead</del> " + "<ins>led </ins>to a loss of a cohesive texture and the degradation of other sensory perceptions such as mouth feel, taste, elasticity, and/or the food product not being tender or appealing to an individual. Another common problem encountered during delayed consumption, extended refrigeration and/or freezing of a formulated fried egg is that the food product exhibits syneresis, or the loss of water when frozen and reheated, or when stored for an extended period of time." + "</p>"; String actual = format.getSimpleHtml(input); assertEquals(expect, actual); }
@Test public void subSupNonUnicode() { String input = "<sub>Z+1</sub> <sup>Z - 1</sup>"; String expect = "<sub>Z+1</sub> \n<sup>Z - 1</sup>"; String actual = format.getSimpleHtml(input); assertEquals(expect, actual); }
@Test public void headingAndParagraphs() { StringBuilder stb = new StringBuilder(); stb.append("<heading lvl=2>HEADING TEXT</heading>\n"); stb.append("<p lvl=1>pargraph text.</p>"); String input = stb.toString(); StringBuilder expectStb = new StringBuilder(); expectStb.append("<h2 level=\"2\">HEADING TEXT</h2> \n"); expectStb.append("<p level=\"1\">pargraph text.</p>"); String expect = expectStb.toString(); String actual = format.getSimpleHtml(input); assertEquals(expect, actual); }
@Test public void pLevel() { Map<String, String> variations = new HashMap<String, String>(); variations.put("<p lvl=\"1\"></p>", "<p level=\"1\"></p>"); variations.put("<p level=\"1\"></p>", "<p level=\"1\"></p>"); for (Entry<String, String> entry : variations.entrySet()) { String actual = format.getSimpleHtml(entry.getKey()); assertEquals(entry.getValue(), actual); } }
@Test public void figRef() { String input = "<figref idref=\"DRAWINGS\">FIG. 1</figref>"; String expect = "<a idref=\"FIG-1\" id=\"FR-0001\" class=\"figref\">FIG. 1</a>"; String actual = format.getSimpleHtml(input); assertEquals(expect, actual); }