@Test public void subSupNonUnicode() { String input = "<sub>Z+1</sub> <sup>Z - 1</sup>"; String expect = "<sub>Z+1</sub> \n<sup>Z - 1</sup>"; String actual = format.getSimpleHtml(input); assertEquals(expect, actual); }
@Override public String getSimpleHtml(String rawText) { rawText = rawText.replaceAll("<\\?in-line-formulae description=\"In-line Formulae\" end=\"lead\"\\?>", "<in-line-formula>"); rawText = rawText.replaceAll("<\\?in-line-formulae description=\"In-line Formulae\" end=\"end\"\\?>", "</in-line-formula>"); rawText = rawText.replace("\u201C", "<q>"); rawText = rawText.replace("\u201d", "</q>"); rawText = rawText.replaceAll("<\\?delete-start [^?>]+\\?>", "<del>"); rawText = rawText.replaceAll("<\\?delete-end [^?>]+\\?>", "</del>"); rawText = rawText.replaceAll("<\\?insert-start [^?>]+\\?>", "<ins>"); rawText = rawText.replaceAll("<\\?insert-end [^?>]+\\?>", "</ins>"); Document document = Jsoup.parse("<body>" + rawText + "</body>", "", Parser.xmlParser()); document.select("bold").tagName("b"); for (Element el : document.select("o")) { if (el.hasAttr("ostyle")) { el.attr("class", el.attr("ostyle")); } else { el.attr("class", el.attr("single")); } } Elements forEls = document.select("in-line-formula"); for (int i = 1; i <= forEls.size(); i++) { Element element = forEls.get(i - 1); element.attr("id", "FOR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.tagName("span"); element.addClass("formula"); } for (Element heading : document.select("heading")) { if (heading.hasAttr("lvl")) { heading.attr("level", heading.attr("lvl")); } heading.tagName("h2"); } document.select("p[id^=h-]").tagName("h4"); for (Element par : document.select("p")) { if (par.hasAttr("lvl")) { par.attr("level", par.attr("lvl")); } } Elements figRefEls = document.select("figref"); for (int i = 1; i <= figRefEls.size(); i++) { Element element = figRefEls.get(i - 1); element.attr("id", "FR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.attr("idref", ReferenceTagger.createFigId(element.text())); element.tagName("a"); element.addClass("figref"); } Elements clmRefEls = document.select("claim-ref"); for (int i = 1; i <= clmRefEls.size(); i++) { Element element = clmRefEls.get(i - 1); element.attr("id", "CR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.tagName("a"); element.addClass("claim"); } for (Element element : document.select("patcit")) { element.tagName("a"); element.addClass("patcite"); } for (Element element : document.select("nplcit")) { element.tagName("a"); element.addClass("nplcite"); } for (Element element : document.select("crossref")) { element.tagName("a"); element.addClass("crossref"); } fixFigrefLists(document); for (Element element : document.select("a")) { figrefMergeTrailing(element); } boolean mathFound = false; Elements mathEls = document.select("math"); if (mathEls.size() > 0) { mathFound = true; } for (int i = 1; i <= mathEls.size(); i++) { Element element = mathEls.get(i - 1); String mathml = ""; try { mathml = Base64.getEncoder().encodeToString(element.outerHtml().getBytes("utf-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } Element newEl = new Element(Tag.valueOf("span"), ""); newEl.attr("id", "MTH-" + Strings.padStart(String.valueOf(i), 4, '0')); newEl.addClass("math"); newEl.attr("format", "mathml"); newEl.appendChild(new TextNode(mathml)); element.replaceWith(newEl); figrefMergeTrailing(newEl); } for (Element el : document.select("sub")) { try { String unicode = UnicodeUtil.toSubscript(el.html()); el.replaceWith(new TextNode(unicode)); } catch (ParseException e) { } } for (Element el : document.select("sup")) { try { String unicode = UnicodeUtil.toSuperscript(el.html()); el.replaceWith(new TextNode(unicode)); } catch (ParseException e) { } } for (Element el : document.select("ul")) { if (el.hasAttr("list-style")) { if (el.attr("list-style").equals("none")) { el.attr("style", "list-style-type:none"); } else if (el.attr("list-style").equals("bullet")) { el.attr("style", "list-style-type:disc"); } else if (el.attr("list-style").equals("dash")) { el.attr("class", "ul-dash"); } } } for (Element el : document.select("ol")) { String type = el.hasAttr("style") ? el.attr("style") : el.attr("ol-style"); el.attr("type", type); } Elements tableEls = document.select("table"); for (int i = 1; i <= tableEls.size(); i++) { Element element = tableEls.get(i - 1); element.attr("id", "TBL-" + Strings.padStart(String.valueOf(i), 4, '0')); Element colGroup = element.prependElement("colgroup"); for (Element spec : element.select("colspec")) { colGroup.appendElement("col").attr("width", spec.attr("colwidth")).attr("align", spec.attr("align")); } for (Element row : element.select("thead row")) { for (Element cell : row.select("entry")) { cell.tagName("th"); } row.tagName("tr"); } for (Element row : element.select("tbody row")) { for (Element cell : row.select("entry")) { String rowSpanSt = cell.attr("morerows"); int rowspan = !rowSpanSt.isEmpty() ? Integer.parseInt(rowSpanSt) + 1 : 1; if (rowspan > 1) { cell.attr("rowspan", String.valueOf(rowspan)); } cell.tagName("td"); } row.tagName("tr"); } } Whitelist whitelist = Whitelist.none(); whitelist.addTags(HTML_WHITELIST_TAGS); whitelist.addAttributes(":all", HTML_WHITELIST_ATTRIB); OutputSettings outSettings = new Document.OutputSettings(); outSettings.charset(Charsets.UTF_16); outSettings.syntax(Syntax.xml); outSettings.outline(false); outSettings.prettyPrint(true); Cleaner cleaner = new Cleaner(whitelist); document = cleaner.clean(document); document.outputSettings(outSettings); String docStr = document.select("body").html(); if (mathFound) { document = Jsoup.parse("<body>" + docStr + "</body>", "", Parser.xmlParser()); document.outputSettings(outSettings); for (Element el : document.select("span.math[format=mathml]")) { String html = new String(Base64.getDecoder().decode(el.html())); el.text(""); el.append(html); } docStr = document.select("body").html(); } return docStr; }
FormattedText implements TextProcessor { @Override public String getSimpleHtml(String rawText) { rawText = rawText.replaceAll("<\\?in-line-formulae description=\"In-line Formulae\" end=\"lead\"\\?>", "<in-line-formula>"); rawText = rawText.replaceAll("<\\?in-line-formulae description=\"In-line Formulae\" end=\"end\"\\?>", "</in-line-formula>"); rawText = rawText.replace("\u201C", "<q>"); rawText = rawText.replace("\u201d", "</q>"); rawText = rawText.replaceAll("<\\?delete-start [^?>]+\\?>", "<del>"); rawText = rawText.replaceAll("<\\?delete-end [^?>]+\\?>", "</del>"); rawText = rawText.replaceAll("<\\?insert-start [^?>]+\\?>", "<ins>"); rawText = rawText.replaceAll("<\\?insert-end [^?>]+\\?>", "</ins>"); Document document = Jsoup.parse("<body>" + rawText + "</body>", "", Parser.xmlParser()); document.select("bold").tagName("b"); for (Element el : document.select("o")) { if (el.hasAttr("ostyle")) { el.attr("class", el.attr("ostyle")); } else { el.attr("class", el.attr("single")); } } Elements forEls = document.select("in-line-formula"); for (int i = 1; i <= forEls.size(); i++) { Element element = forEls.get(i - 1); element.attr("id", "FOR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.tagName("span"); element.addClass("formula"); } for (Element heading : document.select("heading")) { if (heading.hasAttr("lvl")) { heading.attr("level", heading.attr("lvl")); } heading.tagName("h2"); } document.select("p[id^=h-]").tagName("h4"); for (Element par : document.select("p")) { if (par.hasAttr("lvl")) { par.attr("level", par.attr("lvl")); } } Elements figRefEls = document.select("figref"); for (int i = 1; i <= figRefEls.size(); i++) { Element element = figRefEls.get(i - 1); element.attr("id", "FR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.attr("idref", ReferenceTagger.createFigId(element.text())); element.tagName("a"); element.addClass("figref"); } Elements clmRefEls = document.select("claim-ref"); for (int i = 1; i <= clmRefEls.size(); i++) { Element element = clmRefEls.get(i - 1); element.attr("id", "CR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.tagName("a"); element.addClass("claim"); } for (Element element : document.select("patcit")) { element.tagName("a"); element.addClass("patcite"); } for (Element element : document.select("nplcit")) { element.tagName("a"); element.addClass("nplcite"); } for (Element element : document.select("crossref")) { element.tagName("a"); element.addClass("crossref"); } fixFigrefLists(document); for (Element element : document.select("a")) { figrefMergeTrailing(element); } boolean mathFound = false; Elements mathEls = document.select("math"); if (mathEls.size() > 0) { mathFound = true; } for (int i = 1; i <= mathEls.size(); i++) { Element element = mathEls.get(i - 1); String mathml = ""; try { mathml = Base64.getEncoder().encodeToString(element.outerHtml().getBytes("utf-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } Element newEl = new Element(Tag.valueOf("span"), ""); newEl.attr("id", "MTH-" + Strings.padStart(String.valueOf(i), 4, '0')); newEl.addClass("math"); newEl.attr("format", "mathml"); newEl.appendChild(new TextNode(mathml)); element.replaceWith(newEl); figrefMergeTrailing(newEl); } for (Element el : document.select("sub")) { try { String unicode = UnicodeUtil.toSubscript(el.html()); el.replaceWith(new TextNode(unicode)); } catch (ParseException e) { } } for (Element el : document.select("sup")) { try { String unicode = UnicodeUtil.toSuperscript(el.html()); el.replaceWith(new TextNode(unicode)); } catch (ParseException e) { } } for (Element el : document.select("ul")) { if (el.hasAttr("list-style")) { if (el.attr("list-style").equals("none")) { el.attr("style", "list-style-type:none"); } else if (el.attr("list-style").equals("bullet")) { el.attr("style", "list-style-type:disc"); } else if (el.attr("list-style").equals("dash")) { el.attr("class", "ul-dash"); } } } for (Element el : document.select("ol")) { String type = el.hasAttr("style") ? el.attr("style") : el.attr("ol-style"); el.attr("type", type); } Elements tableEls = document.select("table"); for (int i = 1; i <= tableEls.size(); i++) { Element element = tableEls.get(i - 1); element.attr("id", "TBL-" + Strings.padStart(String.valueOf(i), 4, '0')); Element colGroup = element.prependElement("colgroup"); for (Element spec : element.select("colspec")) { colGroup.appendElement("col").attr("width", spec.attr("colwidth")).attr("align", spec.attr("align")); } for (Element row : element.select("thead row")) { for (Element cell : row.select("entry")) { cell.tagName("th"); } row.tagName("tr"); } for (Element row : element.select("tbody row")) { for (Element cell : row.select("entry")) { String rowSpanSt = cell.attr("morerows"); int rowspan = !rowSpanSt.isEmpty() ? Integer.parseInt(rowSpanSt) + 1 : 1; if (rowspan > 1) { cell.attr("rowspan", String.valueOf(rowspan)); } cell.tagName("td"); } row.tagName("tr"); } } Whitelist whitelist = Whitelist.none(); whitelist.addTags(HTML_WHITELIST_TAGS); whitelist.addAttributes(":all", HTML_WHITELIST_ATTRIB); OutputSettings outSettings = new Document.OutputSettings(); outSettings.charset(Charsets.UTF_16); outSettings.syntax(Syntax.xml); outSettings.outline(false); outSettings.prettyPrint(true); Cleaner cleaner = new Cleaner(whitelist); document = cleaner.clean(document); document.outputSettings(outSettings); String docStr = document.select("body").html(); if (mathFound) { document = Jsoup.parse("<body>" + docStr + "</body>", "", Parser.xmlParser()); document.outputSettings(outSettings); for (Element el : document.select("span.math[format=mathml]")) { String html = new String(Base64.getDecoder().decode(el.html())); el.text(""); el.append(html); } docStr = document.select("body").html(); } return docStr; } }
FormattedText implements TextProcessor { @Override public String getSimpleHtml(String rawText) { rawText = rawText.replaceAll("<\\?in-line-formulae description=\"In-line Formulae\" end=\"lead\"\\?>", "<in-line-formula>"); rawText = rawText.replaceAll("<\\?in-line-formulae description=\"In-line Formulae\" end=\"end\"\\?>", "</in-line-formula>"); rawText = rawText.replace("\u201C", "<q>"); rawText = rawText.replace("\u201d", "</q>"); rawText = rawText.replaceAll("<\\?delete-start [^?>]+\\?>", "<del>"); rawText = rawText.replaceAll("<\\?delete-end [^?>]+\\?>", "</del>"); rawText = rawText.replaceAll("<\\?insert-start [^?>]+\\?>", "<ins>"); rawText = rawText.replaceAll("<\\?insert-end [^?>]+\\?>", "</ins>"); Document document = Jsoup.parse("<body>" + rawText + "</body>", "", Parser.xmlParser()); document.select("bold").tagName("b"); for (Element el : document.select("o")) { if (el.hasAttr("ostyle")) { el.attr("class", el.attr("ostyle")); } else { el.attr("class", el.attr("single")); } } Elements forEls = document.select("in-line-formula"); for (int i = 1; i <= forEls.size(); i++) { Element element = forEls.get(i - 1); element.attr("id", "FOR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.tagName("span"); element.addClass("formula"); } for (Element heading : document.select("heading")) { if (heading.hasAttr("lvl")) { heading.attr("level", heading.attr("lvl")); } heading.tagName("h2"); } document.select("p[id^=h-]").tagName("h4"); for (Element par : document.select("p")) { if (par.hasAttr("lvl")) { par.attr("level", par.attr("lvl")); } } Elements figRefEls = document.select("figref"); for (int i = 1; i <= figRefEls.size(); i++) { Element element = figRefEls.get(i - 1); element.attr("id", "FR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.attr("idref", ReferenceTagger.createFigId(element.text())); element.tagName("a"); element.addClass("figref"); } Elements clmRefEls = document.select("claim-ref"); for (int i = 1; i <= clmRefEls.size(); i++) { Element element = clmRefEls.get(i - 1); element.attr("id", "CR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.tagName("a"); element.addClass("claim"); } for (Element element : document.select("patcit")) { element.tagName("a"); element.addClass("patcite"); } for (Element element : document.select("nplcit")) { element.tagName("a"); element.addClass("nplcite"); } for (Element element : document.select("crossref")) { element.tagName("a"); element.addClass("crossref"); } fixFigrefLists(document); for (Element element : document.select("a")) { figrefMergeTrailing(element); } boolean mathFound = false; Elements mathEls = document.select("math"); if (mathEls.size() > 0) { mathFound = true; } for (int i = 1; i <= mathEls.size(); i++) { Element element = mathEls.get(i - 1); String mathml = ""; try { mathml = Base64.getEncoder().encodeToString(element.outerHtml().getBytes("utf-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } Element newEl = new Element(Tag.valueOf("span"), ""); newEl.attr("id", "MTH-" + Strings.padStart(String.valueOf(i), 4, '0')); newEl.addClass("math"); newEl.attr("format", "mathml"); newEl.appendChild(new TextNode(mathml)); element.replaceWith(newEl); figrefMergeTrailing(newEl); } for (Element el : document.select("sub")) { try { String unicode = UnicodeUtil.toSubscript(el.html()); el.replaceWith(new TextNode(unicode)); } catch (ParseException e) { } } for (Element el : document.select("sup")) { try { String unicode = UnicodeUtil.toSuperscript(el.html()); el.replaceWith(new TextNode(unicode)); } catch (ParseException e) { } } for (Element el : document.select("ul")) { if (el.hasAttr("list-style")) { if (el.attr("list-style").equals("none")) { el.attr("style", "list-style-type:none"); } else if (el.attr("list-style").equals("bullet")) { el.attr("style", "list-style-type:disc"); } else if (el.attr("list-style").equals("dash")) { el.attr("class", "ul-dash"); } } } for (Element el : document.select("ol")) { String type = el.hasAttr("style") ? el.attr("style") : el.attr("ol-style"); el.attr("type", type); } Elements tableEls = document.select("table"); for (int i = 1; i <= tableEls.size(); i++) { Element element = tableEls.get(i - 1); element.attr("id", "TBL-" + Strings.padStart(String.valueOf(i), 4, '0')); Element colGroup = element.prependElement("colgroup"); for (Element spec : element.select("colspec")) { colGroup.appendElement("col").attr("width", spec.attr("colwidth")).attr("align", spec.attr("align")); } for (Element row : element.select("thead row")) { for (Element cell : row.select("entry")) { cell.tagName("th"); } row.tagName("tr"); } for (Element row : element.select("tbody row")) { for (Element cell : row.select("entry")) { String rowSpanSt = cell.attr("morerows"); int rowspan = !rowSpanSt.isEmpty() ? Integer.parseInt(rowSpanSt) + 1 : 1; if (rowspan > 1) { cell.attr("rowspan", String.valueOf(rowspan)); } cell.tagName("td"); } row.tagName("tr"); } } Whitelist whitelist = Whitelist.none(); whitelist.addTags(HTML_WHITELIST_TAGS); whitelist.addAttributes(":all", HTML_WHITELIST_ATTRIB); OutputSettings outSettings = new Document.OutputSettings(); outSettings.charset(Charsets.UTF_16); outSettings.syntax(Syntax.xml); outSettings.outline(false); outSettings.prettyPrint(true); Cleaner cleaner = new Cleaner(whitelist); document = cleaner.clean(document); document.outputSettings(outSettings); String docStr = document.select("body").html(); if (mathFound) { document = Jsoup.parse("<body>" + docStr + "</body>", "", Parser.xmlParser()); document.outputSettings(outSettings); for (Element el : document.select("span.math[format=mathml]")) { String html = new String(Base64.getDecoder().decode(el.html())); el.text(""); el.append(html); } docStr = document.select("body").html(); } return docStr; }  }
FormattedText implements TextProcessor { @Override public String getSimpleHtml(String rawText) { rawText = rawText.replaceAll("<\\?in-line-formulae description=\"In-line Formulae\" end=\"lead\"\\?>", "<in-line-formula>"); rawText = rawText.replaceAll("<\\?in-line-formulae description=\"In-line Formulae\" end=\"end\"\\?>", "</in-line-formula>"); rawText = rawText.replace("\u201C", "<q>"); rawText = rawText.replace("\u201d", "</q>"); rawText = rawText.replaceAll("<\\?delete-start [^?>]+\\?>", "<del>"); rawText = rawText.replaceAll("<\\?delete-end [^?>]+\\?>", "</del>"); rawText = rawText.replaceAll("<\\?insert-start [^?>]+\\?>", "<ins>"); rawText = rawText.replaceAll("<\\?insert-end [^?>]+\\?>", "</ins>"); Document document = Jsoup.parse("<body>" + rawText + "</body>", "", Parser.xmlParser()); document.select("bold").tagName("b"); for (Element el : document.select("o")) { if (el.hasAttr("ostyle")) { el.attr("class", el.attr("ostyle")); } else { el.attr("class", el.attr("single")); } } Elements forEls = document.select("in-line-formula"); for (int i = 1; i <= forEls.size(); i++) { Element element = forEls.get(i - 1); element.attr("id", "FOR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.tagName("span"); element.addClass("formula"); } for (Element heading : document.select("heading")) { if (heading.hasAttr("lvl")) { heading.attr("level", heading.attr("lvl")); } heading.tagName("h2"); } document.select("p[id^=h-]").tagName("h4"); for (Element par : document.select("p")) { if (par.hasAttr("lvl")) { par.attr("level", par.attr("lvl")); } } Elements figRefEls = document.select("figref"); for (int i = 1; i <= figRefEls.size(); i++) { Element element = figRefEls.get(i - 1); element.attr("id", "FR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.attr("idref", ReferenceTagger.createFigId(element.text())); element.tagName("a"); element.addClass("figref"); } Elements clmRefEls = document.select("claim-ref"); for (int i = 1; i <= clmRefEls.size(); i++) { Element element = clmRefEls.get(i - 1); element.attr("id", "CR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.tagName("a"); element.addClass("claim"); } for (Element element : document.select("patcit")) { element.tagName("a"); element.addClass("patcite"); } for (Element element : document.select("nplcit")) { element.tagName("a"); element.addClass("nplcite"); } for (Element element : document.select("crossref")) { element.tagName("a"); element.addClass("crossref"); } fixFigrefLists(document); for (Element element : document.select("a")) { figrefMergeTrailing(element); } boolean mathFound = false; Elements mathEls = document.select("math"); if (mathEls.size() > 0) { mathFound = true; } for (int i = 1; i <= mathEls.size(); i++) { Element element = mathEls.get(i - 1); String mathml = ""; try { mathml = Base64.getEncoder().encodeToString(element.outerHtml().getBytes("utf-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } Element newEl = new Element(Tag.valueOf("span"), ""); newEl.attr("id", "MTH-" + Strings.padStart(String.valueOf(i), 4, '0')); newEl.addClass("math"); newEl.attr("format", "mathml"); newEl.appendChild(new TextNode(mathml)); element.replaceWith(newEl); figrefMergeTrailing(newEl); } for (Element el : document.select("sub")) { try { String unicode = UnicodeUtil.toSubscript(el.html()); el.replaceWith(new TextNode(unicode)); } catch (ParseException e) { } } for (Element el : document.select("sup")) { try { String unicode = UnicodeUtil.toSuperscript(el.html()); el.replaceWith(new TextNode(unicode)); } catch (ParseException e) { } } for (Element el : document.select("ul")) { if (el.hasAttr("list-style")) { if (el.attr("list-style").equals("none")) { el.attr("style", "list-style-type:none"); } else if (el.attr("list-style").equals("bullet")) { el.attr("style", "list-style-type:disc"); } else if (el.attr("list-style").equals("dash")) { el.attr("class", "ul-dash"); } } } for (Element el : document.select("ol")) { String type = el.hasAttr("style") ? el.attr("style") : el.attr("ol-style"); el.attr("type", type); } Elements tableEls = document.select("table"); for (int i = 1; i <= tableEls.size(); i++) { Element element = tableEls.get(i - 1); element.attr("id", "TBL-" + Strings.padStart(String.valueOf(i), 4, '0')); Element colGroup = element.prependElement("colgroup"); for (Element spec : element.select("colspec")) { colGroup.appendElement("col").attr("width", spec.attr("colwidth")).attr("align", spec.attr("align")); } for (Element row : element.select("thead row")) { for (Element cell : row.select("entry")) { cell.tagName("th"); } row.tagName("tr"); } for (Element row : element.select("tbody row")) { for (Element cell : row.select("entry")) { String rowSpanSt = cell.attr("morerows"); int rowspan = !rowSpanSt.isEmpty() ? Integer.parseInt(rowSpanSt) + 1 : 1; if (rowspan > 1) { cell.attr("rowspan", String.valueOf(rowspan)); } cell.tagName("td"); } row.tagName("tr"); } } Whitelist whitelist = Whitelist.none(); whitelist.addTags(HTML_WHITELIST_TAGS); whitelist.addAttributes(":all", HTML_WHITELIST_ATTRIB); OutputSettings outSettings = new Document.OutputSettings(); outSettings.charset(Charsets.UTF_16); outSettings.syntax(Syntax.xml); outSettings.outline(false); outSettings.prettyPrint(true); Cleaner cleaner = new Cleaner(whitelist); document = cleaner.clean(document); document.outputSettings(outSettings); String docStr = document.select("body").html(); if (mathFound) { document = Jsoup.parse("<body>" + docStr + "</body>", "", Parser.xmlParser()); document.outputSettings(outSettings); for (Element el : document.select("span.math[format=mathml]")) { String html = new String(Base64.getDecoder().decode(el.html())); el.text(""); el.append(html); } docStr = document.select("body").html(); } return docStr; }  @Override String getPlainText(String rawText, FreetextConfig textConfig); @Override String getSimpleHtml(String rawText); void figrefMergeTrailing(Element element); void fixFigrefLists(Document document); void fixFigrefListItem(Element element); @Override List<String> getParagraphText(String rawText); }
FormattedText implements TextProcessor { @Override public String getSimpleHtml(String rawText) { rawText = rawText.replaceAll("<\\?in-line-formulae description=\"In-line Formulae\" end=\"lead\"\\?>", "<in-line-formula>"); rawText = rawText.replaceAll("<\\?in-line-formulae description=\"In-line Formulae\" end=\"end\"\\?>", "</in-line-formula>"); rawText = rawText.replace("\u201C", "<q>"); rawText = rawText.replace("\u201d", "</q>"); rawText = rawText.replaceAll("<\\?delete-start [^?>]+\\?>", "<del>"); rawText = rawText.replaceAll("<\\?delete-end [^?>]+\\?>", "</del>"); rawText = rawText.replaceAll("<\\?insert-start [^?>]+\\?>", "<ins>"); rawText = rawText.replaceAll("<\\?insert-end [^?>]+\\?>", "</ins>"); Document document = Jsoup.parse("<body>" + rawText + "</body>", "", Parser.xmlParser()); document.select("bold").tagName("b"); for (Element el : document.select("o")) { if (el.hasAttr("ostyle")) { el.attr("class", el.attr("ostyle")); } else { el.attr("class", el.attr("single")); } } Elements forEls = document.select("in-line-formula"); for (int i = 1; i <= forEls.size(); i++) { Element element = forEls.get(i - 1); element.attr("id", "FOR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.tagName("span"); element.addClass("formula"); } for (Element heading : document.select("heading")) { if (heading.hasAttr("lvl")) { heading.attr("level", heading.attr("lvl")); } heading.tagName("h2"); } document.select("p[id^=h-]").tagName("h4"); for (Element par : document.select("p")) { if (par.hasAttr("lvl")) { par.attr("level", par.attr("lvl")); } } Elements figRefEls = document.select("figref"); for (int i = 1; i <= figRefEls.size(); i++) { Element element = figRefEls.get(i - 1); element.attr("id", "FR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.attr("idref", ReferenceTagger.createFigId(element.text())); element.tagName("a"); element.addClass("figref"); } Elements clmRefEls = document.select("claim-ref"); for (int i = 1; i <= clmRefEls.size(); i++) { Element element = clmRefEls.get(i - 1); element.attr("id", "CR-" + Strings.padStart(String.valueOf(i), 4, '0')); element.tagName("a"); element.addClass("claim"); } for (Element element : document.select("patcit")) { element.tagName("a"); element.addClass("patcite"); } for (Element element : document.select("nplcit")) { element.tagName("a"); element.addClass("nplcite"); } for (Element element : document.select("crossref")) { element.tagName("a"); element.addClass("crossref"); } fixFigrefLists(document); for (Element element : document.select("a")) { figrefMergeTrailing(element); } boolean mathFound = false; Elements mathEls = document.select("math"); if (mathEls.size() > 0) { mathFound = true; } for (int i = 1; i <= mathEls.size(); i++) { Element element = mathEls.get(i - 1); String mathml = ""; try { mathml = Base64.getEncoder().encodeToString(element.outerHtml().getBytes("utf-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } Element newEl = new Element(Tag.valueOf("span"), ""); newEl.attr("id", "MTH-" + Strings.padStart(String.valueOf(i), 4, '0')); newEl.addClass("math"); newEl.attr("format", "mathml"); newEl.appendChild(new TextNode(mathml)); element.replaceWith(newEl); figrefMergeTrailing(newEl); } for (Element el : document.select("sub")) { try { String unicode = UnicodeUtil.toSubscript(el.html()); el.replaceWith(new TextNode(unicode)); } catch (ParseException e) { } } for (Element el : document.select("sup")) { try { String unicode = UnicodeUtil.toSuperscript(el.html()); el.replaceWith(new TextNode(unicode)); } catch (ParseException e) { } } for (Element el : document.select("ul")) { if (el.hasAttr("list-style")) { if (el.attr("list-style").equals("none")) { el.attr("style", "list-style-type:none"); } else if (el.attr("list-style").equals("bullet")) { el.attr("style", "list-style-type:disc"); } else if (el.attr("list-style").equals("dash")) { el.attr("class", "ul-dash"); } } } for (Element el : document.select("ol")) { String type = el.hasAttr("style") ? el.attr("style") : el.attr("ol-style"); el.attr("type", type); } Elements tableEls = document.select("table"); for (int i = 1; i <= tableEls.size(); i++) { Element element = tableEls.get(i - 1); element.attr("id", "TBL-" + Strings.padStart(String.valueOf(i), 4, '0')); Element colGroup = element.prependElement("colgroup"); for (Element spec : element.select("colspec")) { colGroup.appendElement("col").attr("width", spec.attr("colwidth")).attr("align", spec.attr("align")); } for (Element row : element.select("thead row")) { for (Element cell : row.select("entry")) { cell.tagName("th"); } row.tagName("tr"); } for (Element row : element.select("tbody row")) { for (Element cell : row.select("entry")) { String rowSpanSt = cell.attr("morerows"); int rowspan = !rowSpanSt.isEmpty() ? Integer.parseInt(rowSpanSt) + 1 : 1; if (rowspan > 1) { cell.attr("rowspan", String.valueOf(rowspan)); } cell.tagName("td"); } row.tagName("tr"); } } Whitelist whitelist = Whitelist.none(); whitelist.addTags(HTML_WHITELIST_TAGS); whitelist.addAttributes(":all", HTML_WHITELIST_ATTRIB); OutputSettings outSettings = new Document.OutputSettings(); outSettings.charset(Charsets.UTF_16); outSettings.syntax(Syntax.xml); outSettings.outline(false); outSettings.prettyPrint(true); Cleaner cleaner = new Cleaner(whitelist); document = cleaner.clean(document); document.outputSettings(outSettings); String docStr = document.select("body").html(); if (mathFound) { document = Jsoup.parse("<body>" + docStr + "</body>", "", Parser.xmlParser()); document.outputSettings(outSettings); for (Element el : document.select("span.math[format=mathml]")) { String html = new String(Base64.getDecoder().decode(el.html())); el.text(""); el.append(html); } docStr = document.select("body").html(); } return docStr; }  @Override String getPlainText(String rawText, FreetextConfig textConfig); @Override String getSimpleHtml(String rawText); void figrefMergeTrailing(Element element); void fixFigrefLists(Document document); void fixFigrefListItem(Element element); @Override List<String> getParagraphText(String rawText); static final ImmutableSet<String> HEADER_ELEMENTS; static final ImmutableSet<String> TABLE_ELEMENTS; static final ImmutableSet<String> LIST_ELEMENTS; }