@Test public void htmlBodyContentHtmlDecodingHeuristicPolicyTest() throws Exception { URL url = HtmlConverter.class.getClassLoader().getResource("HtmlConverter.xml"); if (url == null) { url = HtmlConverter.class.getClassLoader() .getResource("org/apache/uima/ruta/engine/HtmlConverter.xml"); } XMLInputSource in = new XMLInputSource(url); ResourceSpecifier specifier = UIMAFramework.getXMLParser().parseResourceSpecifier(in); AnalysisEngine ae = UIMAFramework.produceAnalysisEngine(specifier); CAS cas = ae.newCAS(); ae.setConfigParameterValue(HtmlConverter.PARAM_OUTPUT_VIEW, outputViewName); ae.reconfigure(); cas.reset(); cas.setDocumentText(htmlDecoding); ae.process(cas); CAS modifiedView = cas.getView(outputViewName); String text = modifiedView.getDocumentText(); String expectedText = "start of body\nnormal\u00A0normal bold\nend of body 3\u20AC\u00A0äöü"; AnnotationFS documentAnnotation = modifiedView.getDocumentAnnotation(); assertEquals(String.format("begin of annotation is %d, %d expected.", documentAnnotation.getBegin(), 0), documentAnnotation.getBegin(), 0); assertEquals(documentAnnotation.getEnd(), text.length()); assertEquals(expectedText, text); cas.release(); }
@Override public void process(JCas jcaz) throws AnalysisEngineProcessException { JCas jcas; try { if (inputViewName != null) { jcas = jcaz.getView(inputViewName); } else { jcas = jcaz; } } catch (CASException e1) { throw new AnalysisEngineProcessException(e1.getCause()); } String documentText = jcas.getDocumentText(); String splitSeq = documentText.contains("\r\n") ? "\r\n" : "\n"; map = new int[documentText.length() + 1]; JCas modview = null; try { Iterator<JCas> viewIterator = jcas.getViewIterator(); while (viewIterator.hasNext()) { JCas jCas2 = viewIterator.next(); if (jCas2.getViewName().equals(modifiedViewName)) { modview = jCas2; getContext().getLogger().log(Level.WARNING, "view with name \"" + modifiedViewName + "\" already exists."); } } if (modview == null) { modview = jcas.createView(modifiedViewName); } } catch (CASException e) { e.printStackTrace(); return; } SortedSet<HtmlConverterPSpan> visibleSpansSoFar = new TreeSet<HtmlConverterPSpan>(); SortedSet<HtmlConverterPSpan> linebreaksFromHtmlTags = new TreeSet<HtmlConverterPSpan>(); SortedSet<HtmlConverterPSpan> gapsFromHtmlTags = new TreeSet<HtmlConverterPSpan>(); try { Parser parser = new Parser(documentText); NodeList list = parser.parse(null); HtmlConverterVisitor visitor = new HtmlConverterVisitor(newlineInducingTags, newlineInducingTagRegExp, gapInducingTags, gapText, skipWhitespaces, processAll); list.visitAllNodesWith(visitor); visibleSpansSoFar = visitor.getTextSpans(); linebreaksFromHtmlTags = visitor.getLinebreaksFromHtmlTags(); gapsFromHtmlTags = visitor.getGapsFromHtmlTags(); } catch (ParserException e) { throw new AnalysisEngineProcessException(e); } if (replaceLinebreaks) { visibleSpansSoFar = this.handleLinebreaksInDocumentText(visibleSpansSoFar, splitSeq); } if (conversionPolicy.equals("heuristic")) { visibleSpansSoFar = this.htmlDecoding(visibleSpansSoFar); } else if (conversionPolicy.equals("explicit")) { for (int i = 0; i < conversionPatterns.length; i++) { String pat = conversionPatterns[i]; String rep = conversionReplacements[i]; visibleSpansSoFar = this.handleConversion(visibleSpansSoFar, pat, rep); } } visibleSpansSoFar.addAll(linebreaksFromHtmlTags); visibleSpansSoFar.addAll(gapsFromHtmlTags); StringBuffer sbu = new StringBuffer(documentText.length()); int originalOffsetI = 0; int outOffset = 0; for (HtmlConverterPSpan vis : visibleSpansSoFar) { final int begin = vis.getBegin(); final int end = vis.getEnd(); while (originalOffsetI < begin) { map[originalOffsetI++] = outOffset; } String s = ""; if (vis instanceof HtmlConverterPSpanReplacement) { s = vis.getTxt(); while (originalOffsetI < begin + s.length()) { map[originalOffsetI++] = outOffset++; } while (originalOffsetI < end) { map[originalOffsetI++] = outOffset; } } else { s = documentText.substring(begin, end); while (originalOffsetI < end) { map[originalOffsetI++] = outOffset++; } } sbu.append(s); } while (originalOffsetI < documentText.length()) { map[originalOffsetI++] = outOffset; } map[documentText.length()] = outOffset + 1; String modTxt = sbu.toString(); modview.setDocumentText(modTxt); try { mapAnnotations(jcas, map, modifiedViewName); } catch (CASException e) { e.printStackTrace(); } }
HtmlConverter extends JCasAnnotator_ImplBase { @Override public void process(JCas jcaz) throws AnalysisEngineProcessException { JCas jcas; try { if (inputViewName != null) { jcas = jcaz.getView(inputViewName); } else { jcas = jcaz; } } catch (CASException e1) { throw new AnalysisEngineProcessException(e1.getCause()); } String documentText = jcas.getDocumentText(); String splitSeq = documentText.contains("\r\n") ? "\r\n" : "\n"; map = new int[documentText.length() + 1]; JCas modview = null; try { Iterator<JCas> viewIterator = jcas.getViewIterator(); while (viewIterator.hasNext()) { JCas jCas2 = viewIterator.next(); if (jCas2.getViewName().equals(modifiedViewName)) { modview = jCas2; getContext().getLogger().log(Level.WARNING, "view with name \"" + modifiedViewName + "\" already exists."); } } if (modview == null) { modview = jcas.createView(modifiedViewName); } } catch (CASException e) { e.printStackTrace(); return; } SortedSet<HtmlConverterPSpan> visibleSpansSoFar = new TreeSet<HtmlConverterPSpan>(); SortedSet<HtmlConverterPSpan> linebreaksFromHtmlTags = new TreeSet<HtmlConverterPSpan>(); SortedSet<HtmlConverterPSpan> gapsFromHtmlTags = new TreeSet<HtmlConverterPSpan>(); try { Parser parser = new Parser(documentText); NodeList list = parser.parse(null); HtmlConverterVisitor visitor = new HtmlConverterVisitor(newlineInducingTags, newlineInducingTagRegExp, gapInducingTags, gapText, skipWhitespaces, processAll); list.visitAllNodesWith(visitor); visibleSpansSoFar = visitor.getTextSpans(); linebreaksFromHtmlTags = visitor.getLinebreaksFromHtmlTags(); gapsFromHtmlTags = visitor.getGapsFromHtmlTags(); } catch (ParserException e) { throw new AnalysisEngineProcessException(e); } if (replaceLinebreaks) { visibleSpansSoFar = this.handleLinebreaksInDocumentText(visibleSpansSoFar, splitSeq); } if (conversionPolicy.equals("heuristic")) { visibleSpansSoFar = this.htmlDecoding(visibleSpansSoFar); } else if (conversionPolicy.equals("explicit")) { for (int i = 0; i < conversionPatterns.length; i++) { String pat = conversionPatterns[i]; String rep = conversionReplacements[i]; visibleSpansSoFar = this.handleConversion(visibleSpansSoFar, pat, rep); } } visibleSpansSoFar.addAll(linebreaksFromHtmlTags); visibleSpansSoFar.addAll(gapsFromHtmlTags); StringBuffer sbu = new StringBuffer(documentText.length()); int originalOffsetI = 0; int outOffset = 0; for (HtmlConverterPSpan vis : visibleSpansSoFar) { final int begin = vis.getBegin(); final int end = vis.getEnd(); while (originalOffsetI < begin) { map[originalOffsetI++] = outOffset; } String s = ""; if (vis instanceof HtmlConverterPSpanReplacement) { s = vis.getTxt(); while (originalOffsetI < begin + s.length()) { map[originalOffsetI++] = outOffset++; } while (originalOffsetI < end) { map[originalOffsetI++] = outOffset; } } else { s = documentText.substring(begin, end); while (originalOffsetI < end) { map[originalOffsetI++] = outOffset++; } } sbu.append(s); } while (originalOffsetI < documentText.length()) { map[originalOffsetI++] = outOffset; } map[documentText.length()] = outOffset + 1; String modTxt = sbu.toString(); modview.setDocumentText(modTxt); try { mapAnnotations(jcas, map, modifiedViewName); } catch (CASException e) { e.printStackTrace(); } } }
HtmlConverter extends JCasAnnotator_ImplBase { @Override public void process(JCas jcaz) throws AnalysisEngineProcessException { JCas jcas; try { if (inputViewName != null) { jcas = jcaz.getView(inputViewName); } else { jcas = jcaz; } } catch (CASException e1) { throw new AnalysisEngineProcessException(e1.getCause()); } String documentText = jcas.getDocumentText(); String splitSeq = documentText.contains("\r\n") ? "\r\n" : "\n"; map = new int[documentText.length() + 1]; JCas modview = null; try { Iterator<JCas> viewIterator = jcas.getViewIterator(); while (viewIterator.hasNext()) { JCas jCas2 = viewIterator.next(); if (jCas2.getViewName().equals(modifiedViewName)) { modview = jCas2; getContext().getLogger().log(Level.WARNING, "view with name \"" + modifiedViewName + "\" already exists."); } } if (modview == null) { modview = jcas.createView(modifiedViewName); } } catch (CASException e) { e.printStackTrace(); return; } SortedSet<HtmlConverterPSpan> visibleSpansSoFar = new TreeSet<HtmlConverterPSpan>(); SortedSet<HtmlConverterPSpan> linebreaksFromHtmlTags = new TreeSet<HtmlConverterPSpan>(); SortedSet<HtmlConverterPSpan> gapsFromHtmlTags = new TreeSet<HtmlConverterPSpan>(); try { Parser parser = new Parser(documentText); NodeList list = parser.parse(null); HtmlConverterVisitor visitor = new HtmlConverterVisitor(newlineInducingTags, newlineInducingTagRegExp, gapInducingTags, gapText, skipWhitespaces, processAll); list.visitAllNodesWith(visitor); visibleSpansSoFar = visitor.getTextSpans(); linebreaksFromHtmlTags = visitor.getLinebreaksFromHtmlTags(); gapsFromHtmlTags = visitor.getGapsFromHtmlTags(); } catch (ParserException e) { throw new AnalysisEngineProcessException(e); } if (replaceLinebreaks) { visibleSpansSoFar = this.handleLinebreaksInDocumentText(visibleSpansSoFar, splitSeq); } if (conversionPolicy.equals("heuristic")) { visibleSpansSoFar = this.htmlDecoding(visibleSpansSoFar); } else if (conversionPolicy.equals("explicit")) { for (int i = 0; i < conversionPatterns.length; i++) { String pat = conversionPatterns[i]; String rep = conversionReplacements[i]; visibleSpansSoFar = this.handleConversion(visibleSpansSoFar, pat, rep); } } visibleSpansSoFar.addAll(linebreaksFromHtmlTags); visibleSpansSoFar.addAll(gapsFromHtmlTags); StringBuffer sbu = new StringBuffer(documentText.length()); int originalOffsetI = 0; int outOffset = 0; for (HtmlConverterPSpan vis : visibleSpansSoFar) { final int begin = vis.getBegin(); final int end = vis.getEnd(); while (originalOffsetI < begin) { map[originalOffsetI++] = outOffset; } String s = ""; if (vis instanceof HtmlConverterPSpanReplacement) { s = vis.getTxt(); while (originalOffsetI < begin + s.length()) { map[originalOffsetI++] = outOffset++; } while (originalOffsetI < end) { map[originalOffsetI++] = outOffset; } } else { s = documentText.substring(begin, end); while (originalOffsetI < end) { map[originalOffsetI++] = outOffset++; } } sbu.append(s); } while (originalOffsetI < documentText.length()) { map[originalOffsetI++] = outOffset; } map[documentText.length()] = outOffset + 1; String modTxt = sbu.toString(); modview.setDocumentText(modTxt); try { mapAnnotations(jcas, map, modifiedViewName); } catch (CASException e) { e.printStackTrace(); } }  }
HtmlConverter extends JCasAnnotator_ImplBase { @Override public void process(JCas jcaz) throws AnalysisEngineProcessException { JCas jcas; try { if (inputViewName != null) { jcas = jcaz.getView(inputViewName); } else { jcas = jcaz; } } catch (CASException e1) { throw new AnalysisEngineProcessException(e1.getCause()); } String documentText = jcas.getDocumentText(); String splitSeq = documentText.contains("\r\n") ? "\r\n" : "\n"; map = new int[documentText.length() + 1]; JCas modview = null; try { Iterator<JCas> viewIterator = jcas.getViewIterator(); while (viewIterator.hasNext()) { JCas jCas2 = viewIterator.next(); if (jCas2.getViewName().equals(modifiedViewName)) { modview = jCas2; getContext().getLogger().log(Level.WARNING, "view with name \"" + modifiedViewName + "\" already exists."); } } if (modview == null) { modview = jcas.createView(modifiedViewName); } } catch (CASException e) { e.printStackTrace(); return; } SortedSet<HtmlConverterPSpan> visibleSpansSoFar = new TreeSet<HtmlConverterPSpan>(); SortedSet<HtmlConverterPSpan> linebreaksFromHtmlTags = new TreeSet<HtmlConverterPSpan>(); SortedSet<HtmlConverterPSpan> gapsFromHtmlTags = new TreeSet<HtmlConverterPSpan>(); try { Parser parser = new Parser(documentText); NodeList list = parser.parse(null); HtmlConverterVisitor visitor = new HtmlConverterVisitor(newlineInducingTags, newlineInducingTagRegExp, gapInducingTags, gapText, skipWhitespaces, processAll); list.visitAllNodesWith(visitor); visibleSpansSoFar = visitor.getTextSpans(); linebreaksFromHtmlTags = visitor.getLinebreaksFromHtmlTags(); gapsFromHtmlTags = visitor.getGapsFromHtmlTags(); } catch (ParserException e) { throw new AnalysisEngineProcessException(e); } if (replaceLinebreaks) { visibleSpansSoFar = this.handleLinebreaksInDocumentText(visibleSpansSoFar, splitSeq); } if (conversionPolicy.equals("heuristic")) { visibleSpansSoFar = this.htmlDecoding(visibleSpansSoFar); } else if (conversionPolicy.equals("explicit")) { for (int i = 0; i < conversionPatterns.length; i++) { String pat = conversionPatterns[i]; String rep = conversionReplacements[i]; visibleSpansSoFar = this.handleConversion(visibleSpansSoFar, pat, rep); } } visibleSpansSoFar.addAll(linebreaksFromHtmlTags); visibleSpansSoFar.addAll(gapsFromHtmlTags); StringBuffer sbu = new StringBuffer(documentText.length()); int originalOffsetI = 0; int outOffset = 0; for (HtmlConverterPSpan vis : visibleSpansSoFar) { final int begin = vis.getBegin(); final int end = vis.getEnd(); while (originalOffsetI < begin) { map[originalOffsetI++] = outOffset; } String s = ""; if (vis instanceof HtmlConverterPSpanReplacement) { s = vis.getTxt(); while (originalOffsetI < begin + s.length()) { map[originalOffsetI++] = outOffset++; } while (originalOffsetI < end) { map[originalOffsetI++] = outOffset; } } else { s = documentText.substring(begin, end); while (originalOffsetI < end) { map[originalOffsetI++] = outOffset++; } } sbu.append(s); } while (originalOffsetI < documentText.length()) { map[originalOffsetI++] = outOffset; } map[documentText.length()] = outOffset + 1; String modTxt = sbu.toString(); modview.setDocumentText(modTxt); try { mapAnnotations(jcas, map, modifiedViewName); } catch (CASException e) { e.printStackTrace(); } }  @Override void initialize(UimaContext aContext); @Override void process(JCas jcaz); }
HtmlConverter extends JCasAnnotator_ImplBase { @Override public void process(JCas jcaz) throws AnalysisEngineProcessException { JCas jcas; try { if (inputViewName != null) { jcas = jcaz.getView(inputViewName); } else { jcas = jcaz; } } catch (CASException e1) { throw new AnalysisEngineProcessException(e1.getCause()); } String documentText = jcas.getDocumentText(); String splitSeq = documentText.contains("\r\n") ? "\r\n" : "\n"; map = new int[documentText.length() + 1]; JCas modview = null; try { Iterator<JCas> viewIterator = jcas.getViewIterator(); while (viewIterator.hasNext()) { JCas jCas2 = viewIterator.next(); if (jCas2.getViewName().equals(modifiedViewName)) { modview = jCas2; getContext().getLogger().log(Level.WARNING, "view with name \"" + modifiedViewName + "\" already exists."); } } if (modview == null) { modview = jcas.createView(modifiedViewName); } } catch (CASException e) { e.printStackTrace(); return; } SortedSet<HtmlConverterPSpan> visibleSpansSoFar = new TreeSet<HtmlConverterPSpan>(); SortedSet<HtmlConverterPSpan> linebreaksFromHtmlTags = new TreeSet<HtmlConverterPSpan>(); SortedSet<HtmlConverterPSpan> gapsFromHtmlTags = new TreeSet<HtmlConverterPSpan>(); try { Parser parser = new Parser(documentText); NodeList list = parser.parse(null); HtmlConverterVisitor visitor = new HtmlConverterVisitor(newlineInducingTags, newlineInducingTagRegExp, gapInducingTags, gapText, skipWhitespaces, processAll); list.visitAllNodesWith(visitor); visibleSpansSoFar = visitor.getTextSpans(); linebreaksFromHtmlTags = visitor.getLinebreaksFromHtmlTags(); gapsFromHtmlTags = visitor.getGapsFromHtmlTags(); } catch (ParserException e) { throw new AnalysisEngineProcessException(e); } if (replaceLinebreaks) { visibleSpansSoFar = this.handleLinebreaksInDocumentText(visibleSpansSoFar, splitSeq); } if (conversionPolicy.equals("heuristic")) { visibleSpansSoFar = this.htmlDecoding(visibleSpansSoFar); } else if (conversionPolicy.equals("explicit")) { for (int i = 0; i < conversionPatterns.length; i++) { String pat = conversionPatterns[i]; String rep = conversionReplacements[i]; visibleSpansSoFar = this.handleConversion(visibleSpansSoFar, pat, rep); } } visibleSpansSoFar.addAll(linebreaksFromHtmlTags); visibleSpansSoFar.addAll(gapsFromHtmlTags); StringBuffer sbu = new StringBuffer(documentText.length()); int originalOffsetI = 0; int outOffset = 0; for (HtmlConverterPSpan vis : visibleSpansSoFar) { final int begin = vis.getBegin(); final int end = vis.getEnd(); while (originalOffsetI < begin) { map[originalOffsetI++] = outOffset; } String s = ""; if (vis instanceof HtmlConverterPSpanReplacement) { s = vis.getTxt(); while (originalOffsetI < begin + s.length()) { map[originalOffsetI++] = outOffset++; } while (originalOffsetI < end) { map[originalOffsetI++] = outOffset; } } else { s = documentText.substring(begin, end); while (originalOffsetI < end) { map[originalOffsetI++] = outOffset++; } } sbu.append(s); } while (originalOffsetI < documentText.length()) { map[originalOffsetI++] = outOffset; } map[documentText.length()] = outOffset + 1; String modTxt = sbu.toString(); modview.setDocumentText(modTxt); try { mapAnnotations(jcas, map, modifiedViewName); } catch (CASException e) { e.printStackTrace(); } }  @Override void initialize(UimaContext aContext); @Override void process(JCas jcaz); static final String NAMESPACE; static final String DEFAULT_MODIFIED_VIEW; static final String LINEBREAK; static final String PARAM_OUTPUT_VIEW; static final String PARAM_INPUT_VIEW; static final String PARAM_REPLACE_LINEBREAKS; static final String PARAM_SKIP_WHITESPACES; static final String PARAM_PROCESS_ALL; static final String PARAM_EXPAND_OFFSETS; static final String PARAM_LINEBREAK_REPLACEMENT; static final String PARAM_NEWLINE_INDUCING_TAGS; static final String PARAM_NEWLINE_INDUCING_TAG_REGEXP; static final String PARAM_GAP_INDUCING_TAGS; static final String PARAM_GAP_TEXT; static final String PARAM_USE_SPACE_GAP; static final String PARAM_CONVERSION_PATTERNS; static final String PARAM_CONVERSION_POLICY; static final String PARAM_CONVERSION_REPLACEMENTS; }