@Test public void testFieldDontTouchRaw() throws Exception { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = factory.newDocumentBuilder(); ByteArrayInputStream input = new ByteArrayInputStream(XML_CONFIG.getBytes("UTF-8")); org.w3c.dom.Document doc = builder.parse(input); TikaProcessor proc = new TikaProcessor.Builder().named("foo").truncatingTextTo(20).replacingRawData(false).intoField("extracted") .configuredWith(doc) .build(); expect(mockDocument.getRawData()).andReturn(XML.getBytes()).anyTimes(); expect(mockDocument.put("extracted"," The title This is")).andReturn(true); expect(mockDocument.put("X_Parsed_By", "org.apache.tika.parser.CompositeParser")).andReturn(true); expect(mockDocument.put("Content_Type", "application/xml")).andReturn(true); replay(); proc.processDocument(mockDocument); }
@Override public Document[] processDocument(Document document) { try { byte[] rawData = document.getRawData(); if (rawData == null) { log.debug("Skipping document without data in " + getName()); return new Document[]{document}; } Tika tika = new Tika(tikaConfig); tika.setMaxStringLength(document.getRawData().length); Metadata metadata = new Metadata(); try (ByteArrayInputStream bais = new ByteArrayInputStream(rawData)) { String textContent = tika.parseToString(bais, metadata, maxLength); if (replaceRaw) { document.setRawData(textContent.getBytes(StandardCharsets.UTF_8)); } if (destField != null) { document.put(destField,textContent); } for (String name : metadata.names()) { document.put(sanitize(name) + plusSuffix(), metadata.get(name)); } } catch (IOException | TikaException e) { log.debug("Tika processing failure!", e); document.setStatus(Status.ERROR); } } catch (Throwable t) { boolean isAccessControl = t instanceof AccessControlException; boolean isSecurity = t instanceof SecurityException; if (!isAccessControl && !isSecurity) { throw t; } } return new Document[]{document}; }
TikaProcessor implements DocumentProcessor { @Override public Document[] processDocument(Document document) { try { byte[] rawData = document.getRawData(); if (rawData == null) { log.debug("Skipping document without data in " + getName()); return new Document[]{document}; } Tika tika = new Tika(tikaConfig); tika.setMaxStringLength(document.getRawData().length); Metadata metadata = new Metadata(); try (ByteArrayInputStream bais = new ByteArrayInputStream(rawData)) { String textContent = tika.parseToString(bais, metadata, maxLength); if (replaceRaw) { document.setRawData(textContent.getBytes(StandardCharsets.UTF_8)); } if (destField != null) { document.put(destField,textContent); } for (String name : metadata.names()) { document.put(sanitize(name) + plusSuffix(), metadata.get(name)); } } catch (IOException | TikaException e) { log.debug("Tika processing failure!", e); document.setStatus(Status.ERROR); } } catch (Throwable t) { boolean isAccessControl = t instanceof AccessControlException; boolean isSecurity = t instanceof SecurityException; if (!isAccessControl && !isSecurity) { throw t; } } return new Document[]{document}; } }
TikaProcessor implements DocumentProcessor { @Override public Document[] processDocument(Document document) { try { byte[] rawData = document.getRawData(); if (rawData == null) { log.debug("Skipping document without data in " + getName()); return new Document[]{document}; } Tika tika = new Tika(tikaConfig); tika.setMaxStringLength(document.getRawData().length); Metadata metadata = new Metadata(); try (ByteArrayInputStream bais = new ByteArrayInputStream(rawData)) { String textContent = tika.parseToString(bais, metadata, maxLength); if (replaceRaw) { document.setRawData(textContent.getBytes(StandardCharsets.UTF_8)); } if (destField != null) { document.put(destField,textContent); } for (String name : metadata.names()) { document.put(sanitize(name) + plusSuffix(), metadata.get(name)); } } catch (IOException | TikaException e) { log.debug("Tika processing failure!", e); document.setStatus(Status.ERROR); } } catch (Throwable t) { boolean isAccessControl = t instanceof AccessControlException; boolean isSecurity = t instanceof SecurityException; if (!isAccessControl && !isSecurity) { throw t; } } return new Document[]{document}; }  }
TikaProcessor implements DocumentProcessor { @Override public Document[] processDocument(Document document) { try { byte[] rawData = document.getRawData(); if (rawData == null) { log.debug("Skipping document without data in " + getName()); return new Document[]{document}; } Tika tika = new Tika(tikaConfig); tika.setMaxStringLength(document.getRawData().length); Metadata metadata = new Metadata(); try (ByteArrayInputStream bais = new ByteArrayInputStream(rawData)) { String textContent = tika.parseToString(bais, metadata, maxLength); if (replaceRaw) { document.setRawData(textContent.getBytes(StandardCharsets.UTF_8)); } if (destField != null) { document.put(destField,textContent); } for (String name : metadata.names()) { document.put(sanitize(name) + plusSuffix(), metadata.get(name)); } } catch (IOException | TikaException e) { log.debug("Tika processing failure!", e); document.setStatus(Status.ERROR); } } catch (Throwable t) { boolean isAccessControl = t instanceof AccessControlException; boolean isSecurity = t instanceof SecurityException; if (!isAccessControl && !isSecurity) { throw t; } } return new Document[]{document}; }  @Override Document[] processDocument(Document document); @Override String getName(); }
TikaProcessor implements DocumentProcessor { @Override public Document[] processDocument(Document document) { try { byte[] rawData = document.getRawData(); if (rawData == null) { log.debug("Skipping document without data in " + getName()); return new Document[]{document}; } Tika tika = new Tika(tikaConfig); tika.setMaxStringLength(document.getRawData().length); Metadata metadata = new Metadata(); try (ByteArrayInputStream bais = new ByteArrayInputStream(rawData)) { String textContent = tika.parseToString(bais, metadata, maxLength); if (replaceRaw) { document.setRawData(textContent.getBytes(StandardCharsets.UTF_8)); } if (destField != null) { document.put(destField,textContent); } for (String name : metadata.names()) { document.put(sanitize(name) + plusSuffix(), metadata.get(name)); } } catch (IOException | TikaException e) { log.debug("Tika processing failure!", e); document.setStatus(Status.ERROR); } } catch (Throwable t) { boolean isAccessControl = t instanceof AccessControlException; boolean isSecurity = t instanceof SecurityException; if (!isAccessControl && !isSecurity) { throw t; } } return new Document[]{document}; }  @Override Document[] processDocument(Document document); @Override String getName();  }