@Test public void testTextBasic() throws IOException { ExtractReader extractReader = new ExtractReader(); List<Metadata> metadataList = extractReader.loadExtract(testTxtFile); assertEquals(1, metadataList.size()); Metadata m = metadataList.get(0); assertEquals(1, m.getValues(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT).length); assertContains("the quick brown fox fox fox jumped over the lazy lazy dog", m.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT)); assertEquals("application/msword", m.get(Metadata.CONTENT_TYPE)); }
public List<Metadata> loadExtract(Path extractFile) throws ExtractReaderException { List<Metadata> metadataList = null; if (extractFile == null || !Files.isRegularFile(extractFile)) { throw new ExtractReaderException(ExtractReaderException.TYPE.NO_EXTRACT_FILE); } FileSuffixes fileSuffixes = parseSuffixes(extractFile.getFileName().toString()); if (fileSuffixes.format == null) { throw new ExtractReaderException(ExtractReaderException.TYPE.INCORRECT_EXTRACT_FILE_SUFFIX); } if (! Files.isRegularFile(extractFile)) { throw new ExtractReaderException(ExtractReaderException.TYPE.NO_EXTRACT_FILE); } long length = -1L; try { length = Files.size(extractFile); } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } if (length == 0L) { throw new ExtractReaderException(ExtractReaderException.TYPE.ZERO_BYTE_EXTRACT_FILE); } if (minExtractLength > IGNORE_LENGTH && length < minExtractLength) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_FILE_TOO_SHORT); } if (maxExtractLength > IGNORE_LENGTH && length > maxExtractLength) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_FILE_TOO_LONG); } Reader reader = null; InputStream is = null; try { is = Files.newInputStream(extractFile); if (fileSuffixes.compression != null) { if (fileSuffixes.compression.equals("bz2")) { is = new BZip2CompressorInputStream(is); } else if (fileSuffixes.compression.equals("gz") || fileSuffixes.compression.equals("gzip")) { is = new GzipCompressorInputStream(is); } else if (fileSuffixes.compression.equals("zip")) { is = new ZCompressorInputStream(is); } else { LOG.warn("Can't yet process compression of type: {}", fileSuffixes.compression); return metadataList; } } reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } try { if (fileSuffixes.format == FileSuffixes.FORMAT.JSON) { metadataList = JsonMetadataList.fromJson(reader); if (alterMetadataList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) { while (metadataList.size() > 1) { metadataList.remove(metadataList.size()-1); } } else if (alterMetadataList.equals(ALTER_METADATA_LIST.AS_IS.CONCATENATE_CONTENT_INTO_FIRST) && metadataList.size() > 1) { StringBuilder sb = new StringBuilder(); Metadata containerMetadata = metadataList.get(0); for (int i = 0; i < metadataList.size(); i++) { Metadata m = metadataList.get(i); String c = m.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT); if (c != null) { sb.append(c); sb.append(" "); } } containerMetadata.set(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT, sb.toString()); while (metadataList.size() > 1) { metadataList.remove(metadataList.size()-1); } } } else { metadataList = generateListFromTextFile(reader, fileSuffixes); } } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } catch (TikaException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_PARSE_EXCEPTION); } finally { IOUtils.closeQuietly(reader); IOUtils.closeQuietly(is); } return metadataList; }
ExtractReader { public List<Metadata> loadExtract(Path extractFile) throws ExtractReaderException { List<Metadata> metadataList = null; if (extractFile == null || !Files.isRegularFile(extractFile)) { throw new ExtractReaderException(ExtractReaderException.TYPE.NO_EXTRACT_FILE); } FileSuffixes fileSuffixes = parseSuffixes(extractFile.getFileName().toString()); if (fileSuffixes.format == null) { throw new ExtractReaderException(ExtractReaderException.TYPE.INCORRECT_EXTRACT_FILE_SUFFIX); } if (! Files.isRegularFile(extractFile)) { throw new ExtractReaderException(ExtractReaderException.TYPE.NO_EXTRACT_FILE); } long length = -1L; try { length = Files.size(extractFile); } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } if (length == 0L) { throw new ExtractReaderException(ExtractReaderException.TYPE.ZERO_BYTE_EXTRACT_FILE); } if (minExtractLength > IGNORE_LENGTH && length < minExtractLength) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_FILE_TOO_SHORT); } if (maxExtractLength > IGNORE_LENGTH && length > maxExtractLength) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_FILE_TOO_LONG); } Reader reader = null; InputStream is = null; try { is = Files.newInputStream(extractFile); if (fileSuffixes.compression != null) { if (fileSuffixes.compression.equals("bz2")) { is = new BZip2CompressorInputStream(is); } else if (fileSuffixes.compression.equals("gz") || fileSuffixes.compression.equals("gzip")) { is = new GzipCompressorInputStream(is); } else if (fileSuffixes.compression.equals("zip")) { is = new ZCompressorInputStream(is); } else { LOG.warn("Can't yet process compression of type: {}", fileSuffixes.compression); return metadataList; } } reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } try { if (fileSuffixes.format == FileSuffixes.FORMAT.JSON) { metadataList = JsonMetadataList.fromJson(reader); if (alterMetadataList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) { while (metadataList.size() > 1) { metadataList.remove(metadataList.size()-1); } } else if (alterMetadataList.equals(ALTER_METADATA_LIST.AS_IS.CONCATENATE_CONTENT_INTO_FIRST) && metadataList.size() > 1) { StringBuilder sb = new StringBuilder(); Metadata containerMetadata = metadataList.get(0); for (int i = 0; i < metadataList.size(); i++) { Metadata m = metadataList.get(i); String c = m.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT); if (c != null) { sb.append(c); sb.append(" "); } } containerMetadata.set(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT, sb.toString()); while (metadataList.size() > 1) { metadataList.remove(metadataList.size()-1); } } } else { metadataList = generateListFromTextFile(reader, fileSuffixes); } } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } catch (TikaException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_PARSE_EXCEPTION); } finally { IOUtils.closeQuietly(reader); IOUtils.closeQuietly(is); } return metadataList; } }
ExtractReader { public List<Metadata> loadExtract(Path extractFile) throws ExtractReaderException { List<Metadata> metadataList = null; if (extractFile == null || !Files.isRegularFile(extractFile)) { throw new ExtractReaderException(ExtractReaderException.TYPE.NO_EXTRACT_FILE); } FileSuffixes fileSuffixes = parseSuffixes(extractFile.getFileName().toString()); if (fileSuffixes.format == null) { throw new ExtractReaderException(ExtractReaderException.TYPE.INCORRECT_EXTRACT_FILE_SUFFIX); } if (! Files.isRegularFile(extractFile)) { throw new ExtractReaderException(ExtractReaderException.TYPE.NO_EXTRACT_FILE); } long length = -1L; try { length = Files.size(extractFile); } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } if (length == 0L) { throw new ExtractReaderException(ExtractReaderException.TYPE.ZERO_BYTE_EXTRACT_FILE); } if (minExtractLength > IGNORE_LENGTH && length < minExtractLength) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_FILE_TOO_SHORT); } if (maxExtractLength > IGNORE_LENGTH && length > maxExtractLength) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_FILE_TOO_LONG); } Reader reader = null; InputStream is = null; try { is = Files.newInputStream(extractFile); if (fileSuffixes.compression != null) { if (fileSuffixes.compression.equals("bz2")) { is = new BZip2CompressorInputStream(is); } else if (fileSuffixes.compression.equals("gz") || fileSuffixes.compression.equals("gzip")) { is = new GzipCompressorInputStream(is); } else if (fileSuffixes.compression.equals("zip")) { is = new ZCompressorInputStream(is); } else { LOG.warn("Can't yet process compression of type: {}", fileSuffixes.compression); return metadataList; } } reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } try { if (fileSuffixes.format == FileSuffixes.FORMAT.JSON) { metadataList = JsonMetadataList.fromJson(reader); if (alterMetadataList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) { while (metadataList.size() > 1) { metadataList.remove(metadataList.size()-1); } } else if (alterMetadataList.equals(ALTER_METADATA_LIST.AS_IS.CONCATENATE_CONTENT_INTO_FIRST) && metadataList.size() > 1) { StringBuilder sb = new StringBuilder(); Metadata containerMetadata = metadataList.get(0); for (int i = 0; i < metadataList.size(); i++) { Metadata m = metadataList.get(i); String c = m.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT); if (c != null) { sb.append(c); sb.append(" "); } } containerMetadata.set(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT, sb.toString()); while (metadataList.size() > 1) { metadataList.remove(metadataList.size()-1); } } } else { metadataList = generateListFromTextFile(reader, fileSuffixes); } } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } catch (TikaException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_PARSE_EXCEPTION); } finally { IOUtils.closeQuietly(reader); IOUtils.closeQuietly(is); } return metadataList; } ExtractReader();  ExtractReader(ALTER_METADATA_LIST alterMetadataList);  ExtractReader(ALTER_METADATA_LIST alterMetadataList, long minExtractLength, long maxExtractLength); }
ExtractReader { public List<Metadata> loadExtract(Path extractFile) throws ExtractReaderException { List<Metadata> metadataList = null; if (extractFile == null || !Files.isRegularFile(extractFile)) { throw new ExtractReaderException(ExtractReaderException.TYPE.NO_EXTRACT_FILE); } FileSuffixes fileSuffixes = parseSuffixes(extractFile.getFileName().toString()); if (fileSuffixes.format == null) { throw new ExtractReaderException(ExtractReaderException.TYPE.INCORRECT_EXTRACT_FILE_SUFFIX); } if (! Files.isRegularFile(extractFile)) { throw new ExtractReaderException(ExtractReaderException.TYPE.NO_EXTRACT_FILE); } long length = -1L; try { length = Files.size(extractFile); } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } if (length == 0L) { throw new ExtractReaderException(ExtractReaderException.TYPE.ZERO_BYTE_EXTRACT_FILE); } if (minExtractLength > IGNORE_LENGTH && length < minExtractLength) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_FILE_TOO_SHORT); } if (maxExtractLength > IGNORE_LENGTH && length > maxExtractLength) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_FILE_TOO_LONG); } Reader reader = null; InputStream is = null; try { is = Files.newInputStream(extractFile); if (fileSuffixes.compression != null) { if (fileSuffixes.compression.equals("bz2")) { is = new BZip2CompressorInputStream(is); } else if (fileSuffixes.compression.equals("gz") || fileSuffixes.compression.equals("gzip")) { is = new GzipCompressorInputStream(is); } else if (fileSuffixes.compression.equals("zip")) { is = new ZCompressorInputStream(is); } else { LOG.warn("Can't yet process compression of type: {}", fileSuffixes.compression); return metadataList; } } reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } try { if (fileSuffixes.format == FileSuffixes.FORMAT.JSON) { metadataList = JsonMetadataList.fromJson(reader); if (alterMetadataList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) { while (metadataList.size() > 1) { metadataList.remove(metadataList.size()-1); } } else if (alterMetadataList.equals(ALTER_METADATA_LIST.AS_IS.CONCATENATE_CONTENT_INTO_FIRST) && metadataList.size() > 1) { StringBuilder sb = new StringBuilder(); Metadata containerMetadata = metadataList.get(0); for (int i = 0; i < metadataList.size(); i++) { Metadata m = metadataList.get(i); String c = m.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT); if (c != null) { sb.append(c); sb.append(" "); } } containerMetadata.set(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT, sb.toString()); while (metadataList.size() > 1) { metadataList.remove(metadataList.size()-1); } } } else { metadataList = generateListFromTextFile(reader, fileSuffixes); } } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } catch (TikaException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_PARSE_EXCEPTION); } finally { IOUtils.closeQuietly(reader); IOUtils.closeQuietly(is); } return metadataList; } ExtractReader();  ExtractReader(ALTER_METADATA_LIST alterMetadataList);  ExtractReader(ALTER_METADATA_LIST alterMetadataList, long minExtractLength, long maxExtractLength); List<Metadata> loadExtract(Path extractFile); }
ExtractReader { public List<Metadata> loadExtract(Path extractFile) throws ExtractReaderException { List<Metadata> metadataList = null; if (extractFile == null || !Files.isRegularFile(extractFile)) { throw new ExtractReaderException(ExtractReaderException.TYPE.NO_EXTRACT_FILE); } FileSuffixes fileSuffixes = parseSuffixes(extractFile.getFileName().toString()); if (fileSuffixes.format == null) { throw new ExtractReaderException(ExtractReaderException.TYPE.INCORRECT_EXTRACT_FILE_SUFFIX); } if (! Files.isRegularFile(extractFile)) { throw new ExtractReaderException(ExtractReaderException.TYPE.NO_EXTRACT_FILE); } long length = -1L; try { length = Files.size(extractFile); } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } if (length == 0L) { throw new ExtractReaderException(ExtractReaderException.TYPE.ZERO_BYTE_EXTRACT_FILE); } if (minExtractLength > IGNORE_LENGTH && length < minExtractLength) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_FILE_TOO_SHORT); } if (maxExtractLength > IGNORE_LENGTH && length > maxExtractLength) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_FILE_TOO_LONG); } Reader reader = null; InputStream is = null; try { is = Files.newInputStream(extractFile); if (fileSuffixes.compression != null) { if (fileSuffixes.compression.equals("bz2")) { is = new BZip2CompressorInputStream(is); } else if (fileSuffixes.compression.equals("gz") || fileSuffixes.compression.equals("gzip")) { is = new GzipCompressorInputStream(is); } else if (fileSuffixes.compression.equals("zip")) { is = new ZCompressorInputStream(is); } else { LOG.warn("Can't yet process compression of type: {}", fileSuffixes.compression); return metadataList; } } reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } try { if (fileSuffixes.format == FileSuffixes.FORMAT.JSON) { metadataList = JsonMetadataList.fromJson(reader); if (alterMetadataList.equals(ALTER_METADATA_LIST.FIRST_ONLY) && metadataList.size() > 1) { while (metadataList.size() > 1) { metadataList.remove(metadataList.size()-1); } } else if (alterMetadataList.equals(ALTER_METADATA_LIST.AS_IS.CONCATENATE_CONTENT_INTO_FIRST) && metadataList.size() > 1) { StringBuilder sb = new StringBuilder(); Metadata containerMetadata = metadataList.get(0); for (int i = 0; i < metadataList.size(); i++) { Metadata m = metadataList.get(i); String c = m.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT); if (c != null) { sb.append(c); sb.append(" "); } } containerMetadata.set(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT, sb.toString()); while (metadataList.size() > 1) { metadataList.remove(metadataList.size()-1); } } } else { metadataList = generateListFromTextFile(reader, fileSuffixes); } } catch (IOException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION); } catch (TikaException e) { throw new ExtractReaderException(ExtractReaderException.TYPE.EXTRACT_PARSE_EXCEPTION); } finally { IOUtils.closeQuietly(reader); IOUtils.closeQuietly(is); } return metadataList; } ExtractReader();  ExtractReader(ALTER_METADATA_LIST alterMetadataList);  ExtractReader(ALTER_METADATA_LIST alterMetadataList, long minExtractLength, long maxExtractLength); List<Metadata> loadExtract(Path extractFile); static final long IGNORE_LENGTH; }