@Test public void filter() throws IOException { final String s = UUID.randomUUID().toString().replace("-", "") + "0000"; final ObjectMetadata o1 = LegacyObjectMetadata.fromMetadata(s, 1, 1, (byte)0, -1); final ObjectMetadata o2 = LegacyObjectMetadata.fromMetadata(s, 2, 2, (byte)0, -1); final ObjectMetadata o3 = LegacyObjectMetadata.fromMetadata(s, 3, 3, (byte)0, -1); final ByteArrayOutputStream source = new ByteArrayOutputStream(); ObjectFileUtil.writeObjectFileVersion(source); source.write(o1.toBytes()); source.write(o2.toBytes()); source.write(o3.toBytes()); final ByteArrayInputStream in = new ByteArrayInputStream(source.toByteArray()); final ByteArrayOutputStream out = new ByteArrayOutputStream(); final Set<Integer> pointSuffixList = new HashSet<Integer>(); ObjectFile.filter(in, out, 1, 2, 2, 3, new HashSet<Integer>(), 0, 100, -1L, -1L, true); assertThat(out.size(), is(ObjectFileVersion.VERSION_HEADER_LENGTH + LegacyObjectMetadata.OBJECT_SIZE)); ByteBuffer outputBuffer = ByteBuffer.wrap(out.toByteArray()); outputBuffer.position(ObjectFileVersion.VERSION_HEADER_LENGTH); byte[] filteredBytes = new byte[LegacyObjectMetadata.OBJECT_SIZE]; outputBuffer.get(filteredBytes, 0, LegacyObjectMetadata.OBJECT_SIZE); final ObjectMetadata filtered = LegacyObjectMetadata.fromBytes(filteredBytes); assertThat(filtered.getName(), is(s)); assertThat(filtered.getSize(), is(2L)); }
public static void filter(final InputStream in, final OutputStream out, final long minFilesize, final long maxFilesize, final int minContainerSuffix, final int maxContainerSuffix, final Set<Integer> containerSuffixes, final int minLegalholds, final int maxLegalHolds, final long minRetention, final long maxRetention, boolean writeVersionHeader) throws IOException { checkNotNull(in); checkNotNull(out); checkArgument(minFilesize >= 0, "minFilesize must be >= 0 [%s]", minFilesize); checkArgument(minFilesize <= maxFilesize, "minFilesize must be <= maxFilesize [%s, %s]", minFilesize, maxFilesize); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minContainerSuffix <= maxContainerSuffix, "minContainerSuffix must be <= maxContainerSuffix [%s, %s]", minContainerSuffix, maxContainerSuffix); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minLegalholds >= 0, "minLegalHolds must be >= 0 [%s]", minLegalholds); checkArgument(minLegalholds <= maxLegalHolds, "minLegalHolds must be <= maxLegalHolds [%s, %s]", minLegalholds, maxLegalHolds); final MutableObjectMetadata object = new MutableObjectMetadata(); final ByteBuffer objectBuffer = ByteBuffer.allocate(LegacyObjectMetadata.OBJECT_SIZE); ObjectFileVersion version = ObjectFileUtil.readObjectFileVersion(in); final byte[] readBytes = ObjectFileUtil.allocateObjectBuffer(version.getMajorVersion(), version.getMinorVersion(), in); int skipLength = 0; if (version.getMajorVersion() == LegacyObjectMetadata.MAJOR_VERSION && version.getMinorVersion() == LegacyObjectMetadata.MINOR_VERSION) { skipLength = ObjectFileVersion.VERSION_HEADER_LENGTH; } else { skipLength = 0; } in.skip(skipLength); if (writeVersionHeader) { ObjectFileUtil.writeObjectFileVersion(out); } while (readFully(in, readBytes)) { final ObjectMetadata objectName = ObjectFileUtil.getObjectFromInputBuffer(version.getMajorVersion(), version.getMinorVersion(), readBytes, objectBuffer.array()); object.setBytes(objectName.toBytes()); if (object.getSize() < minFilesize || object.getSize() > maxFilesize) { continue; } if (object.getContainerSuffix() < minContainerSuffix || object.getContainerSuffix() > maxContainerSuffix) { continue; } if (!containerSuffixes.isEmpty() && !containerSuffixes.contains(object.getContainerSuffix())) { continue; } if (object.getRetention() < minRetention || object.getRetention() > maxRetention) { continue; } if (object.getNumberOfLegalHolds() < minLegalholds || object.getNumberOfLegalHolds() > maxLegalHolds) { continue; } out.write(object.toBytes()); } }
ObjectFile { public static void filter(final InputStream in, final OutputStream out, final long minFilesize, final long maxFilesize, final int minContainerSuffix, final int maxContainerSuffix, final Set<Integer> containerSuffixes, final int minLegalholds, final int maxLegalHolds, final long minRetention, final long maxRetention, boolean writeVersionHeader) throws IOException { checkNotNull(in); checkNotNull(out); checkArgument(minFilesize >= 0, "minFilesize must be >= 0 [%s]", minFilesize); checkArgument(minFilesize <= maxFilesize, "minFilesize must be <= maxFilesize [%s, %s]", minFilesize, maxFilesize); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minContainerSuffix <= maxContainerSuffix, "minContainerSuffix must be <= maxContainerSuffix [%s, %s]", minContainerSuffix, maxContainerSuffix); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minLegalholds >= 0, "minLegalHolds must be >= 0 [%s]", minLegalholds); checkArgument(minLegalholds <= maxLegalHolds, "minLegalHolds must be <= maxLegalHolds [%s, %s]", minLegalholds, maxLegalHolds); final MutableObjectMetadata object = new MutableObjectMetadata(); final ByteBuffer objectBuffer = ByteBuffer.allocate(LegacyObjectMetadata.OBJECT_SIZE); ObjectFileVersion version = ObjectFileUtil.readObjectFileVersion(in); final byte[] readBytes = ObjectFileUtil.allocateObjectBuffer(version.getMajorVersion(), version.getMinorVersion(), in); int skipLength = 0; if (version.getMajorVersion() == LegacyObjectMetadata.MAJOR_VERSION && version.getMinorVersion() == LegacyObjectMetadata.MINOR_VERSION) { skipLength = ObjectFileVersion.VERSION_HEADER_LENGTH; } else { skipLength = 0; } in.skip(skipLength); if (writeVersionHeader) { ObjectFileUtil.writeObjectFileVersion(out); } while (readFully(in, readBytes)) { final ObjectMetadata objectName = ObjectFileUtil.getObjectFromInputBuffer(version.getMajorVersion(), version.getMinorVersion(), readBytes, objectBuffer.array()); object.setBytes(objectName.toBytes()); if (object.getSize() < minFilesize || object.getSize() > maxFilesize) { continue; } if (object.getContainerSuffix() < minContainerSuffix || object.getContainerSuffix() > maxContainerSuffix) { continue; } if (!containerSuffixes.isEmpty() && !containerSuffixes.contains(object.getContainerSuffix())) { continue; } if (object.getRetention() < minRetention || object.getRetention() > maxRetention) { continue; } if (object.getNumberOfLegalHolds() < minLegalholds || object.getNumberOfLegalHolds() > maxLegalHolds) { continue; } out.write(object.toBytes()); } } }
ObjectFile { public static void filter(final InputStream in, final OutputStream out, final long minFilesize, final long maxFilesize, final int minContainerSuffix, final int maxContainerSuffix, final Set<Integer> containerSuffixes, final int minLegalholds, final int maxLegalHolds, final long minRetention, final long maxRetention, boolean writeVersionHeader) throws IOException { checkNotNull(in); checkNotNull(out); checkArgument(minFilesize >= 0, "minFilesize must be >= 0 [%s]", minFilesize); checkArgument(minFilesize <= maxFilesize, "minFilesize must be <= maxFilesize [%s, %s]", minFilesize, maxFilesize); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minContainerSuffix <= maxContainerSuffix, "minContainerSuffix must be <= maxContainerSuffix [%s, %s]", minContainerSuffix, maxContainerSuffix); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minLegalholds >= 0, "minLegalHolds must be >= 0 [%s]", minLegalholds); checkArgument(minLegalholds <= maxLegalHolds, "minLegalHolds must be <= maxLegalHolds [%s, %s]", minLegalholds, maxLegalHolds); final MutableObjectMetadata object = new MutableObjectMetadata(); final ByteBuffer objectBuffer = ByteBuffer.allocate(LegacyObjectMetadata.OBJECT_SIZE); ObjectFileVersion version = ObjectFileUtil.readObjectFileVersion(in); final byte[] readBytes = ObjectFileUtil.allocateObjectBuffer(version.getMajorVersion(), version.getMinorVersion(), in); int skipLength = 0; if (version.getMajorVersion() == LegacyObjectMetadata.MAJOR_VERSION && version.getMinorVersion() == LegacyObjectMetadata.MINOR_VERSION) { skipLength = ObjectFileVersion.VERSION_HEADER_LENGTH; } else { skipLength = 0; } in.skip(skipLength); if (writeVersionHeader) { ObjectFileUtil.writeObjectFileVersion(out); } while (readFully(in, readBytes)) { final ObjectMetadata objectName = ObjectFileUtil.getObjectFromInputBuffer(version.getMajorVersion(), version.getMinorVersion(), readBytes, objectBuffer.array()); object.setBytes(objectName.toBytes()); if (object.getSize() < minFilesize || object.getSize() > maxFilesize) { continue; } if (object.getContainerSuffix() < minContainerSuffix || object.getContainerSuffix() > maxContainerSuffix) { continue; } if (!containerSuffixes.isEmpty() && !containerSuffixes.contains(object.getContainerSuffix())) { continue; } if (object.getRetention() < minRetention || object.getRetention() > maxRetention) { continue; } if (object.getNumberOfLegalHolds() < minLegalholds || object.getNumberOfLegalHolds() > maxLegalHolds) { continue; } out.write(object.toBytes()); } } private  ObjectFile(); }
ObjectFile { public static void filter(final InputStream in, final OutputStream out, final long minFilesize, final long maxFilesize, final int minContainerSuffix, final int maxContainerSuffix, final Set<Integer> containerSuffixes, final int minLegalholds, final int maxLegalHolds, final long minRetention, final long maxRetention, boolean writeVersionHeader) throws IOException { checkNotNull(in); checkNotNull(out); checkArgument(minFilesize >= 0, "minFilesize must be >= 0 [%s]", minFilesize); checkArgument(minFilesize <= maxFilesize, "minFilesize must be <= maxFilesize [%s, %s]", minFilesize, maxFilesize); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minContainerSuffix <= maxContainerSuffix, "minContainerSuffix must be <= maxContainerSuffix [%s, %s]", minContainerSuffix, maxContainerSuffix); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minLegalholds >= 0, "minLegalHolds must be >= 0 [%s]", minLegalholds); checkArgument(minLegalholds <= maxLegalHolds, "minLegalHolds must be <= maxLegalHolds [%s, %s]", minLegalholds, maxLegalHolds); final MutableObjectMetadata object = new MutableObjectMetadata(); final ByteBuffer objectBuffer = ByteBuffer.allocate(LegacyObjectMetadata.OBJECT_SIZE); ObjectFileVersion version = ObjectFileUtil.readObjectFileVersion(in); final byte[] readBytes = ObjectFileUtil.allocateObjectBuffer(version.getMajorVersion(), version.getMinorVersion(), in); int skipLength = 0; if (version.getMajorVersion() == LegacyObjectMetadata.MAJOR_VERSION && version.getMinorVersion() == LegacyObjectMetadata.MINOR_VERSION) { skipLength = ObjectFileVersion.VERSION_HEADER_LENGTH; } else { skipLength = 0; } in.skip(skipLength); if (writeVersionHeader) { ObjectFileUtil.writeObjectFileVersion(out); } while (readFully(in, readBytes)) { final ObjectMetadata objectName = ObjectFileUtil.getObjectFromInputBuffer(version.getMajorVersion(), version.getMinorVersion(), readBytes, objectBuffer.array()); object.setBytes(objectName.toBytes()); if (object.getSize() < minFilesize || object.getSize() > maxFilesize) { continue; } if (object.getContainerSuffix() < minContainerSuffix || object.getContainerSuffix() > maxContainerSuffix) { continue; } if (!containerSuffixes.isEmpty() && !containerSuffixes.contains(object.getContainerSuffix())) { continue; } if (object.getRetention() < minRetention || object.getRetention() > maxRetention) { continue; } if (object.getNumberOfLegalHolds() < minLegalholds || object.getNumberOfLegalHolds() > maxLegalHolds) { continue; } out.write(object.toBytes()); } } private  ObjectFile(); static void main(final String[] args); static InputStream getInputStream(final File input); static OutputStream getOutputStream(final boolean split, final int splitSize,
      final String output); static OutputStream getOutputStream(final String output); static void write(final InputStream in, final OutputStream out, boolean writeVersionHeader); static void read(final InputStream in, final OutputStream out, boolean writeVersionHeader); static void filter(final InputStream in, final OutputStream out, final long minFilesize,
      final long maxFilesize, final int minContainerSuffix, final int maxContainerSuffix,
      final Set<Integer> containerSuffixes, final int minLegalholds, final int maxLegalHolds,
      final long minRetention, final long maxRetention, boolean writeVersionHeader); static void split(final InputStream in, final OutputStream out); static void upgrade(final InputStream in, final OutputStream out); static boolean readFully(final InputStream in, final byte[] b); static void compactObjectFiles(final HashMap<File, Integer> objectFiles, final int maxcount); static void combineFiles(File f1, File f2); static void randomShuffleObjects(final InputStream in, final OutputStream out); static void shuffle(String prefix, String directory, int maxObjects); }
ObjectFile { public static void filter(final InputStream in, final OutputStream out, final long minFilesize, final long maxFilesize, final int minContainerSuffix, final int maxContainerSuffix, final Set<Integer> containerSuffixes, final int minLegalholds, final int maxLegalHolds, final long minRetention, final long maxRetention, boolean writeVersionHeader) throws IOException { checkNotNull(in); checkNotNull(out); checkArgument(minFilesize >= 0, "minFilesize must be >= 0 [%s]", minFilesize); checkArgument(minFilesize <= maxFilesize, "minFilesize must be <= maxFilesize [%s, %s]", minFilesize, maxFilesize); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minContainerSuffix <= maxContainerSuffix, "minContainerSuffix must be <= maxContainerSuffix [%s, %s]", minContainerSuffix, maxContainerSuffix); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minContainerSuffix >= -1, "minContainerSuffix must be >= -1 [%s]", minContainerSuffix); checkArgument(minLegalholds >= 0, "minLegalHolds must be >= 0 [%s]", minLegalholds); checkArgument(minLegalholds <= maxLegalHolds, "minLegalHolds must be <= maxLegalHolds [%s, %s]", minLegalholds, maxLegalHolds); final MutableObjectMetadata object = new MutableObjectMetadata(); final ByteBuffer objectBuffer = ByteBuffer.allocate(LegacyObjectMetadata.OBJECT_SIZE); ObjectFileVersion version = ObjectFileUtil.readObjectFileVersion(in); final byte[] readBytes = ObjectFileUtil.allocateObjectBuffer(version.getMajorVersion(), version.getMinorVersion(), in); int skipLength = 0; if (version.getMajorVersion() == LegacyObjectMetadata.MAJOR_VERSION && version.getMinorVersion() == LegacyObjectMetadata.MINOR_VERSION) { skipLength = ObjectFileVersion.VERSION_HEADER_LENGTH; } else { skipLength = 0; } in.skip(skipLength); if (writeVersionHeader) { ObjectFileUtil.writeObjectFileVersion(out); } while (readFully(in, readBytes)) { final ObjectMetadata objectName = ObjectFileUtil.getObjectFromInputBuffer(version.getMajorVersion(), version.getMinorVersion(), readBytes, objectBuffer.array()); object.setBytes(objectName.toBytes()); if (object.getSize() < minFilesize || object.getSize() > maxFilesize) { continue; } if (object.getContainerSuffix() < minContainerSuffix || object.getContainerSuffix() > maxContainerSuffix) { continue; } if (!containerSuffixes.isEmpty() && !containerSuffixes.contains(object.getContainerSuffix())) { continue; } if (object.getRetention() < minRetention || object.getRetention() > maxRetention) { continue; } if (object.getNumberOfLegalHolds() < minLegalholds || object.getNumberOfLegalHolds() > maxLegalHolds) { continue; } out.write(object.toBytes()); } } private  ObjectFile(); static void main(final String[] args); static InputStream getInputStream(final File input); static OutputStream getOutputStream(final boolean split, final int splitSize,
      final String output); static OutputStream getOutputStream(final String output); static void write(final InputStream in, final OutputStream out, boolean writeVersionHeader); static void read(final InputStream in, final OutputStream out, boolean writeVersionHeader); static void filter(final InputStream in, final OutputStream out, final long minFilesize,
      final long maxFilesize, final int minContainerSuffix, final int maxContainerSuffix,
      final Set<Integer> containerSuffixes, final int minLegalholds, final int maxLegalHolds,
      final long minRetention, final long maxRetention, boolean writeVersionHeader); static void split(final InputStream in, final OutputStream out); static void upgrade(final InputStream in, final OutputStream out); static boolean readFully(final InputStream in, final byte[] b); static void compactObjectFiles(final HashMap<File, Integer> objectFiles, final int maxcount); static void combineFiles(File f1, File f2); static void randomShuffleObjects(final InputStream in, final OutputStream out); static void shuffle(String prefix, String directory, int maxObjects);  }