@Test @SuppressWarnings("deprecation") public void testGetPartitionReturnsNullIfNoAutoCreate() throws IOException { PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash( "username", 2).build(); FileSystemDataset<Record> ds = new FileSystemDataset.Builder<Record>() .namespace("ns") .name("partitioned-users") .configuration(getConfiguration()) .descriptor(new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .format(format) .location(testDirectory) .partitionStrategy(partitionStrategy) .build()) .type(Record.class) .build(); Assert .assertNull(ds.getPartition(new PartitionKey(1), false)); }
@Override @Nullable @SuppressWarnings("deprecation") public PartitionedDataset<E> getPartition(PartitionKey key, boolean allowCreate) { Preconditions.checkState(descriptor.isPartitioned(), "Attempt to get a partition on a non-partitioned dataset (name:%s)", name); LOG.debug("Loading partition for key {}, allowCreate:{}", new Object[]{ key, allowCreate}); Path partitionDirectory = fileSystem.makeQualified( toDirectoryName(directory, key)); try { if (!fileSystem.exists(partitionDirectory)) { if (allowCreate) { if (partitionListener != null) { partitionListener.partitionAdded(namespace, name, toRelativeDirectory(key).toString()); } fileSystem.mkdirs(partitionDirectory); } else { return null; } } } catch (IOException e) { throw new DatasetIOException("Unable to locate or create dataset partition directory " + partitionDirectory, e); } int partitionDepth = key.getLength(); PartitionStrategy subpartitionStrategy = Accessor.getDefault() .getSubpartitionStrategy(partitionStrategy, partitionDepth); return new FileSystemDataset.Builder<E>() .namespace(namespace) .name(name) .fileSystem(fileSystem) .uri(uri) .descriptor(new DatasetDescriptor.Builder(descriptor) .location(partitionDirectory) .partitionStrategy(subpartitionStrategy) .build()) .type(type) .partitionKey(key) .partitionListener(partitionListener) .build(); }
FileSystemDataset extends AbstractDataset<E> implements
    Mergeable<FileSystemDataset<E>>, InputFormatAccessor<E>, LastModifiedAccessor,
    PartitionedDataset<E>, SizeAccessor, Signalable<E>, Replaceable<View<E>> { @Override @Nullable @SuppressWarnings("deprecation") public PartitionedDataset<E> getPartition(PartitionKey key, boolean allowCreate) { Preconditions.checkState(descriptor.isPartitioned(), "Attempt to get a partition on a non-partitioned dataset (name:%s)", name); LOG.debug("Loading partition for key {}, allowCreate:{}", new Object[]{ key, allowCreate}); Path partitionDirectory = fileSystem.makeQualified( toDirectoryName(directory, key)); try { if (!fileSystem.exists(partitionDirectory)) { if (allowCreate) { if (partitionListener != null) { partitionListener.partitionAdded(namespace, name, toRelativeDirectory(key).toString()); } fileSystem.mkdirs(partitionDirectory); } else { return null; } } } catch (IOException e) { throw new DatasetIOException("Unable to locate or create dataset partition directory " + partitionDirectory, e); } int partitionDepth = key.getLength(); PartitionStrategy subpartitionStrategy = Accessor.getDefault() .getSubpartitionStrategy(partitionStrategy, partitionDepth); return new FileSystemDataset.Builder<E>() .namespace(namespace) .name(name) .fileSystem(fileSystem) .uri(uri) .descriptor(new DatasetDescriptor.Builder(descriptor) .location(partitionDirectory) .partitionStrategy(subpartitionStrategy) .build()) .type(type) .partitionKey(key) .partitionListener(partitionListener) .build(); } }
FileSystemDataset extends AbstractDataset<E> implements
    Mergeable<FileSystemDataset<E>>, InputFormatAccessor<E>, LastModifiedAccessor,
    PartitionedDataset<E>, SizeAccessor, Signalable<E>, Replaceable<View<E>> { @Override @Nullable @SuppressWarnings("deprecation") public PartitionedDataset<E> getPartition(PartitionKey key, boolean allowCreate) { Preconditions.checkState(descriptor.isPartitioned(), "Attempt to get a partition on a non-partitioned dataset (name:%s)", name); LOG.debug("Loading partition for key {}, allowCreate:{}", new Object[]{ key, allowCreate}); Path partitionDirectory = fileSystem.makeQualified( toDirectoryName(directory, key)); try { if (!fileSystem.exists(partitionDirectory)) { if (allowCreate) { if (partitionListener != null) { partitionListener.partitionAdded(namespace, name, toRelativeDirectory(key).toString()); } fileSystem.mkdirs(partitionDirectory); } else { return null; } } } catch (IOException e) { throw new DatasetIOException("Unable to locate or create dataset partition directory " + partitionDirectory, e); } int partitionDepth = key.getLength(); PartitionStrategy subpartitionStrategy = Accessor.getDefault() .getSubpartitionStrategy(partitionStrategy, partitionDepth); return new FileSystemDataset.Builder<E>() .namespace(namespace) .name(name) .fileSystem(fileSystem) .uri(uri) .descriptor(new DatasetDescriptor.Builder(descriptor) .location(partitionDirectory) .partitionStrategy(subpartitionStrategy) .build()) .type(type) .partitionKey(key) .partitionListener(partitionListener) .build(); } FileSystemDataset(FileSystem fileSystem, Path directory,
                    String namespace, String name,
                    DatasetDescriptor descriptor, URI uri,
                    @Nullable PartitionListener partitionListener,
                    Class<E> type);   FileSystemDataset(FileSystem fileSystem, Path directory,
                    String namespace, String name,
                    DatasetDescriptor descriptor, URI uri,
                    @Nullable PartitionKey partitionKey,
                    @Nullable PartitionListener partitionListener,
                    Class<E> type); private  FileSystemDataset(FileSystemDataset<?> toCopy, Class<E> type); }
FileSystemDataset extends AbstractDataset<E> implements
    Mergeable<FileSystemDataset<E>>, InputFormatAccessor<E>, LastModifiedAccessor,
    PartitionedDataset<E>, SizeAccessor, Signalable<E>, Replaceable<View<E>> { @Override @Nullable @SuppressWarnings("deprecation") public PartitionedDataset<E> getPartition(PartitionKey key, boolean allowCreate) { Preconditions.checkState(descriptor.isPartitioned(), "Attempt to get a partition on a non-partitioned dataset (name:%s)", name); LOG.debug("Loading partition for key {}, allowCreate:{}", new Object[]{ key, allowCreate}); Path partitionDirectory = fileSystem.makeQualified( toDirectoryName(directory, key)); try { if (!fileSystem.exists(partitionDirectory)) { if (allowCreate) { if (partitionListener != null) { partitionListener.partitionAdded(namespace, name, toRelativeDirectory(key).toString()); } fileSystem.mkdirs(partitionDirectory); } else { return null; } } } catch (IOException e) { throw new DatasetIOException("Unable to locate or create dataset partition directory " + partitionDirectory, e); } int partitionDepth = key.getLength(); PartitionStrategy subpartitionStrategy = Accessor.getDefault() .getSubpartitionStrategy(partitionStrategy, partitionDepth); return new FileSystemDataset.Builder<E>() .namespace(namespace) .name(name) .fileSystem(fileSystem) .uri(uri) .descriptor(new DatasetDescriptor.Builder(descriptor) .location(partitionDirectory) .partitionStrategy(subpartitionStrategy) .build()) .type(type) .partitionKey(key) .partitionListener(partitionListener) .build(); } FileSystemDataset(FileSystem fileSystem, Path directory,
                    String namespace, String name,
                    DatasetDescriptor descriptor, URI uri,
                    @Nullable PartitionListener partitionListener,
                    Class<E> type);   FileSystemDataset(FileSystem fileSystem, Path directory,
                    String namespace, String name,
                    DatasetDescriptor descriptor, URI uri,
                    @Nullable PartitionKey partitionKey,
                    @Nullable PartitionListener partitionListener,
                    Class<E> type); private  FileSystemDataset(FileSystemDataset<?> toCopy, Class<E> type); @Override @SuppressWarnings("unchecked") Dataset<T> asType(Class<T> type); @Override URI getUri(); String getNamespace(); @Override String getName(); @Override DatasetDescriptor getDescriptor(); @Override Iterable<PartitionView<E>> getCoveringPartitions(); boolean deleteAll(); @Override boolean moveToTrash(); PathIterator pathIterator(); Iterator<Path> dirIterator(); @Override FileSystemView<E> filter(Constraints c); @Override @Nullable @SuppressWarnings("deprecation") PartitionedDataset<E> getPartition(PartitionKey key, boolean allowCreate); @Override @SuppressWarnings("deprecation") void dropPartition(PartitionKey key); @Override @SuppressWarnings("deprecation") Iterable<PartitionedDataset<E>> getPartitions(); void addExistingPartitions(); @Override String toString(); @Override void merge(FileSystemDataset<E> update); @Override boolean canReplace(View<E> part); @Override void replace(View<E> target, View<E> replacement); @Override InputFormat<E, Void> getInputFormat(Configuration conf); @SuppressWarnings("unchecked") PartitionKey keyFromDirectory(Path dir); @Override long getSize(); @Override long getLastModified(); @Override boolean isEmpty(); @Override void signalReady(); @Override boolean isReady(); }
FileSystemDataset extends AbstractDataset<E> implements
    Mergeable<FileSystemDataset<E>>, InputFormatAccessor<E>, LastModifiedAccessor,
    PartitionedDataset<E>, SizeAccessor, Signalable<E>, Replaceable<View<E>> { @Override @Nullable @SuppressWarnings("deprecation") public PartitionedDataset<E> getPartition(PartitionKey key, boolean allowCreate) { Preconditions.checkState(descriptor.isPartitioned(), "Attempt to get a partition on a non-partitioned dataset (name:%s)", name); LOG.debug("Loading partition for key {}, allowCreate:{}", new Object[]{ key, allowCreate}); Path partitionDirectory = fileSystem.makeQualified( toDirectoryName(directory, key)); try { if (!fileSystem.exists(partitionDirectory)) { if (allowCreate) { if (partitionListener != null) { partitionListener.partitionAdded(namespace, name, toRelativeDirectory(key).toString()); } fileSystem.mkdirs(partitionDirectory); } else { return null; } } } catch (IOException e) { throw new DatasetIOException("Unable to locate or create dataset partition directory " + partitionDirectory, e); } int partitionDepth = key.getLength(); PartitionStrategy subpartitionStrategy = Accessor.getDefault() .getSubpartitionStrategy(partitionStrategy, partitionDepth); return new FileSystemDataset.Builder<E>() .namespace(namespace) .name(name) .fileSystem(fileSystem) .uri(uri) .descriptor(new DatasetDescriptor.Builder(descriptor) .location(partitionDirectory) .partitionStrategy(subpartitionStrategy) .build()) .type(type) .partitionKey(key) .partitionListener(partitionListener) .build(); } FileSystemDataset(FileSystem fileSystem, Path directory,
                    String namespace, String name,
                    DatasetDescriptor descriptor, URI uri,
                    @Nullable PartitionListener partitionListener,
                    Class<E> type);   FileSystemDataset(FileSystem fileSystem, Path directory,
                    String namespace, String name,
                    DatasetDescriptor descriptor, URI uri,
                    @Nullable PartitionKey partitionKey,
                    @Nullable PartitionListener partitionListener,
                    Class<E> type); private  FileSystemDataset(FileSystemDataset<?> toCopy, Class<E> type); @Override @SuppressWarnings("unchecked") Dataset<T> asType(Class<T> type); @Override URI getUri(); String getNamespace(); @Override String getName(); @Override DatasetDescriptor getDescriptor(); @Override Iterable<PartitionView<E>> getCoveringPartitions(); boolean deleteAll(); @Override boolean moveToTrash(); PathIterator pathIterator(); Iterator<Path> dirIterator(); @Override FileSystemView<E> filter(Constraints c); @Override @Nullable @SuppressWarnings("deprecation") PartitionedDataset<E> getPartition(PartitionKey key, boolean allowCreate); @Override @SuppressWarnings("deprecation") void dropPartition(PartitionKey key); @Override @SuppressWarnings("deprecation") Iterable<PartitionedDataset<E>> getPartitions(); void addExistingPartitions(); @Override String toString(); @Override void merge(FileSystemDataset<E> update); @Override boolean canReplace(View<E> part); @Override void replace(View<E> target, View<E> replacement); @Override InputFormat<E, Void> getInputFormat(Configuration conf); @SuppressWarnings("unchecked") PartitionKey keyFromDirectory(Path dir); @Override long getSize(); @Override long getLastModified(); @Override boolean isEmpty(); @Override void signalReady(); @Override boolean isReady();  }