@Test public void testReplicatePaths() { Properties properties = new Properties(); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.<Partition>absent(), true), new Path(TABLE_ROOT, "partition/file1")); }
public Path getTargetPath(Path sourcePath, FileSystem targetFs, Optional<Partition> partition, boolean isConcreteFile) { if (this.relocateDataFiles) { Preconditions.checkArgument(this.targetTableRoot.isPresent(), "Must define %s to relocate data files.", COPY_TARGET_TABLE_ROOT); Path path = this.targetTableRoot.get(); if (partition.isPresent()) { path = addPartitionToPath(path, partition.get()); } if (!isConcreteFile) { return targetFs.makeQualified(path); } return targetFs.makeQualified(new Path(path, sourcePath.getName())); } if (this.targetTablePrefixTobeReplaced.isPresent() || this.targetTablePrefixReplacement.isPresent()) { Preconditions.checkState(this.targetTablePrefixTobeReplaced.isPresent(), String.format("Must specify both %s option and %s option together", COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Preconditions.checkState(this.targetTablePrefixReplacement.isPresent(), String.format("Must specify both %s option and %s option together", COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Preconditions.checkState(!this.targetTableRoot.isPresent(), String.format("Can not specify the option %s with option %s ", COPY_TARGET_TABLE_ROOT, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Path targetPathWithoutSchemeAndAuthority = HiveCopyEntityHelper.replacedPrefix(sourcePath, this.targetTablePrefixTobeReplaced.get(), this.targetTablePrefixReplacement.get()); return targetFs.makeQualified(targetPathWithoutSchemeAndAuthority); } else if (this.targetTableRoot.isPresent()) { Preconditions.checkArgument(this.dataset.getTableRootPath().isPresent(), "Cannot move paths to a new root unless table has exactly one location."); Preconditions.checkArgument(PathUtils.isAncestor(this.dataset.getTableRootPath().get(), sourcePath), "When moving paths to a new root, all locations must be descendants of the table root location. " + "Table root location: %s, file location: %s.", this.dataset.getTableRootPath(), sourcePath); Path relativePath = PathUtils.relativizePath(sourcePath, this.dataset.getTableRootPath().get()); return targetFs.makeQualified(new Path(this.targetTableRoot.get(), relativePath)); } else { return targetFs.makeQualified(PathUtils.getPathWithoutSchemeAndAuthority(sourcePath)); } }
HiveTargetPathHelper { public Path getTargetPath(Path sourcePath, FileSystem targetFs, Optional<Partition> partition, boolean isConcreteFile) { if (this.relocateDataFiles) { Preconditions.checkArgument(this.targetTableRoot.isPresent(), "Must define %s to relocate data files.", COPY_TARGET_TABLE_ROOT); Path path = this.targetTableRoot.get(); if (partition.isPresent()) { path = addPartitionToPath(path, partition.get()); } if (!isConcreteFile) { return targetFs.makeQualified(path); } return targetFs.makeQualified(new Path(path, sourcePath.getName())); } if (this.targetTablePrefixTobeReplaced.isPresent() || this.targetTablePrefixReplacement.isPresent()) { Preconditions.checkState(this.targetTablePrefixTobeReplaced.isPresent(), String.format("Must specify both %s option and %s option together", COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Preconditions.checkState(this.targetTablePrefixReplacement.isPresent(), String.format("Must specify both %s option and %s option together", COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Preconditions.checkState(!this.targetTableRoot.isPresent(), String.format("Can not specify the option %s with option %s ", COPY_TARGET_TABLE_ROOT, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Path targetPathWithoutSchemeAndAuthority = HiveCopyEntityHelper.replacedPrefix(sourcePath, this.targetTablePrefixTobeReplaced.get(), this.targetTablePrefixReplacement.get()); return targetFs.makeQualified(targetPathWithoutSchemeAndAuthority); } else if (this.targetTableRoot.isPresent()) { Preconditions.checkArgument(this.dataset.getTableRootPath().isPresent(), "Cannot move paths to a new root unless table has exactly one location."); Preconditions.checkArgument(PathUtils.isAncestor(this.dataset.getTableRootPath().get(), sourcePath), "When moving paths to a new root, all locations must be descendants of the table root location. " + "Table root location: %s, file location: %s.", this.dataset.getTableRootPath(), sourcePath); Path relativePath = PathUtils.relativizePath(sourcePath, this.dataset.getTableRootPath().get()); return targetFs.makeQualified(new Path(this.targetTableRoot.get(), relativePath)); } else { return targetFs.makeQualified(PathUtils.getPathWithoutSchemeAndAuthority(sourcePath)); } } }
HiveTargetPathHelper { public Path getTargetPath(Path sourcePath, FileSystem targetFs, Optional<Partition> partition, boolean isConcreteFile) { if (this.relocateDataFiles) { Preconditions.checkArgument(this.targetTableRoot.isPresent(), "Must define %s to relocate data files.", COPY_TARGET_TABLE_ROOT); Path path = this.targetTableRoot.get(); if (partition.isPresent()) { path = addPartitionToPath(path, partition.get()); } if (!isConcreteFile) { return targetFs.makeQualified(path); } return targetFs.makeQualified(new Path(path, sourcePath.getName())); } if (this.targetTablePrefixTobeReplaced.isPresent() || this.targetTablePrefixReplacement.isPresent()) { Preconditions.checkState(this.targetTablePrefixTobeReplaced.isPresent(), String.format("Must specify both %s option and %s option together", COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Preconditions.checkState(this.targetTablePrefixReplacement.isPresent(), String.format("Must specify both %s option and %s option together", COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Preconditions.checkState(!this.targetTableRoot.isPresent(), String.format("Can not specify the option %s with option %s ", COPY_TARGET_TABLE_ROOT, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Path targetPathWithoutSchemeAndAuthority = HiveCopyEntityHelper.replacedPrefix(sourcePath, this.targetTablePrefixTobeReplaced.get(), this.targetTablePrefixReplacement.get()); return targetFs.makeQualified(targetPathWithoutSchemeAndAuthority); } else if (this.targetTableRoot.isPresent()) { Preconditions.checkArgument(this.dataset.getTableRootPath().isPresent(), "Cannot move paths to a new root unless table has exactly one location."); Preconditions.checkArgument(PathUtils.isAncestor(this.dataset.getTableRootPath().get(), sourcePath), "When moving paths to a new root, all locations must be descendants of the table root location. " + "Table root location: %s, file location: %s.", this.dataset.getTableRootPath(), sourcePath); Path relativePath = PathUtils.relativizePath(sourcePath, this.dataset.getTableRootPath().get()); return targetFs.makeQualified(new Path(this.targetTableRoot.get(), relativePath)); } else { return targetFs.makeQualified(PathUtils.getPathWithoutSchemeAndAuthority(sourcePath)); } } HiveTargetPathHelper(HiveDataset dataset); }
HiveTargetPathHelper { public Path getTargetPath(Path sourcePath, FileSystem targetFs, Optional<Partition> partition, boolean isConcreteFile) { if (this.relocateDataFiles) { Preconditions.checkArgument(this.targetTableRoot.isPresent(), "Must define %s to relocate data files.", COPY_TARGET_TABLE_ROOT); Path path = this.targetTableRoot.get(); if (partition.isPresent()) { path = addPartitionToPath(path, partition.get()); } if (!isConcreteFile) { return targetFs.makeQualified(path); } return targetFs.makeQualified(new Path(path, sourcePath.getName())); } if (this.targetTablePrefixTobeReplaced.isPresent() || this.targetTablePrefixReplacement.isPresent()) { Preconditions.checkState(this.targetTablePrefixTobeReplaced.isPresent(), String.format("Must specify both %s option and %s option together", COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Preconditions.checkState(this.targetTablePrefixReplacement.isPresent(), String.format("Must specify both %s option and %s option together", COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Preconditions.checkState(!this.targetTableRoot.isPresent(), String.format("Can not specify the option %s with option %s ", COPY_TARGET_TABLE_ROOT, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Path targetPathWithoutSchemeAndAuthority = HiveCopyEntityHelper.replacedPrefix(sourcePath, this.targetTablePrefixTobeReplaced.get(), this.targetTablePrefixReplacement.get()); return targetFs.makeQualified(targetPathWithoutSchemeAndAuthority); } else if (this.targetTableRoot.isPresent()) { Preconditions.checkArgument(this.dataset.getTableRootPath().isPresent(), "Cannot move paths to a new root unless table has exactly one location."); Preconditions.checkArgument(PathUtils.isAncestor(this.dataset.getTableRootPath().get(), sourcePath), "When moving paths to a new root, all locations must be descendants of the table root location. " + "Table root location: %s, file location: %s.", this.dataset.getTableRootPath(), sourcePath); Path relativePath = PathUtils.relativizePath(sourcePath, this.dataset.getTableRootPath().get()); return targetFs.makeQualified(new Path(this.targetTableRoot.get(), relativePath)); } else { return targetFs.makeQualified(PathUtils.getPathWithoutSchemeAndAuthority(sourcePath)); } } HiveTargetPathHelper(HiveDataset dataset); Path getTargetPath(Path sourcePath, FileSystem targetFs, Optional<Partition> partition, boolean isConcreteFile); }
HiveTargetPathHelper { public Path getTargetPath(Path sourcePath, FileSystem targetFs, Optional<Partition> partition, boolean isConcreteFile) { if (this.relocateDataFiles) { Preconditions.checkArgument(this.targetTableRoot.isPresent(), "Must define %s to relocate data files.", COPY_TARGET_TABLE_ROOT); Path path = this.targetTableRoot.get(); if (partition.isPresent()) { path = addPartitionToPath(path, partition.get()); } if (!isConcreteFile) { return targetFs.makeQualified(path); } return targetFs.makeQualified(new Path(path, sourcePath.getName())); } if (this.targetTablePrefixTobeReplaced.isPresent() || this.targetTablePrefixReplacement.isPresent()) { Preconditions.checkState(this.targetTablePrefixTobeReplaced.isPresent(), String.format("Must specify both %s option and %s option together", COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Preconditions.checkState(this.targetTablePrefixReplacement.isPresent(), String.format("Must specify both %s option and %s option together", COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Preconditions.checkState(!this.targetTableRoot.isPresent(), String.format("Can not specify the option %s with option %s ", COPY_TARGET_TABLE_ROOT, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Path targetPathWithoutSchemeAndAuthority = HiveCopyEntityHelper.replacedPrefix(sourcePath, this.targetTablePrefixTobeReplaced.get(), this.targetTablePrefixReplacement.get()); return targetFs.makeQualified(targetPathWithoutSchemeAndAuthority); } else if (this.targetTableRoot.isPresent()) { Preconditions.checkArgument(this.dataset.getTableRootPath().isPresent(), "Cannot move paths to a new root unless table has exactly one location."); Preconditions.checkArgument(PathUtils.isAncestor(this.dataset.getTableRootPath().get(), sourcePath), "When moving paths to a new root, all locations must be descendants of the table root location. " + "Table root location: %s, file location: %s.", this.dataset.getTableRootPath(), sourcePath); Path relativePath = PathUtils.relativizePath(sourcePath, this.dataset.getTableRootPath().get()); return targetFs.makeQualified(new Path(this.targetTableRoot.get(), relativePath)); } else { return targetFs.makeQualified(PathUtils.getPathWithoutSchemeAndAuthority(sourcePath)); } } HiveTargetPathHelper(HiveDataset dataset); Path getTargetPath(Path sourcePath, FileSystem targetFs, Optional<Partition> partition, boolean isConcreteFile); static final String COPY_TARGET_TABLE_ROOT; static final String COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED; static final String COPY_TARGET_TABLE_PREFIX_REPLACEMENT; static final String RELOCATE_DATA_FILES_KEY; static final String DEFAULT_RELOCATE_DATA_FILES; }