@Test public void testGetRelativePartitionPathRepeatedFolderNameBasePath() { Path basePath = new Path("/test/apache/apache"); Path partitionPath = new Path("/test/apache/apache/hudi"); assertEquals("hudi", FSUtils.getRelativePartitionPath(basePath, partitionPath)); }
public static String getRelativePartitionPath(Path basePath, Path fullPartitionPath) { basePath = Path.getPathWithoutSchemeAndAuthority(basePath); fullPartitionPath = Path.getPathWithoutSchemeAndAuthority(fullPartitionPath); String fullPartitionPathStr = fullPartitionPath.toString(); int partitionStartIndex = fullPartitionPathStr.indexOf(basePath.getName(), basePath.getParent() == null ? 0 : basePath.getParent().toString().length()); return partitionStartIndex + basePath.getName().length() == fullPartitionPathStr.length() ? "" : fullPartitionPathStr.substring(partitionStartIndex + basePath.getName().length() + 1); }
FSUtils { public static String getRelativePartitionPath(Path basePath, Path fullPartitionPath) { basePath = Path.getPathWithoutSchemeAndAuthority(basePath); fullPartitionPath = Path.getPathWithoutSchemeAndAuthority(fullPartitionPath); String fullPartitionPathStr = fullPartitionPath.toString(); int partitionStartIndex = fullPartitionPathStr.indexOf(basePath.getName(), basePath.getParent() == null ? 0 : basePath.getParent().toString().length()); return partitionStartIndex + basePath.getName().length() == fullPartitionPathStr.length() ? "" : fullPartitionPathStr.substring(partitionStartIndex + basePath.getName().length() + 1); } }
FSUtils { public static String getRelativePartitionPath(Path basePath, Path fullPartitionPath) { basePath = Path.getPathWithoutSchemeAndAuthority(basePath); fullPartitionPath = Path.getPathWithoutSchemeAndAuthority(fullPartitionPath); String fullPartitionPathStr = fullPartitionPath.toString(); int partitionStartIndex = fullPartitionPathStr.indexOf(basePath.getName(), basePath.getParent() == null ? 0 : basePath.getParent().toString().length()); return partitionStartIndex + basePath.getName().length() == fullPartitionPathStr.length() ? "" : fullPartitionPathStr.substring(partitionStartIndex + basePath.getName().length() + 1); }  }
FSUtils { public static String getRelativePartitionPath(Path basePath, Path fullPartitionPath) { basePath = Path.getPathWithoutSchemeAndAuthority(basePath); fullPartitionPath = Path.getPathWithoutSchemeAndAuthority(fullPartitionPath); String fullPartitionPathStr = fullPartitionPath.toString(); int partitionStartIndex = fullPartitionPathStr.indexOf(basePath.getName(), basePath.getParent() == null ? 0 : basePath.getParent().toString().length()); return partitionStartIndex + basePath.getName().length() == fullPartitionPathStr.length() ? "" : fullPartitionPathStr.substring(partitionStartIndex + basePath.getName().length() + 1); }  static Configuration prepareHadoopConf(Configuration conf); static FileSystem getFs(String path, Configuration conf); static FileSystem getFs(String path, Configuration conf, boolean localByDefault); static Path addSchemeIfLocalPath(String path); static String makeWriteToken(int taskPartitionId, int stageId, long taskAttemptId); static String makeDataFileName(String instantTime, String writeToken, String fileId); static String makeDataFileName(String instantTime, String writeToken, String fileId, String fileExtension); static String makeBootstrapIndexFileName(String instantTime, String fileId, String ext); static String maskWithoutFileId(String instantTime, int taskPartitionId); static String getCommitFromCommitFile(String commitFileName); static String getCommitTime(String fullFileName); static long getFileSize(FileSystem fs, Path path); static String getFileId(String fullFileName); static List<String> getAllPartitionFoldersThreeLevelsDown(FileSystem fs, String basePath); static String getRelativePartitionPath(Path basePath, Path fullPartitionPath); static List<String> getAllFoldersWithPartitionMetaFile(FileSystem fs, String basePathStr); static void processFiles(FileSystem fs, String basePathStr, Function<FileStatus, Boolean> consumer,
                                  boolean excludeMetaFolder); static List<String> getAllPartitionPaths(FileSystem fs, String basePathStr, boolean assumeDatePartitioning); static String getFileExtension(String fullName); static String createNewFileIdPfx(); static String getFileExtensionFromLog(Path logPath); static String getFileIdFromLogPath(Path path); static String getFileIdFromFilePath(Path filePath); static String getBaseCommitTimeFromLogPath(Path path); static Integer getTaskPartitionIdFromLogPath(Path path); static String getWriteTokenFromLogPath(Path path); static Integer getStageIdFromLogPath(Path path); static Integer getTaskAttemptIdFromLogPath(Path path); static int getFileVersionFromLog(Path logPath); static String makeLogFileName(String fileId, String logFileExtension, String baseCommitTime, int version,
      String writeToken); static boolean isLogFile(Path logPath); static Option<HoodieLogFile> getLatestLogFile(Stream<HoodieLogFile> logFiles); static Stream<HoodieLogFile> getAllLogFiles(FileSystem fs, Path partitionPath, final String fileId,
      final String logFileExtension, final String baseCommitTime); static Option<Pair<Integer, String>> getLatestLogVersion(FileSystem fs, Path partitionPath,
      final String fileId, final String logFileExtension, final String baseCommitTime); static int computeNextLogVersion(FileSystem fs, Path partitionPath, final String fileId,
      final String logFileExtension, final String baseCommitTime); static int getDefaultBufferSize(final FileSystem fs); static Short getDefaultReplication(FileSystem fs, Path path); static boolean recoverDFSFileLease(final DistributedFileSystem dfs, final Path p); static void deleteOlderCleanMetaFiles(FileSystem fs, String metaPath, Stream<HoodieInstant> instants); static void deleteOlderRollbackMetaFiles(FileSystem fs, String metaPath, Stream<HoodieInstant> instants); static void deleteInstantFile(FileSystem fs, String metaPath, HoodieInstant instant); static void createPathIfNotExists(FileSystem fs, Path partitionPath); static Long getSizeInMB(long sizeInBytes); static Path getPartitionPath(String basePath, String partitionPath); static Path getPartitionPath(Path basePath, String partitionPath); static String getDFSFullPartitionPath(FileSystem fs, Path fullPartitionPath); static boolean isGCSInputStream(FSDataInputStream inputStream); static Configuration registerFileSystem(Path file, Configuration conf); static HoodieWrapperFileSystem getFs(String path, SerializableConfiguration hadoopConf,
      ConsistencyGuardConfig consistencyGuardConfig); static List<FileStatus> getGlobStatusExcludingMetaFolder(FileSystem fs, Path globPath); }
FSUtils { public static String getRelativePartitionPath(Path basePath, Path fullPartitionPath) { basePath = Path.getPathWithoutSchemeAndAuthority(basePath); fullPartitionPath = Path.getPathWithoutSchemeAndAuthority(fullPartitionPath); String fullPartitionPathStr = fullPartitionPath.toString(); int partitionStartIndex = fullPartitionPathStr.indexOf(basePath.getName(), basePath.getParent() == null ? 0 : basePath.getParent().toString().length()); return partitionStartIndex + basePath.getName().length() == fullPartitionPathStr.length() ? "" : fullPartitionPathStr.substring(partitionStartIndex + basePath.getName().length() + 1); }  static Configuration prepareHadoopConf(Configuration conf); static FileSystem getFs(String path, Configuration conf); static FileSystem getFs(String path, Configuration conf, boolean localByDefault); static Path addSchemeIfLocalPath(String path); static String makeWriteToken(int taskPartitionId, int stageId, long taskAttemptId); static String makeDataFileName(String instantTime, String writeToken, String fileId); static String makeDataFileName(String instantTime, String writeToken, String fileId, String fileExtension); static String makeBootstrapIndexFileName(String instantTime, String fileId, String ext); static String maskWithoutFileId(String instantTime, int taskPartitionId); static String getCommitFromCommitFile(String commitFileName); static String getCommitTime(String fullFileName); static long getFileSize(FileSystem fs, Path path); static String getFileId(String fullFileName); static List<String> getAllPartitionFoldersThreeLevelsDown(FileSystem fs, String basePath); static String getRelativePartitionPath(Path basePath, Path fullPartitionPath); static List<String> getAllFoldersWithPartitionMetaFile(FileSystem fs, String basePathStr); static void processFiles(FileSystem fs, String basePathStr, Function<FileStatus, Boolean> consumer,
                                  boolean excludeMetaFolder); static List<String> getAllPartitionPaths(FileSystem fs, String basePathStr, boolean assumeDatePartitioning); static String getFileExtension(String fullName); static String createNewFileIdPfx(); static String getFileExtensionFromLog(Path logPath); static String getFileIdFromLogPath(Path path); static String getFileIdFromFilePath(Path filePath); static String getBaseCommitTimeFromLogPath(Path path); static Integer getTaskPartitionIdFromLogPath(Path path); static String getWriteTokenFromLogPath(Path path); static Integer getStageIdFromLogPath(Path path); static Integer getTaskAttemptIdFromLogPath(Path path); static int getFileVersionFromLog(Path logPath); static String makeLogFileName(String fileId, String logFileExtension, String baseCommitTime, int version,
      String writeToken); static boolean isLogFile(Path logPath); static Option<HoodieLogFile> getLatestLogFile(Stream<HoodieLogFile> logFiles); static Stream<HoodieLogFile> getAllLogFiles(FileSystem fs, Path partitionPath, final String fileId,
      final String logFileExtension, final String baseCommitTime); static Option<Pair<Integer, String>> getLatestLogVersion(FileSystem fs, Path partitionPath,
      final String fileId, final String logFileExtension, final String baseCommitTime); static int computeNextLogVersion(FileSystem fs, Path partitionPath, final String fileId,
      final String logFileExtension, final String baseCommitTime); static int getDefaultBufferSize(final FileSystem fs); static Short getDefaultReplication(FileSystem fs, Path path); static boolean recoverDFSFileLease(final DistributedFileSystem dfs, final Path p); static void deleteOlderCleanMetaFiles(FileSystem fs, String metaPath, Stream<HoodieInstant> instants); static void deleteOlderRollbackMetaFiles(FileSystem fs, String metaPath, Stream<HoodieInstant> instants); static void deleteInstantFile(FileSystem fs, String metaPath, HoodieInstant instant); static void createPathIfNotExists(FileSystem fs, Path partitionPath); static Long getSizeInMB(long sizeInBytes); static Path getPartitionPath(String basePath, String partitionPath); static Path getPartitionPath(Path basePath, String partitionPath); static String getDFSFullPartitionPath(FileSystem fs, Path fullPartitionPath); static boolean isGCSInputStream(FSDataInputStream inputStream); static Configuration registerFileSystem(Path file, Configuration conf); static HoodieWrapperFileSystem getFs(String path, SerializableConfiguration hadoopConf,
      ConsistencyGuardConfig consistencyGuardConfig); static List<FileStatus> getGlobStatusExcludingMetaFolder(FileSystem fs, Path globPath);  }