@Test public void testTezBaseStagingPath() throws Exception { Configuration localConf = new Configuration(); localConf.unset(TezConfiguration.TEZ_AM_STAGING_DIR); localConf.set("fs.defaultFS", "file: Path stageDir = TezCommonUtils.getTezBaseStagingPath(localConf); Assert.assertEquals(stageDir.toString(), "file:" + TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT); conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGE_DIR); stageDir = TezCommonUtils.getTezBaseStagingPath(conf); Assert.assertEquals(stageDir.toString(), RESOLVED_STAGE_DIR); }
public static Path getTezBaseStagingPath(Configuration conf) { String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT); Path baseStagingDir; try { Path p = new Path(stagingDirStr); FileSystem fs = p.getFileSystem(conf); if (!fs.exists(p)) { mkDirForAM(fs, p); LOG.info("Stage directory " + p + " doesn't exist and is created"); } baseStagingDir = fs.resolvePath(p); } catch (IOException e) { throw new TezUncheckedException(e); } return baseStagingDir; }
TezCommonUtils { public static Path getTezBaseStagingPath(Configuration conf) { String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT); Path baseStagingDir; try { Path p = new Path(stagingDirStr); FileSystem fs = p.getFileSystem(conf); if (!fs.exists(p)) { mkDirForAM(fs, p); LOG.info("Stage directory " + p + " doesn't exist and is created"); } baseStagingDir = fs.resolvePath(p); } catch (IOException e) { throw new TezUncheckedException(e); } return baseStagingDir; } }
TezCommonUtils { public static Path getTezBaseStagingPath(Configuration conf) { String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT); Path baseStagingDir; try { Path p = new Path(stagingDirStr); FileSystem fs = p.getFileSystem(conf); if (!fs.exists(p)) { mkDirForAM(fs, p); LOG.info("Stage directory " + p + " doesn't exist and is created"); } baseStagingDir = fs.resolvePath(p); } catch (IOException e) { throw new TezUncheckedException(e); } return baseStagingDir; }  }
TezCommonUtils { public static Path getTezBaseStagingPath(Configuration conf) { String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT); Path baseStagingDir; try { Path p = new Path(stagingDirStr); FileSystem fs = p.getFileSystem(conf); if (!fs.exists(p)) { mkDirForAM(fs, p); LOG.info("Stage directory " + p + " doesn't exist and is created"); } baseStagingDir = fs.resolvePath(p); } catch (IOException e) { throw new TezUncheckedException(e); } return baseStagingDir; }  static Path getTezBaseStagingPath(Configuration conf); @Private static Path createTezSystemStagingPath(Configuration conf, String strAppId); @Private static Path getTezSystemStagingPath(Configuration conf, String strAppId); @Private static Path getTezConfStagingPath(Path tezSysStagingPath); @Private static Path getTezSessionJarStagingPath(Path tezSysStagingPath); @Private static Path getTezBinPlanStagingPath(Path tezSysStagingPath); @Private static Path getTezTextPlanStagingPath(Path tezSysStagingPath); @Private static Path getRecoveryPath(Path tezSysStagingPath, Configuration conf); @Private static Path getAttemptRecoveryPath(Path recoveryPath, int attemptID); @Private static Path getDAGRecoveryPath(Path attemptRecoverPath, String dagID); @Private static Path getSummaryRecoveryPath(Path attemptRecoverPath); static void mkDirForAM(FileSystem fs, Path dir); static FSDataOutputStream createFileForAM(FileSystem fs, Path filePath); @Private static ByteString compressByteArrayToByteString(byte[] inBytes); @Private static byte[] decompressByteStringToByteArray(ByteString byteString); }
TezCommonUtils { public static Path getTezBaseStagingPath(Configuration conf) { String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT); Path baseStagingDir; try { Path p = new Path(stagingDirStr); FileSystem fs = p.getFileSystem(conf); if (!fs.exists(p)) { mkDirForAM(fs, p); LOG.info("Stage directory " + p + " doesn't exist and is created"); } baseStagingDir = fs.resolvePath(p); } catch (IOException e) { throw new TezUncheckedException(e); } return baseStagingDir; }  static Path getTezBaseStagingPath(Configuration conf); @Private static Path createTezSystemStagingPath(Configuration conf, String strAppId); @Private static Path getTezSystemStagingPath(Configuration conf, String strAppId); @Private static Path getTezConfStagingPath(Path tezSysStagingPath); @Private static Path getTezSessionJarStagingPath(Path tezSysStagingPath); @Private static Path getTezBinPlanStagingPath(Path tezSysStagingPath); @Private static Path getTezTextPlanStagingPath(Path tezSysStagingPath); @Private static Path getRecoveryPath(Path tezSysStagingPath, Configuration conf); @Private static Path getAttemptRecoveryPath(Path recoveryPath, int attemptID); @Private static Path getDAGRecoveryPath(Path attemptRecoverPath, String dagID); @Private static Path getSummaryRecoveryPath(Path attemptRecoverPath); static void mkDirForAM(FileSystem fs, Path dir); static FSDataOutputStream createFileForAM(FileSystem fs, Path filePath); @Private static ByteString compressByteArrayToByteString(byte[] inBytes); @Private static byte[] decompressByteStringToByteArray(ByteString byteString); static final FsPermission TEZ_AM_DIR_PERMISSION; static final FsPermission TEZ_AM_FILE_PERMISSION; static final String TEZ_SYSTEM_SUB_DIR; }