@Test public void testVerifyAndCreateRemoteDirNonExistence() throws Exception { this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); File aNewFile = new File(String.valueOf("tmp"+System.currentTimeMillis())); this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, aNewFile.getAbsolutePath()); DrainDispatcher dispatcher = createDispatcher(); LogAggregationService logAggregationService = spy( new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler)); logAggregationService.init(this.conf); boolean existsBefore = aNewFile.exists(); assertTrue("The new file already exists!", !existsBefore); logAggregationService.verifyAndCreateRemoteLogDir(this.conf); boolean existsAfter = aNewFile.exists(); assertTrue("The new aggregate file is not successfully created", existsAfter); aNewFile.delete(); }
void verifyAndCreateRemoteLogDir(Configuration conf) { FileSystem remoteFS = null; try { remoteFS = getFileSystem(conf); } catch (IOException e) { throw new YarnRuntimeException("Unable to get Remote FileSystem instance", e); } boolean remoteExists = true; try { FsPermission perms = remoteFS.getFileStatus(this.remoteRootLogDir).getPermission(); if (!perms.equals(TLDIR_PERMISSIONS)) { LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir + "] already exist, but with incorrect permissions. " + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms + "]." + " The cluster may have problems with multiple users."); } } catch (FileNotFoundException e) { remoteExists = false; } catch (IOException e) { throw new YarnRuntimeException( "Failed to check permissions for dir [" + this.remoteRootLogDir + "]", e); } if (!remoteExists) { LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir + "] does not exist. Attempting to create it."); try { Path qualified = this.remoteRootLogDir.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()); remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS)); remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); } catch (IOException e) { throw new YarnRuntimeException("Failed to create remoteLogDir [" + this.remoteRootLogDir + "]", e); } } }
LogAggregationService extends AbstractService implements
    LogHandler { void verifyAndCreateRemoteLogDir(Configuration conf) { FileSystem remoteFS = null; try { remoteFS = getFileSystem(conf); } catch (IOException e) { throw new YarnRuntimeException("Unable to get Remote FileSystem instance", e); } boolean remoteExists = true; try { FsPermission perms = remoteFS.getFileStatus(this.remoteRootLogDir).getPermission(); if (!perms.equals(TLDIR_PERMISSIONS)) { LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir + "] already exist, but with incorrect permissions. " + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms + "]." + " The cluster may have problems with multiple users."); } } catch (FileNotFoundException e) { remoteExists = false; } catch (IOException e) { throw new YarnRuntimeException( "Failed to check permissions for dir [" + this.remoteRootLogDir + "]", e); } if (!remoteExists) { LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir + "] does not exist. Attempting to create it."); try { Path qualified = this.remoteRootLogDir.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()); remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS)); remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); } catch (IOException e) { throw new YarnRuntimeException("Failed to create remoteLogDir [" + this.remoteRootLogDir + "]", e); } } } }
LogAggregationService extends AbstractService implements
    LogHandler { void verifyAndCreateRemoteLogDir(Configuration conf) { FileSystem remoteFS = null; try { remoteFS = getFileSystem(conf); } catch (IOException e) { throw new YarnRuntimeException("Unable to get Remote FileSystem instance", e); } boolean remoteExists = true; try { FsPermission perms = remoteFS.getFileStatus(this.remoteRootLogDir).getPermission(); if (!perms.equals(TLDIR_PERMISSIONS)) { LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir + "] already exist, but with incorrect permissions. " + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms + "]." + " The cluster may have problems with multiple users."); } } catch (FileNotFoundException e) { remoteExists = false; } catch (IOException e) { throw new YarnRuntimeException( "Failed to check permissions for dir [" + this.remoteRootLogDir + "]", e); } if (!remoteExists) { LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir + "] does not exist. Attempting to create it."); try { Path qualified = this.remoteRootLogDir.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()); remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS)); remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); } catch (IOException e) { throw new YarnRuntimeException("Failed to create remoteLogDir [" + this.remoteRootLogDir + "]", e); } } } LogAggregationService(Dispatcher dispatcher, Context context,
      DeletionService deletionService, LocalDirsHandlerService dirsHandler); }
LogAggregationService extends AbstractService implements
    LogHandler { void verifyAndCreateRemoteLogDir(Configuration conf) { FileSystem remoteFS = null; try { remoteFS = getFileSystem(conf); } catch (IOException e) { throw new YarnRuntimeException("Unable to get Remote FileSystem instance", e); } boolean remoteExists = true; try { FsPermission perms = remoteFS.getFileStatus(this.remoteRootLogDir).getPermission(); if (!perms.equals(TLDIR_PERMISSIONS)) { LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir + "] already exist, but with incorrect permissions. " + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms + "]." + " The cluster may have problems with multiple users."); } } catch (FileNotFoundException e) { remoteExists = false; } catch (IOException e) { throw new YarnRuntimeException( "Failed to check permissions for dir [" + this.remoteRootLogDir + "]", e); } if (!remoteExists) { LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir + "] does not exist. Attempting to create it."); try { Path qualified = this.remoteRootLogDir.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()); remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS)); remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); } catch (IOException e) { throw new YarnRuntimeException("Failed to create remoteLogDir [" + this.remoteRootLogDir + "]", e); } } } LogAggregationService(Dispatcher dispatcher, Context context,
      DeletionService deletionService, LocalDirsHandlerService dirsHandler); @Override void handle(LogHandlerEvent event); @VisibleForTesting ConcurrentMap<ApplicationId, AppLogAggregator> getAppLogAggregators(); @VisibleForTesting NodeId getNodeId(); }
LogAggregationService extends AbstractService implements
    LogHandler { void verifyAndCreateRemoteLogDir(Configuration conf) { FileSystem remoteFS = null; try { remoteFS = getFileSystem(conf); } catch (IOException e) { throw new YarnRuntimeException("Unable to get Remote FileSystem instance", e); } boolean remoteExists = true; try { FsPermission perms = remoteFS.getFileStatus(this.remoteRootLogDir).getPermission(); if (!perms.equals(TLDIR_PERMISSIONS)) { LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir + "] already exist, but with incorrect permissions. " + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms + "]." + " The cluster may have problems with multiple users."); } } catch (FileNotFoundException e) { remoteExists = false; } catch (IOException e) { throw new YarnRuntimeException( "Failed to check permissions for dir [" + this.remoteRootLogDir + "]", e); } if (!remoteExists) { LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir + "] does not exist. Attempting to create it."); try { Path qualified = this.remoteRootLogDir.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory()); remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS)); remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); } catch (IOException e) { throw new YarnRuntimeException("Failed to create remoteLogDir [" + this.remoteRootLogDir + "]", e); } } } LogAggregationService(Dispatcher dispatcher, Context context,
      DeletionService deletionService, LocalDirsHandlerService dirsHandler); @Override void handle(LogHandlerEvent event); @VisibleForTesting ConcurrentMap<ApplicationId, AppLogAggregator> getAppLogAggregators(); @VisibleForTesting NodeId getNodeId();  }