LogsCLI extends Configured implements Tool { @Override public int run(String[] args) throws Exception { try { webServiceClient = WebServiceClient.getWebServiceClient().createClient(); return runCommand(args); } finally { if (yarnClient != null) { yarnClient.close(); } if (webServiceClient != null) { webServiceClient.destroy(); } } }  @Override int run(String[] args); static void main(String[] args); @Private @VisibleForTesting int printContainerLogsFromRunningApplication(Configuration conf,
      ContainerLogsRequest request, LogCLIHelpers logCliHelper,
      boolean useRegex, boolean ignoreSizeLimit); @Private @VisibleForTesting ContainerReport getContainerReport(String containerIdStr); @VisibleForTesting Set<String> getMatchedContainerLogFiles(ContainerLogsRequest request,
      boolean useRegex, boolean ignoreSizeLimit); @VisibleForTesting ClientResponse getResponseFromNMWebService(Configuration conf,
      Client webServiceClient, ContainerLogsRequest request, String logFile); @VisibleForTesting String getNodeHttpAddressFromRMWebString(ContainerLogsRequest request); @Private @VisibleForTesting ContainerLogsRequest getMatchedOptionForRunningApp(
      ContainerLogsRequest container, boolean useRegex,
      boolean ignoreSizeLimit); @Private @VisibleForTesting Map<String, ContainerLogsRequest> getMatchedLogTypesForRunningApp(
      List<ContainerLogsRequest> containerLogRequests, boolean useRegex,
      boolean ignoreSizeLimit); static final String HELP_CMD; }
@Test(timeout = 5000l) public void testFailResultCodes() throws Exception { conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class); LogCLIHelpers cliHelper = new LogCLIHelpers(); cliHelper.setConf(conf); YarnClient mockYarnClient = createMockYarnClient( YarnApplicationState.FINISHED, UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI dumper = new LogsCLIForTest(mockYarnClient); dumper.setConf(conf); int exitCode = dumper.run( new String[] { "-applicationId", "application_0_0" } ); assertTrue("Should return an error code", exitCode != 0); exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0", "nonexistentnode:1234", "nobody"); assertTrue("Should return an error code", exitCode != 0); }
@Test(timeout = 10000l) public void testInvalidOpts() throws Exception { YarnClient mockYarnClient = createMockYarnClient( YarnApplicationState.FINISHED, UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); int exitCode = cli.run( new String[] { "-InvalidOpts"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "options parsing failed: Unrecognized option: -InvalidOpts")); }
@Test(timeout = 5000l) public void testUnknownApplicationId() throws Exception { YarnClient mockYarnClient = createMockYarnClientUnknownApp(); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); int exitCode = cli.run(new String[] { "-applicationId", ApplicationId.newInstance(1, 1).toString() }); assertTrue(exitCode != 0); assertTrue(sysErrStream.toString().startsWith( "Unable to get ApplicationState")); }
@Test(timeout = 5000L) public void testUnknownApplicationAttemptId() throws Exception { YarnClient mockYarnClient = createMockYarnClientUnknownApp(); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); ApplicationId appId = ApplicationId.newInstance(0, 1); int exitCode = cli.run(new String[] {"-applicationAttemptId", ApplicationAttemptId.newInstance(appId, 1).toString() }); assertTrue(exitCode != 0); assertTrue(sysErrStream.toString().contains( "Unable to get ApplicationState.")); }
@Test (timeout = 10000) public void testHelpMessage() throws Exception { YarnClient mockYarnClient = createMockYarnClient( YarnApplicationState.FINISHED, UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI dumper = new LogsCLIForTest(mockYarnClient); dumper.setConf(conf); int exitCode = dumper.run(new String[]{}); assertTrue(exitCode == -1); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); pw.println("Retrieve logs for YARN applications."); pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]"); pw.println(); pw.println("general options are:"); pw.println(" -am <AM Containers> Prints the AM Container logs"); pw.println(" for this application."); pw.println(" Specify comma-separated"); pw.println(" value to get logs for"); pw.println(" related AM Container. For"); pw.println(" example, If we specify -am"); pw.println(" 1,2, we will get the logs"); pw.println(" for the first AM Container"); pw.println(" as well as the second AM"); pw.println(" Container. To get logs for"); pw.println(" all AM Containers, use -am"); pw.println(" ALL. To get logs for the"); pw.println(" latest AM Container, use -am"); pw.println(" -1. By default, it will"); pw.println(" print all available logs."); pw.println(" Work with -log_files to get"); pw.println(" only specific logs."); pw.println(" -appOwner <Application Owner> AppOwner (assumed to be"); pw.println(" current user if not"); pw.println(" specified)"); pw.println(" -client_max_retries <Max Retries> Set max retry number for a"); pw.println(" retry client to get the"); pw.println(" container logs for the"); pw.println(" running applications. Use a"); pw.println(" negative value to make retry"); pw.println(" forever. The default value"); pw.println(" is 30."); pw.println(" -client_retry_interval_ms <Retry Interval> Work with"); pw.println(" --client_max_retries to"); pw.println(" create a retry client. The"); pw.println(" default value is 1000."); pw.println(" -clusterId <Cluster ID> ClusterId. By default, it"); pw.println(" will take default cluster id"); pw.println(" from the RM"); pw.println(" -containerId <Container ID> ContainerId. By default, it"); pw.println(" will print all available"); pw.println(" logs. Work with -log_files"); pw.println(" to get only specific logs."); pw.println(" If specified, the"); pw.println(" applicationId can be omitted"); pw.println(" -help Displays help for all"); pw.println(" commands."); pw.println(" -list_nodes Show the list of nodes that"); pw.println(" successfully aggregated"); pw.println(" logs. This option can only"); pw.println(" be used with finished"); pw.println(" applications."); pw.println(" -log_files <Log File Name> Specify comma-separated"); pw.println(" value to get exact matched"); pw.println(" log files. Use \"ALL\" or \"*\""); pw.println(" to fetch all the log files"); pw.println(" for the container."); pw.println(" -log_files_pattern <Log File Pattern> Specify comma-separated"); pw.println(" value to get matched log"); pw.println(" files by using java regex."); pw.println(" Use \".*\" to fetch all the"); pw.println(" log files for the container."); pw.println(" -nodeAddress <Node Address> NodeAddress in the format"); pw.println(" nodename:port"); pw.println(" -out <Local Directory> Local directory for storing"); pw.println(" individual container logs."); pw.println(" The container logs will be"); pw.println(" stored based on the node the"); pw.println(" container ran on."); pw.println(" -show_application_log_info Show the containerIds which"); pw.println(" belong to the specific"); pw.println(" Application. You can combine"); pw.println(" this with --nodeAddress to"); pw.println(" get containerIds for all the"); pw.println(" containers on the specific"); pw.println(" NodeManager."); pw.println(" -show_container_log_info Show the container log"); pw.println(" metadata, including log-file"); pw.println(" names, the size of the log"); pw.println(" files. You can combine this"); pw.println(" with --containerId to get"); pw.println(" log metadata for the"); pw.println(" specific container, or with"); pw.println(" --nodeAddress to get log"); pw.println(" metadata for all the"); pw.println(" containers on the specific"); pw.println(" NodeManager."); pw.println(" -size <size> Prints the log file's first"); pw.println(" 'n' bytes or the last 'n'"); pw.println(" bytes. Use negative values"); pw.println(" as bytes to read from the"); pw.println(" end and positive values as"); pw.println(" bytes to read from the"); pw.println(" beginning."); pw.println(" -size_limit_mb <Size Limit> Use this option to limit the"); pw.println(" size of the total logs which"); pw.println(" could be fetched. By"); pw.println(" default, we only allow to"); pw.println(" fetch at most 10240 MB logs."); pw.println(" If the total log size is"); pw.println(" larger than the specified"); pw.println(" number, the CLI would fail."); pw.println(" The user could specify -1 to"); pw.println(" ignore the size limit and"); pw.println(" fetch all logs."); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertTrue(sysOutStream.toString().contains(appReportStr)); }
@Test (timeout = 15000) public void testFetchFinishedApplictionLogs() throws Exception { String remoteLogRootDir = "target/logs/"; conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); FileSystem fs = FileSystem.get(conf); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId1 = ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId appAttemptId2 = ApplicationAttemptId.newInstance(appId, 2); ContainerId containerId0 = ContainerId.newContainerId(appAttemptId1, 0); ContainerId containerId1 = ContainerId.newContainerId(appAttemptId1, 1); ContainerId containerId2 = ContainerId.newContainerId(appAttemptId1, 2); ContainerId containerId3 = ContainerId.newContainerId(appAttemptId2, 3); final NodeId nodeId = NodeId.newInstance("localhost", 1234); String rootLogDir = "target/LocalLogs"; Path rootLogDirPath = new Path(rootLogDir); if (fs.exists(rootLogDirPath)) { fs.delete(rootLogDirPath, true); } assertTrue(fs.mkdirs(rootLogDirPath)); Path appLogsDir = new Path(rootLogDirPath, appId.toString()); if (fs.exists(appLogsDir)) { fs.delete(appLogsDir, true); } assertTrue(fs.mkdirs(appLogsDir)); List<String> rootLogDirs = Arrays.asList(rootLogDir); List<String> logTypes = new ArrayList<String>(); logTypes.add("syslog"); createContainerLogInLocalDir(appLogsDir, containerId1, fs, logTypes, ImmutableList.of("empty")); createContainerLogInLocalDir(appLogsDir, containerId2, fs, logTypes, Collections.emptyList()); logTypes.add("stdout"); logTypes.add("stdout1234"); createContainerLogInLocalDir(appLogsDir, containerId3, fs, logTypes, Collections.emptyList()); Path path = new Path(remoteLogRootDir + ugi.getShortUserName() + "/bucket-logs-tfile/0001/application_0_0001"); if (fs.exists(path)) { fs.delete(path, true); } assertTrue(fs.mkdirs(path)); uploadEmptyContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId, containerId0, path, fs); uploadEmptyContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId, containerId1, path, fs); uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId, containerId1, path, fs); uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId, containerId2, path, fs); uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId, containerId3, path, fs); YarnClient mockYarnClient = createMockYarnClient( YarnApplicationState.FINISHED, ugi.getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient) { @Override public ContainerReport getContainerReport(String containerIdStr) throws YarnException, IOException { ContainerReport mockReport = mock(ContainerReport.class); doReturn(nodeId).when(mockReport).getAssignedNode(); doReturn("http: return mockReport; } }; cli.setConf(conf); int exitCode = cli.run(new String[] { "-applicationId", appId.toString() }); LOG.info(sysOutStream.toString()); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId2, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout1234"))); assertTrue(sysOutStream.toString().contains( createEmptyLog("empty"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-applicationAttemptId", appAttemptId1.toString()}); LOG.info(sysOutStream.toString()); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId2, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); assertFalse(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); assertFalse(sysOutStream.toString().contains( logMessage(containerId3, "stdout1234"))); assertTrue(sysOutStream.toString().contains( createEmptyLog("empty"))); sysOutStream.reset(); exitCode = cli.run(new String[] { "-applicationAttemptId", appAttemptId1.toString()}); LOG.info(sysOutStream.toString()); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId2, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); assertFalse(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); assertFalse(sysOutStream.toString().contains( logMessage(containerId3, "stdout1234"))); assertTrue(sysOutStream.toString().contains( createEmptyLog("empty"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-log_files_pattern", ".*"}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId2, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout1234"))); assertTrue(sysOutStream.toString().contains( createEmptyLog("empty"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-log_files", "*"}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId2, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout1234"))); assertTrue(sysOutStream.toString().contains( createEmptyLog("empty"))); int fullSize = sysOutStream.toByteArray().length; sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-log_files", "stdout"}); assertTrue(exitCode == 0); assertFalse(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); assertFalse(sysOutStream.toString().contains( logMessage(containerId2, "syslog"))); assertFalse(sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); assertFalse(sysOutStream.toString().contains( logMessage(containerId3, "stdout1234"))); assertFalse(sysOutStream.toString().contains( createEmptyLog("empty"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-logFiles", "stdout"}); assertTrue("Failed with -logFiles", exitCode == 0); assertFalse("Failed with -logFiles", sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); assertFalse("Failed with -logFiles", sysOutStream.toString().contains( logMessage(containerId2, "syslog"))); assertFalse("Failed with -logFiles", sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); assertTrue("Failed with -logFiles", sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); assertFalse("Failed with -logFiles", sysOutStream.toString().contains( logMessage(containerId3, "stdout1234"))); assertFalse("Failed with -logFiles", sysOutStream.toString().contains( createEmptyLog("empty"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-log_files", "stdout", "-logFiles", "syslog"}); assertTrue("Failed with -logFiles and -log_files", exitCode == 0); assertFalse("Failed with -logFiles and -log_files", sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); assertFalse("Failed with -logFiles and -log_files", sysOutStream.toString().contains( logMessage(containerId2, "syslog"))); assertFalse("Failed with -logFiles and -log_files", sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); assertTrue("Failed with -logFiles and -log_files", sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); assertFalse("Failed with -logFiles and -log_files", sysOutStream.toString().contains( logMessage(containerId3, "stdout1234"))); assertFalse("Failed with -logFiles and -log_files", sysOutStream.toString().contains( createEmptyLog("empty"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-log_files_pattern", "std*"}); assertTrue(exitCode == 0); assertFalse(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); assertFalse(sysOutStream.toString().contains( logMessage(containerId2, "syslog"))); assertFalse(sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout1234"))); assertFalse(sysOutStream.toString().contains( createEmptyLog("empty"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-log_files", "123"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "Can not find any log file matching the pattern: [123] " + "for the application: " + appId.toString())); sysErrStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-log_files", "*", "-size", "10000" }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toByteArray().length == fullSize); sysOutStream.reset(); exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId1.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); assertTrue(sysOutStream.toString().contains("LogLastModifiedTime")); assertTrue(!sysOutStream.toString().contains( "Logs for container " + containerId1.toString() + " are not present in this log-file.")); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "123" }); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "Can not find any log file matching the pattern: [123] " + "for the container: " + containerId3 + " within the application: " + appId.toString())); sysErrStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout" }); assertTrue(exitCode == 0); int fullContextSize = sysOutStream.toByteArray().length; String fullContext = sysOutStream.toString(); sysOutStream.reset(); String logMessage = logMessage(containerId3, "stdout"); int fileContentSize = logMessage.getBytes().length; StringBuilder sb = new StringBuilder(); String endOfFile = "End of LogType:stdout"; sb.append("\n" + endOfFile + "\n"); sb.append(StringUtils.repeat("*", endOfFile.length() + 50) + "\n\n"); int tailContentSize = sb.toString().length(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout", "-size", "5"}); assertTrue(exitCode == 0); Assert.assertEquals(new String(logMessage.getBytes(), 0, 5), new String(sysOutStream.toByteArray(), (fullContextSize - fileContentSize - tailContentSize), 5)); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-containerId", containerId1.toString(), "-log_files", "empty", "-size", "5"}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( createEmptyLog("empty"))); sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout", "-size", "-5"}); assertTrue(exitCode == 0); Assert.assertEquals(new String(logMessage.getBytes(), logMessage.getBytes().length - 5, 5), new String(sysOutStream.toByteArray(), (fullContextSize - fileContentSize - tailContentSize), 5)); sysOutStream.reset(); long negative = (fullContextSize + 1000) * (-1); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-containerId", containerId3.toString(), "-log_files", "stdout", "-size", Long.toString(negative)}); assertTrue(exitCode == 0); Assert.assertEquals(fullContext, sysOutStream.toString()); sysOutStream.reset(); exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId0.toString() }); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "Can not find any log file matching the pattern")); sysErrStream.reset(); exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId3.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); sysOutStream.reset(); exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId3.toString() , "-log_files", "stdout"}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); assertTrue(!sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); sysOutStream.reset(); YarnClient mockYarnClientWithException = createMockYarnClientWithException(); cli = new LogsCLIForTest(mockYarnClientWithException); cli.setConf(conf); exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); assertTrue(sysOutStream.toString().contains( containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId))); sysOutStream.reset(); exitCode = cli.run(new String[] { "-containerId", containerId3.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "syslog"))); assertTrue(sysOutStream.toString().contains( logMessage(containerId3, "stdout"))); assertTrue(sysOutStream.toString().contains( containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId))); sysOutStream.reset(); exitCode = cli.run(new String[] { "-containerId", "invalid_container" }); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "Invalid ContainerId specified")); sysErrStream.reset(); fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); }
@Test (timeout = 15000) public void testFetchApplictionLogsAsAnotherUser() throws Exception { String remoteLogRootDir = "target/logs/"; String rootLogDir = "target/LocalLogs"; String testUser = "test"; UserGroupInformation testUgi = UserGroupInformation .createRemoteUser(testUser); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); FileSystem fs = FileSystem.get(conf); ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerId containerId = ContainerId .newContainerId(appAttemptId, 1); NodeId nodeId = NodeId.newInstance("localhost", 1234); try { Path rootLogDirPath = new Path(rootLogDir); if (fs.exists(rootLogDirPath)) { fs.delete(rootLogDirPath, true); } assertTrue(fs.mkdirs(rootLogDirPath)); final Path appLogsDir = new Path(rootLogDirPath, appId.toString()); if (fs.exists(appLogsDir)) { fs.delete(appLogsDir, true); } assertTrue(fs.mkdirs(appLogsDir)); List<String> rootLogDirs = Arrays.asList(rootLogDir); List<String> logTypes = new ArrayList<String>(); logTypes.add("syslog"); createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes, Collections.emptyList()); Path path = new Path(remoteLogRootDir + testUser + "/bucket-logs-tfile/0001/" + appId); if (fs.exists(path)) { fs.delete(path, true); } assertTrue(fs.mkdirs(path)); uploadContainerLogIntoRemoteDir(testUgi, conf, rootLogDirs, nodeId, containerId, path, fs); YarnClient mockYarnClient = createMockYarnClient( YarnApplicationState.FINISHED, testUgi.getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); int exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-appOwner", testUser}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId, "syslog"))); sysOutStream.reset(); exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-appOwner", "invalid"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains("Can not find the logs " + "for the application: " + appId.toString())); sysErrStream.reset(); YarnClient mockYarnClient2 = createMockYarnClientUnknownApp(); cli = new LogsCLIForTest(mockYarnClient2); cli.setConf(conf); exitCode = cli.run(new String[] { "-applicationId", appId.toString()}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId, "syslog"))); sysOutStream.reset(); path = new Path(remoteLogRootDir + testUser + "/logs/" + appId); if (fs.exists(path)) { fs.delete(path, true); } assertTrue(fs.mkdirs(path)); uploadContainerLogIntoRemoteDir(testUgi, conf, rootLogDirs, nodeId, containerId, path, fs); exitCode = cli.run(new String[] { "-applicationId", appId.toString()}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId, "syslog"))); sysOutStream.reset(); ApplicationId appId2 = ApplicationId.newInstance( System.currentTimeMillis(), 2); exitCode = cli.run(new String[] { "-applicationId", appId2.toString()}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "Can not find the appOwner")); sysErrStream.reset(); ApplicationId appTest = ApplicationId.newInstance( System.currentTimeMillis(), 1000); String priorityUser = "priority"; Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser + "/bucket-logs-tfile/1000/" + appTest); if (fs.exists(pathWithoutPerm)) { fs.delete(pathWithoutPerm, true); } FsPermission permission = FsPermission .createImmutable((short) 01300); assertTrue(fs.mkdirs(pathWithoutPerm, permission)); exitCode = cli.run(new String[] { "-applicationId", appTest.toString()}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "Can not find the logs for the application: " + appTest.toString())); sysErrStream.reset(); } finally { fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); } }
@Test (timeout = 5000) public void testWithInvalidApplicationId() throws Exception { LogsCLI cli = createCli(); int exitCode = cli.run(new String[] {"-applicationId", "123"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "Invalid ApplicationId specified")); }
@Test (timeout = 5000) public void testWithInvalidAppAttemptId() throws Exception { LogsCLI cli = createCli(); int exitCode = cli.run(new String[] {"-applicationAttemptId", "123"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "Invalid AppAttemptId specified")); sysErrStream.reset(); }
@Test (timeout = 5000) public void testWithInvalidContainerId() throws Exception { LogsCLI cli = createCli(); int exitCode = cli.run(new String[] {"-containerId", "123"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "Invalid ContainerId specified")); sysErrStream.reset(); }
@Test (timeout = 5000) public void testWithNonMatchingEntityIds() throws Exception { ApplicationId appId1 = ApplicationId.newInstance(0, 1); ApplicationId appId2 = ApplicationId.newInstance(0, 2); ApplicationAttemptId appAttemptId1 = ApplicationAttemptId.newInstance(appId1, 1); ApplicationAttemptId appAttemptId2 = ApplicationAttemptId.newInstance(appId2, 1); ContainerId containerId0 = ContainerId.newContainerId(appAttemptId1, 0); LogsCLI cli = createCli(); int exitCode = cli.run(new String[] {"-applicationId", appId2.toString(), "-applicationAttemptId", appAttemptId1.toString()}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "The Application:" + appId2.toString() + " does not have the AppAttempt:" + appAttemptId1.toString())); sysErrStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId2.toString(), "-containerId", containerId0.toString()}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "The Application:" + appId2.toString() + " does not have the container:" + containerId0.toString())); sysErrStream.reset(); exitCode = cli.run(new String[] {"-applicationAttemptId", appAttemptId2.toString(), "-containerId", containerId0.toString()}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "The AppAttempt:" + appAttemptId2.toString() + " does not have the container:" + containerId0.toString())); sysErrStream.reset(); }
@Test (timeout = 5000) public void testWithExclusiveArguments() throws Exception { ApplicationId appId1 = ApplicationId.newInstance(0, 1); LogsCLI cli = createCli(); int exitCode = cli.run(new String[] {"-applicationId", appId1.toString(), "-show_container_log_info", "-show_application_log_info"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains("Invalid options. " + "Can only accept one of show_application_log_info/" + "show_container_log_info.")); sysErrStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId1.toString(), "-log_files", "*", "-log_files_pattern", ".*"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains("Invalid options. " + "Can only accept one of log_files/" + "log_files_pattern.")); sysErrStream.reset(); }
@Test (timeout = 5000) public void testWithFileInputForOptionOut() throws Exception { String localDir = "target/SaveLogs"; Path localPath = new Path(localDir); FileSystem fs = FileSystem.get(conf); ApplicationId appId1 = ApplicationId.newInstance(0, 1); LogsCLI cli = createCli(); try { fs.mkdirs(localPath); Path tmpFilePath = new Path(localPath, "tmpFile"); if (!fs.exists(tmpFilePath)) { fs.createNewFile(tmpFilePath); } int exitCode = cli.run(new String[] {"-applicationId", appId1.toString(), "-out" , tmpFilePath.toString()}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().contains( "Invalid value for -out option. Please provide a directory.")); } finally { fs.delete(localPath, true); } }
@Test (timeout = 15000) public void testSaveContainerLogsLocally() throws Exception { String remoteLogRootDir = "target/logs/"; String rootLogDir = "target/LocalLogs"; String localDir = "target/SaveLogs"; Path localPath = new Path(localDir); conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); FileSystem fs = FileSystem.get(conf); ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); List<ContainerId> containerIds = new ArrayList<ContainerId>(); ContainerId containerId1 = ContainerId.newContainerId( appAttemptId, 1); ContainerId containerId2 = ContainerId.newContainerId( appAttemptId, 2); containerIds.add(containerId1); containerIds.add(containerId2); List<NodeId> nodeIds = new ArrayList<NodeId>(); NodeId nodeId = NodeId.newInstance("localhost", 1234); NodeId nodeId2 = NodeId.newInstance("test", 4567); nodeIds.add(nodeId); nodeIds.add(nodeId2); try { createContainerLogs(conf, remoteLogRootDir, rootLogDir, fs, appId, containerIds, nodeIds); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); int exitCode = cli.run(new String[] {"-applicationId", appId.toString(), "-out" , localPath.toString()}); assertTrue(exitCode == 0); FileStatus[] nodeDir = fs.listStatus(localPath); Arrays.sort(nodeDir); assertTrue(nodeDir.length == 2); assertTrue(nodeDir[0].getPath().getName().contains( LogAggregationUtils.getNodeString(nodeId))); assertTrue(nodeDir[1].getPath().getName().contains( LogAggregationUtils.getNodeString(nodeId2))); FileStatus[] container1Dir = fs.listStatus(nodeDir[0].getPath()); assertTrue(container1Dir.length == 1); assertTrue(container1Dir[0].getPath().getName().equals( containerId1.toString())); String container1= readContainerContent(container1Dir[0].getPath(), fs); assertTrue(container1.contains(logMessage(containerId1, "syslog"))); FileStatus[] container2Dir = fs.listStatus(nodeDir[1].getPath()); assertTrue(container2Dir.length == 1); assertTrue(container2Dir[0].getPath().getName().equals( containerId2.toString())); String container2= readContainerContent(container2Dir[0].getPath(), fs); assertTrue(container2.contains(logMessage(containerId2, "syslog"))); } finally { fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); fs.delete(localPath, true); } }
@Test (timeout = 15000) public void testPrintContainerLogMetadata() throws Exception { String remoteLogRootDir = "target/logs/"; conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); FileSystem fs = FileSystem.get(conf); String rootLogDir = "target/LocalLogs"; ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); List<ContainerId> containerIds = new ArrayList<ContainerId>(); ContainerId containerId1 = ContainerId.newContainerId( appAttemptId, 1); ContainerId containerId2 = ContainerId.newContainerId( appAttemptId, 2); containerIds.add(containerId1); containerIds.add(containerId2); List<NodeId> nodeIds = new ArrayList<NodeId>(); NodeId nodeId = NodeId.newInstance("localhost", 1234); nodeIds.add(nodeId); nodeIds.add(nodeId); createContainerLogs(conf, remoteLogRootDir, rootLogDir, fs, appId, containerIds, nodeIds); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); cli.run(new String[] {"-applicationId", appId.toString(), "-show_container_log_info"}); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000001 on localhost_")); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000002 on localhost_")); assertTrue(sysOutStream.toString().contains( "syslog")); assertTrue(sysOutStream.toString().contains( "43")); sysOutStream.reset(); cli.run(new String[] {"-applicationId", appId.toString(), "-show_container_log_info", "-containerId", "container_0_0001_01_000001"}); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000001 on localhost_")); assertFalse(sysOutStream.toString().contains( "Container: container_0_0001_01_000002 on localhost_")); assertTrue(sysOutStream.toString().contains( "syslog")); assertTrue(sysOutStream.toString().contains( "43")); sysOutStream.reset(); cli.run(new String[] {"-applicationId", appId.toString(), "-show_container_log_info", "-nodeAddress", "localhost"}); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000001 on localhost_")); assertTrue(sysOutStream.toString().contains( "Container: container_0_0001_01_000002 on localhost_")); assertTrue(sysOutStream.toString().contains( "syslog")); assertTrue(sysOutStream.toString().contains( "43")); sysOutStream.reset(); cli.run(new String[] {"-applicationId", appId.toString(), "-show_container_log_info", "-nodeAddress", "localhost", "-containerId", "container_1234"}); assertTrue(sysErrStream.toString().contains( "Invalid ContainerId specified")); sysErrStream.reset(); cli.run(new String[] {"-applicationId", appId.toString(), "-show_application_log_info"}); assertTrue(sysOutStream.toString().contains( "Application State: Completed.")); assertTrue(sysOutStream.toString().contains( "container_0_0001_01_000001 on localhost")); assertTrue(sysOutStream.toString().contains( "container_0_0001_01_000002 on localhost")); sysOutStream.reset(); cli.run(new String[] {"-applicationId", appId.toString(), "-show_application_log_info", "-nodeAddress", "localhost"}); assertTrue(sysOutStream.toString().contains( "Application State: Completed.")); assertTrue(sysOutStream.toString().contains( "container_0_0001_01_000001 on localhost")); assertTrue(sysOutStream.toString().contains( "container_0_0001_01_000002 on localhost")); sysOutStream.reset(); fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); }
@Test (timeout = 15000) public void testListNodeInfo() throws Exception { String remoteLogRootDir = "target/logs/"; conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); List<ContainerId> containerIds = new ArrayList<ContainerId>(); ContainerId containerId1 = ContainerId.newContainerId( appAttemptId, 1); ContainerId containerId2 = ContainerId.newContainerId( appAttemptId, 2); containerIds.add(containerId1); containerIds.add(containerId2); List<NodeId> nodeIds = new ArrayList<NodeId>(); NodeId nodeId1 = NodeId.newInstance("localhost1", 1234); NodeId nodeId2 = NodeId.newInstance("localhost2", 2345); nodeIds.add(nodeId1); nodeIds.add(nodeId2); String rootLogDir = "target/LocalLogs"; FileSystem fs = FileSystem.get(conf); createContainerLogs(conf, remoteLogRootDir, rootLogDir, fs, appId, containerIds, nodeIds); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, UserGroupInformation.getCurrentUser().getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); cli.run(new String[] { "-applicationId", appId.toString(), "-list_nodes" }); assertTrue(sysOutStream.toString().contains( LogAggregationUtils.getNodeString(nodeId1))); assertTrue(sysOutStream.toString().contains( LogAggregationUtils.getNodeString(nodeId2))); sysOutStream.reset(); fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); }
@Test (timeout = 15000) public void testFetchApplictionLogsHar() throws Exception { String remoteLogRootDir = "target/logs/"; conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); conf .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); FileSystem fs = FileSystem.get(conf); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); URL harUrl = ClassLoader.getSystemClassLoader() .getResource("application_1440536969523_0001.har"); assertNotNull(harUrl); Path path = new Path(remoteLogRootDir + ugi.getShortUserName() + "/bucket-logs-tfile/0001/application_1440536969523_0001"); if (fs.exists(path)) { fs.delete(path, true); } assertTrue(fs.mkdirs(path)); Path harPath = new Path(path, "application_1440536969523_0001.har"); fs.copyFromLocalFile(false, new Path(harUrl.toURI()), harPath); assertTrue(fs.exists(harPath)); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, ugi.getShortUserName()); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); int exitCode = cli.run(new String[]{"-applicationId", "application_1440536969523_0001"}); assertTrue(exitCode == 0); String out = sysOutStream.toString(); assertTrue( out.contains("container_1440536969523_0001_01_000001 on host1_1111")); assertTrue(out.contains("Hello stderr")); assertTrue(out.contains("Hello stdout")); assertTrue(out.contains("Hello syslog")); assertTrue( out.contains("container_1440536969523_0001_01_000002 on host2_2222")); assertTrue(out.contains("Goodbye stderr")); assertTrue(out.contains("Goodbye stdout")); assertTrue(out.contains("Goodbye syslog")); sysOutStream.reset(); fs.delete(new Path(remoteLogRootDir), true); }