LogsCLI extends Configured implements Tool { @Override public int run(String[] args) throws Exception { Options opts = new Options(); opts.addOption(HELP_CMD, false, "Displays help for all commands."); Option appIdOpt = new Option(APPLICATION_ID_OPTION, true, "ApplicationId (required)"); appIdOpt.setRequired(true); opts.addOption(appIdOpt); opts.addOption(CONTAINER_ID_OPTION, true, "ContainerId. " + "By default, it will only print syslog if the application is runing." + " Work with -logFiles to get other logs."); opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress in the format " + "nodename:port"); opts.addOption(APP_OWNER_OPTION, true, "AppOwner (assumed to be current user if not specified)"); Option amOption = new Option(AM_CONTAINER_OPTION, true, "Prints the AM Container logs for this application. " + "Specify comma-separated value to get logs for related AM Container. " + "For example, If we specify -am 1,2, we will get the logs for " + "the first AM Container as well as the second AM Container. " + "To get logs for all AM Containers, use -am ALL. " + "To get logs for the latest AM Container, use -am -1. " + "By default, it will only print out syslog. Work with -logFiles " + "to get other logs"); amOption.setValueSeparator(','); amOption.setArgs(Option.UNLIMITED_VALUES); amOption.setArgName("AM Containers"); opts.addOption(amOption); Option logFileOpt = new Option(CONTAINER_LOG_FILES, true, "Work with -am/-containerId and specify comma-separated value " + "to get specified container log files. Use \"ALL\" to fetch all the " + "log files for the container."); logFileOpt.setValueSeparator(','); logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); opts.addOption(logFileOpt); opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID"); opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID"); opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address"); opts.getOption(APP_OWNER_OPTION).setArgName("Application Owner"); opts.getOption(AM_CONTAINER_OPTION).setArgName("AM Containers"); Options printOpts = new Options(); printOpts.addOption(opts.getOption(HELP_CMD)); printOpts.addOption(opts.getOption(CONTAINER_ID_OPTION)); printOpts.addOption(opts.getOption(NODE_ADDRESS_OPTION)); printOpts.addOption(opts.getOption(APP_OWNER_OPTION)); printOpts.addOption(opts.getOption(AM_CONTAINER_OPTION)); printOpts.addOption(opts.getOption(CONTAINER_LOG_FILES)); if (args.length < 1) { printHelpMessage(printOpts); return -1; } if (args[0].equals("-help")) { printHelpMessage(printOpts); return 0; } CommandLineParser parser = new GnuParser(); String appIdStr = null; String containerIdStr = null; String nodeAddress = null; String appOwner = null; boolean getAMContainerLogs = false; String[] logFiles = null; List<String> amContainersList = new ArrayList<String>(); try { CommandLine commandLine = parser.parse(opts, args, true); appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION); containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION); nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION); if (getAMContainerLogs) { String[] amContainers = commandLine.getOptionValues(AM_CONTAINER_OPTION); for (String am : amContainers) { boolean errorInput = false; if (!am.trim().equalsIgnoreCase("ALL")) { try { int id = Integer.parseInt(am.trim()); if (id != -1 && id <= 0) { errorInput = true; } } catch (NumberFormatException ex) { errorInput = true; } if (errorInput) { System.err.println( "Invalid input for option -am. Valid inputs are 'ALL', -1 " + "and any other integer which is larger than 0."); printHelpMessage(printOpts); return -1; } amContainersList.add(am.trim()); } else { amContainersList.add("ALL"); break; } } } if (commandLine.hasOption(CONTAINER_LOG_FILES)) { logFiles = commandLine.getOptionValues(CONTAINER_LOG_FILES); } } catch (ParseException e) { System.err.println("options parsing failed: " + e.getMessage()); printHelpMessage(printOpts); return -1; } if (appIdStr == null) { System.err.println("ApplicationId cannot be null!"); printHelpMessage(printOpts); return -1; } ApplicationId appId = null; try { appId = ConverterUtils.toApplicationId(appIdStr); } catch (Exception e) { System.err.println("Invalid ApplicationId specified"); return -1; } LogCLIHelpers logCliHelper = new LogCLIHelpers(); logCliHelper.setConf(getConf()); if (appOwner == null || appOwner.isEmpty()) { appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); } YarnApplicationState appState = YarnApplicationState.NEW; try { appState = getApplicationState(appId); if (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING || appState == YarnApplicationState.SUBMITTED) { System.out.println("Logs are not avaiable right now."); return -1; } } catch (IOException | YarnException e) { System.err.println("Unable to get ApplicationState." + " Attempting to fetch logs directly from the filesystem."); } if (getAMContainerLogs) { if (logFiles == null || logFiles.length == 0) { logFiles = new String[] { "syslog" }; } if (appState == YarnApplicationState.ACCEPTED || appState == YarnApplicationState.RUNNING) { return printAMContainerLogs(getConf(), appIdStr, amContainersList, logFiles, logCliHelper, appOwner, false); } else { if (getConf().getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { return printAMContainerLogs(getConf(), appIdStr, amContainersList, logFiles, logCliHelper, appOwner, true); } else { System.out .println( "Can not get AMContainers logs for the application:" + appId); System.out.println("This application:" + appId + " is finished." + " Please enable the application history service. Or Using " + "yarn logs -applicationId <appId> -containerId <containerId> " + "--nodeAddress <nodeHttpAddress> to get the container logs"); return -1; } } } int resultCode = 0; if (containerIdStr != null) { if (nodeAddress != null && isApplicationFinished(appState)) { List<String> logs; if (logFiles == null) { logs = null; } else if (fetchAllLogFiles(logFiles)) { logs = null; } else { logs = Arrays.asList(logFiles); } return logCliHelper.dumpAContainersLogsForALogType(appIdStr, containerIdStr, nodeAddress, appOwner, logs); } try { ContainerReport report = getContainerReport(containerIdStr); String nodeHttpAddress = report.getNodeHttpAddress().replaceFirst( WebAppUtils.getHttpSchemePrefix(getConf()), ""); String nodeId = report.getAssignedNode().toString(); if (!isApplicationFinished(appState)) { if (logFiles == null || logFiles.length == 0) { logFiles = new String[] { "syslog" }; } printContainerLogsFromRunningApplication(getConf(), appIdStr, containerIdStr, nodeHttpAddress, nodeId, logFiles, logCliHelper, appOwner); } else { String [] requestedLogFiles = logFiles; if(fetchAllLogFiles(logFiles)) { requestedLogFiles = null; } printContainerLogsForFinishedApplication(appIdStr, containerIdStr, nodeId, requestedLogFiles, logCliHelper, appOwner); } return resultCode; } catch (IOException | YarnException ex) { System.err.println("Unable to get logs for this container:" + containerIdStr + "for the application:" + appId); if (!getConf().getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) { System.out.println("Please enable the application history service. Or "); } System.out.println("Using " + "yarn logs -applicationId <appId> -containerId <containerId> " + "--nodeAddress <nodeHttpAddress> to get the container logs"); return -1; } } else { if (nodeAddress == null) { resultCode = logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out); } else { System.out.println("Should at least provide ContainerId!"); printHelpMessage(printOpts); resultCode = -1; } } return resultCode; }  @Override int run(String[] args); static void main(String[] args); static final String HELP_CMD; }
@Test(timeout = 5000l) public void testFailResultCodes() throws Exception { Configuration conf = new YarnConfiguration(); conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class); LogCLIHelpers cliHelper = new LogCLIHelpers(); cliHelper.setConf(conf); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); LogsCLI dumper = new LogsCLIForTest(mockYarnClient); dumper.setConf(conf); int exitCode = dumper.run( new String[] { "-applicationId", "application_0_0" } ); assertTrue("Should return an error code", exitCode != 0); exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0", "nonexistentnode:1234", "nobody"); assertTrue("Should return an error code", exitCode != 0); }
@Test(timeout = 5000l) public void testInvalidApplicationId() throws Exception { Configuration conf = new YarnConfiguration(); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); int exitCode = cli.run( new String[] { "-applicationId", "not_an_app_id"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified")); }
@Test(timeout = 5000l) public void testUnknownApplicationId() throws Exception { Configuration conf = new YarnConfiguration(); YarnClient mockYarnClient = createMockYarnClientUnknownApp(); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); int exitCode = cli.run(new String[] { "-applicationId", ApplicationId.newInstance(1, 1).toString() }); assertTrue(exitCode != 0); assertTrue(sysErrStream.toString().startsWith( "Unable to get ApplicationState")); }
@Test(timeout = 5000l) public void testHelpMessage() throws Exception { Configuration conf = new YarnConfiguration(); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); LogsCLI dumper = new LogsCLIForTest(mockYarnClient); dumper.setConf(conf); int exitCode = dumper.run(new String[]{}); assertTrue(exitCode == -1); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); pw.println("Retrieve logs for completed YARN applications."); pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]"); pw.println(); pw.println("general options are:"); pw.println(" -am <AM Containers> Prints the AM Container logs for this"); pw.println(" application. Specify comma-separated"); pw.println(" value to get logs for related AM"); pw.println(" Container. For example, If we specify -am"); pw.println(" 1,2, we will get the logs for the first"); pw.println(" AM Container as well as the second AM"); pw.println(" Container. To get logs for all AM"); pw.println(" Containers, use -am ALL. To get logs for"); pw.println(" the latest AM Container, use -am -1. By"); pw.println(" default, it will only print out syslog."); pw.println(" Work with -logFiles to get other logs"); pw.println(" -appOwner <Application Owner> AppOwner (assumed to be current user if"); pw.println(" not specified)"); pw.println(" -containerId <Container ID> ContainerId. By default, it will only"); pw.println(" print syslog if the application is"); pw.println(" runing. Work with -logFiles to get other"); pw.println(" logs."); pw.println(" -help Displays help for all commands."); pw.println(" -logFiles <Log File Name> Work with -am/-containerId and specify"); pw.println(" comma-separated value to get specified"); pw.println(" container log files. Use \"ALL\" to fetch"); pw.println(" all the log files for the container."); pw.println(" -nodeAddress <Node Address> NodeAddress in the format nodename:port"); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); }
@Test (timeout = 15000) public void testFetchApplictionLogs() throws Exception { String remoteLogRootDir = "target/logs/"; Configuration configuration = new Configuration(); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); FileSystem fs = FileSystem.get(configuration); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.newInstance(appId, 1); ContainerId containerId0 = ContainerIdPBImpl.newContainerId(appAttemptId, 0); ContainerId containerId1 = ContainerIdPBImpl.newContainerId(appAttemptId, 1); ContainerId containerId2 = ContainerIdPBImpl.newContainerId(appAttemptId, 2); ContainerId containerId3 = ContainerIdPBImpl.newContainerId(appAttemptId, 3); NodeId nodeId = NodeId.newInstance("localhost", 1234); String rootLogDir = "target/LocalLogs"; Path rootLogDirPath = new Path(rootLogDir); if (fs.exists(rootLogDirPath)) { fs.delete(rootLogDirPath, true); } assertTrue(fs.mkdirs(rootLogDirPath)); Path appLogsDir = new Path(rootLogDirPath, appId.toString()); if (fs.exists(appLogsDir)) { fs.delete(appLogsDir, true); } assertTrue(fs.mkdirs(appLogsDir)); List<String> rootLogDirs = Arrays.asList(rootLogDir); List<String> logTypes = new ArrayList<String>(); logTypes.add("syslog"); createContainerLogInLocalDir(appLogsDir, containerId1, fs, logTypes); createContainerLogInLocalDir(appLogsDir, containerId2, fs, logTypes); logTypes.add("stdout"); createContainerLogInLocalDir(appLogsDir, containerId3, fs, logTypes); Path path = new Path(remoteLogRootDir + ugi.getShortUserName() + "/logs/application_0_0001"); if (fs.exists(path)) { fs.delete(path, true); } assertTrue(fs.mkdirs(path)); uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId0, path, fs); uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1, path, fs); uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1, path, fs); uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId2, path, fs); uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId3, path, fs); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(configuration); int exitCode = cli.run(new String[] { "-applicationId", appId.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000001 in syslog!")); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000002 in syslog!")); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000003 in syslog!")); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000003 in stdout!")); sysOutStream.reset(); exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId1.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000001 in syslog!")); assertTrue(sysOutStream.toString().contains("Log Upload Time")); assertTrue(!sysOutStream.toString().contains( "Logs for container " + containerId1.toString() + " are not present in this log-file.")); sysOutStream.reset(); exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId0.toString() }); assertTrue(exitCode == -1); assertTrue(sysOutStream.toString().contains( "Logs for container " + containerId0.toString() + " are not present in this log-file.")); sysOutStream.reset(); exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId3.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000003 in syslog!")); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000003 in stdout!")); sysOutStream.reset(); exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId3.toString() , "-logFiles", "stdout"}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000003 in stdout!")); assertTrue(!sysOutStream.toString().contains( "Hello container_0_0001_01_000003 in syslog!")); sysOutStream.reset(); fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); }
@Test (timeout = 15000) public void testFetchApplictionLogsHar() throws Exception { String remoteLogRootDir = "target/logs/"; Configuration configuration = new Configuration(); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); FileSystem fs = FileSystem.get(configuration); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); URL harUrl = ClassLoader.getSystemClassLoader() .getResource("application_1440536969523_0001.har"); assertNotNull(harUrl); Path path = new Path(remoteLogRootDir + ugi.getShortUserName() + "/logs/application_1440536969523_0001"); if (fs.exists(path)) { fs.delete(path, true); } assertTrue(fs.mkdirs(path)); Path harPath = new Path(path, "application_1440536969523_0001.har"); fs.copyFromLocalFile(false, new Path(harUrl.toURI()), harPath); assertTrue(fs.exists(harPath)); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(configuration); int exitCode = cli.run(new String[]{"-applicationId", "application_1440536969523_0001"}); assertTrue(exitCode == 0); String out = sysOutStream.toString(); assertTrue( out.contains("container_1440536969523_0001_01_000001 on host1_1111")); assertTrue(out.contains("Hello stderr")); assertTrue(out.contains("Hello stdout")); assertTrue(out.contains("Hello syslog")); assertTrue( out.contains("container_1440536969523_0001_01_000002 on host2_2222")); assertTrue(out.contains("Goodbye stderr")); assertTrue(out.contains("Goodbye stdout")); assertTrue(out.contains("Goodbye syslog")); sysOutStream.reset(); fs.delete(new Path(remoteLogRootDir), true); }