LogsCLI extends Configured implements Tool { @Override public int run(String[] args) throws Exception { Options opts = new Options(); opts.addOption(HELP_CMD, false, "Displays help for all commands."); Option appIdOpt = new Option(APPLICATION_ID_OPTION, true, "ApplicationId (required)"); appIdOpt.setRequired(true); opts.addOption(appIdOpt); opts.addOption(CONTAINER_ID_OPTION, true, "ContainerId (must be specified if node address is specified)"); opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress in the format " + "nodename:port (must be specified if container id is specified)"); opts.addOption(APP_OWNER_OPTION, true, "AppOwner (assumed to be current user if not specified)"); opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID"); opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID"); opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address"); opts.getOption(APP_OWNER_OPTION).setArgName("Application Owner"); Options printOpts = new Options(); printOpts.addOption(opts.getOption(HELP_CMD)); printOpts.addOption(opts.getOption(CONTAINER_ID_OPTION)); printOpts.addOption(opts.getOption(NODE_ADDRESS_OPTION)); printOpts.addOption(opts.getOption(APP_OWNER_OPTION)); if (args.length < 1) { printHelpMessage(printOpts); return -1; } if (args[0].equals("-help")) { printHelpMessage(printOpts); return 0; } CommandLineParser parser = new GnuParser(); String appIdStr = null; String containerIdStr = null; String nodeAddress = null; String appOwner = null; try { CommandLine commandLine = parser.parse(opts, args, true); appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION); containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION); nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); } catch (ParseException e) { System.err.println("options parsing failed: " + e.getMessage()); printHelpMessage(printOpts); return -1; } if (appIdStr == null) { System.err.println("ApplicationId cannot be null!"); printHelpMessage(printOpts); return -1; } ApplicationId appId = null; try { appId = ConverterUtils.toApplicationId(appIdStr); } catch (Exception e) { System.err.println("Invalid ApplicationId specified"); return -1; } try { int resultCode = verifyApplicationState(appId); if (resultCode != 0) { System.out.println("Logs are not avaiable right now."); return resultCode; } } catch (Exception e) { System.err.println("Unable to get ApplicationState." + " Attempting to fetch logs directly from the filesystem."); } LogCLIHelpers logCliHelper = new LogCLIHelpers(); logCliHelper.setConf(getConf()); if (appOwner == null || appOwner.isEmpty()) { appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); } int resultCode = 0; if (containerIdStr == null && nodeAddress == null) { resultCode = logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out); } else if ((containerIdStr == null && nodeAddress != null) || (containerIdStr != null && nodeAddress == null)) { System.out.println("ContainerId or NodeAddress cannot be null!"); printHelpMessage(printOpts); resultCode = -1; } else { resultCode = logCliHelper.dumpAContainersLogs(appIdStr, containerIdStr, nodeAddress, appOwner); } return resultCode; }  @Override int run(String[] args); static void main(String[] args); static final String HELP_CMD; }
@Test(timeout = 5000l) public void testFailResultCodes() throws Exception { Configuration conf = new YarnConfiguration(); conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class); LogCLIHelpers cliHelper = new LogCLIHelpers(); cliHelper.setConf(conf); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); LogsCLI dumper = new LogsCLIForTest(mockYarnClient); dumper.setConf(conf); int exitCode = dumper.run( new String[] { "-applicationId", "application_0_0" } ); assertTrue("Should return an error code", exitCode != 0); exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0", "nonexistentnode:1234", "nobody"); assertTrue("Should return an error code", exitCode != 0); }
@Test(timeout = 5000l) public void testInvalidApplicationId() throws Exception { Configuration conf = new YarnConfiguration(); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); int exitCode = cli.run( new String[] { "-applicationId", "not_an_app_id"}); assertTrue(exitCode == -1); assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified")); }
@Test(timeout = 5000l) public void testUnknownApplicationId() throws Exception { Configuration conf = new YarnConfiguration(); YarnClient mockYarnClient = createMockYarnClientUnknownApp(); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(conf); int exitCode = cli.run(new String[] { "-applicationId", ApplicationId.newInstance(1, 1).toString() }); assertTrue(exitCode != 0); assertTrue(sysErrStream.toString().startsWith( "Unable to get ApplicationState")); }
@Test(timeout = 5000l) public void testHelpMessage() throws Exception { Configuration conf = new YarnConfiguration(); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); LogsCLI dumper = new LogsCLIForTest(mockYarnClient); dumper.setConf(conf); int exitCode = dumper.run(new String[]{}); assertTrue(exitCode == -1); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); pw.println("Retrieve logs for completed YARN applications."); pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]"); pw.println(); pw.println("general options are:"); pw.println(" -appOwner <Application Owner> AppOwner (assumed to be current user if"); pw.println(" not specified)"); pw.println(" -containerId <Container ID> ContainerId (must be specified if node"); pw.println(" address is specified)"); pw.println(" -help Displays help for all commands."); pw.println(" -nodeAddress <Node Address> NodeAddress in the format nodename:port"); pw.println(" (must be specified if container id is"); pw.println(" specified)"); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); }
@Test (timeout = 15000) public void testFetchApplictionLogs() throws Exception { String remoteLogRootDir = "target/logs/"; Configuration configuration = new Configuration(); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); FileSystem fs = FileSystem.get(configuration); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.newInstance(appId, 1); ContainerId containerId0 = ContainerIdPBImpl.newContainerId(appAttemptId, 0); ContainerId containerId1 = ContainerIdPBImpl.newContainerId(appAttemptId, 1); ContainerId containerId2 = ContainerIdPBImpl.newContainerId(appAttemptId, 2); NodeId nodeId = NodeId.newInstance("localhost", 1234); String rootLogDir = "target/LocalLogs"; Path rootLogDirPath = new Path(rootLogDir); if (fs.exists(rootLogDirPath)) { fs.delete(rootLogDirPath, true); } assertTrue(fs.mkdirs(rootLogDirPath)); Path appLogsDir = new Path(rootLogDirPath, appId.toString()); if (fs.exists(appLogsDir)) { fs.delete(appLogsDir, true); } assertTrue(fs.mkdirs(appLogsDir)); List<String> rootLogDirs = Arrays.asList(rootLogDir); createContainerLogInLocalDir(appLogsDir, containerId1, fs); createContainerLogInLocalDir(appLogsDir, containerId2, fs); Path path = new Path(remoteLogRootDir + ugi.getShortUserName() + "/logs/application_0_0001"); if (fs.exists(path)) { fs.delete(path, true); } assertTrue(fs.mkdirs(path)); uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId0, path, fs); uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1, path, fs); uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1, path, fs); uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId2, path, fs); YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED); LogsCLI cli = new LogsCLIForTest(mockYarnClient); cli.setConf(configuration); int exitCode = cli.run(new String[] { "-applicationId", appId.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000001!")); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000002!")); sysOutStream.reset(); exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId1.toString() }); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( "Hello container_0_0001_01_000001!")); assertTrue(sysOutStream.toString().contains("Log Upload Time")); assertTrue(!sysOutStream.toString().contains( "Logs for container " + containerId1.toString() + " are not present in this log-file.")); sysOutStream.reset(); exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(), "-containerId", containerId0.toString() }); assertTrue(exitCode == -1); assertTrue(sysOutStream.toString().contains( "Logs for container " + containerId0.toString() + " are not present in this log-file.")); fs.delete(new Path(remoteLogRootDir), true); fs.delete(new Path(rootLogDir), true); }