@Test public void inferDateFromLog() throws Exception { String alluxioLog = "2020-03-19 11:58:10,104 WARN ServerConfiguration - Reloaded properties\n" + "2020-03-19 11:58:10,106 WARN ServerConfiguration - Loaded hostname localhost\n" + "2020-03-19 11:58:10,591 WARN RetryUtils - Failed to load cluster default.."; File alluxioLogFile = new File(mTestDir, "alluxio-worker.log"); writeToFile(alluxioLogFile, alluxioLog); LocalDateTime alluxioLogDatetime = CollectLogCommand.inferFileStartTime(alluxioLogFile); LocalDateTime expectedDatetime = LocalDateTime.of(2020, 3, 19, 11, 58, 10, 104 * MILLISEC_TO_NANOSEC); assertThat(String.format("Expected datetime is %s but inferred %s from file%n", expectedDatetime, alluxioLogDatetime), alluxioLogDatetime, new DatetimeMatcher().setDatetime(expectedDatetime)); String yarnAppLog = "\nLogged in as: user\nApplication\nAbout\nJobs\nTools\n" + "Log Type: container-localizer-syslog\n\n" + "Log Upload Time: Mon May 18 16:11:22 +0800 2020\n\n" + "Log Length: 0\n\n\nLog Type: stderr\n\n" + "Log Upload Time: Mon May 18 16:11:22 +0800 2020\n\n" + "Log Length: 3616\n\n" + "20/05/18 16:11:18 INFO util.SignalUtils: Registered signal handler for TERM\n" + "20/05/18 16:11:18 INFO util.SignalUtils: Registered signal handler for HUP\n" + "20/05/18 16:11:18 INFO util.SignalUtils: Registered signal handler for INT"; File yarnAppLogFile = new File(mTestDir, "yarn-application.log"); writeToFile(yarnAppLogFile, yarnAppLog); LocalDateTime yarnAppDatetime = CollectLogCommand.inferFileStartTime(yarnAppLogFile); expectedDatetime = LocalDateTime.of(2020, 5, 18, 16, 11, 18); assertThat(String.format("Expected datetime is %s but inferred %s from file%n", expectedDatetime, yarnAppDatetime), yarnAppDatetime, new DatetimeMatcher().setDatetime(expectedDatetime)); String yarnLog = "2020-05-16 02:02:25,855 INFO org.apache.hadoop.yarn.server.resourcemanager" + ".rmcontainer.RMContainerImpl: container_e103_1584954066020_230169_01_000004 " + "Container Transitioned from ALLOCATED to ACQUIRED\n" + "2020-05-16 02:02:25,909 INFO org.apache.hadoop.yarn.server.resourcemanager" + ".scheduler.AppSchedulingInfo: checking for deactivate... \n" + "2020-05-16 02:02:26,006 INFO org.apache.hadoop.yarn.server.resourcemanager" + ".rmcontainer.RMContainerImpl: container_e103_1584954066020_230168_01_000047 " + "Container Transitioned from ALLOCATED to ACQUIRED"; File yarnLogFile = new File(mTestDir, "yarn-rm.log"); writeToFile(yarnLogFile, yarnLog); LocalDateTime yarnDatetime = CollectLogCommand.inferFileStartTime(yarnLogFile); expectedDatetime = LocalDateTime.of(2020, 5, 16, 2, 2, 25, 855 * MILLISEC_TO_NANOSEC); assertThat(String.format("Expected datetime is %s but inferred %s from file%n", expectedDatetime, yarnDatetime), yarnDatetime, new DatetimeMatcher().setDatetime(expectedDatetime)); String zkLog = "2020-05-14 21:05:53,822 WARN org.apache.zookeeper.server.NIOServerCnxn: " + "caught end of stream exception\n" + "EndOfStreamException: Unable to read additional data from client sessionid.., " + "likely client has closed socket\n" + "\tat org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:241)\n" + "\tat org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory..)\n" + "\tat java.lang.Thread.run(Thread.java:748)\n" + "2020-05-14 21:05:53,823 INFO org.apache.zookeeper.server.NIOServerCnxn: " + "Closed socket connection for client /10.64.23.190:50120 which had sessionid..\n" + "2020-05-14 21:05:53,911 WARN org.apache.zookeeper.server.NIOServerCnxn: " + "caught end of stream exception\n" + "EndOfStreamException: Unable to read additional data from client sessionid.." + "likely client has closed socket\n"; File zkLogFile = new File(mTestDir, "zk.log"); writeToFile(zkLogFile, zkLog); LocalDateTime zkDatetime = CollectLogCommand.inferFileStartTime(zkLogFile); expectedDatetime = LocalDateTime.of(2020, 5, 14, 21, 5, 53, 822 * MILLISEC_TO_NANOSEC); assertThat(String.format("Expected datetime is %s but inferred %s from file%n", expectedDatetime, zkDatetime), zkDatetime, new DatetimeMatcher().setDatetime(expectedDatetime)); String hdfsLog = "2020-05-15 22:02:27,878 INFO BlockStateChange: BLOCK* addStoredBlock: " + "blockMap updated: 10.64.23.184:1025 is added to blk_1126197354_52572663 size 0..\n" + "2020-05-15 22:02:27,878 INFO BlockStateChange: BLOCK* addStoredBlock: blockMap " + "updated: 10.70.22.117:1025 is added to blk_1126197354_52572663 size 0.."; File hdfsLogFile = new File(mTestDir, "hdfs.log"); writeToFile(hdfsLogFile, hdfsLog); LocalDateTime hdfsDatetime = CollectLogCommand.inferFileStartTime(hdfsLogFile); expectedDatetime = LocalDateTime.of(2020, 5, 15, 22, 2, 27, 878 * MILLISEC_TO_NANOSEC); assertThat(String.format("Expected datetime is %s but inferred %s from file%n", expectedDatetime, hdfsDatetime), hdfsDatetime, new DatetimeMatcher().setDatetime(expectedDatetime)); String prestoLog = "2020-05-16T00:00:01.059+0800\tINFO\tdispatcher-query-7960" + "\tio.prestosql.event.QueryMonitor\tTIMELINE: Query 20200515_155959_06700_6r6b4" + " :: Transaction:[4d30e960-c319-439c-84dd-022ddab6fa5e] :: elapsed 1208ms :: " + "planning 0ms :: waiting 0ms :: scheduling 1208ms :: running 0ms :: finishing 1208ms" + " :: begin 2020-05-15T23:59:59.850+08:00 :: end 2020-05-16T00:00:01.058+08:00\n" + "2020-05-16T00:00:03.530+0800\tINFO\tdispatcher-query-7948\t" + "io.prestosql.event.QueryMonitor\tTIMELINE: Query 20200515_160001_06701_6r6b4" + " :: Transaction:[be9b396e-6697-4ecd-9782-2ca1174c1be1] :: elapsed 2316ms" + " :: planning 0ms :: waiting 0ms :: scheduling 2316ms :: running 0ms" + " :: finishing 2316ms :: begin 2020-05-16T00:00:01.212+08:00" + " :: end 2020-05-16T00:00:03.528+08:00"; File prestoLogFile = new File(mTestDir, "presto.log"); writeToFile(prestoLogFile, prestoLog); LocalDateTime prestoDatetime = CollectLogCommand.inferFileStartTime(prestoLogFile); expectedDatetime = LocalDateTime.of(2020, 5, 16, 0, 0, 1, 59 * MILLISEC_TO_NANOSEC); assertThat(String.format("Expected datetime is %s but inferred %s from file%n", expectedDatetime, prestoDatetime), prestoDatetime, new DatetimeMatcher().setDatetime(expectedDatetime)); String gcLog = "Java HotSpot(TM) 64-Bit Server VM (25.151-b12) for linux-amd64 JRE (1.8.0), " + "built on Sep 5 2017 19:20:58 by \"java_re\" with gcc 4.3.0 20080428..\n" + "Memory: 4k page, physical 1979200k(1940096k free), swap 332799k(332799k free)\n" + "CommandLine flags: -XX:CMSInitiatingOccupancyFraction=65 -XX:InitialHeapSize=..\n" + "2020-05-07T10:01:11.409+0800: 4.304: [GC (GCLocker Initiated GC) " + "2020-05-07T10:01:11.410+0800: 4.304: [ParNew: 25669137K->130531K(28311552K), " + "0.4330954 secs] 25669137K->130531K(147849216K), 0.4332463 secs] " + "[Times: user=4.50 sys=0.08, real=0.44 secs] "; File gcLogFile = new File(mTestDir, "gc.log"); writeToFile(gcLogFile, gcLog); LocalDateTime gcDatetime = CollectLogCommand.inferFileStartTime(gcLogFile); expectedDatetime = LocalDateTime.of(2020, 5, 7, 10, 1, 11, 409 * MILLISEC_TO_NANOSEC); assertThat(String.format("Expected datetime is %s but inferred %s from file%n", expectedDatetime, gcDatetime), gcDatetime, new DatetimeMatcher().setDatetime(expectedDatetime)); }
public static LocalDateTime inferFileStartTime(File f) throws FileNotFoundException { int r = 0; try (Scanner scanner = new Scanner(f)) { while (scanner.hasNextLine() && r < TRY_PARSE_LOG_ROWS) { String line = scanner.nextLine(); LocalDateTime datetime = parseDateTime(line); if (datetime != null) { return datetime; } r++; } } return null; }
CollectLogCommand extends AbstractCollectInfoCommand { public static LocalDateTime inferFileStartTime(File f) throws FileNotFoundException { int r = 0; try (Scanner scanner = new Scanner(f)) { while (scanner.hasNextLine() && r < TRY_PARSE_LOG_ROWS) { String line = scanner.nextLine(); LocalDateTime datetime = parseDateTime(line); if (datetime != null) { return datetime; } r++; } } return null; } }
CollectLogCommand extends AbstractCollectInfoCommand { public static LocalDateTime inferFileStartTime(File f) throws FileNotFoundException { int r = 0; try (Scanner scanner = new Scanner(f)) { while (scanner.hasNextLine() && r < TRY_PARSE_LOG_ROWS) { String line = scanner.nextLine(); LocalDateTime datetime = parseDateTime(line); if (datetime != null) { return datetime; } r++; } } return null; } CollectLogCommand(FileSystemContext fsContext); }
CollectLogCommand extends AbstractCollectInfoCommand { public static LocalDateTime inferFileStartTime(File f) throws FileNotFoundException { int r = 0; try (Scanner scanner = new Scanner(f)) { while (scanner.hasNextLine() && r < TRY_PARSE_LOG_ROWS) { String line = scanner.nextLine(); LocalDateTime datetime = parseDateTime(line); if (datetime != null) { return datetime; } r++; } } return null; } CollectLogCommand(FileSystemContext fsContext); @Override String getCommandName(); @Override boolean hasSubCommand(); @Override int run(CommandLine cl); static LocalDateTime inferFileStartTime(File f); @Override String getUsage(); @Override String getDescription(); @Nullable static LocalDateTime parseDateTime(String s); }
CollectLogCommand extends AbstractCollectInfoCommand { public static LocalDateTime inferFileStartTime(File f) throws FileNotFoundException { int r = 0; try (Scanner scanner = new Scanner(f)) { while (scanner.hasNextLine() && r < TRY_PARSE_LOG_ROWS) { String line = scanner.nextLine(); LocalDateTime datetime = parseDateTime(line); if (datetime != null) { return datetime; } r++; } } return null; } CollectLogCommand(FileSystemContext fsContext); @Override String getCommandName(); @Override boolean hasSubCommand(); @Override int run(CommandLine cl); static LocalDateTime inferFileStartTime(File f); @Override String getUsage(); @Override String getDescription(); @Nullable static LocalDateTime parseDateTime(String s); static final String COMMAND_NAME; static final Set<String> FILE_NAMES_PREFIXES; static final String INCLUDE_OPTION_NAME; static final String EXCLUDE_OPTION_NAME; static final String ADDITIONAL_OPTION_NAME; static final Options OPTIONS; }