JobHistoryEventHandler extends AbstractService implements EventHandler<JobHistoryEvent> { @Private public void handleEvent(JobHistoryEvent event) { synchronized (lock) { if (event.getHistoryEvent().getEventType() == EventType.AM_STARTED) { try { AMStartedEvent amStartedEvent = (AMStartedEvent) event.getHistoryEvent(); setupEventWriter(event.getJobID(), amStartedEvent.getForcedJobStateOnShutDown()); } catch (IOException ioe) { LOG.error("Error JobHistoryEventHandler in handleEvent: " + event, ioe); throw new YarnRuntimeException(ioe); } } MetaInfo mi = fileMap.get(event.getJobID()); try { HistoryEvent historyEvent = event.getHistoryEvent(); if (! (historyEvent instanceof NormalizedResourceEvent)) { mi.writeEvent(historyEvent); } processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(), event.getJobID()); if (timelineClient != null) { processEventForTimelineServer(historyEvent, event.getJobID(), event.getTimestamp()); } if (LOG.isDebugEnabled()) { LOG.debug("In HistoryEventHandler " + event.getHistoryEvent().getEventType()); } } catch (IOException e) { LOG.error("Error writing History Event: " + event.getHistoryEvent(), e); throw new YarnRuntimeException(e); } if (event.getHistoryEvent().getEventType() == EventType.JOB_SUBMITTED) { JobSubmittedEvent jobSubmittedEvent = (JobSubmittedEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setSubmitTime(jobSubmittedEvent.getSubmitTime()); mi.getJobIndexInfo().setQueueName(jobSubmittedEvent.getJobQueueName()); } if(event.getHistoryEvent().getEventType() == EventType.JOB_INITED ){ JobInitedEvent jie = (JobInitedEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setJobStartTime(jie.getLaunchTime()); } if (event.getHistoryEvent().getEventType() == EventType.JOB_QUEUE_CHANGED) { JobQueueChangeEvent jQueueEvent = (JobQueueChangeEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setQueueName(jQueueEvent.getJobQueueName()); } if (event.getHistoryEvent().getEventType() == EventType.JOB_FINISHED) { try { JobFinishedEvent jFinishedEvent = (JobFinishedEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setFinishTime(jFinishedEvent.getFinishTime()); mi.getJobIndexInfo().setNumMaps(jFinishedEvent.getFinishedMaps()); mi.getJobIndexInfo().setNumReduces( jFinishedEvent.getFinishedReduces()); mi.getJobIndexInfo().setJobStatus(JobState.SUCCEEDED.toString()); closeEventWriter(event.getJobID()); processDoneFiles(event.getJobID()); } catch (IOException e) { throw new YarnRuntimeException(e); } } if (event.getHistoryEvent().getEventType() == EventType.JOB_ERROR) { try { JobUnsuccessfulCompletionEvent jucEvent = (JobUnsuccessfulCompletionEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setFinishTime(jucEvent.getFinishTime()); mi.getJobIndexInfo().setNumMaps(jucEvent.getFinishedMaps()); mi.getJobIndexInfo().setNumReduces(jucEvent.getFinishedReduces()); mi.getJobIndexInfo().setJobStatus(jucEvent.getStatus()); closeEventWriter(event.getJobID()); if(context.isLastAMRetry()) processDoneFiles(event.getJobID()); } catch (IOException e) { throw new YarnRuntimeException(e); } } if (event.getHistoryEvent().getEventType() == EventType.JOB_FAILED || event.getHistoryEvent().getEventType() == EventType.JOB_KILLED) { try { JobUnsuccessfulCompletionEvent jucEvent = (JobUnsuccessfulCompletionEvent) event .getHistoryEvent(); mi.getJobIndexInfo().setFinishTime(jucEvent.getFinishTime()); mi.getJobIndexInfo().setNumMaps(jucEvent.getFinishedMaps()); mi.getJobIndexInfo().setNumReduces(jucEvent.getFinishedReduces()); mi.getJobIndexInfo().setJobStatus(jucEvent.getStatus()); closeEventWriter(event.getJobID()); processDoneFiles(event.getJobID()); } catch (IOException e) { throw new YarnRuntimeException(e); } } } } JobHistoryEventHandler(AppContext context, int startCount); void closeWriter(JobId id); @Override void handle(JobHistoryEvent event); @Private void handleEvent(JobHistoryEvent event); void processEventForJobSummary(HistoryEvent event, JobSummary summary, 
      JobId jobId); @Private JsonNode countersToJSON(Counters counters); void setForcejobCompletion(boolean forceJobCompletion);  }
@Test (timeout=50000) public void testFirstFlushOnCompletionEvent() throws Exception { TestParams t = new TestParams(); Configuration conf = new Configuration(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, t.workDir); conf.setLong(MRJobConfig.MR_AM_HISTORY_COMPLETE_EVENT_FLUSH_TIMEOUT_MS, 60 * 1000l); conf.setInt(MRJobConfig.MR_AM_HISTORY_JOB_COMPLETE_UNFLUSHED_MULTIPLIER, 10); conf.setInt(MRJobConfig.MR_AM_HISTORY_MAX_UNFLUSHED_COMPLETE_EVENTS, 10); conf.setInt( MRJobConfig.MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD, 200); JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext, 0); JHEvenHandlerForTest jheh = spy(realJheh); jheh.init(conf); EventWriter mockWriter = null; try { jheh.start(); handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000))); mockWriter = jheh.getEventWriter(); verify(mockWriter).write(any(HistoryEvent.class)); for (int i = 0; i < 100; i++) { queueEvent(jheh, new JobHistoryEvent(t.jobId, new TaskStartedEvent( t.taskID, 0, TaskType.MAP, ""))); } handleNextNEvents(jheh, 100); verify(mockWriter, times(0)).flush(); handleEvent(jheh, new JobHistoryEvent(t.jobId, new TaskFinishedEvent( t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null))); verify(mockWriter).flush(); } finally { jheh.stop(); verify(mockWriter).close(); } }
@Test (timeout=50000) public void testMaxUnflushedCompletionEvents() throws Exception { TestParams t = new TestParams(); Configuration conf = new Configuration(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, t.workDir); conf.setLong(MRJobConfig.MR_AM_HISTORY_COMPLETE_EVENT_FLUSH_TIMEOUT_MS, 60 * 1000l); conf.setInt(MRJobConfig.MR_AM_HISTORY_JOB_COMPLETE_UNFLUSHED_MULTIPLIER, 10); conf.setInt(MRJobConfig.MR_AM_HISTORY_MAX_UNFLUSHED_COMPLETE_EVENTS, 10); conf.setInt( MRJobConfig.MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD, 5); JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext, 0); JHEvenHandlerForTest jheh = spy(realJheh); jheh.init(conf); EventWriter mockWriter = null; try { jheh.start(); handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000))); mockWriter = jheh.getEventWriter(); verify(mockWriter).write(any(HistoryEvent.class)); for (int i = 0 ; i < 100 ; i++) { queueEvent(jheh, new JobHistoryEvent(t.jobId, new TaskFinishedEvent( t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null))); } handleNextNEvents(jheh, 9); verify(mockWriter, times(0)).flush(); handleNextNEvents(jheh, 1); verify(mockWriter).flush(); handleNextNEvents(jheh, 50); verify(mockWriter, times(6)).flush(); } finally { jheh.stop(); verify(mockWriter).close(); } }
@Test (timeout=50000) public void testUnflushedTimer() throws Exception { TestParams t = new TestParams(); Configuration conf = new Configuration(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, t.workDir); conf.setLong(MRJobConfig.MR_AM_HISTORY_COMPLETE_EVENT_FLUSH_TIMEOUT_MS, 2 * 1000l); conf.setInt(MRJobConfig.MR_AM_HISTORY_JOB_COMPLETE_UNFLUSHED_MULTIPLIER, 10); conf.setInt(MRJobConfig.MR_AM_HISTORY_MAX_UNFLUSHED_COMPLETE_EVENTS, 100); conf.setInt( MRJobConfig.MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD, 5); JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext, 0); JHEvenHandlerForTest jheh = spy(realJheh); jheh.init(conf); EventWriter mockWriter = null; try { jheh.start(); handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000))); mockWriter = jheh.getEventWriter(); verify(mockWriter).write(any(HistoryEvent.class)); for (int i = 0 ; i < 100 ; i++) { queueEvent(jheh, new JobHistoryEvent(t.jobId, new TaskFinishedEvent( t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null))); } handleNextNEvents(jheh, 9); verify(mockWriter, times(0)).flush(); Thread.sleep(2 * 4 * 1000l); verify(mockWriter).flush(); } finally { jheh.stop(); verify(mockWriter).close(); } }
@Test (timeout=50000) public void testBatchedFlushJobEndMultiplier() throws Exception { TestParams t = new TestParams(); Configuration conf = new Configuration(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, t.workDir); conf.setLong(MRJobConfig.MR_AM_HISTORY_COMPLETE_EVENT_FLUSH_TIMEOUT_MS, 60 * 1000l); conf.setInt(MRJobConfig.MR_AM_HISTORY_JOB_COMPLETE_UNFLUSHED_MULTIPLIER, 3); conf.setInt(MRJobConfig.MR_AM_HISTORY_MAX_UNFLUSHED_COMPLETE_EVENTS, 10); conf.setInt( MRJobConfig.MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD, 0); JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext, 0); JHEvenHandlerForTest jheh = spy(realJheh); jheh.init(conf); EventWriter mockWriter = null; try { jheh.start(); handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000))); mockWriter = jheh.getEventWriter(); verify(mockWriter).write(any(HistoryEvent.class)); for (int i = 0 ; i < 100 ; i++) { queueEvent(jheh, new JobHistoryEvent(t.jobId, new TaskFinishedEvent( t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null))); } queueEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( TypeConverter.fromYarn(t.jobId), 0, 10, 10, 0, 0, null, null, new Counters()))); handleNextNEvents(jheh, 29); verify(mockWriter, times(0)).flush(); handleNextNEvents(jheh, 72); verify(mockWriter, times(4)).flush(); } finally { jheh.stop(); verify(mockWriter).close(); } }
@Test (timeout=50000) public void testDefaultFsIsUsedForHistory() throws Exception { Configuration conf = new Configuration(); conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, dfsCluster.getURI().toString()); FileOutputStream os = new FileOutputStream(coreSitePath); conf.writeXml(os); os.close(); conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "file: TestParams t = new TestParams(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, t.dfsWorkDir); JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext, 0, false); JHEvenHandlerForTest jheh = spy(realJheh); jheh.init(conf); try { jheh.start(); handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000))); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters()))); FileSystem dfsFileSystem = dfsCluster.getFileSystem(); assertTrue("Minicluster contains some history files", dfsFileSystem.globStatus(new Path(t.dfsWorkDir + "/*")).length != 0); FileSystem localFileSystem = LocalFileSystem.get(conf); assertFalse("No history directory on non-default file system", localFileSystem.exists(new Path(t.dfsWorkDir))); } finally { jheh.stop(); } }
@Test (timeout=50000) public void testTimelineEventHandling() throws Exception { TestParams t = new TestParams(false); Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); MiniYARNCluster yarnCluster = null; long currentTime = System.currentTimeMillis(); try { yarnCluster = new MiniYARNCluster( TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1); yarnCluster.init(conf); yarnCluster.start(); Configuration confJHEH = new YarnConfiguration(conf); confJHEH.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true); confJHEH.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, MiniYARNCluster.getHostname() + ":" + yarnCluster.getApplicationHistoryServer().getPort()); JHEvenHandlerForTest jheh = new JHEvenHandlerForTest(t.mockAppContext, 0); jheh.init(confJHEH); jheh.start(); TimelineStore ts = yarnCluster.getApplicationHistoryServer() .getTimelineStore(); handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000), currentTime - 10)); TimelineEntities entities = ts.getEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null, null, null); Assert.assertEquals(1, entities.getEntities().size()); TimelineEntity tEntity = entities.getEntities().get(0); Assert.assertEquals(t.jobId.toString(), tEntity.getEntityId()); Assert.assertEquals(1, tEntity.getEvents().size()); Assert.assertEquals(EventType.AM_STARTED.toString(), tEntity.getEvents().get(0).getEventType()); Assert.assertEquals(currentTime - 10, tEntity.getEvents().get(0).getTimestamp()); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobSubmittedEvent(TypeConverter.fromYarn(t.jobId), "name", "user", 200, "/foo/job.xml", new HashMap<JobACL, AccessControlList>(), "default"), currentTime + 10)); entities = ts.getEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null, null, null); Assert.assertEquals(1, entities.getEntities().size()); tEntity = entities.getEntities().get(0); Assert.assertEquals(t.jobId.toString(), tEntity.getEntityId()); Assert.assertEquals(2, tEntity.getEvents().size()); Assert.assertEquals(EventType.JOB_SUBMITTED.toString(), tEntity.getEvents().get(0).getEventType()); Assert.assertEquals(EventType.AM_STARTED.toString(), tEntity.getEvents().get(1).getEventType()); Assert.assertEquals(currentTime + 10, tEntity.getEvents().get(0).getTimestamp()); Assert.assertEquals(currentTime - 10, tEntity.getEvents().get(1).getTimestamp()); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobQueueChangeEvent(TypeConverter.fromYarn(t.jobId), "q2"), currentTime - 20)); entities = ts.getEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null, null, null); Assert.assertEquals(1, entities.getEntities().size()); tEntity = entities.getEntities().get(0); Assert.assertEquals(t.jobId.toString(), tEntity.getEntityId()); Assert.assertEquals(3, tEntity.getEvents().size()); Assert.assertEquals(EventType.JOB_SUBMITTED.toString(), tEntity.getEvents().get(0).getEventType()); Assert.assertEquals(EventType.AM_STARTED.toString(), tEntity.getEvents().get(1).getEventType()); Assert.assertEquals(EventType.JOB_QUEUE_CHANGED.toString(), tEntity.getEvents().get(2).getEventType()); Assert.assertEquals(currentTime + 10, tEntity.getEvents().get(0).getTimestamp()); Assert.assertEquals(currentTime - 10, tEntity.getEvents().get(1).getTimestamp()); Assert.assertEquals(currentTime - 20, tEntity.getEvents().get(2).getTimestamp()); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent(TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters()), currentTime)); entities = ts.getEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null, null, null); Assert.assertEquals(1, entities.getEntities().size()); tEntity = entities.getEntities().get(0); Assert.assertEquals(t.jobId.toString(), tEntity.getEntityId()); Assert.assertEquals(4, tEntity.getEvents().size()); Assert.assertEquals(EventType.JOB_SUBMITTED.toString(), tEntity.getEvents().get(0).getEventType()); Assert.assertEquals(EventType.JOB_FINISHED.toString(), tEntity.getEvents().get(1).getEventType()); Assert.assertEquals(EventType.AM_STARTED.toString(), tEntity.getEvents().get(2).getEventType()); Assert.assertEquals(EventType.JOB_QUEUE_CHANGED.toString(), tEntity.getEvents().get(3).getEventType()); Assert.assertEquals(currentTime + 10, tEntity.getEvents().get(0).getTimestamp()); Assert.assertEquals(currentTime, tEntity.getEvents().get(1).getTimestamp()); Assert.assertEquals(currentTime - 10, tEntity.getEvents().get(2).getTimestamp()); Assert.assertEquals(currentTime - 20, tEntity.getEvents().get(3).getTimestamp()); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn(t.jobId), 0, 0, 0, JobStateInternal.KILLED.toString()), currentTime + 20)); entities = ts.getEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null, null, null); Assert.assertEquals(1, entities.getEntities().size()); tEntity = entities.getEntities().get(0); Assert.assertEquals(t.jobId.toString(), tEntity.getEntityId()); Assert.assertEquals(5, tEntity.getEvents().size()); Assert.assertEquals(EventType.JOB_KILLED.toString(), tEntity.getEvents().get(0).getEventType()); Assert.assertEquals(EventType.JOB_SUBMITTED.toString(), tEntity.getEvents().get(1).getEventType()); Assert.assertEquals(EventType.JOB_FINISHED.toString(), tEntity.getEvents().get(2).getEventType()); Assert.assertEquals(EventType.AM_STARTED.toString(), tEntity.getEvents().get(3).getEventType()); Assert.assertEquals(EventType.JOB_QUEUE_CHANGED.toString(), tEntity.getEvents().get(4).getEventType()); Assert.assertEquals(currentTime + 20, tEntity.getEvents().get(0).getTimestamp()); Assert.assertEquals(currentTime + 10, tEntity.getEvents().get(1).getTimestamp()); Assert.assertEquals(currentTime, tEntity.getEvents().get(2).getTimestamp()); Assert.assertEquals(currentTime - 10, tEntity.getEvents().get(3).getTimestamp()); Assert.assertEquals(currentTime - 20, tEntity.getEvents().get(4).getTimestamp()); handleEvent(jheh, new JobHistoryEvent(t.jobId, new TaskStartedEvent(t.taskID, 0, TaskType.MAP, ""))); entities = ts.getEntities("MAPREDUCE_TASK", null, null, null, null, null, null, null, null, null); Assert.assertEquals(1, entities.getEntities().size()); tEntity = entities.getEntities().get(0); Assert.assertEquals(t.taskID.toString(), tEntity.getEntityId()); Assert.assertEquals(1, tEntity.getEvents().size()); Assert.assertEquals(EventType.TASK_STARTED.toString(), tEntity.getEvents().get(0).getEventType()); Assert.assertEquals(TaskType.MAP.toString(), tEntity.getEvents().get(0).getEventInfo().get("TASK_TYPE")); handleEvent(jheh, new JobHistoryEvent(t.jobId, new TaskStartedEvent(t.taskID, 0, TaskType.REDUCE, ""))); entities = ts.getEntities("MAPREDUCE_TASK", null, null, null, null, null, null, null, null, null); Assert.assertEquals(1, entities.getEntities().size()); tEntity = entities.getEntities().get(0); Assert.assertEquals(t.taskID.toString(), tEntity.getEntityId()); Assert.assertEquals(2, tEntity.getEvents().size()); Assert.assertEquals(EventType.TASK_STARTED.toString(), tEntity.getEvents().get(1).getEventType()); Assert.assertEquals(TaskType.REDUCE.toString(), tEntity.getEvents().get(0).getEventInfo().get("TASK_TYPE")); Assert.assertEquals(TaskType.MAP.toString(), tEntity.getEvents().get(1).getEventInfo().get("TASK_TYPE")); } finally { if (yarnCluster != null) { yarnCluster.stop(); } } }