TypeConverter { public static org.apache.hadoop.mapred.JobID fromYarn(JobId id) { String identifier = fromClusterTimeStamp(id.getAppId().getClusterTimestamp()); return new org.apache.hadoop.mapred.JobID(identifier, id.getId()); }  static org.apache.hadoop.mapred.JobID fromYarn(JobId id); static org.apache.hadoop.mapreduce.JobID fromYarn(ApplicationId appID); static JobId toYarn(org.apache.hadoop.mapreduce.JobID id); static int toYarnApplicationPriority(String priority); static org.apache.hadoop.mapreduce.TaskType fromYarn(
      TaskType taskType); static TaskType toYarn(org.apache.hadoop.mapreduce.TaskType taskType); static org.apache.hadoop.mapred.TaskID fromYarn(TaskId id); static TaskId toYarn(org.apache.hadoop.mapreduce.TaskID id); static TaskAttemptState toYarn(
      org.apache.hadoop.mapred.TaskStatus.State state); static Phase toYarn(org.apache.hadoop.mapred.TaskStatus.Phase phase); static TaskCompletionEvent[] fromYarn(
      TaskAttemptCompletionEvent[] newEvents); static TaskCompletionEvent fromYarn(
      TaskAttemptCompletionEvent newEvent); static TaskCompletionEvent.Status fromYarn(
      TaskAttemptCompletionEventStatus newStatus); static org.apache.hadoop.mapred.TaskAttemptID fromYarn(
      TaskAttemptId id); static TaskAttemptId toYarn(
      org.apache.hadoop.mapred.TaskAttemptID id); static TaskAttemptId toYarn(
      org.apache.hadoop.mapreduce.TaskAttemptID id); static org.apache.hadoop.mapreduce.Counters fromYarn(
      Counters yCntrs); static Counters toYarn(org.apache.hadoop.mapred.Counters counters); static Counters toYarn(org.apache.hadoop.mapreduce.Counters counters); static JobStatus fromYarn(JobReport jobreport, String trackingUrl); static org.apache.hadoop.mapreduce.JobPriority fromYarnApplicationPriority(int priority); static org.apache.hadoop.mapreduce.QueueState fromYarn(
      QueueState state); static int fromYarn(JobState state); static org.apache.hadoop.mapred.TIPStatus fromYarn(
      TaskState state); static TaskReport fromYarn(org.apache.hadoop.mapreduce.v2.api.records.TaskReport report); static List<TaskReport> fromYarn(
      List<org.apache.hadoop.mapreduce.v2.api.records.TaskReport> taskReports); static State fromYarn(YarnApplicationState yarnApplicationState,
      FinalApplicationStatus finalApplicationStatus); static TaskTrackerInfo fromYarn(NodeReport node); static TaskTrackerInfo[] fromYarnNodes(List<NodeReport> nodes); static JobStatus fromYarn(ApplicationReport application,
      String jobFile); static JobStatus[] fromYarnApps(List<ApplicationReport> applications,
      Configuration conf); static QueueInfo fromYarn(org.apache.hadoop.yarn.api.records.QueueInfo
      queueInfo, Configuration conf); static QueueInfo[] fromYarnQueueInfo(
      List<org.apache.hadoop.yarn.api.records.QueueInfo> queues,
      Configuration conf); static QueueAclsInfo[] fromYarnQueueUserAclsInfo(
      List<QueueUserACLInfo> userAcls);  }
@Test public void testEnums() throws Exception { for (YarnApplicationState applicationState : YarnApplicationState.values()) { TypeConverter.fromYarn(applicationState, FinalApplicationStatus.FAILED); } Assert.assertEquals(State.PREP, TypeConverter.fromYarn( YarnApplicationState.NEW_SAVING, FinalApplicationStatus.FAILED)); for (TaskType taskType : TaskType.values()) { TypeConverter.fromYarn(taskType); } for (JobState jobState : JobState.values()) { TypeConverter.fromYarn(jobState); } for (QueueState queueState : QueueState.values()) { TypeConverter.fromYarn(queueState); } for (TaskState taskState : TaskState.values()) { TypeConverter.fromYarn(taskState); } }
@Test public void testFromYarn() throws Exception { int appStartTime = 612354; int appFinishTime = 612355; YarnApplicationState state = YarnApplicationState.RUNNING; ApplicationId applicationId = ApplicationId.newInstance(0, 0); ApplicationReport applicationReport = Records .newRecord(ApplicationReport.class); applicationReport.setApplicationId(applicationId); applicationReport.setYarnApplicationState(state); applicationReport.setStartTime(appStartTime); applicationReport.setFinishTime(appFinishTime); applicationReport.setUser("TestTypeConverter-user"); applicationReport.setPriority(Priority.newInstance(3)); ApplicationResourceUsageReport appUsageRpt = Records .newRecord(ApplicationResourceUsageReport.class); Resource r = Records.newRecord(Resource.class); r.setMemorySize(2048); appUsageRpt.setNeededResources(r); appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); appUsageRpt.setReservedResources(r); appUsageRpt.setUsedResources(r); applicationReport.setApplicationResourceUsageReport(appUsageRpt); JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile"); Assert.assertEquals(appStartTime, jobStatus.getStartTime()); Assert.assertEquals(appFinishTime, jobStatus.getFinishTime()); Assert.assertEquals(state.toString(), jobStatus.getState().toString()); Assert.assertEquals(JobPriority.NORMAL, jobStatus.getPriority()); }
@Test public void testFromYarnApplicationReport() { ApplicationId mockAppId = mock(ApplicationId.class); when(mockAppId.getClusterTimestamp()).thenReturn(12345L); when(mockAppId.getId()).thenReturn(6789); ApplicationReport mockReport = mock(ApplicationReport.class); when(mockReport.getTrackingUrl()).thenReturn("dummy-tracking-url"); when(mockReport.getApplicationId()).thenReturn(mockAppId); when(mockReport.getYarnApplicationState()).thenReturn(YarnApplicationState.KILLED); when(mockReport.getUser()).thenReturn("dummy-user"); when(mockReport.getQueue()).thenReturn("dummy-queue"); when(mockReport.getPriority()).thenReturn(Priority.newInstance(4)); String jobFile = "dummy-path/job.xml"; try { JobStatus status = TypeConverter.fromYarn(mockReport, jobFile); } catch (NullPointerException npe) { Assert.fail("Type converstion from YARN fails for jobs without " + "ApplicationUsageReport"); } ApplicationResourceUsageReport appUsageRpt = Records .newRecord(ApplicationResourceUsageReport.class); Resource r = Records.newRecord(Resource.class); r.setMemorySize(2048); appUsageRpt.setNeededResources(r); appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); appUsageRpt.setReservedResources(r); appUsageRpt.setUsedResources(r); when(mockReport.getApplicationResourceUsageReport()).thenReturn(appUsageRpt); JobStatus status = TypeConverter.fromYarn(mockReport, jobFile); Assert.assertNotNull("fromYarn returned null status", status); Assert.assertEquals("jobFile set incorrectly", "dummy-path/job.xml", status.getJobFile()); Assert.assertEquals("queue set incorrectly", "dummy-queue", status.getQueue()); Assert.assertEquals("trackingUrl set incorrectly", "dummy-tracking-url", status.getTrackingUrl()); Assert.assertEquals("user set incorrectly", "dummy-user", status.getUsername()); Assert.assertEquals("schedulingInfo set incorrectly", "dummy-tracking-url", status.getSchedulingInfo()); Assert.assertEquals("jobId set incorrectly", 6789, status.getJobID().getId()); Assert.assertEquals("state set incorrectly", JobStatus.State.KILLED, status.getState()); Assert.assertEquals("needed mem info set incorrectly", 2048, status.getNeededMem()); Assert.assertEquals("num rsvd slots info set incorrectly", 1, status.getNumReservedSlots()); Assert.assertEquals("num used slots info set incorrectly", 3, status.getNumUsedSlots()); Assert.assertEquals("rsvd mem info set incorrectly", 2048, status.getReservedMem()); Assert.assertEquals("used mem info set incorrectly", 2048, status.getUsedMem()); Assert.assertEquals("priority set incorrectly", JobPriority.HIGH, status.getPriority()); }
@Test public void testFromYarnQueue() { org.apache.hadoop.yarn.api.records.QueueInfo child = Mockito.mock(org.apache.hadoop.yarn.api.records.QueueInfo.class); Mockito.when(child.getQueueState()).thenReturn(QueueState.RUNNING); org.apache.hadoop.yarn.api.records.QueueInfo queueInfo = Mockito.mock(org.apache.hadoop.yarn.api.records.QueueInfo.class); List<org.apache.hadoop.yarn.api.records.QueueInfo> children = new ArrayList<org.apache.hadoop.yarn.api.records.QueueInfo>(); children.add(child); Mockito.when(queueInfo.getChildQueues()).thenReturn(children); Mockito.when(queueInfo.getQueueState()).thenReturn(QueueState.RUNNING); org.apache.hadoop.mapreduce.QueueInfo returned = TypeConverter.fromYarn(queueInfo, new Configuration()); assertThat(returned.getQueueChildren().size()) .withFailMessage("QueueInfo children weren't properly converted") .isEqualTo(1); }
@Test public void testFromYarnJobReport() throws Exception { int jobStartTime = 612354; int jobFinishTime = 612355; JobState state = JobState.RUNNING; JobId jobId = Records.newRecord(JobId.class); JobReport jobReport = Records.newRecord(JobReport.class); ApplicationId applicationId = ApplicationId.newInstance(0, 0); jobId.setAppId(applicationId); jobId.setId(0); jobReport.setJobId(jobId); jobReport.setJobState(state); jobReport.setStartTime(jobStartTime); jobReport.setFinishTime(jobFinishTime); jobReport.setUser("TestTypeConverter-user"); jobReport.setJobPriority(Priority.newInstance(0)); JobStatus jobStatus = TypeConverter.fromYarn(jobReport, "dummy-jobfile"); Assert.assertEquals(jobStartTime, jobStatus.getStartTime()); Assert.assertEquals(jobFinishTime, jobStatus.getFinishTime()); Assert.assertEquals(state.toString(), jobStatus.getState().toString()); Assert.assertEquals(JobPriority.DEFAULT, jobStatus.getPriority()); }