@Test public void testUpdateJobCounterCatchesEveryInternalException() throws IOException { Job job = Job.getInstance(new Configuration(false)); HadoopCmdOutput hadoopCmdOutput = new HadoopCmdOutput(job, new StringBuilder()); hadoopCmdOutput.updateJobCounter(); }
public void updateJobCounter() { try { Counters counters = job.getCounters(); if (counters == null) { String errorMsg = "no counters for job " + getMrJobId(); logger.warn(errorMsg); output.append(errorMsg); } else { this.output.append(counters.toString()).append("\n"); logger.debug(counters.toString()); mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue()); rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue()); String outputFolder = job.getConfiguration().get("mapreduce.output.fileoutputformat.outputdir", KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); logger.debug("outputFolder is " + outputFolder); Path outputPath = new Path(outputFolder); String fsScheme = outputPath.getFileSystem(job.getConfiguration()).getScheme(); long bytesWritten = counters.findCounter(fsScheme, FileSystemCounter.BYTES_WRITTEN).getValue(); if (bytesWritten == 0) { logger.debug("Seems no counter found for " + fsScheme); bytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue(); } hdfsBytesWritten = String.valueOf(bytesWritten); } JobStatus jobStatus = job.getStatus(); if (jobStatus.getState() == JobStatus.State.FAILED) { logger.warn("Job Diagnostics:" + jobStatus.getFailureInfo()); output.append("Job Diagnostics:").append(jobStatus.getFailureInfo()).append("\n"); TaskCompletionEvent taskEvent = getOneTaskFailure(job); if (taskEvent != null) { String[] fails = job.getTaskDiagnostics(taskEvent.getTaskAttemptId()); logger.warn("Failure task Diagnostics:"); output.append("Failure task Diagnostics:").append("\n"); for (String failure : fails) { logger.warn(failure); output.append(failure).append("\n"); } } } } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); output.append(e.getLocalizedMessage()); } }
HadoopCmdOutput { public void updateJobCounter() { try { Counters counters = job.getCounters(); if (counters == null) { String errorMsg = "no counters for job " + getMrJobId(); logger.warn(errorMsg); output.append(errorMsg); } else { this.output.append(counters.toString()).append("\n"); logger.debug(counters.toString()); mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue()); rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue()); String outputFolder = job.getConfiguration().get("mapreduce.output.fileoutputformat.outputdir", KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); logger.debug("outputFolder is " + outputFolder); Path outputPath = new Path(outputFolder); String fsScheme = outputPath.getFileSystem(job.getConfiguration()).getScheme(); long bytesWritten = counters.findCounter(fsScheme, FileSystemCounter.BYTES_WRITTEN).getValue(); if (bytesWritten == 0) { logger.debug("Seems no counter found for " + fsScheme); bytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue(); } hdfsBytesWritten = String.valueOf(bytesWritten); } JobStatus jobStatus = job.getStatus(); if (jobStatus.getState() == JobStatus.State.FAILED) { logger.warn("Job Diagnostics:" + jobStatus.getFailureInfo()); output.append("Job Diagnostics:").append(jobStatus.getFailureInfo()).append("\n"); TaskCompletionEvent taskEvent = getOneTaskFailure(job); if (taskEvent != null) { String[] fails = job.getTaskDiagnostics(taskEvent.getTaskAttemptId()); logger.warn("Failure task Diagnostics:"); output.append("Failure task Diagnostics:").append("\n"); for (String failure : fails) { logger.warn(failure); output.append(failure).append("\n"); } } } } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); output.append(e.getLocalizedMessage()); } } }
HadoopCmdOutput { public void updateJobCounter() { try { Counters counters = job.getCounters(); if (counters == null) { String errorMsg = "no counters for job " + getMrJobId(); logger.warn(errorMsg); output.append(errorMsg); } else { this.output.append(counters.toString()).append("\n"); logger.debug(counters.toString()); mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue()); rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue()); String outputFolder = job.getConfiguration().get("mapreduce.output.fileoutputformat.outputdir", KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); logger.debug("outputFolder is " + outputFolder); Path outputPath = new Path(outputFolder); String fsScheme = outputPath.getFileSystem(job.getConfiguration()).getScheme(); long bytesWritten = counters.findCounter(fsScheme, FileSystemCounter.BYTES_WRITTEN).getValue(); if (bytesWritten == 0) { logger.debug("Seems no counter found for " + fsScheme); bytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue(); } hdfsBytesWritten = String.valueOf(bytesWritten); } JobStatus jobStatus = job.getStatus(); if (jobStatus.getState() == JobStatus.State.FAILED) { logger.warn("Job Diagnostics:" + jobStatus.getFailureInfo()); output.append("Job Diagnostics:").append(jobStatus.getFailureInfo()).append("\n"); TaskCompletionEvent taskEvent = getOneTaskFailure(job); if (taskEvent != null) { String[] fails = job.getTaskDiagnostics(taskEvent.getTaskAttemptId()); logger.warn("Failure task Diagnostics:"); output.append("Failure task Diagnostics:").append("\n"); for (String failure : fails) { logger.warn(failure); output.append(failure).append("\n"); } } } } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); output.append(e.getLocalizedMessage()); } } HadoopCmdOutput(Job job, StringBuilder output); }
HadoopCmdOutput { public void updateJobCounter() { try { Counters counters = job.getCounters(); if (counters == null) { String errorMsg = "no counters for job " + getMrJobId(); logger.warn(errorMsg); output.append(errorMsg); } else { this.output.append(counters.toString()).append("\n"); logger.debug(counters.toString()); mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue()); rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue()); String outputFolder = job.getConfiguration().get("mapreduce.output.fileoutputformat.outputdir", KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); logger.debug("outputFolder is " + outputFolder); Path outputPath = new Path(outputFolder); String fsScheme = outputPath.getFileSystem(job.getConfiguration()).getScheme(); long bytesWritten = counters.findCounter(fsScheme, FileSystemCounter.BYTES_WRITTEN).getValue(); if (bytesWritten == 0) { logger.debug("Seems no counter found for " + fsScheme); bytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue(); } hdfsBytesWritten = String.valueOf(bytesWritten); } JobStatus jobStatus = job.getStatus(); if (jobStatus.getState() == JobStatus.State.FAILED) { logger.warn("Job Diagnostics:" + jobStatus.getFailureInfo()); output.append("Job Diagnostics:").append(jobStatus.getFailureInfo()).append("\n"); TaskCompletionEvent taskEvent = getOneTaskFailure(job); if (taskEvent != null) { String[] fails = job.getTaskDiagnostics(taskEvent.getTaskAttemptId()); logger.warn("Failure task Diagnostics:"); output.append("Failure task Diagnostics:").append("\n"); for (String failure : fails) { logger.warn(failure); output.append(failure).append("\n"); } } } } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); output.append(e.getLocalizedMessage()); } } HadoopCmdOutput(Job job, StringBuilder output); String getMrJobId(); Map<String, String> getInfo(); String getMapInputRecords(); String getHdfsBytesWritten(); String getRawInputBytesRead(); void updateJobCounter(); }
HadoopCmdOutput { public void updateJobCounter() { try { Counters counters = job.getCounters(); if (counters == null) { String errorMsg = "no counters for job " + getMrJobId(); logger.warn(errorMsg); output.append(errorMsg); } else { this.output.append(counters.toString()).append("\n"); logger.debug(counters.toString()); mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue()); rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue()); String outputFolder = job.getConfiguration().get("mapreduce.output.fileoutputformat.outputdir", KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); logger.debug("outputFolder is " + outputFolder); Path outputPath = new Path(outputFolder); String fsScheme = outputPath.getFileSystem(job.getConfiguration()).getScheme(); long bytesWritten = counters.findCounter(fsScheme, FileSystemCounter.BYTES_WRITTEN).getValue(); if (bytesWritten == 0) { logger.debug("Seems no counter found for " + fsScheme); bytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue(); } hdfsBytesWritten = String.valueOf(bytesWritten); } JobStatus jobStatus = job.getStatus(); if (jobStatus.getState() == JobStatus.State.FAILED) { logger.warn("Job Diagnostics:" + jobStatus.getFailureInfo()); output.append("Job Diagnostics:").append(jobStatus.getFailureInfo()).append("\n"); TaskCompletionEvent taskEvent = getOneTaskFailure(job); if (taskEvent != null) { String[] fails = job.getTaskDiagnostics(taskEvent.getTaskAttemptId()); logger.warn("Failure task Diagnostics:"); output.append("Failure task Diagnostics:").append("\n"); for (String failure : fails) { logger.warn(failure); output.append(failure).append("\n"); } } } } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); output.append(e.getLocalizedMessage()); } } HadoopCmdOutput(Job job, StringBuilder output); String getMrJobId(); Map<String, String> getInfo(); String getMapInputRecords(); String getHdfsBytesWritten(); String getRawInputBytesRead(); void updateJobCounter();  }