@Test public void parse() throws IOException { String pigCountersJson = ClassPathResourceProvider .getResourceContent("/eu/dnetlib/iis/common/report/pigCounters.json"); PigCounters pigCounters = pigCountersParser.parse(pigCountersJson); Map<String, String> rootLevelCounters = pigCounters.getRootLevelCounters(); assertEquals(15, rootLevelCounters.size()); assertEquals("4", rootLevelCounters.get("NUMBER_JOBS")); assertEquals("0", rootLevelCounters.get("SMM_SPILL_COUNT")); assertEquals("0", rootLevelCounters.get("RETURN_CODE")); assertEquals("job_1467867518322_6688,job_1467867518322_6687,job_1467867518322_6689,job_1467867518322_6690", rootLevelCounters.get("JOB_GRAPH")); assertEquals("3", rootLevelCounters.get("RECORD_WRITTEN")); assertEquals("-1", rootLevelCounters.get("ERROR_CODE")); assertEquals("HASH_JOIN,DISTINCT,FILTER", rootLevelCounters.get("FEATURES")); assertEquals("1bbda10f-3149-49de-9798-77ec97b5e3e1", rootLevelCounters.get("SCRIPT_ID")); assertEquals("0.12.0-cdh5.5.2", rootLevelCounters.get("PIG_VERSION")); assertEquals("3870", rootLevelCounters.get("BYTES_WRITTEN")); assertEquals("0", rootLevelCounters.get("PROACTIVE_SPILL_COUNT_RECORDS")); assertEquals("0", rootLevelCounters.get("PROACTIVE_SPILL_COUNT_OBJECTS")); assertEquals("2.6.0-cdh5.5.2", rootLevelCounters.get("HADOOP_VERSION")); assertEquals("92829", rootLevelCounters.get("DURATION")); assertEquals("PIG", rootLevelCounters.get("ACTION_TYPE")); assertThat(pigCounters.getJobIds(), containsInAnyOrder( "job_1467867518322_6687", "job_1467867518322_6688", "job_1467867518322_6689", "job_1467867518322_6690")); JobCounters jobCounters1 = pigCounters.getJobCounters("job_1467867518322_6687"); assertThat(jobCounters1.getAliases(), containsInAnyOrder("cachedDocumentId", "documentMeta")); assertThat(jobCounters1.getCountersCount(), is(21)); assertThat(jobCounters1.getCounter("JOB_ID"), is("job_1467867518322_6687")); assertThat(jobCounters1.getCounter("Alias"), is("cachedDocumentId,documentMeta")); assertThat(jobCounters1.getCounter("NUMBER_MAPS"), is("1")); assertThat(jobCounters1.getCounter("MAP_INPUT_RECORDS"), is("3")); assertThat(jobCounters1.getCounter("MAP_OUTPUT_RECORDS"), is("3")); assertThat(jobCounters1.getCounter("MIN_MAP_TIME"), is("3149")); assertThat(jobCounters1.getCounter("MAX_MAP_TIME"), is("3149")); assertThat(jobCounters1.getCounter("AVG_MAP_TIME"), is("3149")); assertThat(jobCounters1.getCounter("NUMBER_REDUCES"), is("1")); assertThat(jobCounters1.getCounter("REDUCE_INPUT_RECORDS"), is("3")); assertThat(jobCounters1.getCounter("REDUCE_OUTPUT_RECORDS"), is("0")); assertThat(jobCounters1.getCounter("MIN_REDUCE_TIME"), is("2806")); assertThat(jobCounters1.getCounter("MAX_REDUCE_TIME"), is("2806")); assertThat(jobCounters1.getCounter("AVG_REDUCE_TIME"), is("2806")); assertThat(jobCounters1.getCounter("RECORD_WRITTEN"), is("0")); assertThat(jobCounters1.getCounter("BYTES_WRITTEN"), is("0")); assertThat(jobCounters1.getCounter("HDFS_BYTES_WRITTEN"), is("737")); assertThat(jobCounters1.getCounter("SMMS_SPILL_COUNT"), is("0")); assertThat(jobCounters1.getCounter("PROACTIVE_SPILL_COUNT_RECS"), is("0")); assertThat(jobCounters1.getCounter("PROACTIVE_SPILL_COUNT_OBJECTS"), is("0")); assertThat(jobCounters1.getCounter("FEATURE"), is("DISTINCT,MULTI_QUERY")); JobCounters jobCounters2 = pigCounters.getJobCounters("job_1467867518322_6688"); assertThat(jobCounters2.getAliases(), containsInAnyOrder("documentContent", "documentContentId")); assertThat(jobCounters2.getCountersCount(), is(21)); JobCounters jobCounters3 = pigCounters.getJobCounters("job_1467867518322_6689"); assertThat(jobCounters3.getAliases(), containsInAnyOrder("documentContentFiltered", "joinedDocumentContent", "joinedFilteredDocumentContent")); assertThat(jobCounters3.getCountersCount(), is(21)); JobCounters jobCounters4 = pigCounters.getJobCounters("job_1467867518322_6690"); assertThat(jobCounters4.getAliases(), containsInAnyOrder("documentMetaFiltered", "joinedDocumentMeta")); assertThat(jobCounters4.getCountersCount(), is(21)); }
public PigCounters parse(String pigCountersJson) { JsonObject pigCountersJsonObject = jsonParser.parse(pigCountersJson).getAsJsonObject(); List<String> jobIds = extractJobIds(pigCountersJsonObject); List<JobCounters> jobCountersList = jobIds.stream() .map(jobId -> extractJobCounters(pigCountersJsonObject, jobId)) .collect(toList()); Map<String, String> rootLevelCounters = extractRootLevelCounters(pigCountersJsonObject); return new PigCounters(rootLevelCounters, jobCountersList); }
PigCountersParser { public PigCounters parse(String pigCountersJson) { JsonObject pigCountersJsonObject = jsonParser.parse(pigCountersJson).getAsJsonObject(); List<String> jobIds = extractJobIds(pigCountersJsonObject); List<JobCounters> jobCountersList = jobIds.stream() .map(jobId -> extractJobCounters(pigCountersJsonObject, jobId)) .collect(toList()); Map<String, String> rootLevelCounters = extractRootLevelCounters(pigCountersJsonObject); return new PigCounters(rootLevelCounters, jobCountersList); } }
PigCountersParser { public PigCounters parse(String pigCountersJson) { JsonObject pigCountersJsonObject = jsonParser.parse(pigCountersJson).getAsJsonObject(); List<String> jobIds = extractJobIds(pigCountersJsonObject); List<JobCounters> jobCountersList = jobIds.stream() .map(jobId -> extractJobCounters(pigCountersJsonObject, jobId)) .collect(toList()); Map<String, String> rootLevelCounters = extractRootLevelCounters(pigCountersJsonObject); return new PigCounters(rootLevelCounters, jobCountersList); }  }
PigCountersParser { public PigCounters parse(String pigCountersJson) { JsonObject pigCountersJsonObject = jsonParser.parse(pigCountersJson).getAsJsonObject(); List<String> jobIds = extractJobIds(pigCountersJsonObject); List<JobCounters> jobCountersList = jobIds.stream() .map(jobId -> extractJobCounters(pigCountersJsonObject, jobId)) .collect(toList()); Map<String, String> rootLevelCounters = extractRootLevelCounters(pigCountersJsonObject); return new PigCounters(rootLevelCounters, jobCountersList); }  PigCounters parse(String pigCountersJson); }
PigCountersParser { public PigCounters parse(String pigCountersJson) { JsonObject pigCountersJsonObject = jsonParser.parse(pigCountersJson).getAsJsonObject(); List<String> jobIds = extractJobIds(pigCountersJsonObject); List<JobCounters> jobCountersList = jobIds.stream() .map(jobId -> extractJobCounters(pigCountersJsonObject, jobId)) .collect(toList()); Map<String, String> rootLevelCounters = extractRootLevelCounters(pigCountersJsonObject); return new PigCounters(rootLevelCounters, jobCountersList); }  PigCounters parse(String pigCountersJson);  }