@Test public void testBasicParse() throws Throwable { AbstractDeserializer serDe = getTestSerDe(); Text t = new Text(testLogLine); Object row = serDe.deserialize(t); if (!(row instanceof List)) { fail("row must be instanceof List<>"); } List<?> rowArray = (List<?>)row; LOG.debug("Deserialized row: {}", row); assertEquals("127.0.0.1", rowArray.get(0)); assertEquals(1351112444000L, rowArray.get(1)); assertEquals("Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0", rowArray.get(2)); assertEquals(800L, rowArray.get(3)); assertEquals(600L, rowArray.get(4)); }
@Override public Object deserialize(Writable writable) throws SerDeException { if (!(writable instanceof Text)) { throw new SerDeException("The input MUST be a Text line."); } linesInput++; try { currentValue.clear(); parser.parse(currentValue, writable.toString()); } catch (DissectionFailure dissectionFailure) { linesBad++; if (linesInput >= MINIMAL_FAIL_LINES) { if (100* linesBad > MINIMAL_FAIL_PERCENTAGE * linesInput){ throw new SerDeException("To many bad lines: " + linesBad + " of " + linesInput + " are bad."); } } return null; } catch (InvalidDissectorException |MissingDissectorsException e) { throw new SerDeException("Cannot continue; Fix the Dissectors before retrying", e); } for (ColumnToGetterMapping ctgm: columnToGetterMappings) { switch(ctgm.casts) { case STRING: String currentValueString = currentValue.getString(ctgm.fieldValue); row.set(ctgm.index, currentValueString); break; case LONG: Long currentValueLong = currentValue.getLong(ctgm.fieldValue); row.set(ctgm.index, currentValueLong); break; case DOUBLE: Double currentValueDouble = currentValue.getDouble(ctgm.fieldValue); row.set(ctgm.index, currentValueDouble); break; default: } } return row; }
ApacheHttpdlogDeserializer extends AbstractDeserializer { @Override public Object deserialize(Writable writable) throws SerDeException { if (!(writable instanceof Text)) { throw new SerDeException("The input MUST be a Text line."); } linesInput++; try { currentValue.clear(); parser.parse(currentValue, writable.toString()); } catch (DissectionFailure dissectionFailure) { linesBad++; if (linesInput >= MINIMAL_FAIL_LINES) { if (100* linesBad > MINIMAL_FAIL_PERCENTAGE * linesInput){ throw new SerDeException("To many bad lines: " + linesBad + " of " + linesInput + " are bad."); } } return null; } catch (InvalidDissectorException |MissingDissectorsException e) { throw new SerDeException("Cannot continue; Fix the Dissectors before retrying", e); } for (ColumnToGetterMapping ctgm: columnToGetterMappings) { switch(ctgm.casts) { case STRING: String currentValueString = currentValue.getString(ctgm.fieldValue); row.set(ctgm.index, currentValueString); break; case LONG: Long currentValueLong = currentValue.getLong(ctgm.fieldValue); row.set(ctgm.index, currentValueLong); break; case DOUBLE: Double currentValueDouble = currentValue.getDouble(ctgm.fieldValue); row.set(ctgm.index, currentValueDouble); break; default: } } return row; } }
ApacheHttpdlogDeserializer extends AbstractDeserializer { @Override public Object deserialize(Writable writable) throws SerDeException { if (!(writable instanceof Text)) { throw new SerDeException("The input MUST be a Text line."); } linesInput++; try { currentValue.clear(); parser.parse(currentValue, writable.toString()); } catch (DissectionFailure dissectionFailure) { linesBad++; if (linesInput >= MINIMAL_FAIL_LINES) { if (100* linesBad > MINIMAL_FAIL_PERCENTAGE * linesInput){ throw new SerDeException("To many bad lines: " + linesBad + " of " + linesInput + " are bad."); } } return null; } catch (InvalidDissectorException |MissingDissectorsException e) { throw new SerDeException("Cannot continue; Fix the Dissectors before retrying", e); } for (ColumnToGetterMapping ctgm: columnToGetterMappings) { switch(ctgm.casts) { case STRING: String currentValueString = currentValue.getString(ctgm.fieldValue); row.set(ctgm.index, currentValueString); break; case LONG: Long currentValueLong = currentValue.getLong(ctgm.fieldValue); row.set(ctgm.index, currentValueLong); break; case DOUBLE: Double currentValueDouble = currentValue.getDouble(ctgm.fieldValue); row.set(ctgm.index, currentValueDouble); break; default: } } return row; }  }
ApacheHttpdlogDeserializer extends AbstractDeserializer { @Override public Object deserialize(Writable writable) throws SerDeException { if (!(writable instanceof Text)) { throw new SerDeException("The input MUST be a Text line."); } linesInput++; try { currentValue.clear(); parser.parse(currentValue, writable.toString()); } catch (DissectionFailure dissectionFailure) { linesBad++; if (linesInput >= MINIMAL_FAIL_LINES) { if (100* linesBad > MINIMAL_FAIL_PERCENTAGE * linesInput){ throw new SerDeException("To many bad lines: " + linesBad + " of " + linesInput + " are bad."); } } return null; } catch (InvalidDissectorException |MissingDissectorsException e) { throw new SerDeException("Cannot continue; Fix the Dissectors before retrying", e); } for (ColumnToGetterMapping ctgm: columnToGetterMappings) { switch(ctgm.casts) { case STRING: String currentValueString = currentValue.getString(ctgm.fieldValue); row.set(ctgm.index, currentValueString); break; case LONG: Long currentValueLong = currentValue.getLong(ctgm.fieldValue); row.set(ctgm.index, currentValueLong); break; case DOUBLE: Double currentValueDouble = currentValue.getDouble(ctgm.fieldValue); row.set(ctgm.index, currentValueDouble); break; default: } } return row; }  @Override void initialize(Configuration conf, Properties props); @Override ObjectInspector getObjectInspector(); @Override Object deserialize(Writable writable); @Override SerDeStats getSerDeStats(); }
ApacheHttpdlogDeserializer extends AbstractDeserializer { @Override public Object deserialize(Writable writable) throws SerDeException { if (!(writable instanceof Text)) { throw new SerDeException("The input MUST be a Text line."); } linesInput++; try { currentValue.clear(); parser.parse(currentValue, writable.toString()); } catch (DissectionFailure dissectionFailure) { linesBad++; if (linesInput >= MINIMAL_FAIL_LINES) { if (100* linesBad > MINIMAL_FAIL_PERCENTAGE * linesInput){ throw new SerDeException("To many bad lines: " + linesBad + " of " + linesInput + " are bad."); } } return null; } catch (InvalidDissectorException |MissingDissectorsException e) { throw new SerDeException("Cannot continue; Fix the Dissectors before retrying", e); } for (ColumnToGetterMapping ctgm: columnToGetterMappings) { switch(ctgm.casts) { case STRING: String currentValueString = currentValue.getString(ctgm.fieldValue); row.set(ctgm.index, currentValueString); break; case LONG: Long currentValueLong = currentValue.getLong(ctgm.fieldValue); row.set(ctgm.index, currentValueLong); break; case DOUBLE: Double currentValueDouble = currentValue.getDouble(ctgm.fieldValue); row.set(ctgm.index, currentValueDouble); break; default: } } return row; }  @Override void initialize(Configuration conf, Properties props); @Override ObjectInspector getObjectInspector(); @Override Object deserialize(Writable writable); @Override SerDeStats getSerDeStats();  }