@Test public void testUnionRecordConversionWriter() throws Exception { Schema schema = new Schema.Parser().parse(this.getClass().getClassLoader().getResourceAsStream("union_test/schema.avsc")); TypeDescription orcSchema = AvroOrcSchemaConverter.getOrcSchema(schema); GenericRecordToOrcValueWriter valueWriter = new GenericRecordToOrcValueWriter(orcSchema, schema); VectorizedRowBatch rowBatch = orcSchema.createRowBatch(); List<GenericRecord> recordList = GobblinOrcWriterTest .deserializeAvroRecords(this.getClass(), schema, "union_test/data.json"); for (GenericRecord record : recordList) { valueWriter.write(record, rowBatch); } File tempFile = new File(Files.createTempDir(), "orc"); tempFile.deleteOnExit(); Path filePath = new Path(tempFile.getAbsolutePath()); OrcFile.WriterOptions options = OrcFile.writerOptions(new Properties(), new Configuration()); options.setSchema(orcSchema); Writer orcFileWriter = OrcFile.createWriter(filePath, options); orcFileWriter.addRowBatch(rowBatch); orcFileWriter.close(); FileSystem fs = FileSystem.get(new Configuration()); List<Writable> orcRecords = deserializeOrcRecords(filePath, fs); Assert.assertEquals(orcRecords.size(), 5); List<OrcUnion> unionList = orcRecords.stream().map(this::getUnionFieldFromStruct).collect(Collectors.toList()); TypeDescription unionSchema = orcSchema.getChildren().get(0); OrcUnion union_0 = new OrcUnion(unionSchema); union_0.set((byte) 0, new Text("urn:li:member:3")); Assert.assertTrue(unionList.contains(union_0)); OrcUnion union_1 = new OrcUnion(unionSchema); union_1.set((byte) 0, new Text("urn:li:member:4")); Assert.assertTrue(unionList.contains(union_1)); OrcUnion union_2 = new OrcUnion(unionSchema); union_2.set((byte) 1, new IntWritable(2)); Assert.assertTrue(unionList.contains(union_2)); OrcUnion union_3 = new OrcUnion(unionSchema); union_3.set((byte) 1, new IntWritable(1)); Assert.assertTrue(unionList.contains(union_3)); OrcUnion union_4 = new OrcUnion(unionSchema); union_4.set((byte) 1, new IntWritable(3)); Assert.assertTrue(unionList.contains(union_4)); }
@Override public void write(GenericRecord value, VectorizedRowBatch output) throws IOException { int row = output.size++; for (int c = 0; c < converters.length; ++c) { ColumnVector col = output.cols[c]; if (value.get(c) == null) { col.noNulls = false; col.isNull[row] = true; } else { col.isNull[row] = false; converters[c].addValue(row, c, value.get(c), col); } } }
GenericRecordToOrcValueWriter implements OrcValueWriter<GenericRecord> { @Override public void write(GenericRecord value, VectorizedRowBatch output) throws IOException { int row = output.size++; for (int c = 0; c < converters.length; ++c) { ColumnVector col = output.cols[c]; if (value.get(c) == null) { col.noNulls = false; col.isNull[row] = true; } else { col.isNull[row] = false; converters[c].addValue(row, c, value.get(c), col); } } } }
GenericRecordToOrcValueWriter implements OrcValueWriter<GenericRecord> { @Override public void write(GenericRecord value, VectorizedRowBatch output) throws IOException { int row = output.size++; for (int c = 0; c < converters.length; ++c) { ColumnVector col = output.cols[c]; if (value.get(c) == null) { col.noNulls = false; col.isNull[row] = true; } else { col.isNull[row] = false; converters[c].addValue(row, c, value.get(c), col); } } } GenericRecordToOrcValueWriter(TypeDescription typeDescription, Schema avroSchema);  GenericRecordToOrcValueWriter(TypeDescription typeDescription, Schema avroSchema, State state); }
GenericRecordToOrcValueWriter implements OrcValueWriter<GenericRecord> { @Override public void write(GenericRecord value, VectorizedRowBatch output) throws IOException { int row = output.size++; for (int c = 0; c < converters.length; ++c) { ColumnVector col = output.cols[c]; if (value.get(c) == null) { col.noNulls = false; col.isNull[row] = true; } else { col.isNull[row] = false; converters[c].addValue(row, c, value.get(c), col); } } } GenericRecordToOrcValueWriter(TypeDescription typeDescription, Schema avroSchema);  GenericRecordToOrcValueWriter(TypeDescription typeDescription, Schema avroSchema, State state); @Override void write(GenericRecord value, VectorizedRowBatch output); }
GenericRecordToOrcValueWriter implements OrcValueWriter<GenericRecord> { @Override public void write(GenericRecord value, VectorizedRowBatch output) throws IOException { int row = output.size++; for (int c = 0; c < converters.length; ++c) { ColumnVector col = output.cols[c]; if (value.get(c) == null) { col.noNulls = false; col.isNull[row] = true; } else { col.isNull[row] = false; converters[c].addValue(row, c, value.get(c), col); } } } GenericRecordToOrcValueWriter(TypeDescription typeDescription, Schema avroSchema);  GenericRecordToOrcValueWriter(TypeDescription typeDescription, Schema avroSchema, State state); @Override void write(GenericRecord value, VectorizedRowBatch output); @VisibleForTesting
  public int resizeCount; }