@Test public void testUpConvertSimpleOrcStruct() { TypeDescription baseStructSchema = TypeDescription.fromString("struct<a:int,b:string>"); OrcStruct baseStruct = (OrcStruct) OrcStruct.createValue(baseStructSchema); OrcTestUtils.fillOrcStructWithFixedValue(baseStruct, baseStructSchema, intValue1, stringValue1, boolValue); TypeDescription evolved_baseStructSchema = TypeDescription.fromString("struct<a:int,b:string,c:int>"); OrcStruct evolvedStruct = (OrcStruct) OrcStruct.createValue(evolved_baseStructSchema); OrcUtils.upConvertOrcStruct(baseStruct, evolvedStruct, evolved_baseStructSchema); Assert.assertEquals(((IntWritable) evolvedStruct.getFieldValue("a")).get(), intValue1); Assert.assertEquals(evolvedStruct.getFieldValue("b").toString(), stringValue1); Assert.assertNull(evolvedStruct.getFieldValue("c")); OrcStruct baseStruct_shadow = (OrcStruct) OrcStruct.createValue(baseStructSchema); OrcUtils.upConvertOrcStruct(evolvedStruct, baseStruct_shadow, baseStructSchema); Assert.assertEquals(baseStruct, baseStruct_shadow); }
@VisibleForTesting public static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema) { Preconditions.checkArgument(newStruct.getSchema().equals(targetSchema)); int indexInNewSchema = 0; List<String> oldSchemaFieldNames = oldStruct.getSchema().getFieldNames(); Map<String, Integer> oldSchemaIndex = IntStream.range(0, oldSchemaFieldNames.size()).boxed() .collect(Collectors.toMap(oldSchemaFieldNames::get, Function.identity())); List<TypeDescription> oldSchemaTypes = oldStruct.getSchema().getChildren(); List<TypeDescription> newSchemaTypes = targetSchema.getChildren(); for (String fieldName : targetSchema.getFieldNames()) { if (oldSchemaFieldNames.contains(fieldName) && oldStruct.getFieldValue(fieldName) != null) { int fieldIndex = oldSchemaIndex.get(fieldName); TypeDescription oldFieldSchema = oldSchemaTypes.get(fieldIndex); TypeDescription newFieldSchema = newSchemaTypes.get(indexInNewSchema); if (isEvolutionValid(oldFieldSchema, newFieldSchema)) { WritableComparable oldField = oldStruct.getFieldValue(fieldName); WritableComparable newField = newStruct.getFieldValue(fieldName); newField = (newField == null) ? OrcUtils.createValueRecursively(newFieldSchema) : newField; newStruct.setFieldValue(fieldName, structConversionHelper(oldField, newField, newFieldSchema)); } else { throw new SchemaEvolution.IllegalEvolutionException(String .format("ORC does not support type conversion from file" + " type %s to reader type %s ", oldFieldSchema.toString(), newFieldSchema.toString())); } } else { newStruct.setFieldValue(fieldName, null); } indexInNewSchema++; } }
OrcUtils { @VisibleForTesting public static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema) { Preconditions.checkArgument(newStruct.getSchema().equals(targetSchema)); int indexInNewSchema = 0; List<String> oldSchemaFieldNames = oldStruct.getSchema().getFieldNames(); Map<String, Integer> oldSchemaIndex = IntStream.range(0, oldSchemaFieldNames.size()).boxed() .collect(Collectors.toMap(oldSchemaFieldNames::get, Function.identity())); List<TypeDescription> oldSchemaTypes = oldStruct.getSchema().getChildren(); List<TypeDescription> newSchemaTypes = targetSchema.getChildren(); for (String fieldName : targetSchema.getFieldNames()) { if (oldSchemaFieldNames.contains(fieldName) && oldStruct.getFieldValue(fieldName) != null) { int fieldIndex = oldSchemaIndex.get(fieldName); TypeDescription oldFieldSchema = oldSchemaTypes.get(fieldIndex); TypeDescription newFieldSchema = newSchemaTypes.get(indexInNewSchema); if (isEvolutionValid(oldFieldSchema, newFieldSchema)) { WritableComparable oldField = oldStruct.getFieldValue(fieldName); WritableComparable newField = newStruct.getFieldValue(fieldName); newField = (newField == null) ? OrcUtils.createValueRecursively(newFieldSchema) : newField; newStruct.setFieldValue(fieldName, structConversionHelper(oldField, newField, newFieldSchema)); } else { throw new SchemaEvolution.IllegalEvolutionException(String .format("ORC does not support type conversion from file" + " type %s to reader type %s ", oldFieldSchema.toString(), newFieldSchema.toString())); } } else { newStruct.setFieldValue(fieldName, null); } indexInNewSchema++; } } }
OrcUtils { @VisibleForTesting public static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema) { Preconditions.checkArgument(newStruct.getSchema().equals(targetSchema)); int indexInNewSchema = 0; List<String> oldSchemaFieldNames = oldStruct.getSchema().getFieldNames(); Map<String, Integer> oldSchemaIndex = IntStream.range(0, oldSchemaFieldNames.size()).boxed() .collect(Collectors.toMap(oldSchemaFieldNames::get, Function.identity())); List<TypeDescription> oldSchemaTypes = oldStruct.getSchema().getChildren(); List<TypeDescription> newSchemaTypes = targetSchema.getChildren(); for (String fieldName : targetSchema.getFieldNames()) { if (oldSchemaFieldNames.contains(fieldName) && oldStruct.getFieldValue(fieldName) != null) { int fieldIndex = oldSchemaIndex.get(fieldName); TypeDescription oldFieldSchema = oldSchemaTypes.get(fieldIndex); TypeDescription newFieldSchema = newSchemaTypes.get(indexInNewSchema); if (isEvolutionValid(oldFieldSchema, newFieldSchema)) { WritableComparable oldField = oldStruct.getFieldValue(fieldName); WritableComparable newField = newStruct.getFieldValue(fieldName); newField = (newField == null) ? OrcUtils.createValueRecursively(newFieldSchema) : newField; newStruct.setFieldValue(fieldName, structConversionHelper(oldField, newField, newFieldSchema)); } else { throw new SchemaEvolution.IllegalEvolutionException(String .format("ORC does not support type conversion from file" + " type %s to reader type %s ", oldFieldSchema.toString(), newFieldSchema.toString())); } } else { newStruct.setFieldValue(fieldName, null); } indexInNewSchema++; } } private  OrcUtils(); }
OrcUtils { @VisibleForTesting public static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema) { Preconditions.checkArgument(newStruct.getSchema().equals(targetSchema)); int indexInNewSchema = 0; List<String> oldSchemaFieldNames = oldStruct.getSchema().getFieldNames(); Map<String, Integer> oldSchemaIndex = IntStream.range(0, oldSchemaFieldNames.size()).boxed() .collect(Collectors.toMap(oldSchemaFieldNames::get, Function.identity())); List<TypeDescription> oldSchemaTypes = oldStruct.getSchema().getChildren(); List<TypeDescription> newSchemaTypes = targetSchema.getChildren(); for (String fieldName : targetSchema.getFieldNames()) { if (oldSchemaFieldNames.contains(fieldName) && oldStruct.getFieldValue(fieldName) != null) { int fieldIndex = oldSchemaIndex.get(fieldName); TypeDescription oldFieldSchema = oldSchemaTypes.get(fieldIndex); TypeDescription newFieldSchema = newSchemaTypes.get(indexInNewSchema); if (isEvolutionValid(oldFieldSchema, newFieldSchema)) { WritableComparable oldField = oldStruct.getFieldValue(fieldName); WritableComparable newField = newStruct.getFieldValue(fieldName); newField = (newField == null) ? OrcUtils.createValueRecursively(newFieldSchema) : newField; newStruct.setFieldValue(fieldName, structConversionHelper(oldField, newField, newFieldSchema)); } else { throw new SchemaEvolution.IllegalEvolutionException(String .format("ORC does not support type conversion from file" + " type %s to reader type %s ", oldFieldSchema.toString(), newFieldSchema.toString())); } } else { newStruct.setFieldValue(fieldName, null); } indexInNewSchema++; } } private  OrcUtils(); static TypeDescription getTypeDescriptionFromFile(Configuration conf, Path orcFilePath); static Reader getRecordReaderFromFile(Configuration conf, Path orcFilePath); static TypeDescription getNewestSchemaFromSource(Job job, FileSystem fs); @VisibleForTesting static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema); static void handlePrimitiveWritableComparable(WritableComparable from, WritableComparable to); static WritableComparable createValueRecursively(TypeDescription schema, int elemNum); static WritableComparable createValueRecursively(TypeDescription schema); static boolean eligibleForUpConvertHelper(TypeDescription originalSchema, TypeDescription targetSchema); static boolean eligibleForUpConvert(TypeDescription originalSchema, TypeDescription targetSchema); }
OrcUtils { @VisibleForTesting public static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema) { Preconditions.checkArgument(newStruct.getSchema().equals(targetSchema)); int indexInNewSchema = 0; List<String> oldSchemaFieldNames = oldStruct.getSchema().getFieldNames(); Map<String, Integer> oldSchemaIndex = IntStream.range(0, oldSchemaFieldNames.size()).boxed() .collect(Collectors.toMap(oldSchemaFieldNames::get, Function.identity())); List<TypeDescription> oldSchemaTypes = oldStruct.getSchema().getChildren(); List<TypeDescription> newSchemaTypes = targetSchema.getChildren(); for (String fieldName : targetSchema.getFieldNames()) { if (oldSchemaFieldNames.contains(fieldName) && oldStruct.getFieldValue(fieldName) != null) { int fieldIndex = oldSchemaIndex.get(fieldName); TypeDescription oldFieldSchema = oldSchemaTypes.get(fieldIndex); TypeDescription newFieldSchema = newSchemaTypes.get(indexInNewSchema); if (isEvolutionValid(oldFieldSchema, newFieldSchema)) { WritableComparable oldField = oldStruct.getFieldValue(fieldName); WritableComparable newField = newStruct.getFieldValue(fieldName); newField = (newField == null) ? OrcUtils.createValueRecursively(newFieldSchema) : newField; newStruct.setFieldValue(fieldName, structConversionHelper(oldField, newField, newFieldSchema)); } else { throw new SchemaEvolution.IllegalEvolutionException(String .format("ORC does not support type conversion from file" + " type %s to reader type %s ", oldFieldSchema.toString(), newFieldSchema.toString())); } } else { newStruct.setFieldValue(fieldName, null); } indexInNewSchema++; } } private  OrcUtils(); static TypeDescription getTypeDescriptionFromFile(Configuration conf, Path orcFilePath); static Reader getRecordReaderFromFile(Configuration conf, Path orcFilePath); static TypeDescription getNewestSchemaFromSource(Job job, FileSystem fs); @VisibleForTesting static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema); static void handlePrimitiveWritableComparable(WritableComparable from, WritableComparable to); static WritableComparable createValueRecursively(TypeDescription schema, int elemNum); static WritableComparable createValueRecursively(TypeDescription schema); static boolean eligibleForUpConvertHelper(TypeDescription originalSchema, TypeDescription targetSchema); static boolean eligibleForUpConvert(TypeDescription originalSchema, TypeDescription targetSchema);  }