@Test public void testUpConvertOrcStructOfMap() { TypeDescription structOfMapSchema = TypeDescription.fromString("struct<a:map<string,int>>"); OrcStruct structOfMap = (OrcStruct) OrcStruct.createValue(structOfMapSchema); TypeDescription mapSchema = TypeDescription.createMap(TypeDescription.createString(), TypeDescription.createInt()); OrcMap testMap = new OrcMap(mapSchema); testMap.put(new Text(stringValue1), new IntWritable(intValue1)); testMap.put(new Text(stringValue2), new IntWritable(intValue2)); structOfMap.setFieldValue("a", testMap); TypeDescription evolvedStructOfMapSchema = TypeDescription.fromString("struct<a:map<string,bigint>>"); OrcStruct evolvedStructOfMap = (OrcStruct) OrcStruct.createValue(evolvedStructOfMapSchema); OrcMap evolvedMap = new OrcMap(TypeDescription.createMap(TypeDescription.createString(), TypeDescription.createInt())); evolvedMap.put(new Text(""), new LongWritable()); evolvedStructOfMap.setFieldValue("a", evolvedMap); OrcUtils.upConvertOrcStruct(structOfMap, evolvedStructOfMap, evolvedStructOfMapSchema); Assert.assertEquals(((OrcMap) evolvedStructOfMap.getFieldValue("a")).get(new Text(stringValue1)), new LongWritable(intValue1)); Assert.assertEquals(((OrcMap) evolvedStructOfMap.getFieldValue("a")).get(new Text(stringValue2)), new LongWritable(intValue2)); Assert.assertEquals(((OrcMap) evolvedStructOfMap.getFieldValue("a")).size(), 2); testMap.put(new Text(stringValue3), new IntWritable(intValue3)); Assert.assertEquals(((OrcMap) structOfMap.getFieldValue("a")).size(), 3); OrcUtils.upConvertOrcStruct(structOfMap, evolvedStructOfMap, evolvedStructOfMapSchema); Assert.assertEquals(((OrcMap) evolvedStructOfMap.getFieldValue("a")).size(), 3); Assert.assertEquals(((OrcMap) evolvedStructOfMap.getFieldValue("a")).get(new Text(stringValue1)), new LongWritable(intValue1)); Assert.assertEquals(((OrcMap) evolvedStructOfMap.getFieldValue("a")).get(new Text(stringValue2)), new LongWritable(intValue2)); Assert.assertEquals(((OrcMap) evolvedStructOfMap.getFieldValue("a")).get(new Text(stringValue3)), new LongWritable(intValue3)); }
@VisibleForTesting public static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema) { Preconditions.checkArgument(newStruct.getSchema().equals(targetSchema)); int indexInNewSchema = 0; List<String> oldSchemaFieldNames = oldStruct.getSchema().getFieldNames(); Map<String, Integer> oldSchemaIndex = IntStream.range(0, oldSchemaFieldNames.size()).boxed() .collect(Collectors.toMap(oldSchemaFieldNames::get, Function.identity())); List<TypeDescription> oldSchemaTypes = oldStruct.getSchema().getChildren(); List<TypeDescription> newSchemaTypes = targetSchema.getChildren(); for (String fieldName : targetSchema.getFieldNames()) { if (oldSchemaFieldNames.contains(fieldName) && oldStruct.getFieldValue(fieldName) != null) { int fieldIndex = oldSchemaIndex.get(fieldName); TypeDescription oldFieldSchema = oldSchemaTypes.get(fieldIndex); TypeDescription newFieldSchema = newSchemaTypes.get(indexInNewSchema); if (isEvolutionValid(oldFieldSchema, newFieldSchema)) { WritableComparable oldField = oldStruct.getFieldValue(fieldName); WritableComparable newField = newStruct.getFieldValue(fieldName); newField = (newField == null) ? OrcUtils.createValueRecursively(newFieldSchema) : newField; newStruct.setFieldValue(fieldName, structConversionHelper(oldField, newField, newFieldSchema)); } else { throw new SchemaEvolution.IllegalEvolutionException(String .format("ORC does not support type conversion from file" + " type %s to reader type %s ", oldFieldSchema.toString(), newFieldSchema.toString())); } } else { newStruct.setFieldValue(fieldName, null); } indexInNewSchema++; } }
OrcUtils { @VisibleForTesting public static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema) { Preconditions.checkArgument(newStruct.getSchema().equals(targetSchema)); int indexInNewSchema = 0; List<String> oldSchemaFieldNames = oldStruct.getSchema().getFieldNames(); Map<String, Integer> oldSchemaIndex = IntStream.range(0, oldSchemaFieldNames.size()).boxed() .collect(Collectors.toMap(oldSchemaFieldNames::get, Function.identity())); List<TypeDescription> oldSchemaTypes = oldStruct.getSchema().getChildren(); List<TypeDescription> newSchemaTypes = targetSchema.getChildren(); for (String fieldName : targetSchema.getFieldNames()) { if (oldSchemaFieldNames.contains(fieldName) && oldStruct.getFieldValue(fieldName) != null) { int fieldIndex = oldSchemaIndex.get(fieldName); TypeDescription oldFieldSchema = oldSchemaTypes.get(fieldIndex); TypeDescription newFieldSchema = newSchemaTypes.get(indexInNewSchema); if (isEvolutionValid(oldFieldSchema, newFieldSchema)) { WritableComparable oldField = oldStruct.getFieldValue(fieldName); WritableComparable newField = newStruct.getFieldValue(fieldName); newField = (newField == null) ? OrcUtils.createValueRecursively(newFieldSchema) : newField; newStruct.setFieldValue(fieldName, structConversionHelper(oldField, newField, newFieldSchema)); } else { throw new SchemaEvolution.IllegalEvolutionException(String .format("ORC does not support type conversion from file" + " type %s to reader type %s ", oldFieldSchema.toString(), newFieldSchema.toString())); } } else { newStruct.setFieldValue(fieldName, null); } indexInNewSchema++; } } }
OrcUtils { @VisibleForTesting public static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema) { Preconditions.checkArgument(newStruct.getSchema().equals(targetSchema)); int indexInNewSchema = 0; List<String> oldSchemaFieldNames = oldStruct.getSchema().getFieldNames(); Map<String, Integer> oldSchemaIndex = IntStream.range(0, oldSchemaFieldNames.size()).boxed() .collect(Collectors.toMap(oldSchemaFieldNames::get, Function.identity())); List<TypeDescription> oldSchemaTypes = oldStruct.getSchema().getChildren(); List<TypeDescription> newSchemaTypes = targetSchema.getChildren(); for (String fieldName : targetSchema.getFieldNames()) { if (oldSchemaFieldNames.contains(fieldName) && oldStruct.getFieldValue(fieldName) != null) { int fieldIndex = oldSchemaIndex.get(fieldName); TypeDescription oldFieldSchema = oldSchemaTypes.get(fieldIndex); TypeDescription newFieldSchema = newSchemaTypes.get(indexInNewSchema); if (isEvolutionValid(oldFieldSchema, newFieldSchema)) { WritableComparable oldField = oldStruct.getFieldValue(fieldName); WritableComparable newField = newStruct.getFieldValue(fieldName); newField = (newField == null) ? OrcUtils.createValueRecursively(newFieldSchema) : newField; newStruct.setFieldValue(fieldName, structConversionHelper(oldField, newField, newFieldSchema)); } else { throw new SchemaEvolution.IllegalEvolutionException(String .format("ORC does not support type conversion from file" + " type %s to reader type %s ", oldFieldSchema.toString(), newFieldSchema.toString())); } } else { newStruct.setFieldValue(fieldName, null); } indexInNewSchema++; } } private  OrcUtils(); }
OrcUtils { @VisibleForTesting public static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema) { Preconditions.checkArgument(newStruct.getSchema().equals(targetSchema)); int indexInNewSchema = 0; List<String> oldSchemaFieldNames = oldStruct.getSchema().getFieldNames(); Map<String, Integer> oldSchemaIndex = IntStream.range(0, oldSchemaFieldNames.size()).boxed() .collect(Collectors.toMap(oldSchemaFieldNames::get, Function.identity())); List<TypeDescription> oldSchemaTypes = oldStruct.getSchema().getChildren(); List<TypeDescription> newSchemaTypes = targetSchema.getChildren(); for (String fieldName : targetSchema.getFieldNames()) { if (oldSchemaFieldNames.contains(fieldName) && oldStruct.getFieldValue(fieldName) != null) { int fieldIndex = oldSchemaIndex.get(fieldName); TypeDescription oldFieldSchema = oldSchemaTypes.get(fieldIndex); TypeDescription newFieldSchema = newSchemaTypes.get(indexInNewSchema); if (isEvolutionValid(oldFieldSchema, newFieldSchema)) { WritableComparable oldField = oldStruct.getFieldValue(fieldName); WritableComparable newField = newStruct.getFieldValue(fieldName); newField = (newField == null) ? OrcUtils.createValueRecursively(newFieldSchema) : newField; newStruct.setFieldValue(fieldName, structConversionHelper(oldField, newField, newFieldSchema)); } else { throw new SchemaEvolution.IllegalEvolutionException(String .format("ORC does not support type conversion from file" + " type %s to reader type %s ", oldFieldSchema.toString(), newFieldSchema.toString())); } } else { newStruct.setFieldValue(fieldName, null); } indexInNewSchema++; } } private  OrcUtils(); static TypeDescription getTypeDescriptionFromFile(Configuration conf, Path orcFilePath); static Reader getRecordReaderFromFile(Configuration conf, Path orcFilePath); static TypeDescription getNewestSchemaFromSource(Job job, FileSystem fs); @VisibleForTesting static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema); static void handlePrimitiveWritableComparable(WritableComparable from, WritableComparable to); static WritableComparable createValueRecursively(TypeDescription schema, int elemNum); static WritableComparable createValueRecursively(TypeDescription schema); static boolean eligibleForUpConvertHelper(TypeDescription originalSchema, TypeDescription targetSchema); static boolean eligibleForUpConvert(TypeDescription originalSchema, TypeDescription targetSchema); }
OrcUtils { @VisibleForTesting public static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema) { Preconditions.checkArgument(newStruct.getSchema().equals(targetSchema)); int indexInNewSchema = 0; List<String> oldSchemaFieldNames = oldStruct.getSchema().getFieldNames(); Map<String, Integer> oldSchemaIndex = IntStream.range(0, oldSchemaFieldNames.size()).boxed() .collect(Collectors.toMap(oldSchemaFieldNames::get, Function.identity())); List<TypeDescription> oldSchemaTypes = oldStruct.getSchema().getChildren(); List<TypeDescription> newSchemaTypes = targetSchema.getChildren(); for (String fieldName : targetSchema.getFieldNames()) { if (oldSchemaFieldNames.contains(fieldName) && oldStruct.getFieldValue(fieldName) != null) { int fieldIndex = oldSchemaIndex.get(fieldName); TypeDescription oldFieldSchema = oldSchemaTypes.get(fieldIndex); TypeDescription newFieldSchema = newSchemaTypes.get(indexInNewSchema); if (isEvolutionValid(oldFieldSchema, newFieldSchema)) { WritableComparable oldField = oldStruct.getFieldValue(fieldName); WritableComparable newField = newStruct.getFieldValue(fieldName); newField = (newField == null) ? OrcUtils.createValueRecursively(newFieldSchema) : newField; newStruct.setFieldValue(fieldName, structConversionHelper(oldField, newField, newFieldSchema)); } else { throw new SchemaEvolution.IllegalEvolutionException(String .format("ORC does not support type conversion from file" + " type %s to reader type %s ", oldFieldSchema.toString(), newFieldSchema.toString())); } } else { newStruct.setFieldValue(fieldName, null); } indexInNewSchema++; } } private  OrcUtils(); static TypeDescription getTypeDescriptionFromFile(Configuration conf, Path orcFilePath); static Reader getRecordReaderFromFile(Configuration conf, Path orcFilePath); static TypeDescription getNewestSchemaFromSource(Job job, FileSystem fs); @VisibleForTesting static void upConvertOrcStruct(OrcStruct oldStruct, OrcStruct newStruct, TypeDescription targetSchema); static void handlePrimitiveWritableComparable(WritableComparable from, WritableComparable to); static WritableComparable createValueRecursively(TypeDescription schema, int elemNum); static WritableComparable createValueRecursively(TypeDescription schema); static boolean eligibleForUpConvertHelper(TypeDescription originalSchema, TypeDescription targetSchema); static boolean eligibleForUpConvert(TypeDescription originalSchema, TypeDescription targetSchema);  }