@Test public void testGetBagSubSchema() throws Exception { ResourceFieldSchema[] bagSubFieldSchemas = new ResourceFieldSchema[1]; bagSubFieldSchemas[0] = new ResourceFieldSchema().setName("innertuple") .setDescription("The tuple in the bag").setType(DataType.TUPLE); ResourceFieldSchema[] innerTupleFieldSchemas = new ResourceFieldSchema[1]; innerTupleFieldSchemas[0] = new ResourceFieldSchema().setName("innerfield").setType(DataType.CHARARRAY); bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas)); ResourceSchema expected = new ResourceSchema().setFields(bagSubFieldSchemas); HCatSchema hCatSchema = new HCatSchema(Lists.newArrayList( new HCatFieldSchema("innerLlama", HCatFieldSchema.Type.STRING, null))); HCatFieldSchema hCatFieldSchema = new HCatFieldSchema("llama", HCatFieldSchema.Type.ARRAY, hCatSchema, null); ResourceSchema actual = PigHCatUtil.getBagSubSchema(hCatFieldSchema); Assert.assertEquals(expected.toString(), actual.toString()); }
protected static ResourceSchema getBagSubSchema(HCatFieldSchema hfs) throws IOException { Properties props = UDFContext.getUDFContext().getClientSystemProps(); String innerTupleName = HCatConstants.HCAT_PIG_INNER_TUPLE_NAME_DEFAULT; if (props != null && props.containsKey(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME)) { innerTupleName = props.getProperty(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME) .replaceAll("FIELDNAME", hfs.getName()); } String innerFieldName = HCatConstants.HCAT_PIG_INNER_FIELD_NAME_DEFAULT; if (props != null && props.containsKey(HCatConstants.HCAT_PIG_INNER_FIELD_NAME)) { innerFieldName = props.getProperty(HCatConstants.HCAT_PIG_INNER_FIELD_NAME) .replaceAll("FIELDNAME", hfs.getName()); } ResourceFieldSchema[] bagSubFieldSchemas = new ResourceFieldSchema[1]; bagSubFieldSchemas[0] = new ResourceFieldSchema().setName(innerTupleName) .setDescription("The tuple in the bag") .setType(DataType.TUPLE); HCatFieldSchema arrayElementFieldSchema = hfs.getArrayElementSchema().get(0); if (arrayElementFieldSchema.getType() == Type.STRUCT) { bagSubFieldSchemas[0].setSchema(getTupleSubSchema(arrayElementFieldSchema)); } else if (arrayElementFieldSchema.getType() == Type.ARRAY) { ResourceSchema s = new ResourceSchema(); List<ResourceFieldSchema> lrfs = Arrays.asList(getResourceSchemaFromFieldSchema(arrayElementFieldSchema)); s.setFields(lrfs.toArray(new ResourceFieldSchema[0])); bagSubFieldSchemas[0].setSchema(s); } else { ResourceFieldSchema[] innerTupleFieldSchemas = new ResourceFieldSchema[1]; innerTupleFieldSchemas[0] = new ResourceFieldSchema().setName(innerFieldName) .setDescription("The inner field in the tuple in the bag") .setType(getPigType(arrayElementFieldSchema)) .setSchema(null); bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas)); } ResourceSchema s = new ResourceSchema().setFields(bagSubFieldSchemas); return s; }
PigHCatUtil { protected static ResourceSchema getBagSubSchema(HCatFieldSchema hfs) throws IOException { Properties props = UDFContext.getUDFContext().getClientSystemProps(); String innerTupleName = HCatConstants.HCAT_PIG_INNER_TUPLE_NAME_DEFAULT; if (props != null && props.containsKey(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME)) { innerTupleName = props.getProperty(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME) .replaceAll("FIELDNAME", hfs.getName()); } String innerFieldName = HCatConstants.HCAT_PIG_INNER_FIELD_NAME_DEFAULT; if (props != null && props.containsKey(HCatConstants.HCAT_PIG_INNER_FIELD_NAME)) { innerFieldName = props.getProperty(HCatConstants.HCAT_PIG_INNER_FIELD_NAME) .replaceAll("FIELDNAME", hfs.getName()); } ResourceFieldSchema[] bagSubFieldSchemas = new ResourceFieldSchema[1]; bagSubFieldSchemas[0] = new ResourceFieldSchema().setName(innerTupleName) .setDescription("The tuple in the bag") .setType(DataType.TUPLE); HCatFieldSchema arrayElementFieldSchema = hfs.getArrayElementSchema().get(0); if (arrayElementFieldSchema.getType() == Type.STRUCT) { bagSubFieldSchemas[0].setSchema(getTupleSubSchema(arrayElementFieldSchema)); } else if (arrayElementFieldSchema.getType() == Type.ARRAY) { ResourceSchema s = new ResourceSchema(); List<ResourceFieldSchema> lrfs = Arrays.asList(getResourceSchemaFromFieldSchema(arrayElementFieldSchema)); s.setFields(lrfs.toArray(new ResourceFieldSchema[0])); bagSubFieldSchemas[0].setSchema(s); } else { ResourceFieldSchema[] innerTupleFieldSchemas = new ResourceFieldSchema[1]; innerTupleFieldSchemas[0] = new ResourceFieldSchema().setName(innerFieldName) .setDescription("The inner field in the tuple in the bag") .setType(getPigType(arrayElementFieldSchema)) .setSchema(null); bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas)); } ResourceSchema s = new ResourceSchema().setFields(bagSubFieldSchemas); return s; } }
PigHCatUtil { protected static ResourceSchema getBagSubSchema(HCatFieldSchema hfs) throws IOException { Properties props = UDFContext.getUDFContext().getClientSystemProps(); String innerTupleName = HCatConstants.HCAT_PIG_INNER_TUPLE_NAME_DEFAULT; if (props != null && props.containsKey(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME)) { innerTupleName = props.getProperty(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME) .replaceAll("FIELDNAME", hfs.getName()); } String innerFieldName = HCatConstants.HCAT_PIG_INNER_FIELD_NAME_DEFAULT; if (props != null && props.containsKey(HCatConstants.HCAT_PIG_INNER_FIELD_NAME)) { innerFieldName = props.getProperty(HCatConstants.HCAT_PIG_INNER_FIELD_NAME) .replaceAll("FIELDNAME", hfs.getName()); } ResourceFieldSchema[] bagSubFieldSchemas = new ResourceFieldSchema[1]; bagSubFieldSchemas[0] = new ResourceFieldSchema().setName(innerTupleName) .setDescription("The tuple in the bag") .setType(DataType.TUPLE); HCatFieldSchema arrayElementFieldSchema = hfs.getArrayElementSchema().get(0); if (arrayElementFieldSchema.getType() == Type.STRUCT) { bagSubFieldSchemas[0].setSchema(getTupleSubSchema(arrayElementFieldSchema)); } else if (arrayElementFieldSchema.getType() == Type.ARRAY) { ResourceSchema s = new ResourceSchema(); List<ResourceFieldSchema> lrfs = Arrays.asList(getResourceSchemaFromFieldSchema(arrayElementFieldSchema)); s.setFields(lrfs.toArray(new ResourceFieldSchema[0])); bagSubFieldSchemas[0].setSchema(s); } else { ResourceFieldSchema[] innerTupleFieldSchemas = new ResourceFieldSchema[1]; innerTupleFieldSchemas[0] = new ResourceFieldSchema().setName(innerFieldName) .setDescription("The inner field in the tuple in the bag") .setType(getPigType(arrayElementFieldSchema)) .setSchema(null); bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas)); } ResourceSchema s = new ResourceSchema().setFields(bagSubFieldSchemas); return s; }  }
PigHCatUtil { protected static ResourceSchema getBagSubSchema(HCatFieldSchema hfs) throws IOException { Properties props = UDFContext.getUDFContext().getClientSystemProps(); String innerTupleName = HCatConstants.HCAT_PIG_INNER_TUPLE_NAME_DEFAULT; if (props != null && props.containsKey(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME)) { innerTupleName = props.getProperty(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME) .replaceAll("FIELDNAME", hfs.getName()); } String innerFieldName = HCatConstants.HCAT_PIG_INNER_FIELD_NAME_DEFAULT; if (props != null && props.containsKey(HCatConstants.HCAT_PIG_INNER_FIELD_NAME)) { innerFieldName = props.getProperty(HCatConstants.HCAT_PIG_INNER_FIELD_NAME) .replaceAll("FIELDNAME", hfs.getName()); } ResourceFieldSchema[] bagSubFieldSchemas = new ResourceFieldSchema[1]; bagSubFieldSchemas[0] = new ResourceFieldSchema().setName(innerTupleName) .setDescription("The tuple in the bag") .setType(DataType.TUPLE); HCatFieldSchema arrayElementFieldSchema = hfs.getArrayElementSchema().get(0); if (arrayElementFieldSchema.getType() == Type.STRUCT) { bagSubFieldSchemas[0].setSchema(getTupleSubSchema(arrayElementFieldSchema)); } else if (arrayElementFieldSchema.getType() == Type.ARRAY) { ResourceSchema s = new ResourceSchema(); List<ResourceFieldSchema> lrfs = Arrays.asList(getResourceSchemaFromFieldSchema(arrayElementFieldSchema)); s.setFields(lrfs.toArray(new ResourceFieldSchema[0])); bagSubFieldSchemas[0].setSchema(s); } else { ResourceFieldSchema[] innerTupleFieldSchemas = new ResourceFieldSchema[1]; innerTupleFieldSchemas[0] = new ResourceFieldSchema().setName(innerFieldName) .setDescription("The inner field in the tuple in the bag") .setType(getPigType(arrayElementFieldSchema)) .setSchema(null); bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas)); } ResourceSchema s = new ResourceSchema().setFields(bagSubFieldSchemas); return s; }  static boolean pigHasBooleanSupport(); static Pair<String, String> getDBTableNames(String location); static String getHCatServerUri(Job job); static String getHCatServerPrincipal(Job job); Table getTable(String location, String hcatServerUri, String hcatServerPrincipal); static ResourceSchema getResourceSchema(HCatSchema hcatSchema); static byte getPigType(HCatFieldSchema hfs); static byte getPigType(Type type); static Tuple transformToTuple(HCatRecord hr, HCatSchema hs); @SuppressWarnings("unchecked") static Object extractPigObject(Object o, HCatFieldSchema hfs); static void validateHCatTableSchemaFollowsPigRules(HCatSchema hcatTableSchema); static void getConfigFromUDFProperties(Properties p, Configuration config, String propName); static void saveConfigIntoUDFProperties(Properties p, Configuration config, String propName); }
PigHCatUtil { protected static ResourceSchema getBagSubSchema(HCatFieldSchema hfs) throws IOException { Properties props = UDFContext.getUDFContext().getClientSystemProps(); String innerTupleName = HCatConstants.HCAT_PIG_INNER_TUPLE_NAME_DEFAULT; if (props != null && props.containsKey(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME)) { innerTupleName = props.getProperty(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME) .replaceAll("FIELDNAME", hfs.getName()); } String innerFieldName = HCatConstants.HCAT_PIG_INNER_FIELD_NAME_DEFAULT; if (props != null && props.containsKey(HCatConstants.HCAT_PIG_INNER_FIELD_NAME)) { innerFieldName = props.getProperty(HCatConstants.HCAT_PIG_INNER_FIELD_NAME) .replaceAll("FIELDNAME", hfs.getName()); } ResourceFieldSchema[] bagSubFieldSchemas = new ResourceFieldSchema[1]; bagSubFieldSchemas[0] = new ResourceFieldSchema().setName(innerTupleName) .setDescription("The tuple in the bag") .setType(DataType.TUPLE); HCatFieldSchema arrayElementFieldSchema = hfs.getArrayElementSchema().get(0); if (arrayElementFieldSchema.getType() == Type.STRUCT) { bagSubFieldSchemas[0].setSchema(getTupleSubSchema(arrayElementFieldSchema)); } else if (arrayElementFieldSchema.getType() == Type.ARRAY) { ResourceSchema s = new ResourceSchema(); List<ResourceFieldSchema> lrfs = Arrays.asList(getResourceSchemaFromFieldSchema(arrayElementFieldSchema)); s.setFields(lrfs.toArray(new ResourceFieldSchema[0])); bagSubFieldSchemas[0].setSchema(s); } else { ResourceFieldSchema[] innerTupleFieldSchemas = new ResourceFieldSchema[1]; innerTupleFieldSchemas[0] = new ResourceFieldSchema().setName(innerFieldName) .setDescription("The inner field in the tuple in the bag") .setType(getPigType(arrayElementFieldSchema)) .setSchema(null); bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas)); } ResourceSchema s = new ResourceSchema().setFields(bagSubFieldSchemas); return s; }  static boolean pigHasBooleanSupport(); static Pair<String, String> getDBTableNames(String location); static String getHCatServerUri(Job job); static String getHCatServerPrincipal(Job job); Table getTable(String location, String hcatServerUri, String hcatServerPrincipal); static ResourceSchema getResourceSchema(HCatSchema hcatSchema); static byte getPigType(HCatFieldSchema hfs); static byte getPigType(Type type); static Tuple transformToTuple(HCatRecord hr, HCatSchema hs); @SuppressWarnings("unchecked") static Object extractPigObject(Object o, HCatFieldSchema hfs); static void validateHCatTableSchemaFollowsPigRules(HCatSchema hcatTableSchema); static void getConfigFromUDFProperties(Properties p, Configuration config, String propName); static void saveConfigIntoUDFProperties(Properties p, Configuration config, String propName);  }