@Test public void testSerializeAsPath() throws Exception { Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"test\", " + "\"fields\":[" + "{\"name\": \"name\", \"type\": \"string\"}, " + "{\"name\": \"title\", \"type\": \"string\"}" + "]}"); GenericRecord partition = new GenericData.Record(schema); partition.put("name", "a/b:c\\d e"); partition.put("title", "title"); Assert.assertEquals(AvroUtils.serializeAsPath(partition, true, true), new Path("name=a_b_c_d_e/title=title")); Assert.assertEquals(AvroUtils.serializeAsPath(partition, false, true), new Path("a_b_c_d_e/title")); Assert.assertEquals(AvroUtils.serializeAsPath(partition, false, false), new Path("a/b_c_d_e/title")); }
public static Path serializeAsPath(GenericRecord record, boolean includeFieldNames, boolean replacePathSeparators) { if (record == null) { return new Path(""); } List<String> tokens = Lists.newArrayList(); for (Schema.Field field : record.getSchema().getFields()) { String sanitizedName = HadoopUtils.sanitizePath(field.name(), "_"); String sanitizedValue = HadoopUtils.sanitizePath(record.get(field.name()).toString(), "_"); if (replacePathSeparators) { sanitizedName = sanitizedName.replaceAll(Path.SEPARATOR, "_"); sanitizedValue = sanitizedValue.replaceAll(Path.SEPARATOR, "_"); } if (includeFieldNames) { tokens.add(String.format("%s=%s", sanitizedName, sanitizedValue)); } else if (!Strings.isNullOrEmpty(sanitizedValue)) { tokens.add(sanitizedValue); } } return new Path(Joiner.on(Path.SEPARATOR).join(tokens)); }
AvroUtils { public static Path serializeAsPath(GenericRecord record, boolean includeFieldNames, boolean replacePathSeparators) { if (record == null) { return new Path(""); } List<String> tokens = Lists.newArrayList(); for (Schema.Field field : record.getSchema().getFields()) { String sanitizedName = HadoopUtils.sanitizePath(field.name(), "_"); String sanitizedValue = HadoopUtils.sanitizePath(record.get(field.name()).toString(), "_"); if (replacePathSeparators) { sanitizedName = sanitizedName.replaceAll(Path.SEPARATOR, "_"); sanitizedValue = sanitizedValue.replaceAll(Path.SEPARATOR, "_"); } if (includeFieldNames) { tokens.add(String.format("%s=%s", sanitizedName, sanitizedValue)); } else if (!Strings.isNullOrEmpty(sanitizedValue)) { tokens.add(sanitizedValue); } } return new Path(Joiner.on(Path.SEPARATOR).join(tokens)); } }
AvroUtils { public static Path serializeAsPath(GenericRecord record, boolean includeFieldNames, boolean replacePathSeparators) { if (record == null) { return new Path(""); } List<String> tokens = Lists.newArrayList(); for (Schema.Field field : record.getSchema().getFields()) { String sanitizedName = HadoopUtils.sanitizePath(field.name(), "_"); String sanitizedValue = HadoopUtils.sanitizePath(record.get(field.name()).toString(), "_"); if (replacePathSeparators) { sanitizedName = sanitizedName.replaceAll(Path.SEPARATOR, "_"); sanitizedValue = sanitizedValue.replaceAll(Path.SEPARATOR, "_"); } if (includeFieldNames) { tokens.add(String.format("%s=%s", sanitizedName, sanitizedValue)); } else if (!Strings.isNullOrEmpty(sanitizedValue)) { tokens.add(sanitizedValue); } } return new Path(Joiner.on(Path.SEPARATOR).join(tokens)); }  }
AvroUtils { public static Path serializeAsPath(GenericRecord record, boolean includeFieldNames, boolean replacePathSeparators) { if (record == null) { return new Path(""); } List<String> tokens = Lists.newArrayList(); for (Schema.Field field : record.getSchema().getFields()) { String sanitizedName = HadoopUtils.sanitizePath(field.name(), "_"); String sanitizedValue = HadoopUtils.sanitizePath(record.get(field.name()).toString(), "_"); if (replacePathSeparators) { sanitizedName = sanitizedName.replaceAll(Path.SEPARATOR, "_"); sanitizedValue = sanitizedValue.replaceAll(Path.SEPARATOR, "_"); } if (includeFieldNames) { tokens.add(String.format("%s=%s", sanitizedName, sanitizedValue)); } else if (!Strings.isNullOrEmpty(sanitizedValue)) { tokens.add(sanitizedValue); } } return new Path(Joiner.on(Path.SEPARATOR).join(tokens)); }  static boolean checkReaderWriterCompatibility(Schema readerSchema, Schema writerSchema, boolean ignoreNamespace); static Schema addSchemaCreationTime(Schema inputSchema, Schema outputSchema); static String getSchemaCreationTime(Schema inputSchema); static Schema setSchemaCreationTime(Schema inputSchema, String creationTime); static List<Field> deepCopySchemaFields(Schema readerSchema); static void convertFieldToSchemaWithProps(Map<String,JsonNode> fieldProps, Schema targetSchemaObj); static Optional<Schema> getFieldSchema(Schema schema, String fieldLocation); static Optional<Field> getField(Schema schema, String fieldLocation); static Optional<Object> getFieldValue(GenericRecord record, String fieldLocation); static Map<String, Object> getMultiFieldValue(GenericRecord record, String fieldLocation); @SuppressWarnings("unchecked") static Map<String, String> toStringMap(Object map); static GenericRecord convertRecordSchema(GenericRecord record, Schema newSchema); static byte[] recordToByteArray(GenericRecord record); static Schema getSchemaFromDataFile(Path dataFile, FileSystem fs); static Schema parseSchemaFromFile(Path filePath, FileSystem fs); static void writeSchemaToFile(Schema schema, Path filePath, FileSystem fs, boolean overwrite); static void writeSchemaToFile(Schema schema, Path filePath, Path tempFilePath, FileSystem fs, boolean overwrite); static void writeSchemaToFile(Schema schema, Path filePath, FileSystem fs, boolean overwrite, FsPermission perm); static void writeSchemaToFile(Schema schema, Path filePath, Path tempFilePath, FileSystem fs, boolean overwrite,
      FsPermission perm); static Schema getDirectorySchema(Path directory, FileSystem fs, boolean latest); static Schema getDirectorySchema(Path directory, Configuration conf, boolean latest); static Schema nullifyFieldsForSchemaMerge(Schema oldSchema, Schema newSchema); static Optional<Schema> removeUncomparableFields(Schema schema); static Schema switchName(Schema schema, String newName); static Schema switchNamespace(Schema schema, Map<String, String> namespaceOverride); static Path serializeAsPath(GenericRecord record, boolean includeFieldNames, boolean replacePathSeparators); static String sanitizeSchemaString(String schemaString); static GenericRecord slowDeserializeGenericRecord(byte[] serializedRecord, Schema schema); static Schema decorateRecordSchema(Schema inputSchema, @Nonnull List<Field> fieldList); static GenericRecord decorateRecord(GenericRecord inputRecord, @Nonnull Map<String, Object> fieldMap,
      Schema outputSchema); static GenericRecord overrideNameAndNamespace(GenericRecord input, String nameOverride, Optional<Map<String, String>> namespaceOverride); static Schema overrideNameAndNamespace(Schema input, String nameOverride, Optional<Map<String, String>> namespaceOverride); static boolean isSchemaRecursive(Schema schema, Optional<Logger> logger); static Pair<Schema, List<SchemaEntry>> dropRecursiveFields(Schema schema); }
AvroUtils { public static Path serializeAsPath(GenericRecord record, boolean includeFieldNames, boolean replacePathSeparators) { if (record == null) { return new Path(""); } List<String> tokens = Lists.newArrayList(); for (Schema.Field field : record.getSchema().getFields()) { String sanitizedName = HadoopUtils.sanitizePath(field.name(), "_"); String sanitizedValue = HadoopUtils.sanitizePath(record.get(field.name()).toString(), "_"); if (replacePathSeparators) { sanitizedName = sanitizedName.replaceAll(Path.SEPARATOR, "_"); sanitizedValue = sanitizedValue.replaceAll(Path.SEPARATOR, "_"); } if (includeFieldNames) { tokens.add(String.format("%s=%s", sanitizedName, sanitizedValue)); } else if (!Strings.isNullOrEmpty(sanitizedValue)) { tokens.add(sanitizedValue); } } return new Path(Joiner.on(Path.SEPARATOR).join(tokens)); }  static boolean checkReaderWriterCompatibility(Schema readerSchema, Schema writerSchema, boolean ignoreNamespace); static Schema addSchemaCreationTime(Schema inputSchema, Schema outputSchema); static String getSchemaCreationTime(Schema inputSchema); static Schema setSchemaCreationTime(Schema inputSchema, String creationTime); static List<Field> deepCopySchemaFields(Schema readerSchema); static void convertFieldToSchemaWithProps(Map<String,JsonNode> fieldProps, Schema targetSchemaObj); static Optional<Schema> getFieldSchema(Schema schema, String fieldLocation); static Optional<Field> getField(Schema schema, String fieldLocation); static Optional<Object> getFieldValue(GenericRecord record, String fieldLocation); static Map<String, Object> getMultiFieldValue(GenericRecord record, String fieldLocation); @SuppressWarnings("unchecked") static Map<String, String> toStringMap(Object map); static GenericRecord convertRecordSchema(GenericRecord record, Schema newSchema); static byte[] recordToByteArray(GenericRecord record); static Schema getSchemaFromDataFile(Path dataFile, FileSystem fs); static Schema parseSchemaFromFile(Path filePath, FileSystem fs); static void writeSchemaToFile(Schema schema, Path filePath, FileSystem fs, boolean overwrite); static void writeSchemaToFile(Schema schema, Path filePath, Path tempFilePath, FileSystem fs, boolean overwrite); static void writeSchemaToFile(Schema schema, Path filePath, FileSystem fs, boolean overwrite, FsPermission perm); static void writeSchemaToFile(Schema schema, Path filePath, Path tempFilePath, FileSystem fs, boolean overwrite,
      FsPermission perm); static Schema getDirectorySchema(Path directory, FileSystem fs, boolean latest); static Schema getDirectorySchema(Path directory, Configuration conf, boolean latest); static Schema nullifyFieldsForSchemaMerge(Schema oldSchema, Schema newSchema); static Optional<Schema> removeUncomparableFields(Schema schema); static Schema switchName(Schema schema, String newName); static Schema switchNamespace(Schema schema, Map<String, String> namespaceOverride); static Path serializeAsPath(GenericRecord record, boolean includeFieldNames, boolean replacePathSeparators); static String sanitizeSchemaString(String schemaString); static GenericRecord slowDeserializeGenericRecord(byte[] serializedRecord, Schema schema); static Schema decorateRecordSchema(Schema inputSchema, @Nonnull List<Field> fieldList); static GenericRecord decorateRecord(GenericRecord inputRecord, @Nonnull Map<String, Object> fieldMap,
      Schema outputSchema); static GenericRecord overrideNameAndNamespace(GenericRecord input, String nameOverride, Optional<Map<String, String>> namespaceOverride); static Schema overrideNameAndNamespace(Schema input, String nameOverride, Optional<Map<String, String>> namespaceOverride); static boolean isSchemaRecursive(Schema schema, Optional<Logger> logger); static Pair<Schema, List<SchemaEntry>> dropRecursiveFields(Schema schema); static final String FIELD_LOCATION_DELIMITER; }