Hive13DdlGenerator extends DdlGenerator { public List<HivePartitionDto> getHivePartitions(BusinessObjectDataKey businessObjectDataKey, List<SchemaColumn> autoDiscoverableSubPartitionColumns, String s3KeyPrefix, Collection<String> storageFiles, String storageName) { LinkedHashMap<List<String>, HivePartitionDto> linkedHashMap = new LinkedHashMap<>(); Pattern pattern = getHivePathPattern(autoDiscoverableSubPartitionColumns); for (String storageFile : storageFiles) { String relativeFilePath = storageFile.substring(s3KeyPrefix.length()); Matcher matcher = pattern.matcher(relativeFilePath); Assert.isTrue(matcher.matches(), String.format("Registered storage file or directory does not match the expected Hive sub-directory pattern. " + "Storage: {%s}, file/directory: {%s}, business object data: {%s}, S3 key prefix: {%s}, pattern: {%s}", storageName, storageFile, businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey), s3KeyPrefix, pattern.pattern())); List<String> partitionValues = new ArrayList<>(); partitionValues.add(businessObjectDataKey.getPartitionValue()); partitionValues.addAll(businessObjectDataKey.getSubPartitionValues()); for (int i = 1; i <= matcher.groupCount() - 1; i++) { partitionValues.add(matcher.group(i)); } String partitionPathTrailingPart = matcher.group(matcher.groupCount()); if (!partitionValues.contains(null)) { String partitionPath = relativeFilePath.substring(0, relativeFilePath.length() - StringUtils.length(partitionPathTrailingPart)); HivePartitionDto hivePartition = linkedHashMap.get(partitionValues); if (hivePartition != null) { Assert.isTrue(hivePartition.getPath().equals(partitionPath), String.format( "Found two different locations for the same Hive partition. Storage: {%s}, business object data: {%s}, " + "S3 key prefix: {%s}, path[1]: {%s}, path[2]: {%s}", storageName, businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey), s3KeyPrefix, hivePartition.getPath(), partitionPath)); } else { linkedHashMap.put(partitionValues, new HivePartitionDto(partitionPath, partitionValues)); } } } List<HivePartitionDto> hivePartitions = new ArrayList<>(); hivePartitions.addAll(linkedHashMap.values()); return hivePartitions; }  String escapeSingleQuotes(String string); @Override String generateCreateTableDdl(BusinessObjectDataDdlRequest request, BusinessObjectFormatEntity businessObjectFormatEntity,
        CustomDdlEntity customDdlEntity, List<String> storageNames, List<StorageEntity> requestedStorageEntities,
        Map<String, StorageEntity> cachedStorageEntities, Map<String, String> cachedS3BucketNames); @Override String generateCreateTableDdl(BusinessObjectFormatDdlRequest request, BusinessObjectFormatEntity businessObjectFormatEntity,
        CustomDdlEntity customDdlEntity); @Override String generateReplaceColumnsStatement(BusinessObjectFormatDdlRequest request, BusinessObjectFormatEntity businessObjectFormatEntity); String getDdlCharacterValue(String string); String getDdlCharacterValue(String string, boolean escapeSingleBackslash); @Override BusinessObjectDataDdlOutputFormatEnum getDdlOutputFormat(); List<HivePartitionDto> getHivePartitions(BusinessObjectDataKey businessObjectDataKey, List<SchemaColumn> autoDiscoverableSubPartitionColumns,
        String s3KeyPrefix, Collection<String> storageFiles, String storageName); Pattern getHivePathPattern(List<SchemaColumn> partitionColumns); String getHivePathRegex(List<SchemaColumn> partitionColumns); static final String NO_PARTITIONING_PARTITION_KEY; static final String NO_PARTITIONING_PARTITION_VALUE; static final String ORC_HIVE_FILE_FORMAT; static final String PARQUET_HIVE_FILE_FORMAT; static final String TEXT_HIVE_FILE_FORMAT; static final String JSON_HIVE_FILE_FORMAT; static final String REGEX_S3_EMPTY_PARTITION; }
@Test public void testGetHivePartitions() { List<SchemaColumn> autoDiscoverableSubPartitionColumns; List<String> storageFilePaths; List<HivePartitionDto> expectedHivePartitions; List<HivePartitionDto> resultHivePartitions; BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION); autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column1", "column2")); storageFilePaths = new ArrayList<>(); expectedHivePartitions = new ArrayList<>(); resultHivePartitions = hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); assertEquals(expectedHivePartitions, resultHivePartitions); autoDiscoverableSubPartitionColumns = new ArrayList<>(); storageFilePaths = getStorageFilePaths(Arrays.asList("/file1.dat", "/file2.dat", "/")); expectedHivePartitions = Collections.singletonList(HivePartitionDto.builder().withPath("").withPartitionValues(Collections.singletonList(PARTITION_VALUE)).build()); resultHivePartitions = hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); assertEquals(expectedHivePartitions, resultHivePartitions); autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column1", "column2")); storageFilePaths = getStorageFilePaths(Arrays.asList("/COLUMN1=111/COLUMN2=222/file.dat", "/column1=aa/column2=bb/")); expectedHivePartitions = Arrays .asList(HivePartitionDto.builder().withPath("/COLUMN1=111/COLUMN2=222").withPartitionValues(Arrays.asList(PARTITION_VALUE, "111", "222")).build(), HivePartitionDto.builder().withPath("/column1=aa/column2=bb").withPartitionValues(Arrays.asList(PARTITION_VALUE, "aa", "bb")).build()); resultHivePartitions = hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); assertEquals(expectedHivePartitions, resultHivePartitions); }
@Test public void testGetHivePartitionEmptyPartitions() { List<SchemaColumn> autoDiscoverableSubPartitionColumns; List<String> storageFilePaths; List<HivePartitionDto> expectedHivePartitions; List<HivePartitionDto> resultHivePartitions; BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION); autoDiscoverableSubPartitionColumns = new ArrayList<>(); storageFilePaths = getStorageFilePaths(Collections.singletonList("/")); expectedHivePartitions = Collections.singletonList(HivePartitionDto.builder().withPath("").withPartitionValues(Collections.singletonList(PARTITION_VALUE)).build()); resultHivePartitions = hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); assertEquals(expectedHivePartitions, resultHivePartitions); autoDiscoverableSubPartitionColumns = new ArrayList<>(); storageFilePaths = getStorageFilePaths(Collections.singletonList("_$folder$")); expectedHivePartitions = Collections.singletonList(HivePartitionDto.builder().withPath("").withPartitionValues(Collections.singletonList(PARTITION_VALUE)).build()); resultHivePartitions = hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); assertEquals(expectedHivePartitions, resultHivePartitions); autoDiscoverableSubPartitionColumns = getPartitionColumns(Collections.singletonList("column1")); storageFilePaths = getStorageFilePaths(Collections.singletonList("/column1=aa/")); expectedHivePartitions = Collections.singletonList(HivePartitionDto.builder().withPath("/column1=aa").withPartitionValues(Arrays.asList(PARTITION_VALUE, "aa")).build()); resultHivePartitions = hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); assertEquals(expectedHivePartitions, resultHivePartitions); autoDiscoverableSubPartitionColumns = getPartitionColumns(Collections.singletonList("column1")); storageFilePaths = getStorageFilePaths(Collections.singletonList("/column1=aa_$folder$")); expectedHivePartitions = Collections.singletonList(HivePartitionDto.builder().withPath("/column1=aa").withPartitionValues(Arrays.asList(PARTITION_VALUE, "aa")).build()); resultHivePartitions = hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); assertEquals(expectedHivePartitions, resultHivePartitions); autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column1", "column2")); storageFilePaths = getStorageFilePaths(Collections.singletonList("/column1=aa/column2=bb/")); expectedHivePartitions = Collections.singletonList( HivePartitionDto.builder().withPath("/column1=aa/column2=bb").withPartitionValues(Arrays.asList(PARTITION_VALUE, "aa", "bb")).build()); resultHivePartitions = hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); assertEquals(expectedHivePartitions, resultHivePartitions); autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column1", "column2")); storageFilePaths = getStorageFilePaths(Collections.singletonList("/column1=aa/column2=bb_$folder$")); expectedHivePartitions = Collections.singletonList( HivePartitionDto.builder().withPath("/column1=aa/column2=bb").withPartitionValues(Arrays.asList(PARTITION_VALUE, "aa", "bb")).build()); resultHivePartitions = hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); assertEquals(expectedHivePartitions, resultHivePartitions); autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column1", "column2")); storageFilePaths = getStorageFilePaths(Arrays.asList("/column1=aa/column2=bb/", "/column1=aa/column2=bb/file.dat")); expectedHivePartitions = Collections.singletonList( HivePartitionDto.builder().withPath("/column1=aa/column2=bb").withPartitionValues(Arrays.asList(PARTITION_VALUE, "aa", "bb")).build()); resultHivePartitions = hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); assertEquals(expectedHivePartitions, resultHivePartitions); autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column1", "column2")); storageFilePaths = getStorageFilePaths(Arrays.asList("/column1=aa/column2=bb_$folder$", "/column1=aa/column2=bb/file.dat")); expectedHivePartitions = Collections.singletonList( HivePartitionDto.builder().withPath("/column1=aa/column2=bb").withPartitionValues(Arrays.asList(PARTITION_VALUE, "aa", "bb")).build()); resultHivePartitions = hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); assertEquals(expectedHivePartitions, resultHivePartitions); autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column_1", "Column_2", "Column_3", "Column_4")); storageFilePaths = getStorageFilePaths(Arrays.asList("/", "_$folder$", "/column_1=a01/", "/column_1=a02_$folder$", "/column_1=a03/column-2=b01/", "/column_1=a04/column-2=b02_$folder$", "/column_1=a05/column-2=b03/COLUMN_3=c01/", "/column_1=a06/column-2=b04/COLUMN_3=c02_$folder$", "/column_1=a07/column-2=b05/COLUMN_3=c03/COLUMN-4=d01/", "/column_1=a08/column-2=b06/COLUMN_3=c04/COLUMN-4=d02_$folder$", "/column_1=a09/column-2=b07/COLUMN_3=c05/COLUMN-4=d03/file.dat", "/column_1=a10/column-2=b08/COLUMN_3=c06/COLUMN-4=d04_$folder$/", "/column_1=a11/column-2=b09/COLUMN_3=c07/COLUMN-4=d05_$folder$/file.dat", "/column_1=a12/column-2=b10/COLUMN_3=c08/COLUMN-4=d06/_$folder$file.dat", "/column_1=a13/column-2=b11/COLUMN_3=c09/COLUMN-4=d07/file_$folder$.dat", "/column_1=a14/column-2=b12/COLUMN_3=c10/COLUMN-4=d08/file.dat_$folder$", "/column_1=a15/column-2=b13/COLUMN_3=c11/COLUMN-4=d09/_$folder$" )); expectedHivePartitions = Arrays.asList(HivePartitionDto.builder().withPath("/column_1=a07/column-2=b05/COLUMN_3=c03/COLUMN-4=d01") .withPartitionValues(Arrays.asList(PARTITION_VALUE, "a07", "b05", "c03", "d01")).build(), HivePartitionDto.builder().withPath("/column_1=a08/column-2=b06/COLUMN_3=c04/COLUMN-4=d02") .withPartitionValues(Arrays.asList(PARTITION_VALUE, "a08", "b06", "c04", "d02")).build(), HivePartitionDto.builder().withPath("/column_1=a09/column-2=b07/COLUMN_3=c05/COLUMN-4=d03") .withPartitionValues(Arrays.asList(PARTITION_VALUE, "a09", "b07", "c05", "d03")).build(), HivePartitionDto.builder().withPath("/column_1=a10/column-2=b08/COLUMN_3=c06/COLUMN-4=d04_$folder$") .withPartitionValues(Arrays.asList(PARTITION_VALUE, "a10", "b08", "c06", "d04_$folder$")).build(), HivePartitionDto.builder().withPath("/column_1=a11/column-2=b09/COLUMN_3=c07/COLUMN-4=d05_$folder$") .withPartitionValues(Arrays.asList(PARTITION_VALUE, "a11", "b09", "c07", "d05_$folder$")).build(), HivePartitionDto.builder().withPath("/column_1=a12/column-2=b10/COLUMN_3=c08/COLUMN-4=d06") .withPartitionValues(Arrays.asList(PARTITION_VALUE, "a12", "b10", "c08", "d06")).build(), HivePartitionDto.builder().withPath("/column_1=a13/column-2=b11/COLUMN_3=c09/COLUMN-4=d07") .withPartitionValues(Arrays.asList(PARTITION_VALUE, "a13", "b11", "c09", "d07")).build(), HivePartitionDto.builder().withPath("/column_1=a14/column-2=b12/COLUMN_3=c10/COLUMN-4=d08") .withPartitionValues(Arrays.asList(PARTITION_VALUE, "a14", "b12", "c10", "d08")).build(), HivePartitionDto.builder().withPath("/column_1=a15/column-2=b13/COLUMN_3=c11/COLUMN-4=d09") .withPartitionValues(Arrays.asList(PARTITION_VALUE, "a15", "b13", "c11", "d09")).build()); resultHivePartitions = hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); assertEquals(expectedHivePartitions, resultHivePartitions); }
@Test public void testGetHivePartitionsMultiplePathsFound() { List<SchemaColumn> autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column1", "column2")); List<String> partitionPaths = Arrays.asList("/COLUMN1=111/COLUMN2=222", "/column1=111/COLUMN2=222"); List<String> storageFilePaths = getStorageFilePaths(Arrays.asList(partitionPaths.get(0) + "/file.dat", partitionPaths.get(1) + "/file.dat")); BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION); try { hive13DdlGenerator .getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, STORAGE_NAME); fail("Should throw an IllegalArgumentException when multiple locations exist for the same Hive partition."); } catch (IllegalArgumentException e) { assertEquals(String.format("Found two different locations for the same Hive partition. " + "Storage: {%s}, business object data: {%s}, S3 key prefix: {%s}, path[1]: {%s}, path[2]: {%s}", STORAGE_NAME, businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey), TEST_S3_KEY_PREFIX, partitionPaths.get(0), partitionPaths.get(1)), e.getMessage()); } }