@Test public void testResolveConfig() throws IOException { HiveDatasetFinder.DbAndTable realDbAndTable = new HiveDatasetFinder.DbAndTable("realDb", "realTable"); HiveDatasetFinder.DbAndTable logicalDbAndTable = new HiveDatasetFinder.DbAndTable("logicalDb", "logicalTable"); Config resolvedConfig = HiveDataset.resolveConfig(config, realDbAndTable, logicalDbAndTable); Assert.assertEquals(resolvedConfig.getString(DUMMY_CONFIG_KEY_WITH_DB_TOKEN), "resPrefix_realDb_resPostfix", "Real DB not resolved correctly"); Assert.assertEquals(resolvedConfig.getString(DUMMY_CONFIG_KEY_WITH_TABLE_TOKEN), "resPrefix_realTable_resPostfix", "Real Table not resolved correctly"); Assert.assertEquals(resolvedConfig.getString(HiveDatasetVersionCleaner.REPLACEMENT_HIVE_DB_NAME_KEY), "resPrefix_logicalDb_resPostfix", "Logical DB not resolved correctly"); Assert.assertEquals(resolvedConfig.getString(HiveDatasetVersionCleaner.REPLACEMENT_HIVE_TABLE_NAME_KEY), "resPrefix_logicalTable_resPostfix", "Logical Table not resolved correctly"); Assert.assertEquals(resolvedConfig.getString(DUMMY_CONFIG_KEY_WITH_STRIP_SUFFIX),"testRoot"); }
@VisibleForTesting protected static Config resolveConfig(Config datasetConfig, DbAndTable realDbAndTable, DbAndTable logicalDbAndTable) { Preconditions.checkNotNull(datasetConfig, "Dataset config should not be null"); Preconditions.checkNotNull(realDbAndTable, "Real DB and table should not be null"); Preconditions.checkNotNull(logicalDbAndTable, "Logical DB and table should not be null"); ImmutableMap.Builder<String, Object> immutableMapBuilder = ImmutableMap.builder(); Config resolvedConfig = datasetConfig.resolve(); for (Map.Entry<String, ConfigValue> entry : resolvedConfig.entrySet()) { if (ConfigValueType.LIST.equals(entry.getValue().valueType())) { List<String> rawValueList = resolvedConfig.getStringList(entry.getKey()); List<String> resolvedValueList = Lists.newArrayList(); for (String rawValue : rawValueList) { String resolvedValue = StringUtils.replaceEach(rawValue, new String[] { DATABASE_TOKEN, TABLE_TOKEN, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN }, new String[] { realDbAndTable.getDb(), realDbAndTable.getTable(), logicalDbAndTable.getDb(), logicalDbAndTable.getTable() }); resolvedValueList.add(resolvedValue); } StringBuilder listToStringWithQuotes = new StringBuilder(); for (String resolvedValueStr : resolvedValueList) { if (listToStringWithQuotes.length() > 0) { listToStringWithQuotes.append(","); } listToStringWithQuotes.append("\"").append(resolvedValueStr).append("\""); } immutableMapBuilder.put(entry.getKey(), listToStringWithQuotes.toString()); } else { String resolvedValue = StringUtils.replaceEach(resolvedConfig.getString(entry.getKey()), new String[] { DATABASE_TOKEN, TABLE_TOKEN, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN }, new String[] { realDbAndTable.getDb(), realDbAndTable.getTable(), logicalDbAndTable.getDb(), logicalDbAndTable.getTable() }); immutableMapBuilder.put(entry.getKey(), resolvedValue); } } return ConfigFactory.parseMap(immutableMapBuilder.build()); }
HiveDataset implements PrioritizedCopyableDataset { @VisibleForTesting protected static Config resolveConfig(Config datasetConfig, DbAndTable realDbAndTable, DbAndTable logicalDbAndTable) { Preconditions.checkNotNull(datasetConfig, "Dataset config should not be null"); Preconditions.checkNotNull(realDbAndTable, "Real DB and table should not be null"); Preconditions.checkNotNull(logicalDbAndTable, "Logical DB and table should not be null"); ImmutableMap.Builder<String, Object> immutableMapBuilder = ImmutableMap.builder(); Config resolvedConfig = datasetConfig.resolve(); for (Map.Entry<String, ConfigValue> entry : resolvedConfig.entrySet()) { if (ConfigValueType.LIST.equals(entry.getValue().valueType())) { List<String> rawValueList = resolvedConfig.getStringList(entry.getKey()); List<String> resolvedValueList = Lists.newArrayList(); for (String rawValue : rawValueList) { String resolvedValue = StringUtils.replaceEach(rawValue, new String[] { DATABASE_TOKEN, TABLE_TOKEN, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN }, new String[] { realDbAndTable.getDb(), realDbAndTable.getTable(), logicalDbAndTable.getDb(), logicalDbAndTable.getTable() }); resolvedValueList.add(resolvedValue); } StringBuilder listToStringWithQuotes = new StringBuilder(); for (String resolvedValueStr : resolvedValueList) { if (listToStringWithQuotes.length() > 0) { listToStringWithQuotes.append(","); } listToStringWithQuotes.append("\"").append(resolvedValueStr).append("\""); } immutableMapBuilder.put(entry.getKey(), listToStringWithQuotes.toString()); } else { String resolvedValue = StringUtils.replaceEach(resolvedConfig.getString(entry.getKey()), new String[] { DATABASE_TOKEN, TABLE_TOKEN, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN }, new String[] { realDbAndTable.getDb(), realDbAndTable.getTable(), logicalDbAndTable.getDb(), logicalDbAndTable.getTable() }); immutableMapBuilder.put(entry.getKey(), resolvedValue); } } return ConfigFactory.parseMap(immutableMapBuilder.build()); } }
HiveDataset implements PrioritizedCopyableDataset { @VisibleForTesting protected static Config resolveConfig(Config datasetConfig, DbAndTable realDbAndTable, DbAndTable logicalDbAndTable) { Preconditions.checkNotNull(datasetConfig, "Dataset config should not be null"); Preconditions.checkNotNull(realDbAndTable, "Real DB and table should not be null"); Preconditions.checkNotNull(logicalDbAndTable, "Logical DB and table should not be null"); ImmutableMap.Builder<String, Object> immutableMapBuilder = ImmutableMap.builder(); Config resolvedConfig = datasetConfig.resolve(); for (Map.Entry<String, ConfigValue> entry : resolvedConfig.entrySet()) { if (ConfigValueType.LIST.equals(entry.getValue().valueType())) { List<String> rawValueList = resolvedConfig.getStringList(entry.getKey()); List<String> resolvedValueList = Lists.newArrayList(); for (String rawValue : rawValueList) { String resolvedValue = StringUtils.replaceEach(rawValue, new String[] { DATABASE_TOKEN, TABLE_TOKEN, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN }, new String[] { realDbAndTable.getDb(), realDbAndTable.getTable(), logicalDbAndTable.getDb(), logicalDbAndTable.getTable() }); resolvedValueList.add(resolvedValue); } StringBuilder listToStringWithQuotes = new StringBuilder(); for (String resolvedValueStr : resolvedValueList) { if (listToStringWithQuotes.length() > 0) { listToStringWithQuotes.append(","); } listToStringWithQuotes.append("\"").append(resolvedValueStr).append("\""); } immutableMapBuilder.put(entry.getKey(), listToStringWithQuotes.toString()); } else { String resolvedValue = StringUtils.replaceEach(resolvedConfig.getString(entry.getKey()), new String[] { DATABASE_TOKEN, TABLE_TOKEN, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN }, new String[] { realDbAndTable.getDb(), realDbAndTable.getTable(), logicalDbAndTable.getDb(), logicalDbAndTable.getTable() }); immutableMapBuilder.put(entry.getKey(), resolvedValue); } } return ConfigFactory.parseMap(immutableMapBuilder.build()); } HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties);  HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Config datasetConfig);  HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties, Config datasetConfig); }
HiveDataset implements PrioritizedCopyableDataset { @VisibleForTesting protected static Config resolveConfig(Config datasetConfig, DbAndTable realDbAndTable, DbAndTable logicalDbAndTable) { Preconditions.checkNotNull(datasetConfig, "Dataset config should not be null"); Preconditions.checkNotNull(realDbAndTable, "Real DB and table should not be null"); Preconditions.checkNotNull(logicalDbAndTable, "Logical DB and table should not be null"); ImmutableMap.Builder<String, Object> immutableMapBuilder = ImmutableMap.builder(); Config resolvedConfig = datasetConfig.resolve(); for (Map.Entry<String, ConfigValue> entry : resolvedConfig.entrySet()) { if (ConfigValueType.LIST.equals(entry.getValue().valueType())) { List<String> rawValueList = resolvedConfig.getStringList(entry.getKey()); List<String> resolvedValueList = Lists.newArrayList(); for (String rawValue : rawValueList) { String resolvedValue = StringUtils.replaceEach(rawValue, new String[] { DATABASE_TOKEN, TABLE_TOKEN, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN }, new String[] { realDbAndTable.getDb(), realDbAndTable.getTable(), logicalDbAndTable.getDb(), logicalDbAndTable.getTable() }); resolvedValueList.add(resolvedValue); } StringBuilder listToStringWithQuotes = new StringBuilder(); for (String resolvedValueStr : resolvedValueList) { if (listToStringWithQuotes.length() > 0) { listToStringWithQuotes.append(","); } listToStringWithQuotes.append("\"").append(resolvedValueStr).append("\""); } immutableMapBuilder.put(entry.getKey(), listToStringWithQuotes.toString()); } else { String resolvedValue = StringUtils.replaceEach(resolvedConfig.getString(entry.getKey()), new String[] { DATABASE_TOKEN, TABLE_TOKEN, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN }, new String[] { realDbAndTable.getDb(), realDbAndTable.getTable(), logicalDbAndTable.getDb(), logicalDbAndTable.getTable() }); immutableMapBuilder.put(entry.getKey(), resolvedValue); } } return ConfigFactory.parseMap(immutableMapBuilder.build()); } HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties);  HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Config datasetConfig);  HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties, Config datasetConfig); @Override Iterator<FileSet<CopyEntity>> getFileSetIterator(FileSystem targetFs, CopyConfiguration configuration); @Override Iterator<FileSet<CopyEntity>> getFileSetIterator(FileSystem targetFs, CopyConfiguration configuration,
      Comparator<FileSet<CopyEntity>> prioritizer, PushDownRequestor<FileSet<CopyEntity>> requestor); @Override String datasetURN(); static String resolveTemplate(String rawString, Table table); static List<Partition> sortPartitions(List<Partition> partitions); List<Partition> getPartitionsFromDataset(); }
HiveDataset implements PrioritizedCopyableDataset { @VisibleForTesting protected static Config resolveConfig(Config datasetConfig, DbAndTable realDbAndTable, DbAndTable logicalDbAndTable) { Preconditions.checkNotNull(datasetConfig, "Dataset config should not be null"); Preconditions.checkNotNull(realDbAndTable, "Real DB and table should not be null"); Preconditions.checkNotNull(logicalDbAndTable, "Logical DB and table should not be null"); ImmutableMap.Builder<String, Object> immutableMapBuilder = ImmutableMap.builder(); Config resolvedConfig = datasetConfig.resolve(); for (Map.Entry<String, ConfigValue> entry : resolvedConfig.entrySet()) { if (ConfigValueType.LIST.equals(entry.getValue().valueType())) { List<String> rawValueList = resolvedConfig.getStringList(entry.getKey()); List<String> resolvedValueList = Lists.newArrayList(); for (String rawValue : rawValueList) { String resolvedValue = StringUtils.replaceEach(rawValue, new String[] { DATABASE_TOKEN, TABLE_TOKEN, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN }, new String[] { realDbAndTable.getDb(), realDbAndTable.getTable(), logicalDbAndTable.getDb(), logicalDbAndTable.getTable() }); resolvedValueList.add(resolvedValue); } StringBuilder listToStringWithQuotes = new StringBuilder(); for (String resolvedValueStr : resolvedValueList) { if (listToStringWithQuotes.length() > 0) { listToStringWithQuotes.append(","); } listToStringWithQuotes.append("\"").append(resolvedValueStr).append("\""); } immutableMapBuilder.put(entry.getKey(), listToStringWithQuotes.toString()); } else { String resolvedValue = StringUtils.replaceEach(resolvedConfig.getString(entry.getKey()), new String[] { DATABASE_TOKEN, TABLE_TOKEN, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN }, new String[] { realDbAndTable.getDb(), realDbAndTable.getTable(), logicalDbAndTable.getDb(), logicalDbAndTable.getTable() }); immutableMapBuilder.put(entry.getKey(), resolvedValue); } } return ConfigFactory.parseMap(immutableMapBuilder.build()); } HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties);  HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Config datasetConfig);  HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties, Config datasetConfig); @Override Iterator<FileSet<CopyEntity>> getFileSetIterator(FileSystem targetFs, CopyConfiguration configuration); @Override Iterator<FileSet<CopyEntity>> getFileSetIterator(FileSystem targetFs, CopyConfiguration configuration,
      Comparator<FileSet<CopyEntity>> prioritizer, PushDownRequestor<FileSet<CopyEntity>> requestor); @Override String datasetURN(); static String resolveTemplate(String rawString, Table table); static List<Partition> sortPartitions(List<Partition> partitions); List<Partition> getPartitionsFromDataset(); static final ImmutableSet<TableType> COPYABLE_TABLES; static final String REGISTERER; static final String REGISTRATION_GENERATION_TIME_MILLIS; static final String DATASET_NAME_PATTERN_KEY; static final String DATABASE; static final String TABLE; static final String DATASET_STAGING_PATH; static final String DATABASE_TOKEN; static final String TABLE_TOKEN; static final String LOGICAL_DB_TOKEN; static final String LOGICAL_TABLE_TOKEN; }