@Test public void testAddMetadataToTargetTable() throws Exception { org.apache.hadoop.hive.ql.metadata.Table meta_table = new Table(Table.getEmptyTable("testDB", "testTable")); Map<String, String> storageParams = new HashMap<>(); storageParams.put("path", "randomPath"); meta_table.getSd().getSerdeInfo().setParameters(storageParams); HiveCopyEntityHelper.addMetadataToTargetTable(meta_table, new Path("newPath"), "testDB", 10L); Assert.assertEquals(meta_table.getSd().getSerdeInfo().getParameters().get("path"), "newPath"); storageParams.clear(); meta_table.getSd().getSerdeInfo().setParameters(storageParams); HiveCopyEntityHelper.addMetadataToTargetTable(meta_table, new Path("newPath"), "testDB", 10L); Assert.assertFalse(meta_table.getSd().getSerdeInfo().getParameters().containsKey("path")); }
@VisibleForTesting static void addMetadataToTargetTable(Table targetTable, Path targetLocation, String targetDatabase, long startTime) throws IOException { targetTable.setDbName(targetDatabase); targetTable.setDataLocation(targetLocation); targetTable.setOwner(UserGroupInformation.getCurrentUser().getShortUserName()); targetTable.getTTable().putToParameters(HiveDataset.REGISTERER, GOBBLIN_DISTCP); targetTable.getTTable().putToParameters(HiveDataset.REGISTRATION_GENERATION_TIME_MILLIS, Long.toString(startTime)); targetTable.getTTable().getSd().getSerdeInfo().getParameters() .computeIfPresent(HiveConstants.PATH, (k,v) -> targetLocation.toString()); targetTable.getTTable().unsetCreateTime(); }
HiveCopyEntityHelper { @VisibleForTesting static void addMetadataToTargetTable(Table targetTable, Path targetLocation, String targetDatabase, long startTime) throws IOException { targetTable.setDbName(targetDatabase); targetTable.setDataLocation(targetLocation); targetTable.setOwner(UserGroupInformation.getCurrentUser().getShortUserName()); targetTable.getTTable().putToParameters(HiveDataset.REGISTERER, GOBBLIN_DISTCP); targetTable.getTTable().putToParameters(HiveDataset.REGISTRATION_GENERATION_TIME_MILLIS, Long.toString(startTime)); targetTable.getTTable().getSd().getSerdeInfo().getParameters() .computeIfPresent(HiveConstants.PATH, (k,v) -> targetLocation.toString()); targetTable.getTTable().unsetCreateTime(); } }
HiveCopyEntityHelper { @VisibleForTesting static void addMetadataToTargetTable(Table targetTable, Path targetLocation, String targetDatabase, long startTime) throws IOException { targetTable.setDbName(targetDatabase); targetTable.setDataLocation(targetLocation); targetTable.setOwner(UserGroupInformation.getCurrentUser().getShortUserName()); targetTable.getTTable().putToParameters(HiveDataset.REGISTERER, GOBBLIN_DISTCP); targetTable.getTTable().putToParameters(HiveDataset.REGISTRATION_GENERATION_TIME_MILLIS, Long.toString(startTime)); targetTable.getTTable().getSd().getSerdeInfo().getParameters() .computeIfPresent(HiveConstants.PATH, (k,v) -> targetLocation.toString()); targetTable.getTTable().unsetCreateTime(); } HiveCopyEntityHelper(HiveDataset dataset, CopyConfiguration configuration, FileSystem targetFs); }
HiveCopyEntityHelper { @VisibleForTesting static void addMetadataToTargetTable(Table targetTable, Path targetLocation, String targetDatabase, long startTime) throws IOException { targetTable.setDbName(targetDatabase); targetTable.setDataLocation(targetLocation); targetTable.setOwner(UserGroupInformation.getCurrentUser().getShortUserName()); targetTable.getTTable().putToParameters(HiveDataset.REGISTERER, GOBBLIN_DISTCP); targetTable.getTTable().putToParameters(HiveDataset.REGISTRATION_GENERATION_TIME_MILLIS, Long.toString(startTime)); targetTable.getTTable().getSd().getSerdeInfo().getParameters() .computeIfPresent(HiveConstants.PATH, (k,v) -> targetLocation.toString()); targetTable.getTTable().unsetCreateTime(); } HiveCopyEntityHelper(HiveDataset dataset, CopyConfiguration configuration, FileSystem targetFs); FileSystem getTargetFileSystem(); }
HiveCopyEntityHelper { @VisibleForTesting static void addMetadataToTargetTable(Table targetTable, Path targetLocation, String targetDatabase, long startTime) throws IOException { targetTable.setDbName(targetDatabase); targetTable.setDataLocation(targetLocation); targetTable.setOwner(UserGroupInformation.getCurrentUser().getShortUserName()); targetTable.getTTable().putToParameters(HiveDataset.REGISTERER, GOBBLIN_DISTCP); targetTable.getTTable().putToParameters(HiveDataset.REGISTRATION_GENERATION_TIME_MILLIS, Long.toString(startTime)); targetTable.getTTable().getSd().getSerdeInfo().getParameters() .computeIfPresent(HiveConstants.PATH, (k,v) -> targetLocation.toString()); targetTable.getTTable().unsetCreateTime(); } HiveCopyEntityHelper(HiveDataset dataset, CopyConfiguration configuration, FileSystem targetFs); FileSystem getTargetFileSystem(); static final String EXISTING_ENTITY_POLICY_KEY; static final String DEFAULT_EXISTING_ENTITY_POLICY; static final String UNMANAGED_DATA_POLICY_KEY; static final String DEFAULT_UNMANAGED_DATA_POLICY; static final String TARGET_METASTORE_URI_KEY; static final String TARGET_DATABASE_KEY; static final String COPY_PARTITIONS_FILTER_CONSTANT; static final String COPY_PARTITION_FILTER_GENERATOR; static final String FAST_PARTITION_SKIP_PREDICATE; static final String FAST_TABLE_SKIP_PREDICATE; static final String DELETE_FILES_ON_DEREGISTER; static final DeregisterFileDeleteMethod DEFAULT_DEREGISTER_DELETE_METHOD; static final String HIVE_PARTITION_EXTENDED_FILTER_TYPE; static final String GOBBLIN_DISTCP; }