@Test public void testAddTableDeregisterSteps() throws Exception { HiveDataset dataset = Mockito.mock(HiveDataset.class); Mockito.when(dataset.getProperties()).thenReturn(new Properties()); HiveCopyEntityHelper helper = Mockito.mock(HiveCopyEntityHelper.class); Mockito.when(helper.getDeleteMethod()).thenReturn(DeregisterFileDeleteMethod.NO_DELETE); Mockito.when(helper.getTargetURI()).thenReturn(Optional.of("/targetURI")); Mockito.when(helper.getHiveRegProps()).thenReturn(new HiveRegProps(new State())); Mockito.when(helper.getDataset()).thenReturn(dataset); Mockito.when(helper.addTableDeregisterSteps(Mockito.any(List.class), Mockito.any(String.class), Mockito.anyInt(), Mockito.any(org.apache.hadoop.hive.ql.metadata.Table.class))).thenCallRealMethod(); org.apache.hadoop.hive.ql.metadata.Table meta_table = Mockito.mock(org.apache.hadoop.hive.ql.metadata.Table.class); org.apache.hadoop.hive.metastore.api.Table api_table = Mockito.mock(org.apache.hadoop.hive.metastore.api.Table.class); Mockito.when(api_table.getDbName()).thenReturn("TestDB"); Mockito.when(api_table.getTableName()).thenReturn("TestTable"); Mockito.when(meta_table.getTTable()).thenReturn(api_table); List<CopyEntity> copyEntities = new ArrayList<CopyEntity>(); String fileSet = "testFileSet"; int initialPriority = 0; int priority = helper.addTableDeregisterSteps(copyEntities, fileSet, initialPriority, meta_table); Assert.assertTrue(priority == 1); Assert.assertTrue(copyEntities.size() == 1); Assert.assertTrue(copyEntities.get(0) instanceof PostPublishStep); PostPublishStep p = (PostPublishStep) (copyEntities.get(0)); Assert .assertTrue(p.getStep().toString().contains("Deregister table TestDB.TestTable on Hive metastore /targetURI")); }
@VisibleForTesting protected int addTableDeregisterSteps(List<CopyEntity> copyEntities, String fileSet, int initialPriority, Table table) throws IOException { int stepPriority = initialPriority; Collection<Path> tablePaths = Lists.newArrayList(); switch (this.getDeleteMethod()) { case RECURSIVE: tablePaths = Lists.newArrayList(table.getDataLocation()); break; case INPUT_FORMAT: InputFormat<?, ?> inputFormat = HiveUtils.getInputFormat(table.getSd()); HiveLocationDescriptor targetLocation = new HiveLocationDescriptor(table.getDataLocation(), inputFormat, this.getTargetFs(), this.getDataset().getProperties()); tablePaths = targetLocation.getPaths().keySet(); break; case NO_DELETE: tablePaths = Lists.newArrayList(); break; default: tablePaths = Lists.newArrayList(); } if (!tablePaths.isEmpty()) { DeleteFileCommitStep deletePaths = DeleteFileCommitStep.fromPaths(this.getTargetFs(), tablePaths, this.getDataset().getProperties(), table.getDataLocation()); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deletePaths, stepPriority++)); } TableDeregisterStep deregister = new TableDeregisterStep(table.getTTable(), this.getTargetURI(), this.getHiveRegProps()); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deregister, stepPriority++)); return stepPriority; }
HiveCopyEntityHelper { @VisibleForTesting protected int addTableDeregisterSteps(List<CopyEntity> copyEntities, String fileSet, int initialPriority, Table table) throws IOException { int stepPriority = initialPriority; Collection<Path> tablePaths = Lists.newArrayList(); switch (this.getDeleteMethod()) { case RECURSIVE: tablePaths = Lists.newArrayList(table.getDataLocation()); break; case INPUT_FORMAT: InputFormat<?, ?> inputFormat = HiveUtils.getInputFormat(table.getSd()); HiveLocationDescriptor targetLocation = new HiveLocationDescriptor(table.getDataLocation(), inputFormat, this.getTargetFs(), this.getDataset().getProperties()); tablePaths = targetLocation.getPaths().keySet(); break; case NO_DELETE: tablePaths = Lists.newArrayList(); break; default: tablePaths = Lists.newArrayList(); } if (!tablePaths.isEmpty()) { DeleteFileCommitStep deletePaths = DeleteFileCommitStep.fromPaths(this.getTargetFs(), tablePaths, this.getDataset().getProperties(), table.getDataLocation()); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deletePaths, stepPriority++)); } TableDeregisterStep deregister = new TableDeregisterStep(table.getTTable(), this.getTargetURI(), this.getHiveRegProps()); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deregister, stepPriority++)); return stepPriority; } }
HiveCopyEntityHelper { @VisibleForTesting protected int addTableDeregisterSteps(List<CopyEntity> copyEntities, String fileSet, int initialPriority, Table table) throws IOException { int stepPriority = initialPriority; Collection<Path> tablePaths = Lists.newArrayList(); switch (this.getDeleteMethod()) { case RECURSIVE: tablePaths = Lists.newArrayList(table.getDataLocation()); break; case INPUT_FORMAT: InputFormat<?, ?> inputFormat = HiveUtils.getInputFormat(table.getSd()); HiveLocationDescriptor targetLocation = new HiveLocationDescriptor(table.getDataLocation(), inputFormat, this.getTargetFs(), this.getDataset().getProperties()); tablePaths = targetLocation.getPaths().keySet(); break; case NO_DELETE: tablePaths = Lists.newArrayList(); break; default: tablePaths = Lists.newArrayList(); } if (!tablePaths.isEmpty()) { DeleteFileCommitStep deletePaths = DeleteFileCommitStep.fromPaths(this.getTargetFs(), tablePaths, this.getDataset().getProperties(), table.getDataLocation()); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deletePaths, stepPriority++)); } TableDeregisterStep deregister = new TableDeregisterStep(table.getTTable(), this.getTargetURI(), this.getHiveRegProps()); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deregister, stepPriority++)); return stepPriority; } HiveCopyEntityHelper(HiveDataset dataset, CopyConfiguration configuration, FileSystem targetFs); }
HiveCopyEntityHelper { @VisibleForTesting protected int addTableDeregisterSteps(List<CopyEntity> copyEntities, String fileSet, int initialPriority, Table table) throws IOException { int stepPriority = initialPriority; Collection<Path> tablePaths = Lists.newArrayList(); switch (this.getDeleteMethod()) { case RECURSIVE: tablePaths = Lists.newArrayList(table.getDataLocation()); break; case INPUT_FORMAT: InputFormat<?, ?> inputFormat = HiveUtils.getInputFormat(table.getSd()); HiveLocationDescriptor targetLocation = new HiveLocationDescriptor(table.getDataLocation(), inputFormat, this.getTargetFs(), this.getDataset().getProperties()); tablePaths = targetLocation.getPaths().keySet(); break; case NO_DELETE: tablePaths = Lists.newArrayList(); break; default: tablePaths = Lists.newArrayList(); } if (!tablePaths.isEmpty()) { DeleteFileCommitStep deletePaths = DeleteFileCommitStep.fromPaths(this.getTargetFs(), tablePaths, this.getDataset().getProperties(), table.getDataLocation()); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deletePaths, stepPriority++)); } TableDeregisterStep deregister = new TableDeregisterStep(table.getTTable(), this.getTargetURI(), this.getHiveRegProps()); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deregister, stepPriority++)); return stepPriority; } HiveCopyEntityHelper(HiveDataset dataset, CopyConfiguration configuration, FileSystem targetFs); FileSystem getTargetFileSystem(); }
HiveCopyEntityHelper { @VisibleForTesting protected int addTableDeregisterSteps(List<CopyEntity> copyEntities, String fileSet, int initialPriority, Table table) throws IOException { int stepPriority = initialPriority; Collection<Path> tablePaths = Lists.newArrayList(); switch (this.getDeleteMethod()) { case RECURSIVE: tablePaths = Lists.newArrayList(table.getDataLocation()); break; case INPUT_FORMAT: InputFormat<?, ?> inputFormat = HiveUtils.getInputFormat(table.getSd()); HiveLocationDescriptor targetLocation = new HiveLocationDescriptor(table.getDataLocation(), inputFormat, this.getTargetFs(), this.getDataset().getProperties()); tablePaths = targetLocation.getPaths().keySet(); break; case NO_DELETE: tablePaths = Lists.newArrayList(); break; default: tablePaths = Lists.newArrayList(); } if (!tablePaths.isEmpty()) { DeleteFileCommitStep deletePaths = DeleteFileCommitStep.fromPaths(this.getTargetFs(), tablePaths, this.getDataset().getProperties(), table.getDataLocation()); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deletePaths, stepPriority++)); } TableDeregisterStep deregister = new TableDeregisterStep(table.getTTable(), this.getTargetURI(), this.getHiveRegProps()); copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String> newHashMap(), deregister, stepPriority++)); return stepPriority; } HiveCopyEntityHelper(HiveDataset dataset, CopyConfiguration configuration, FileSystem targetFs); FileSystem getTargetFileSystem(); static final String EXISTING_ENTITY_POLICY_KEY; static final String DEFAULT_EXISTING_ENTITY_POLICY; static final String UNMANAGED_DATA_POLICY_KEY; static final String DEFAULT_UNMANAGED_DATA_POLICY; static final String TARGET_METASTORE_URI_KEY; static final String TARGET_DATABASE_KEY; static final String COPY_PARTITIONS_FILTER_CONSTANT; static final String COPY_PARTITION_FILTER_GENERATOR; static final String FAST_PARTITION_SKIP_PREDICATE; static final String FAST_TABLE_SKIP_PREDICATE; static final String DELETE_FILES_ON_DEREGISTER; static final DeregisterFileDeleteMethod DEFAULT_DEREGISTER_DELETE_METHOD; static final String HIVE_PARTITION_EXTENDED_FILTER_TYPE; static final String GOBBLIN_DISTCP; }