@Test public void copyTableQueryTest() throws Exception { Map<String, String> partitionsDMLInfo = Maps.newHashMap(); String partitionName = "datepartition"; String partitionValue = "2017-07-15-08"; partitionsDMLInfo.put(partitionName, partitionValue); String expectedQuery = "INSERT OVERWRITE TABLE `" + outputDatabaseName + "`.`" + outputTableName + "` \n" + "PARTITION (`" + partitionName + "`) \n" + "SELECT * FROM `" + inputDbName + "`.`" + inputTableName + "` WHERE " + "`" + partitionName + "`='" + partitionsDMLInfo.get(partitionName) + "'"; String actualQuery = HiveConverterUtils.generateTableCopy(inputTableName, outputTableName, inputDbName, outputDatabaseName, Optional.of(partitionsDMLInfo)); Assert.assertEquals(expectedQuery, actualQuery); }
public static String generateTableCopy( String inputTblName, String outputTblName, String inputDbName, String outputDbName, Optional<Map<String, String>> optionalPartitionDMLInfo) { Preconditions.checkArgument(StringUtils.isNotBlank(inputTblName)); Preconditions.checkArgument(StringUtils.isNotBlank(outputTblName)); Preconditions.checkArgument(StringUtils.isNotBlank(inputDbName)); Preconditions.checkArgument(StringUtils.isNotBlank(outputDbName)); StringBuilder dmlQuery = new StringBuilder(); dmlQuery.append(String.format("INSERT OVERWRITE TABLE `%s`.`%s` %n", outputDbName, outputTblName)); if (optionalPartitionDMLInfo.isPresent() && optionalPartitionDMLInfo.get().size() > 0) { dmlQuery.append(partitionKeyValues(optionalPartitionDMLInfo)); } dmlQuery.append(String.format("SELECT * FROM `%s`.`%s`", inputDbName, inputTblName)); if (optionalPartitionDMLInfo.isPresent()) { if (optionalPartitionDMLInfo.get().size() > 0) { dmlQuery.append(" WHERE "); String partitionsAndValues = optionalPartitionDMLInfo.get().entrySet().stream() .map(e -> "`" + e.getKey() + "`='" + e.getValue() + "'") .collect(joining(" AND ")); dmlQuery.append(partitionsAndValues); } } return dmlQuery.toString(); }
HiveConverterUtils { public static String generateTableCopy( String inputTblName, String outputTblName, String inputDbName, String outputDbName, Optional<Map<String, String>> optionalPartitionDMLInfo) { Preconditions.checkArgument(StringUtils.isNotBlank(inputTblName)); Preconditions.checkArgument(StringUtils.isNotBlank(outputTblName)); Preconditions.checkArgument(StringUtils.isNotBlank(inputDbName)); Preconditions.checkArgument(StringUtils.isNotBlank(outputDbName)); StringBuilder dmlQuery = new StringBuilder(); dmlQuery.append(String.format("INSERT OVERWRITE TABLE `%s`.`%s` %n", outputDbName, outputTblName)); if (optionalPartitionDMLInfo.isPresent() && optionalPartitionDMLInfo.get().size() > 0) { dmlQuery.append(partitionKeyValues(optionalPartitionDMLInfo)); } dmlQuery.append(String.format("SELECT * FROM `%s`.`%s`", inputDbName, inputTblName)); if (optionalPartitionDMLInfo.isPresent()) { if (optionalPartitionDMLInfo.get().size() > 0) { dmlQuery.append(" WHERE "); String partitionsAndValues = optionalPartitionDMLInfo.get().entrySet().stream() .map(e -> "`" + e.getKey() + "`='" + e.getValue() + "'") .collect(joining(" AND ")); dmlQuery.append(partitionsAndValues); } } return dmlQuery.toString(); } }
HiveConverterUtils { public static String generateTableCopy( String inputTblName, String outputTblName, String inputDbName, String outputDbName, Optional<Map<String, String>> optionalPartitionDMLInfo) { Preconditions.checkArgument(StringUtils.isNotBlank(inputTblName)); Preconditions.checkArgument(StringUtils.isNotBlank(outputTblName)); Preconditions.checkArgument(StringUtils.isNotBlank(inputDbName)); Preconditions.checkArgument(StringUtils.isNotBlank(outputDbName)); StringBuilder dmlQuery = new StringBuilder(); dmlQuery.append(String.format("INSERT OVERWRITE TABLE `%s`.`%s` %n", outputDbName, outputTblName)); if (optionalPartitionDMLInfo.isPresent() && optionalPartitionDMLInfo.get().size() > 0) { dmlQuery.append(partitionKeyValues(optionalPartitionDMLInfo)); } dmlQuery.append(String.format("SELECT * FROM `%s`.`%s`", inputDbName, inputTblName)); if (optionalPartitionDMLInfo.isPresent()) { if (optionalPartitionDMLInfo.get().size() > 0) { dmlQuery.append(" WHERE "); String partitionsAndValues = optionalPartitionDMLInfo.get().entrySet().stream() .map(e -> "`" + e.getKey() + "`='" + e.getValue() + "'") .collect(joining(" AND ")); dmlQuery.append(partitionsAndValues); } } return dmlQuery.toString(); }  }
HiveConverterUtils { public static String generateTableCopy( String inputTblName, String outputTblName, String inputDbName, String outputDbName, Optional<Map<String, String>> optionalPartitionDMLInfo) { Preconditions.checkArgument(StringUtils.isNotBlank(inputTblName)); Preconditions.checkArgument(StringUtils.isNotBlank(outputTblName)); Preconditions.checkArgument(StringUtils.isNotBlank(inputDbName)); Preconditions.checkArgument(StringUtils.isNotBlank(outputDbName)); StringBuilder dmlQuery = new StringBuilder(); dmlQuery.append(String.format("INSERT OVERWRITE TABLE `%s`.`%s` %n", outputDbName, outputTblName)); if (optionalPartitionDMLInfo.isPresent() && optionalPartitionDMLInfo.get().size() > 0) { dmlQuery.append(partitionKeyValues(optionalPartitionDMLInfo)); } dmlQuery.append(String.format("SELECT * FROM `%s`.`%s`", inputDbName, inputTblName)); if (optionalPartitionDMLInfo.isPresent()) { if (optionalPartitionDMLInfo.get().size() > 0) { dmlQuery.append(" WHERE "); String partitionsAndValues = optionalPartitionDMLInfo.get().entrySet().stream() .map(e -> "`" + e.getKey() + "`='" + e.getValue() + "'") .collect(joining(" AND ")); dmlQuery.append(partitionsAndValues); } } return dmlQuery.toString(); }  static String getStagingTableName(String stagingTableNamePrefix); static String getOutputDataLocation(String outputDataLocation); static String getStagingDataLocation(String outputDataLocation, String stagingTableName); static String generateCreateDuplicateTableDDL(
      String inputDbName,
      String inputTblName,
      String tblName,
      String tblLocation,
      Optional<String> optionalDbName); static String generateAlterTblPropsDML(
      String tableName,
      Optional<String> optionalDbName,
      Schema schema
      ); static Pair<String, String> getORCSchemaPropsFromAvroSchema(Schema avroSchema); static String generateStagingCTASStatementFromSelectStar(HiveDatasetFinder.DbAndTable outputDbAndTable,
      HiveDatasetFinder.DbAndTable sourceEntity, Map<String, String> partitionDMLInfo,
      StorageFormat storageFormat, String outputTableLocation); static String generateStagingCTASStatement(HiveDatasetFinder.DbAndTable outputDbAndTable,
      String sourceQuery, StorageFormat storageFormat, String outputTableLocation); static String generateTableCopy(
      String inputTblName,
      String outputTblName,
      String inputDbName,
      String outputDbName,
      Optional<Map<String, String>> optionalPartitionDMLInfo); static void populatePartitionInfo(HiveProcessingEntity conversionEntity, Map<String, String> partitionsDDLInfo,
      Map<String, String> partitionsDMLInfo); static void createStagingDirectory(FileSystem fs, String destination, HiveProcessingEntity conversionEntity,
      WorkUnitState workUnit); static String getStagingDataPartitionDirName(HiveProcessingEntity conversionEntity,
      List<String> sourceDataPathIdentifier); static Optional<Path> getDestinationPartitionLocation(Optional<Table> table, WorkUnitState state,
      String partitionName); static String updatePartitionLocation(String outputDataPartitionLocation, WorkUnitState workUnitState,
      Optional<Path> destPartitionLocation); static Pair<Optional<Table>, Optional<List<Partition>>> getDestinationTableMeta(String dbName,
      String tableName, Properties props); }
HiveConverterUtils { public static String generateTableCopy( String inputTblName, String outputTblName, String inputDbName, String outputDbName, Optional<Map<String, String>> optionalPartitionDMLInfo) { Preconditions.checkArgument(StringUtils.isNotBlank(inputTblName)); Preconditions.checkArgument(StringUtils.isNotBlank(outputTblName)); Preconditions.checkArgument(StringUtils.isNotBlank(inputDbName)); Preconditions.checkArgument(StringUtils.isNotBlank(outputDbName)); StringBuilder dmlQuery = new StringBuilder(); dmlQuery.append(String.format("INSERT OVERWRITE TABLE `%s`.`%s` %n", outputDbName, outputTblName)); if (optionalPartitionDMLInfo.isPresent() && optionalPartitionDMLInfo.get().size() > 0) { dmlQuery.append(partitionKeyValues(optionalPartitionDMLInfo)); } dmlQuery.append(String.format("SELECT * FROM `%s`.`%s`", inputDbName, inputTblName)); if (optionalPartitionDMLInfo.isPresent()) { if (optionalPartitionDMLInfo.get().size() > 0) { dmlQuery.append(" WHERE "); String partitionsAndValues = optionalPartitionDMLInfo.get().entrySet().stream() .map(e -> "`" + e.getKey() + "`='" + e.getValue() + "'") .collect(joining(" AND ")); dmlQuery.append(partitionsAndValues); } } return dmlQuery.toString(); }  static String getStagingTableName(String stagingTableNamePrefix); static String getOutputDataLocation(String outputDataLocation); static String getStagingDataLocation(String outputDataLocation, String stagingTableName); static String generateCreateDuplicateTableDDL(
      String inputDbName,
      String inputTblName,
      String tblName,
      String tblLocation,
      Optional<String> optionalDbName); static String generateAlterTblPropsDML(
      String tableName,
      Optional<String> optionalDbName,
      Schema schema
      ); static Pair<String, String> getORCSchemaPropsFromAvroSchema(Schema avroSchema); static String generateStagingCTASStatementFromSelectStar(HiveDatasetFinder.DbAndTable outputDbAndTable,
      HiveDatasetFinder.DbAndTable sourceEntity, Map<String, String> partitionDMLInfo,
      StorageFormat storageFormat, String outputTableLocation); static String generateStagingCTASStatement(HiveDatasetFinder.DbAndTable outputDbAndTable,
      String sourceQuery, StorageFormat storageFormat, String outputTableLocation); static String generateTableCopy(
      String inputTblName,
      String outputTblName,
      String inputDbName,
      String outputDbName,
      Optional<Map<String, String>> optionalPartitionDMLInfo); static void populatePartitionInfo(HiveProcessingEntity conversionEntity, Map<String, String> partitionsDDLInfo,
      Map<String, String> partitionsDMLInfo); static void createStagingDirectory(FileSystem fs, String destination, HiveProcessingEntity conversionEntity,
      WorkUnitState workUnit); static String getStagingDataPartitionDirName(HiveProcessingEntity conversionEntity,
      List<String> sourceDataPathIdentifier); static Optional<Path> getDestinationPartitionLocation(Optional<Table> table, WorkUnitState state,
      String partitionName); static String updatePartitionLocation(String outputDataPartitionLocation, WorkUnitState workUnitState,
      Optional<Path> destPartitionLocation); static Pair<Optional<Table>, Optional<List<Partition>>> getDestinationTableMeta(String dbName,
      String tableName, Properties props); static final String HIVE_DATASET_PARTITION_OVERWRITE; static final boolean DEFAULT_HIVE_DATASET_PARTITION_OVERWRITE; static final String HIVE_DATASET_DESTINATION_SKIP_SETGROUP; static final boolean DEFAULT_HIVE_DATASET_DESTINATION_SKIP_SETGROUP; }