@Test public void testParseLogicalDbAndTable() throws IOException { String datasetNamePattern; HiveDatasetFinder.DbAndTable logicalDbAndTable; datasetNamePattern = "dbPrefix_$LOGICAL_DB_dbPostfix.tablePrefix_$LOGICAL_TABLE_tablePostfix"; logicalDbAndTable = HiveDataset.parseLogicalDbAndTable(datasetNamePattern, new HiveDatasetFinder.DbAndTable("dbPrefix_myDB_dbPostfix", "tablePrefix_myTable_tablePostfix"), HiveDataset.LOGICAL_DB_TOKEN, HiveDataset.LOGICAL_TABLE_TOKEN); Assert.assertEquals(logicalDbAndTable.getDb(), "myDB", "DB name not parsed correctly"); Assert.assertEquals(logicalDbAndTable.getTable(), "myTable", "Table name not parsed correctly"); datasetNamePattern = "$LOGICAL_DB_dbPostfix.$LOGICAL_TABLE_tablePostfix"; logicalDbAndTable = HiveDataset.parseLogicalDbAndTable(datasetNamePattern, new HiveDatasetFinder.DbAndTable("myDB_dbPostfix", "myTable_tablePostfix"), HiveDataset.LOGICAL_DB_TOKEN, HiveDataset.LOGICAL_TABLE_TOKEN); Assert.assertEquals(logicalDbAndTable.getDb(), "myDB", "DB name not parsed correctly"); Assert.assertEquals(logicalDbAndTable.getTable(), "myTable", "Table name not parsed correctly"); datasetNamePattern = "dbPrefix_$LOGICAL_DB.tablePrefix_$LOGICAL_TABLE"; logicalDbAndTable = HiveDataset.parseLogicalDbAndTable(datasetNamePattern, new HiveDatasetFinder.DbAndTable("dbPrefix_myDB", "tablePrefix_myTable"), HiveDataset.LOGICAL_DB_TOKEN, HiveDataset.LOGICAL_TABLE_TOKEN); Assert.assertEquals(logicalDbAndTable.getDb(), "myDB", "DB name not parsed correctly"); Assert.assertEquals(logicalDbAndTable.getTable(), "myTable", "Table name not parsed correctly"); datasetNamePattern = "$LOGICAL_DB.$LOGICAL_TABLE"; logicalDbAndTable = HiveDataset.parseLogicalDbAndTable(datasetNamePattern, new HiveDatasetFinder.DbAndTable("myDB", "myTable"), HiveDataset.LOGICAL_DB_TOKEN, HiveDataset.LOGICAL_TABLE_TOKEN); Assert.assertEquals(logicalDbAndTable.getDb(), "myDB", "DB name not parsed correctly"); Assert.assertEquals(logicalDbAndTable.getTable(), "myTable", "Table name not parsed correctly"); datasetNamePattern = ""; try { logicalDbAndTable = HiveDataset.parseLogicalDbAndTable(datasetNamePattern, new HiveDatasetFinder.DbAndTable("dbPrefix_myDB_dbPostfix", "tablePrefix_myTable_tablePostfix"), HiveDataset.LOGICAL_DB_TOKEN, HiveDataset.LOGICAL_TABLE_TOKEN); Assert.fail("Dataset name pattern is missing, code should have thrown exception"); } catch (IllegalArgumentException e) { } datasetNamePattern = "dbPrefix_$LOGICAL_DB_dbPostfixtablePrefix_$LOGICAL_TABLE_tablePostfix"; try { logicalDbAndTable = HiveDataset.parseLogicalDbAndTable(datasetNamePattern, new HiveDatasetFinder.DbAndTable("dbPrefix_myDB_dbPostfix", "tablePrefix_myTable_tablePostfix"), HiveDataset.LOGICAL_DB_TOKEN, HiveDataset.LOGICAL_TABLE_TOKEN); Assert.fail("Dataset name pattern is missing, code should have thrown exception"); } catch (IllegalArgumentException e) { } }
@VisibleForTesting protected static DbAndTable parseLogicalDbAndTable(String datasetNamePattern, DbAndTable dbAndTable, String logicalDbToken, String logicalTableToken) { Preconditions.checkArgument(StringUtils.isNotBlank(datasetNamePattern), "Dataset name pattern must not be empty."); List<String> datasetNameSplit = Lists.newArrayList(SPLIT_ON_DOT.split(datasetNamePattern)); Preconditions.checkArgument(datasetNameSplit.size() == 2, "Dataset name pattern must of the format: " + "dbPrefix_$LOGICAL_DB_dbPostfix.tablePrefix_$LOGICAL_TABLE_tablePostfix (prefix / postfix are optional)"); String dbNamePattern = datasetNameSplit.get(0); String tableNamePattern = datasetNameSplit.get(1); String logicalDb = extractTokenValueFromEntity(dbAndTable.getDb(), dbNamePattern, logicalDbToken); String logicalTable = extractTokenValueFromEntity(dbAndTable.getTable(), tableNamePattern, logicalTableToken); return new DbAndTable(logicalDb, logicalTable); }
HiveDataset implements PrioritizedCopyableDataset { @VisibleForTesting protected static DbAndTable parseLogicalDbAndTable(String datasetNamePattern, DbAndTable dbAndTable, String logicalDbToken, String logicalTableToken) { Preconditions.checkArgument(StringUtils.isNotBlank(datasetNamePattern), "Dataset name pattern must not be empty."); List<String> datasetNameSplit = Lists.newArrayList(SPLIT_ON_DOT.split(datasetNamePattern)); Preconditions.checkArgument(datasetNameSplit.size() == 2, "Dataset name pattern must of the format: " + "dbPrefix_$LOGICAL_DB_dbPostfix.tablePrefix_$LOGICAL_TABLE_tablePostfix (prefix / postfix are optional)"); String dbNamePattern = datasetNameSplit.get(0); String tableNamePattern = datasetNameSplit.get(1); String logicalDb = extractTokenValueFromEntity(dbAndTable.getDb(), dbNamePattern, logicalDbToken); String logicalTable = extractTokenValueFromEntity(dbAndTable.getTable(), tableNamePattern, logicalTableToken); return new DbAndTable(logicalDb, logicalTable); } }
HiveDataset implements PrioritizedCopyableDataset { @VisibleForTesting protected static DbAndTable parseLogicalDbAndTable(String datasetNamePattern, DbAndTable dbAndTable, String logicalDbToken, String logicalTableToken) { Preconditions.checkArgument(StringUtils.isNotBlank(datasetNamePattern), "Dataset name pattern must not be empty."); List<String> datasetNameSplit = Lists.newArrayList(SPLIT_ON_DOT.split(datasetNamePattern)); Preconditions.checkArgument(datasetNameSplit.size() == 2, "Dataset name pattern must of the format: " + "dbPrefix_$LOGICAL_DB_dbPostfix.tablePrefix_$LOGICAL_TABLE_tablePostfix (prefix / postfix are optional)"); String dbNamePattern = datasetNameSplit.get(0); String tableNamePattern = datasetNameSplit.get(1); String logicalDb = extractTokenValueFromEntity(dbAndTable.getDb(), dbNamePattern, logicalDbToken); String logicalTable = extractTokenValueFromEntity(dbAndTable.getTable(), tableNamePattern, logicalTableToken); return new DbAndTable(logicalDb, logicalTable); } HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties);  HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Config datasetConfig);  HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties, Config datasetConfig); }
HiveDataset implements PrioritizedCopyableDataset { @VisibleForTesting protected static DbAndTable parseLogicalDbAndTable(String datasetNamePattern, DbAndTable dbAndTable, String logicalDbToken, String logicalTableToken) { Preconditions.checkArgument(StringUtils.isNotBlank(datasetNamePattern), "Dataset name pattern must not be empty."); List<String> datasetNameSplit = Lists.newArrayList(SPLIT_ON_DOT.split(datasetNamePattern)); Preconditions.checkArgument(datasetNameSplit.size() == 2, "Dataset name pattern must of the format: " + "dbPrefix_$LOGICAL_DB_dbPostfix.tablePrefix_$LOGICAL_TABLE_tablePostfix (prefix / postfix are optional)"); String dbNamePattern = datasetNameSplit.get(0); String tableNamePattern = datasetNameSplit.get(1); String logicalDb = extractTokenValueFromEntity(dbAndTable.getDb(), dbNamePattern, logicalDbToken); String logicalTable = extractTokenValueFromEntity(dbAndTable.getTable(), tableNamePattern, logicalTableToken); return new DbAndTable(logicalDb, logicalTable); } HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties);  HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Config datasetConfig);  HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties, Config datasetConfig); @Override Iterator<FileSet<CopyEntity>> getFileSetIterator(FileSystem targetFs, CopyConfiguration configuration); @Override Iterator<FileSet<CopyEntity>> getFileSetIterator(FileSystem targetFs, CopyConfiguration configuration,
      Comparator<FileSet<CopyEntity>> prioritizer, PushDownRequestor<FileSet<CopyEntity>> requestor); @Override String datasetURN(); static String resolveTemplate(String rawString, Table table); static List<Partition> sortPartitions(List<Partition> partitions); List<Partition> getPartitionsFromDataset(); }
HiveDataset implements PrioritizedCopyableDataset { @VisibleForTesting protected static DbAndTable parseLogicalDbAndTable(String datasetNamePattern, DbAndTable dbAndTable, String logicalDbToken, String logicalTableToken) { Preconditions.checkArgument(StringUtils.isNotBlank(datasetNamePattern), "Dataset name pattern must not be empty."); List<String> datasetNameSplit = Lists.newArrayList(SPLIT_ON_DOT.split(datasetNamePattern)); Preconditions.checkArgument(datasetNameSplit.size() == 2, "Dataset name pattern must of the format: " + "dbPrefix_$LOGICAL_DB_dbPostfix.tablePrefix_$LOGICAL_TABLE_tablePostfix (prefix / postfix are optional)"); String dbNamePattern = datasetNameSplit.get(0); String tableNamePattern = datasetNameSplit.get(1); String logicalDb = extractTokenValueFromEntity(dbAndTable.getDb(), dbNamePattern, logicalDbToken); String logicalTable = extractTokenValueFromEntity(dbAndTable.getTable(), tableNamePattern, logicalTableToken); return new DbAndTable(logicalDb, logicalTable); } HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties);  HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Config datasetConfig);  HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties, Config datasetConfig); @Override Iterator<FileSet<CopyEntity>> getFileSetIterator(FileSystem targetFs, CopyConfiguration configuration); @Override Iterator<FileSet<CopyEntity>> getFileSetIterator(FileSystem targetFs, CopyConfiguration configuration,
      Comparator<FileSet<CopyEntity>> prioritizer, PushDownRequestor<FileSet<CopyEntity>> requestor); @Override String datasetURN(); static String resolveTemplate(String rawString, Table table); static List<Partition> sortPartitions(List<Partition> partitions); List<Partition> getPartitionsFromDataset(); static final ImmutableSet<TableType> COPYABLE_TABLES; static final String REGISTERER; static final String REGISTRATION_GENERATION_TIME_MILLIS; static final String DATASET_NAME_PATTERN_KEY; static final String DATABASE; static final String TABLE; static final String DATASET_STAGING_PATH; static final String DATABASE_TOKEN; static final String TABLE_TOKEN; static final String LOGICAL_DB_TOKEN; static final String LOGICAL_TABLE_TOKEN; }