@Test public void testRejectsDuplicatePartitionNames() { final DatasetDescriptor descriptor = new DatasetDescriptor.Builder(testDescriptor) .partitionStrategy(new PartitionStrategy.Builder() .identity("timestamp", "timestamp") .build()) .build(); TestHelpers.assertThrows("Should reject duplicate field and partition name", ValidationException.class, new Runnable() { @Override public void run() { provider.create(NAMESPACE, "reject", descriptor); } }); }
@Override public DatasetDescriptor create(String namespace, String name, DatasetDescriptor descriptor) { Compatibility.checkDatasetName(namespace, name); Compatibility.checkDescriptor(descriptor); String resolved = resolveNamespace( namespace, name, descriptor.getLocation()); if (resolved != null) { if (resolved.equals(namespace)) { throw new DatasetExistsException( "Metadata already exists for dataset: " + namespace + "." + name); } else { LOG.warn("Creating table {}.{} for {}: replaces default.{}", new Object[]{ namespace, name, pathForDataset(namespace, name), name}); Compatibility.checkUpdate(load(resolved, name), descriptor); } } LOG.info("Creating an external Hive table: {}.{}", namespace, name); DatasetDescriptor newDescriptor = descriptor; if (descriptor.getLocation() == null) { newDescriptor = new DatasetDescriptor.Builder(descriptor) .location(pathForDataset(namespace, name)) .build(); } Path managerPath = new Path(new Path(newDescriptor.getLocation().toString()), SCHEMA_DIRECTORY); SchemaManager manager = SchemaManager.create(conf, managerPath); URI managedSchemaUri = manager.writeSchema(descriptor.getSchema()); try { newDescriptor = new DatasetDescriptor.Builder(newDescriptor) .schemaUri(managedSchemaUri) .build(); } catch (IOException e) { throw new DatasetIOException("Unable to load schema", e); } FileSystemUtil.ensureLocationExists(newDescriptor, conf); Table table = HiveUtils.tableForDescriptor( namespace, name, newDescriptor, true ); getMetaStoreUtil().createTable(table); return newDescriptor; }
HiveExternalMetadataProvider extends HiveAbstractMetadataProvider { @Override public DatasetDescriptor create(String namespace, String name, DatasetDescriptor descriptor) { Compatibility.checkDatasetName(namespace, name); Compatibility.checkDescriptor(descriptor); String resolved = resolveNamespace( namespace, name, descriptor.getLocation()); if (resolved != null) { if (resolved.equals(namespace)) { throw new DatasetExistsException( "Metadata already exists for dataset: " + namespace + "." + name); } else { LOG.warn("Creating table {}.{} for {}: replaces default.{}", new Object[]{ namespace, name, pathForDataset(namespace, name), name}); Compatibility.checkUpdate(load(resolved, name), descriptor); } } LOG.info("Creating an external Hive table: {}.{}", namespace, name); DatasetDescriptor newDescriptor = descriptor; if (descriptor.getLocation() == null) { newDescriptor = new DatasetDescriptor.Builder(descriptor) .location(pathForDataset(namespace, name)) .build(); } Path managerPath = new Path(new Path(newDescriptor.getLocation().toString()), SCHEMA_DIRECTORY); SchemaManager manager = SchemaManager.create(conf, managerPath); URI managedSchemaUri = manager.writeSchema(descriptor.getSchema()); try { newDescriptor = new DatasetDescriptor.Builder(newDescriptor) .schemaUri(managedSchemaUri) .build(); } catch (IOException e) { throw new DatasetIOException("Unable to load schema", e); } FileSystemUtil.ensureLocationExists(newDescriptor, conf); Table table = HiveUtils.tableForDescriptor( namespace, name, newDescriptor, true ); getMetaStoreUtil().createTable(table); return newDescriptor; } }
HiveExternalMetadataProvider extends HiveAbstractMetadataProvider { @Override public DatasetDescriptor create(String namespace, String name, DatasetDescriptor descriptor) { Compatibility.checkDatasetName(namespace, name); Compatibility.checkDescriptor(descriptor); String resolved = resolveNamespace( namespace, name, descriptor.getLocation()); if (resolved != null) { if (resolved.equals(namespace)) { throw new DatasetExistsException( "Metadata already exists for dataset: " + namespace + "." + name); } else { LOG.warn("Creating table {}.{} for {}: replaces default.{}", new Object[]{ namespace, name, pathForDataset(namespace, name), name}); Compatibility.checkUpdate(load(resolved, name), descriptor); } } LOG.info("Creating an external Hive table: {}.{}", namespace, name); DatasetDescriptor newDescriptor = descriptor; if (descriptor.getLocation() == null) { newDescriptor = new DatasetDescriptor.Builder(descriptor) .location(pathForDataset(namespace, name)) .build(); } Path managerPath = new Path(new Path(newDescriptor.getLocation().toString()), SCHEMA_DIRECTORY); SchemaManager manager = SchemaManager.create(conf, managerPath); URI managedSchemaUri = manager.writeSchema(descriptor.getSchema()); try { newDescriptor = new DatasetDescriptor.Builder(newDescriptor) .schemaUri(managedSchemaUri) .build(); } catch (IOException e) { throw new DatasetIOException("Unable to load schema", e); } FileSystemUtil.ensureLocationExists(newDescriptor, conf); Table table = HiveUtils.tableForDescriptor( namespace, name, newDescriptor, true ); getMetaStoreUtil().createTable(table); return newDescriptor; } HiveExternalMetadataProvider(Configuration conf, Path rootDirectory); }
HiveExternalMetadataProvider extends HiveAbstractMetadataProvider { @Override public DatasetDescriptor create(String namespace, String name, DatasetDescriptor descriptor) { Compatibility.checkDatasetName(namespace, name); Compatibility.checkDescriptor(descriptor); String resolved = resolveNamespace( namespace, name, descriptor.getLocation()); if (resolved != null) { if (resolved.equals(namespace)) { throw new DatasetExistsException( "Metadata already exists for dataset: " + namespace + "." + name); } else { LOG.warn("Creating table {}.{} for {}: replaces default.{}", new Object[]{ namespace, name, pathForDataset(namespace, name), name}); Compatibility.checkUpdate(load(resolved, name), descriptor); } } LOG.info("Creating an external Hive table: {}.{}", namespace, name); DatasetDescriptor newDescriptor = descriptor; if (descriptor.getLocation() == null) { newDescriptor = new DatasetDescriptor.Builder(descriptor) .location(pathForDataset(namespace, name)) .build(); } Path managerPath = new Path(new Path(newDescriptor.getLocation().toString()), SCHEMA_DIRECTORY); SchemaManager manager = SchemaManager.create(conf, managerPath); URI managedSchemaUri = manager.writeSchema(descriptor.getSchema()); try { newDescriptor = new DatasetDescriptor.Builder(newDescriptor) .schemaUri(managedSchemaUri) .build(); } catch (IOException e) { throw new DatasetIOException("Unable to load schema", e); } FileSystemUtil.ensureLocationExists(newDescriptor, conf); Table table = HiveUtils.tableForDescriptor( namespace, name, newDescriptor, true ); getMetaStoreUtil().createTable(table); return newDescriptor; } HiveExternalMetadataProvider(Configuration conf, Path rootDirectory); @Override DatasetDescriptor create(String namespace, String name, DatasetDescriptor descriptor); @Override void partitionAdded(String namespace, String name, String path); }
HiveExternalMetadataProvider extends HiveAbstractMetadataProvider { @Override public DatasetDescriptor create(String namespace, String name, DatasetDescriptor descriptor) { Compatibility.checkDatasetName(namespace, name); Compatibility.checkDescriptor(descriptor); String resolved = resolveNamespace( namespace, name, descriptor.getLocation()); if (resolved != null) { if (resolved.equals(namespace)) { throw new DatasetExistsException( "Metadata already exists for dataset: " + namespace + "." + name); } else { LOG.warn("Creating table {}.{} for {}: replaces default.{}", new Object[]{ namespace, name, pathForDataset(namespace, name), name}); Compatibility.checkUpdate(load(resolved, name), descriptor); } } LOG.info("Creating an external Hive table: {}.{}", namespace, name); DatasetDescriptor newDescriptor = descriptor; if (descriptor.getLocation() == null) { newDescriptor = new DatasetDescriptor.Builder(descriptor) .location(pathForDataset(namespace, name)) .build(); } Path managerPath = new Path(new Path(newDescriptor.getLocation().toString()), SCHEMA_DIRECTORY); SchemaManager manager = SchemaManager.create(conf, managerPath); URI managedSchemaUri = manager.writeSchema(descriptor.getSchema()); try { newDescriptor = new DatasetDescriptor.Builder(newDescriptor) .schemaUri(managedSchemaUri) .build(); } catch (IOException e) { throw new DatasetIOException("Unable to load schema", e); } FileSystemUtil.ensureLocationExists(newDescriptor, conf); Table table = HiveUtils.tableForDescriptor( namespace, name, newDescriptor, true ); getMetaStoreUtil().createTable(table); return newDescriptor; } HiveExternalMetadataProvider(Configuration conf, Path rootDirectory); @Override DatasetDescriptor create(String namespace, String name, DatasetDescriptor descriptor); @Override void partitionAdded(String namespace, String name, String path);  }