@Test public void testUnrecognizedFormat() throws Exception { command.avroSchemaFile = "resource:test-schemas/user.avsc"; command.datasets = Lists.newArrayList("users"); command.format = "nosuchformat"; TestHelpers.assertThrows("Should fail on invalid format", IllegalArgumentException.class, new Callable() { @Override public Void call() throws Exception { command.run(); return null; } }); verifyZeroInteractions(console); }
@Override public int run() throws IOException { if (datasets == null || datasets.size() != 1) { throw new IllegalArgumentException( "Exactly one dataset name or URI must be specified."); } String dataset = datasets.get(0); DatasetRepository repo; String namespace; String name; if (isDatasetOrViewUri(dataset)) { URI uri = URI.create(URI.create(dataset).getSchemeSpecificPart()); Pair<DatasetRepository, Map<String, String>> reg = Registration .lookupDatasetUri(uri); repo = reg.first(); namespace = reg.second().get(URIBuilder.NAMESPACE_OPTION); name = reg.second().get(URIBuilder.DATASET_NAME_OPTION); if (location == null) { location = reg.second().get("location"); } } else { repo = getDatasetRepository(); namespace = URIBuilder.NAMESPACE_DEFAULT; name = dataset; } boolean isFileDataset = (repo instanceof FileSystemDatasetRepository); DatasetDescriptor.Builder descriptorBuilder = new DatasetDescriptor.Builder(); Path dataPath = null; if (location != null) { dataPath = qualifiedPath(location); descriptorBuilder.location(dataPath); } else if (isFileDataset) { dataPath = ((FileSystemDatasetRepository) repo) .pathForDataset(namespace, name); } FileSystem fs = isFileDataset ? dataPath.getFileSystem(getConf()) : null; Format existingFormat = null; if (isFileDataset) { existingFormat = FileSystemUtil.format(fs, dataPath); } if (existingFormat != null) { ValidationException.check(format == null || Formats.fromString(format).equals(existingFormat), "Found %s data, but --format is %s", existingFormat.getName(), format); descriptorBuilder.format(existingFormat); } else if (format != null) { descriptorBuilder.format(format); } Schema existingSchema = null; if (isFileDataset) { existingSchema = FileSystemUtil.schema("record", fs, dataPath); } Schema schema = existingSchema; if (existingSchema != null) { if (avroSchemaFile != null) { schema = Schemas.fromAvsc(open(avroSchemaFile)); ValidationException.check( SchemaValidationUtil.canRead(existingSchema, schema), "Schema %s cannot read existing data schema: %s", avroSchemaFile, existingSchema.toString(true)); descriptorBuilder.schemaUri(qualifiedURI(avroSchemaFile)); } else { descriptorBuilder.schema(existingSchema); } } else if (avroSchemaFile != null) { descriptorBuilder.schemaUri(qualifiedURI(avroSchemaFile)); } else { throw new ValidationException("Schema is missing: use --schema"); } PartitionStrategy existingStrategy = null; if (isFileDataset && existingSchema != null) { existingStrategy = FileSystemUtil.strategy(fs, dataPath); } if (existingStrategy != null) { if (partitionStrategyFile != null) { PartitionStrategy strategy = PartitionStrategyParser .parse(open(partitionStrategyFile)); Compatibility.checkStrategyUpdate(existingStrategy, strategy, schema); descriptorBuilder.partitionStrategy(strategy); } else { descriptorBuilder.partitionStrategy(existingStrategy); } } else if (partitionStrategyFile != null) { descriptorBuilder.partitionStrategyUri(qualifiedURI(partitionStrategyFile)); } if (columnMappingFile != null) { descriptorBuilder.columnMappingUri(qualifiedURI(columnMappingFile)); } if (properties != null) { for (Map.Entry<String, String> entry : properties.entrySet()) { descriptorBuilder.property(entry.getKey(), entry.getValue()); } } repo.create(namespace, name, descriptorBuilder.build()); console.debug("Created {}", dataset); return 0; }
CreateDatasetCommand extends BaseDatasetCommand { @Override public int run() throws IOException { if (datasets == null || datasets.size() != 1) { throw new IllegalArgumentException( "Exactly one dataset name or URI must be specified."); } String dataset = datasets.get(0); DatasetRepository repo; String namespace; String name; if (isDatasetOrViewUri(dataset)) { URI uri = URI.create(URI.create(dataset).getSchemeSpecificPart()); Pair<DatasetRepository, Map<String, String>> reg = Registration .lookupDatasetUri(uri); repo = reg.first(); namespace = reg.second().get(URIBuilder.NAMESPACE_OPTION); name = reg.second().get(URIBuilder.DATASET_NAME_OPTION); if (location == null) { location = reg.second().get("location"); } } else { repo = getDatasetRepository(); namespace = URIBuilder.NAMESPACE_DEFAULT; name = dataset; } boolean isFileDataset = (repo instanceof FileSystemDatasetRepository); DatasetDescriptor.Builder descriptorBuilder = new DatasetDescriptor.Builder(); Path dataPath = null; if (location != null) { dataPath = qualifiedPath(location); descriptorBuilder.location(dataPath); } else if (isFileDataset) { dataPath = ((FileSystemDatasetRepository) repo) .pathForDataset(namespace, name); } FileSystem fs = isFileDataset ? dataPath.getFileSystem(getConf()) : null; Format existingFormat = null; if (isFileDataset) { existingFormat = FileSystemUtil.format(fs, dataPath); } if (existingFormat != null) { ValidationException.check(format == null || Formats.fromString(format).equals(existingFormat), "Found %s data, but --format is %s", existingFormat.getName(), format); descriptorBuilder.format(existingFormat); } else if (format != null) { descriptorBuilder.format(format); } Schema existingSchema = null; if (isFileDataset) { existingSchema = FileSystemUtil.schema("record", fs, dataPath); } Schema schema = existingSchema; if (existingSchema != null) { if (avroSchemaFile != null) { schema = Schemas.fromAvsc(open(avroSchemaFile)); ValidationException.check( SchemaValidationUtil.canRead(existingSchema, schema), "Schema %s cannot read existing data schema: %s", avroSchemaFile, existingSchema.toString(true)); descriptorBuilder.schemaUri(qualifiedURI(avroSchemaFile)); } else { descriptorBuilder.schema(existingSchema); } } else if (avroSchemaFile != null) { descriptorBuilder.schemaUri(qualifiedURI(avroSchemaFile)); } else { throw new ValidationException("Schema is missing: use --schema"); } PartitionStrategy existingStrategy = null; if (isFileDataset && existingSchema != null) { existingStrategy = FileSystemUtil.strategy(fs, dataPath); } if (existingStrategy != null) { if (partitionStrategyFile != null) { PartitionStrategy strategy = PartitionStrategyParser .parse(open(partitionStrategyFile)); Compatibility.checkStrategyUpdate(existingStrategy, strategy, schema); descriptorBuilder.partitionStrategy(strategy); } else { descriptorBuilder.partitionStrategy(existingStrategy); } } else if (partitionStrategyFile != null) { descriptorBuilder.partitionStrategyUri(qualifiedURI(partitionStrategyFile)); } if (columnMappingFile != null) { descriptorBuilder.columnMappingUri(qualifiedURI(columnMappingFile)); } if (properties != null) { for (Map.Entry<String, String> entry : properties.entrySet()) { descriptorBuilder.property(entry.getKey(), entry.getValue()); } } repo.create(namespace, name, descriptorBuilder.build()); console.debug("Created {}", dataset); return 0; } }
CreateDatasetCommand extends BaseDatasetCommand { @Override public int run() throws IOException { if (datasets == null || datasets.size() != 1) { throw new IllegalArgumentException( "Exactly one dataset name or URI must be specified."); } String dataset = datasets.get(0); DatasetRepository repo; String namespace; String name; if (isDatasetOrViewUri(dataset)) { URI uri = URI.create(URI.create(dataset).getSchemeSpecificPart()); Pair<DatasetRepository, Map<String, String>> reg = Registration .lookupDatasetUri(uri); repo = reg.first(); namespace = reg.second().get(URIBuilder.NAMESPACE_OPTION); name = reg.second().get(URIBuilder.DATASET_NAME_OPTION); if (location == null) { location = reg.second().get("location"); } } else { repo = getDatasetRepository(); namespace = URIBuilder.NAMESPACE_DEFAULT; name = dataset; } boolean isFileDataset = (repo instanceof FileSystemDatasetRepository); DatasetDescriptor.Builder descriptorBuilder = new DatasetDescriptor.Builder(); Path dataPath = null; if (location != null) { dataPath = qualifiedPath(location); descriptorBuilder.location(dataPath); } else if (isFileDataset) { dataPath = ((FileSystemDatasetRepository) repo) .pathForDataset(namespace, name); } FileSystem fs = isFileDataset ? dataPath.getFileSystem(getConf()) : null; Format existingFormat = null; if (isFileDataset) { existingFormat = FileSystemUtil.format(fs, dataPath); } if (existingFormat != null) { ValidationException.check(format == null || Formats.fromString(format).equals(existingFormat), "Found %s data, but --format is %s", existingFormat.getName(), format); descriptorBuilder.format(existingFormat); } else if (format != null) { descriptorBuilder.format(format); } Schema existingSchema = null; if (isFileDataset) { existingSchema = FileSystemUtil.schema("record", fs, dataPath); } Schema schema = existingSchema; if (existingSchema != null) { if (avroSchemaFile != null) { schema = Schemas.fromAvsc(open(avroSchemaFile)); ValidationException.check( SchemaValidationUtil.canRead(existingSchema, schema), "Schema %s cannot read existing data schema: %s", avroSchemaFile, existingSchema.toString(true)); descriptorBuilder.schemaUri(qualifiedURI(avroSchemaFile)); } else { descriptorBuilder.schema(existingSchema); } } else if (avroSchemaFile != null) { descriptorBuilder.schemaUri(qualifiedURI(avroSchemaFile)); } else { throw new ValidationException("Schema is missing: use --schema"); } PartitionStrategy existingStrategy = null; if (isFileDataset && existingSchema != null) { existingStrategy = FileSystemUtil.strategy(fs, dataPath); } if (existingStrategy != null) { if (partitionStrategyFile != null) { PartitionStrategy strategy = PartitionStrategyParser .parse(open(partitionStrategyFile)); Compatibility.checkStrategyUpdate(existingStrategy, strategy, schema); descriptorBuilder.partitionStrategy(strategy); } else { descriptorBuilder.partitionStrategy(existingStrategy); } } else if (partitionStrategyFile != null) { descriptorBuilder.partitionStrategyUri(qualifiedURI(partitionStrategyFile)); } if (columnMappingFile != null) { descriptorBuilder.columnMappingUri(qualifiedURI(columnMappingFile)); } if (properties != null) { for (Map.Entry<String, String> entry : properties.entrySet()) { descriptorBuilder.property(entry.getKey(), entry.getValue()); } } repo.create(namespace, name, descriptorBuilder.build()); console.debug("Created {}", dataset); return 0; } CreateDatasetCommand(Logger console); }
CreateDatasetCommand extends BaseDatasetCommand { @Override public int run() throws IOException { if (datasets == null || datasets.size() != 1) { throw new IllegalArgumentException( "Exactly one dataset name or URI must be specified."); } String dataset = datasets.get(0); DatasetRepository repo; String namespace; String name; if (isDatasetOrViewUri(dataset)) { URI uri = URI.create(URI.create(dataset).getSchemeSpecificPart()); Pair<DatasetRepository, Map<String, String>> reg = Registration .lookupDatasetUri(uri); repo = reg.first(); namespace = reg.second().get(URIBuilder.NAMESPACE_OPTION); name = reg.second().get(URIBuilder.DATASET_NAME_OPTION); if (location == null) { location = reg.second().get("location"); } } else { repo = getDatasetRepository(); namespace = URIBuilder.NAMESPACE_DEFAULT; name = dataset; } boolean isFileDataset = (repo instanceof FileSystemDatasetRepository); DatasetDescriptor.Builder descriptorBuilder = new DatasetDescriptor.Builder(); Path dataPath = null; if (location != null) { dataPath = qualifiedPath(location); descriptorBuilder.location(dataPath); } else if (isFileDataset) { dataPath = ((FileSystemDatasetRepository) repo) .pathForDataset(namespace, name); } FileSystem fs = isFileDataset ? dataPath.getFileSystem(getConf()) : null; Format existingFormat = null; if (isFileDataset) { existingFormat = FileSystemUtil.format(fs, dataPath); } if (existingFormat != null) { ValidationException.check(format == null || Formats.fromString(format).equals(existingFormat), "Found %s data, but --format is %s", existingFormat.getName(), format); descriptorBuilder.format(existingFormat); } else if (format != null) { descriptorBuilder.format(format); } Schema existingSchema = null; if (isFileDataset) { existingSchema = FileSystemUtil.schema("record", fs, dataPath); } Schema schema = existingSchema; if (existingSchema != null) { if (avroSchemaFile != null) { schema = Schemas.fromAvsc(open(avroSchemaFile)); ValidationException.check( SchemaValidationUtil.canRead(existingSchema, schema), "Schema %s cannot read existing data schema: %s", avroSchemaFile, existingSchema.toString(true)); descriptorBuilder.schemaUri(qualifiedURI(avroSchemaFile)); } else { descriptorBuilder.schema(existingSchema); } } else if (avroSchemaFile != null) { descriptorBuilder.schemaUri(qualifiedURI(avroSchemaFile)); } else { throw new ValidationException("Schema is missing: use --schema"); } PartitionStrategy existingStrategy = null; if (isFileDataset && existingSchema != null) { existingStrategy = FileSystemUtil.strategy(fs, dataPath); } if (existingStrategy != null) { if (partitionStrategyFile != null) { PartitionStrategy strategy = PartitionStrategyParser .parse(open(partitionStrategyFile)); Compatibility.checkStrategyUpdate(existingStrategy, strategy, schema); descriptorBuilder.partitionStrategy(strategy); } else { descriptorBuilder.partitionStrategy(existingStrategy); } } else if (partitionStrategyFile != null) { descriptorBuilder.partitionStrategyUri(qualifiedURI(partitionStrategyFile)); } if (columnMappingFile != null) { descriptorBuilder.columnMappingUri(qualifiedURI(columnMappingFile)); } if (properties != null) { for (Map.Entry<String, String> entry : properties.entrySet()) { descriptorBuilder.property(entry.getKey(), entry.getValue()); } } repo.create(namespace, name, descriptorBuilder.build()); console.debug("Created {}", dataset); return 0; } CreateDatasetCommand(Logger console); @Override int run(); @Override List<String> getExamples(); }
CreateDatasetCommand extends BaseDatasetCommand { @Override public int run() throws IOException { if (datasets == null || datasets.size() != 1) { throw new IllegalArgumentException( "Exactly one dataset name or URI must be specified."); } String dataset = datasets.get(0); DatasetRepository repo; String namespace; String name; if (isDatasetOrViewUri(dataset)) { URI uri = URI.create(URI.create(dataset).getSchemeSpecificPart()); Pair<DatasetRepository, Map<String, String>> reg = Registration .lookupDatasetUri(uri); repo = reg.first(); namespace = reg.second().get(URIBuilder.NAMESPACE_OPTION); name = reg.second().get(URIBuilder.DATASET_NAME_OPTION); if (location == null) { location = reg.second().get("location"); } } else { repo = getDatasetRepository(); namespace = URIBuilder.NAMESPACE_DEFAULT; name = dataset; } boolean isFileDataset = (repo instanceof FileSystemDatasetRepository); DatasetDescriptor.Builder descriptorBuilder = new DatasetDescriptor.Builder(); Path dataPath = null; if (location != null) { dataPath = qualifiedPath(location); descriptorBuilder.location(dataPath); } else if (isFileDataset) { dataPath = ((FileSystemDatasetRepository) repo) .pathForDataset(namespace, name); } FileSystem fs = isFileDataset ? dataPath.getFileSystem(getConf()) : null; Format existingFormat = null; if (isFileDataset) { existingFormat = FileSystemUtil.format(fs, dataPath); } if (existingFormat != null) { ValidationException.check(format == null || Formats.fromString(format).equals(existingFormat), "Found %s data, but --format is %s", existingFormat.getName(), format); descriptorBuilder.format(existingFormat); } else if (format != null) { descriptorBuilder.format(format); } Schema existingSchema = null; if (isFileDataset) { existingSchema = FileSystemUtil.schema("record", fs, dataPath); } Schema schema = existingSchema; if (existingSchema != null) { if (avroSchemaFile != null) { schema = Schemas.fromAvsc(open(avroSchemaFile)); ValidationException.check( SchemaValidationUtil.canRead(existingSchema, schema), "Schema %s cannot read existing data schema: %s", avroSchemaFile, existingSchema.toString(true)); descriptorBuilder.schemaUri(qualifiedURI(avroSchemaFile)); } else { descriptorBuilder.schema(existingSchema); } } else if (avroSchemaFile != null) { descriptorBuilder.schemaUri(qualifiedURI(avroSchemaFile)); } else { throw new ValidationException("Schema is missing: use --schema"); } PartitionStrategy existingStrategy = null; if (isFileDataset && existingSchema != null) { existingStrategy = FileSystemUtil.strategy(fs, dataPath); } if (existingStrategy != null) { if (partitionStrategyFile != null) { PartitionStrategy strategy = PartitionStrategyParser .parse(open(partitionStrategyFile)); Compatibility.checkStrategyUpdate(existingStrategy, strategy, schema); descriptorBuilder.partitionStrategy(strategy); } else { descriptorBuilder.partitionStrategy(existingStrategy); } } else if (partitionStrategyFile != null) { descriptorBuilder.partitionStrategyUri(qualifiedURI(partitionStrategyFile)); } if (columnMappingFile != null) { descriptorBuilder.columnMappingUri(qualifiedURI(columnMappingFile)); } if (properties != null) { for (Map.Entry<String, String> entry : properties.entrySet()) { descriptorBuilder.property(entry.getKey(), entry.getValue()); } } repo.create(namespace, name, descriptorBuilder.build()); console.debug("Created {}", dataset); return 0; } CreateDatasetCommand(Logger console); @Override int run(); @Override List<String> getExamples();  }