@Test public void testLoadSetsLocation() throws IOException { ensureCreated(); DatasetDescriptor loaded = provider.load(NAMESPACE, NAME); Assert.assertNotNull("Loaded descriptor should have a location", loaded.getLocation()); if (distributed) { Assert.assertEquals( getDFS(), FileSystem.get(loaded.getLocation(), new Configuration())); Assert.assertEquals( "hdfs", loaded.getLocation().getScheme()); Assert.assertEquals( getDFS().getUri().getAuthority(), loaded.getLocation().getAuthority()); } else { Assert.assertEquals( getFS(), FileSystem.get(loaded.getLocation(), new Configuration())); Assert.assertEquals( "file", loaded.getLocation().getScheme()); Assert.assertEquals( getFS().getUri().getAuthority(), loaded.getLocation().getAuthority()); } }
@Override public DatasetDescriptor load(String namespace, String name) { Preconditions.checkNotNull(namespace, "Namespace cannot be null"); Preconditions.checkNotNull(name, "Dataset name cannot be null"); LOG.debug("Loading dataset metadata name: {}", name); Path metadataPath = find(namespace, name); InputStream inputStream = null; Properties properties = new Properties(); DatasetDescriptor.Builder builder = new DatasetDescriptor.Builder(); Path descriptorPath = new Path(metadataPath, DESCRIPTOR_FILE_NAME); boolean threw = true; try { inputStream = rootFileSystem.open(descriptorPath); properties.load(inputStream); threw = false; } catch (IOException e) { throw new DatasetIOException( "Unable to load descriptor file:" + descriptorPath + " for dataset:" + name, e); } finally { try { Closeables.close(inputStream, threw); } catch (IOException e) { throw new DatasetIOException("Cannot close", e); } } if (properties.containsKey(FORMAT_FIELD_NAME)) { builder.format(Accessor.getDefault().newFormat( properties.getProperty(FORMAT_FIELD_NAME))); } if (properties.containsKey(COMPRESSION_TYPE_FIELD_NAME)) { builder.compressionType(properties.getProperty(COMPRESSION_TYPE_FIELD_NAME)); } if (properties.containsKey(PARTITION_EXPRESSION_FIELD_NAME)) { builder.partitionStrategy(Accessor.getDefault().fromExpression(properties .getProperty(PARTITION_EXPRESSION_FIELD_NAME))); } SchemaManager manager = SchemaManager.load(conf, new Path(metadataPath, SCHEMA_DIRECTORY_NAME)); URI schemaURI; if (manager == null) { Path schemaPath = new Path(metadataPath, SCHEMA_FILE_NAME); checkExists(getFileSytem(), schemaPath); schemaURI = rootFileSystem.makeQualified(schemaPath).toUri(); } else { schemaURI = manager.getNewestSchemaURI(); } if (schemaURI == null) { throw new DatasetException("Maformed dataset metadata at " + metadataPath + ". No schemas are present."); } try { builder.schemaUri(schemaURI); } catch (IOException e) { throw new DatasetIOException( "Unable to load schema file:" + schemaURI + " for dataset:" + name, e); } final Path location; if (properties.containsKey(LOCATION_FIELD_NAME)) { location = new Path(properties.getProperty(LOCATION_FIELD_NAME)); } else { location = expectedPathForDataset(namespace, name); } builder.location(location); for (String property : properties.stringPropertyNames()) { if (!RESERVED_PROPERTIES.contains(property)) { builder.property(property, properties.getProperty(property)); } } return builder.build(); }
FileSystemMetadataProvider extends AbstractMetadataProvider { @Override public DatasetDescriptor load(String namespace, String name) { Preconditions.checkNotNull(namespace, "Namespace cannot be null"); Preconditions.checkNotNull(name, "Dataset name cannot be null"); LOG.debug("Loading dataset metadata name: {}", name); Path metadataPath = find(namespace, name); InputStream inputStream = null; Properties properties = new Properties(); DatasetDescriptor.Builder builder = new DatasetDescriptor.Builder(); Path descriptorPath = new Path(metadataPath, DESCRIPTOR_FILE_NAME); boolean threw = true; try { inputStream = rootFileSystem.open(descriptorPath); properties.load(inputStream); threw = false; } catch (IOException e) { throw new DatasetIOException( "Unable to load descriptor file:" + descriptorPath + " for dataset:" + name, e); } finally { try { Closeables.close(inputStream, threw); } catch (IOException e) { throw new DatasetIOException("Cannot close", e); } } if (properties.containsKey(FORMAT_FIELD_NAME)) { builder.format(Accessor.getDefault().newFormat( properties.getProperty(FORMAT_FIELD_NAME))); } if (properties.containsKey(COMPRESSION_TYPE_FIELD_NAME)) { builder.compressionType(properties.getProperty(COMPRESSION_TYPE_FIELD_NAME)); } if (properties.containsKey(PARTITION_EXPRESSION_FIELD_NAME)) { builder.partitionStrategy(Accessor.getDefault().fromExpression(properties .getProperty(PARTITION_EXPRESSION_FIELD_NAME))); } SchemaManager manager = SchemaManager.load(conf, new Path(metadataPath, SCHEMA_DIRECTORY_NAME)); URI schemaURI; if (manager == null) { Path schemaPath = new Path(metadataPath, SCHEMA_FILE_NAME); checkExists(getFileSytem(), schemaPath); schemaURI = rootFileSystem.makeQualified(schemaPath).toUri(); } else { schemaURI = manager.getNewestSchemaURI(); } if (schemaURI == null) { throw new DatasetException("Maformed dataset metadata at " + metadataPath + ". No schemas are present."); } try { builder.schemaUri(schemaURI); } catch (IOException e) { throw new DatasetIOException( "Unable to load schema file:" + schemaURI + " for dataset:" + name, e); } final Path location; if (properties.containsKey(LOCATION_FIELD_NAME)) { location = new Path(properties.getProperty(LOCATION_FIELD_NAME)); } else { location = expectedPathForDataset(namespace, name); } builder.location(location); for (String property : properties.stringPropertyNames()) { if (!RESERVED_PROPERTIES.contains(property)) { builder.property(property, properties.getProperty(property)); } } return builder.build(); } }
FileSystemMetadataProvider extends AbstractMetadataProvider { @Override public DatasetDescriptor load(String namespace, String name) { Preconditions.checkNotNull(namespace, "Namespace cannot be null"); Preconditions.checkNotNull(name, "Dataset name cannot be null"); LOG.debug("Loading dataset metadata name: {}", name); Path metadataPath = find(namespace, name); InputStream inputStream = null; Properties properties = new Properties(); DatasetDescriptor.Builder builder = new DatasetDescriptor.Builder(); Path descriptorPath = new Path(metadataPath, DESCRIPTOR_FILE_NAME); boolean threw = true; try { inputStream = rootFileSystem.open(descriptorPath); properties.load(inputStream); threw = false; } catch (IOException e) { throw new DatasetIOException( "Unable to load descriptor file:" + descriptorPath + " for dataset:" + name, e); } finally { try { Closeables.close(inputStream, threw); } catch (IOException e) { throw new DatasetIOException("Cannot close", e); } } if (properties.containsKey(FORMAT_FIELD_NAME)) { builder.format(Accessor.getDefault().newFormat( properties.getProperty(FORMAT_FIELD_NAME))); } if (properties.containsKey(COMPRESSION_TYPE_FIELD_NAME)) { builder.compressionType(properties.getProperty(COMPRESSION_TYPE_FIELD_NAME)); } if (properties.containsKey(PARTITION_EXPRESSION_FIELD_NAME)) { builder.partitionStrategy(Accessor.getDefault().fromExpression(properties .getProperty(PARTITION_EXPRESSION_FIELD_NAME))); } SchemaManager manager = SchemaManager.load(conf, new Path(metadataPath, SCHEMA_DIRECTORY_NAME)); URI schemaURI; if (manager == null) { Path schemaPath = new Path(metadataPath, SCHEMA_FILE_NAME); checkExists(getFileSytem(), schemaPath); schemaURI = rootFileSystem.makeQualified(schemaPath).toUri(); } else { schemaURI = manager.getNewestSchemaURI(); } if (schemaURI == null) { throw new DatasetException("Maformed dataset metadata at " + metadataPath + ". No schemas are present."); } try { builder.schemaUri(schemaURI); } catch (IOException e) { throw new DatasetIOException( "Unable to load schema file:" + schemaURI + " for dataset:" + name, e); } final Path location; if (properties.containsKey(LOCATION_FIELD_NAME)) { location = new Path(properties.getProperty(LOCATION_FIELD_NAME)); } else { location = expectedPathForDataset(namespace, name); } builder.location(location); for (String property : properties.stringPropertyNames()) { if (!RESERVED_PROPERTIES.contains(property)) { builder.property(property, properties.getProperty(property)); } } return builder.build(); } FileSystemMetadataProvider(Configuration conf, Path rootDirectory); }
FileSystemMetadataProvider extends AbstractMetadataProvider { @Override public DatasetDescriptor load(String namespace, String name) { Preconditions.checkNotNull(namespace, "Namespace cannot be null"); Preconditions.checkNotNull(name, "Dataset name cannot be null"); LOG.debug("Loading dataset metadata name: {}", name); Path metadataPath = find(namespace, name); InputStream inputStream = null; Properties properties = new Properties(); DatasetDescriptor.Builder builder = new DatasetDescriptor.Builder(); Path descriptorPath = new Path(metadataPath, DESCRIPTOR_FILE_NAME); boolean threw = true; try { inputStream = rootFileSystem.open(descriptorPath); properties.load(inputStream); threw = false; } catch (IOException e) { throw new DatasetIOException( "Unable to load descriptor file:" + descriptorPath + " for dataset:" + name, e); } finally { try { Closeables.close(inputStream, threw); } catch (IOException e) { throw new DatasetIOException("Cannot close", e); } } if (properties.containsKey(FORMAT_FIELD_NAME)) { builder.format(Accessor.getDefault().newFormat( properties.getProperty(FORMAT_FIELD_NAME))); } if (properties.containsKey(COMPRESSION_TYPE_FIELD_NAME)) { builder.compressionType(properties.getProperty(COMPRESSION_TYPE_FIELD_NAME)); } if (properties.containsKey(PARTITION_EXPRESSION_FIELD_NAME)) { builder.partitionStrategy(Accessor.getDefault().fromExpression(properties .getProperty(PARTITION_EXPRESSION_FIELD_NAME))); } SchemaManager manager = SchemaManager.load(conf, new Path(metadataPath, SCHEMA_DIRECTORY_NAME)); URI schemaURI; if (manager == null) { Path schemaPath = new Path(metadataPath, SCHEMA_FILE_NAME); checkExists(getFileSytem(), schemaPath); schemaURI = rootFileSystem.makeQualified(schemaPath).toUri(); } else { schemaURI = manager.getNewestSchemaURI(); } if (schemaURI == null) { throw new DatasetException("Maformed dataset metadata at " + metadataPath + ". No schemas are present."); } try { builder.schemaUri(schemaURI); } catch (IOException e) { throw new DatasetIOException( "Unable to load schema file:" + schemaURI + " for dataset:" + name, e); } final Path location; if (properties.containsKey(LOCATION_FIELD_NAME)) { location = new Path(properties.getProperty(LOCATION_FIELD_NAME)); } else { location = expectedPathForDataset(namespace, name); } builder.location(location); for (String property : properties.stringPropertyNames()) { if (!RESERVED_PROPERTIES.contains(property)) { builder.property(property, properties.getProperty(property)); } } return builder.build(); } FileSystemMetadataProvider(Configuration conf, Path rootDirectory); @Override DatasetDescriptor load(String namespace, String name); @Override DatasetDescriptor create(String namespace, String name, DatasetDescriptor descriptor); @Override DatasetDescriptor update(String namespace, String name, DatasetDescriptor descriptor); @Override boolean delete(String namespace, String name); @Override boolean moveToTrash(String namespace, String name); @Override boolean exists(String namespace, String name); @SuppressWarnings("deprecation") @Override Set<String> namespaces(); @SuppressWarnings("deprecation") @Override Set<String> datasets(String namespace); @Override String toString(); }
FileSystemMetadataProvider extends AbstractMetadataProvider { @Override public DatasetDescriptor load(String namespace, String name) { Preconditions.checkNotNull(namespace, "Namespace cannot be null"); Preconditions.checkNotNull(name, "Dataset name cannot be null"); LOG.debug("Loading dataset metadata name: {}", name); Path metadataPath = find(namespace, name); InputStream inputStream = null; Properties properties = new Properties(); DatasetDescriptor.Builder builder = new DatasetDescriptor.Builder(); Path descriptorPath = new Path(metadataPath, DESCRIPTOR_FILE_NAME); boolean threw = true; try { inputStream = rootFileSystem.open(descriptorPath); properties.load(inputStream); threw = false; } catch (IOException e) { throw new DatasetIOException( "Unable to load descriptor file:" + descriptorPath + " for dataset:" + name, e); } finally { try { Closeables.close(inputStream, threw); } catch (IOException e) { throw new DatasetIOException("Cannot close", e); } } if (properties.containsKey(FORMAT_FIELD_NAME)) { builder.format(Accessor.getDefault().newFormat( properties.getProperty(FORMAT_FIELD_NAME))); } if (properties.containsKey(COMPRESSION_TYPE_FIELD_NAME)) { builder.compressionType(properties.getProperty(COMPRESSION_TYPE_FIELD_NAME)); } if (properties.containsKey(PARTITION_EXPRESSION_FIELD_NAME)) { builder.partitionStrategy(Accessor.getDefault().fromExpression(properties .getProperty(PARTITION_EXPRESSION_FIELD_NAME))); } SchemaManager manager = SchemaManager.load(conf, new Path(metadataPath, SCHEMA_DIRECTORY_NAME)); URI schemaURI; if (manager == null) { Path schemaPath = new Path(metadataPath, SCHEMA_FILE_NAME); checkExists(getFileSytem(), schemaPath); schemaURI = rootFileSystem.makeQualified(schemaPath).toUri(); } else { schemaURI = manager.getNewestSchemaURI(); } if (schemaURI == null) { throw new DatasetException("Maformed dataset metadata at " + metadataPath + ". No schemas are present."); } try { builder.schemaUri(schemaURI); } catch (IOException e) { throw new DatasetIOException( "Unable to load schema file:" + schemaURI + " for dataset:" + name, e); } final Path location; if (properties.containsKey(LOCATION_FIELD_NAME)) { location = new Path(properties.getProperty(LOCATION_FIELD_NAME)); } else { location = expectedPathForDataset(namespace, name); } builder.location(location); for (String property : properties.stringPropertyNames()) { if (!RESERVED_PROPERTIES.contains(property)) { builder.property(property, properties.getProperty(property)); } } return builder.build(); } FileSystemMetadataProvider(Configuration conf, Path rootDirectory); @Override DatasetDescriptor load(String namespace, String name); @Override DatasetDescriptor create(String namespace, String name, DatasetDescriptor descriptor); @Override DatasetDescriptor update(String namespace, String name, DatasetDescriptor descriptor); @Override boolean delete(String namespace, String name); @Override boolean moveToTrash(String namespace, String name); @Override boolean exists(String namespace, String name); @SuppressWarnings("deprecation") @Override Set<String> namespaces(); @SuppressWarnings("deprecation") @Override Set<String> datasets(String namespace); @Override String toString();  }