GoogleHadoopFileSystemConfiguration { static GoogleCloudStorageFileSystemOptions.Builder getGcsFsOptionsBuilder(Configuration config) { return GoogleCloudStorageFileSystemOptions.builder() .setBucketDeleteEnabled(GCE_BUCKET_DELETE_ENABLE.get(config, config::getBoolean)) .setMarkerFilePattern(GCS_MARKER_FILE_PATTERN.get(config, config::get)) .setPerformanceCacheEnabled(GCS_PERFORMANCE_CACHE_ENABLE.get(config, config::getBoolean)) .setCooperativeLockingEnabled( GCS_COOPERATIVE_LOCKING_ENABLE.get(config, config::getBoolean)) .setPerformanceCacheOptions(getPerformanceCachingOptions(config)) .setStatusParallelEnabled(GCS_STATUS_PARALLEL_ENABLE.get(config, config::getBoolean)) .setCloudStorageOptions(getGcsOptionsBuilder(config).build()); }   static final String GCS_CONFIG_PREFIX; static final List<String> CONFIG_KEY_PREFIXES; static final HadoopConfigurationProperty<String> GCS_ROOT_URL; static final HadoopConfigurationProperty<String> GCS_SERVICE_PATH; static final HadoopConfigurationProperty<String> PERMISSIONS_TO_REPORT; static final HadoopConfigurationProperty<Long> BLOCK_SIZE; static final HadoopConfigurationProperty<String> DELEGATION_TOKEN_BINDING_CLASS; static final HadoopConfigurationProperty<String> GCS_PROJECT_ID; static final HadoopConfigurationProperty<String> GCS_WORKING_DIRECTORY; static final HadoopConfigurationProperty<Boolean> GCE_BUCKET_DELETE_ENABLE; static final HadoopConfigurationProperty<RequesterPaysMode> GCS_REQUESTER_PAYS_MODE; static final HadoopConfigurationProperty<String> GCS_REQUESTER_PAYS_PROJECT_ID; static final HadoopConfigurationProperty<Collection<String>> GCS_REQUESTER_PAYS_BUCKETS; static final HadoopConfigurationProperty<GcsFileChecksumType> GCS_FILE_CHECKSUM_TYPE; static final HadoopConfigurationProperty<Boolean> GCS_PERFORMANCE_CACHE_ENABLE; static final HadoopConfigurationProperty<Long> GCS_PERFORMANCE_CACHE_MAX_ENTRY_AGE_MILLIS; static final HadoopConfigurationProperty<Boolean> GCS_STATUS_PARALLEL_ENABLE; static final HadoopConfigurationProperty<Boolean> GCS_LAZY_INITIALIZATION_ENABLE; static final HadoopConfigurationProperty<Boolean> GCS_REPAIR_IMPLICIT_DIRECTORIES_ENABLE; static final HadoopConfigurationProperty<Boolean> GCS_INFER_IMPLICIT_DIRECTORIES_ENABLE; static final HadoopConfigurationProperty<Boolean> GCS_FLAT_GLOB_ENABLE; static final HadoopConfigurationProperty<Boolean> GCS_CONCURRENT_GLOB_ENABLE; static final HadoopConfigurationProperty<String> GCS_MARKER_FILE_PATTERN; static final HadoopConfigurationProperty<Long> GCS_MAX_REQUESTS_PER_BATCH; static final HadoopConfigurationProperty<Integer> GCS_BATCH_THREADS; static final HadoopConfigurationProperty<Long> GCS_COPY_MAX_REQUESTS_PER_BATCH; static final HadoopConfigurationProperty<Integer> GCS_COPY_BATCH_THREADS; static final HadoopConfigurationProperty<Boolean> GCS_COPY_WITH_REWRITE_ENABLE; static final HadoopConfigurationProperty<Long> GCS_REWRITE_MAX_BYTES_PER_CALL; static final HadoopConfigurationProperty<Boolean> GCS_OVERWRITE_GENERATION_MISMATCH_IGNORE; static final HadoopConfigurationProperty<Long> GCS_MAX_LIST_ITEMS_PER_CALL; static final HadoopConfigurationProperty<Integer> GCS_HTTP_MAX_RETRY; static final HadoopConfigurationProperty<Integer> GCS_HTTP_CONNECT_TIMEOUT; static final HadoopConfigurationProperty<Integer> GCS_HTTP_READ_TIMEOUT; static final HadoopConfigurationProperty<String> GCS_APPLICATION_NAME_SUFFIX; static final HadoopConfigurationProperty<Integer> GCS_MAX_WAIT_MILLIS_EMPTY_OBJECT_CREATE; static final HadoopConfigurationProperty<OutputStreamType> GCS_OUTPUT_STREAM_TYPE; static final HadoopConfigurationProperty<Integer> GCS_OUTPUT_STREAM_BUFFER_SIZE; static final HadoopConfigurationProperty<Integer> GCS_OUTPUT_STREAM_PIPE_BUFFER_SIZE; static final HadoopConfigurationProperty<PipeType> GCS_OUTPUT_STREAM_PIPE_TYPE; static final HadoopConfigurationProperty<Integer> GCS_OUTPUT_STREAM_UPLOAD_CHUNK_SIZE; static final HadoopConfigurationProperty<Integer> GCS_OUTPUT_STREAM_UPLOAD_CACHE_SIZE; static final HadoopConfigurationProperty<Boolean> GCS_OUTPUT_STREAM_DIRECT_UPLOAD_ENABLE; static final HadoopConfigurationProperty<Integer> GCS_OUTPUT_STREAM_SYNC_MIN_INTERVAL_MS; static final HadoopConfigurationProperty<Integer> GCS_INPUT_STREAM_BUFFER_SIZE; static final HadoopConfigurationProperty<Boolean> GCS_INPUT_STREAM_FAST_FAIL_ON_NOT_FOUND_ENABLE; static final HadoopConfigurationProperty<Boolean> GCS_INPUT_STREAM_SUPPORT_GZIP_ENCODING_ENABLE; static final HadoopConfigurationProperty<Long> GCS_INPUT_STREAM_INPLACE_SEEK_LIMIT; static final HadoopConfigurationProperty<Fadvise> GCS_INPUT_STREAM_FADVISE; static final HadoopConfigurationProperty<Integer> GCS_INPUT_STREAM_MIN_RANGE_REQUEST_SIZE; static final HadoopConfigurationProperty<Boolean> GCS_GRPC_ENABLE; static final HadoopConfigurationProperty<Boolean> GCS_GRPC_CHECKSUMS_ENABLE; static final HadoopConfigurationProperty<String> GCS_GRPC_SERVER_ADDRESS; static final HadoopConfigurationProperty<String> GCS_CONFIG_OVERRIDE_FILE; static final HadoopConfigurationProperty<Boolean> GCS_COOPERATIVE_LOCKING_ENABLE; static final HadoopConfigurationProperty<Long> GCS_COOPERATIVE_LOCKING_EXPIRATION_TIMEOUT_MS; static final HadoopConfigurationProperty<Integer> GCS_COOPERATIVE_LOCKING_MAX_CONCURRENT_OPERATIONS; static final HadoopConfigurationProperty<Map<String, String>> GCS_HTTP_HEADERS; static final HadoopConfigurationProperty<String> GCS_ENCRYPTION_ALGORITHM; static final HadoopConfigurationProperty<RedactedString> GCS_ENCRYPTION_KEY; static final HadoopConfigurationProperty<RedactedString> GCS_ENCRYPTION_KEY_HASH; static final HadoopConfigurationProperty<Class<? extends AuthorizationHandler>> GCS_AUTHORIZATION_HANDLER_IMPL; static final HadoopConfigurationProperty<Map<String, String>> GCS_AUTHORIZATION_HANDLER_PROPERTIES_PREFIX; }
@Test public void testHttpHeadersProperties_multipleHeaders() { Configuration config = new Configuration(); config.set(GCS_HTTP_HEADERS.getKey() + "test-header", "test-VAL"); config.set(GCS_HTTP_HEADERS.getKey() + "key-in-header", "+G2Ap33m5NVOgmXznSGTEvG0I="); GoogleCloudStorageFileSystemOptions options = GoogleHadoopFileSystemConfiguration.getGcsFsOptionsBuilder(config).build(); assertThat(options.getCloudStorageOptions().getHttpRequestHeaders()) .containsExactly("test-header", "test-VAL", "key-in-header", "+G2Ap33m5NVOgmXznSGTEvG0I="); }
@Test public void testEncryptionProperties() { Configuration config = new Configuration(); config.set(GCS_ENCRYPTION_ALGORITHM.getKey(), "AES256"); config.set(GCS_ENCRYPTION_KEY.getKey(), "+G2Ap33m5NVOgmXznSGTEvG0I="); config.set(GCS_ENCRYPTION_KEY_HASH.getKey(), "LpH4y6BkG/1B+n3FwORpdoyQ="); GoogleCloudStorageFileSystemOptions options = GoogleHadoopFileSystemConfiguration.getGcsFsOptionsBuilder(config).build(); assertThat(options.getCloudStorageOptions().getEncryptionAlgorithm()).isEqualTo("AES256"); assertThat(options.getCloudStorageOptions().getEncryptionKey()).isNotNull(); assertThat(options.getCloudStorageOptions().getEncryptionKey().toString()) .isEqualTo("<redacted>"); assertThat(options.getCloudStorageOptions().getEncryptionKey().value()) .isEqualTo("+G2Ap33m5NVOgmXznSGTEvG0I="); assertThat(options.getCloudStorageOptions().getEncryptionKeyHash()).isNotNull(); assertThat(options.getCloudStorageOptions().getEncryptionKeyHash().toString()) .isEqualTo("<redacted>"); assertThat(options.getCloudStorageOptions().getEncryptionKeyHash().value()) .isEqualTo("LpH4y6BkG/1B+n3FwORpdoyQ="); }
@Test public void testProxyPropertiesAll() { HadoopConfigurationProperty<String> gcsProxyUsername = new HadoopConfigurationProperty<>( GCS_CONFIG_PREFIX + PROXY_USERNAME_SUFFIX.getKey(), "proxy-user"); HadoopConfigurationProperty<String> gcsProxyPassword = new HadoopConfigurationProperty<>( GCS_CONFIG_PREFIX + PROXY_PASSWORD_SUFFIX.getKey(), "proxy-pass"); HadoopConfigurationProperty<String> gcsProxyAddress = new HadoopConfigurationProperty<>( GCS_CONFIG_PREFIX + PROXY_ADDRESS_SUFFIX.getKey(), "proxy-address"); Configuration config = new Configuration(); config.set(gcsProxyUsername.getKey(), gcsProxyUsername.getDefault()); config.set(gcsProxyPassword.getKey(), gcsProxyPassword.getDefault()); config.set(gcsProxyAddress.getKey(), gcsProxyAddress.getDefault()); GoogleCloudStorageFileSystemOptions options = GoogleHadoopFileSystemConfiguration.getGcsFsOptionsBuilder(config).build(); assertThat(options.getCloudStorageOptions().getProxyUsername()).isNotNull(); assertThat(options.getCloudStorageOptions().getProxyUsername().value()).isEqualTo("proxy-user"); assertThat(options.getCloudStorageOptions().getProxyUsername().toString()) .isEqualTo("<redacted>"); assertThat(options.getCloudStorageOptions().getProxyPassword()).isNotNull(); assertThat(options.getCloudStorageOptions().getProxyPassword().value()).isEqualTo("proxy-pass"); assertThat(options.getCloudStorageOptions().getProxyPassword().toString()) .isEqualTo("<redacted>"); assertThat(options.getCloudStorageOptions().getProxyAddress()).isEqualTo("proxy-address"); }
@Test public void testHttpHeadersProperties_singleHeader() { Configuration config = new Configuration(); config.set(GCS_HTTP_HEADERS.getKey() + "header-key", "val=ue"); GoogleCloudStorageFileSystemOptions options = GoogleHadoopFileSystemConfiguration.getGcsFsOptionsBuilder(config).build(); assertThat(options.getCloudStorageOptions().getHttpRequestHeaders()) .containsExactly("header-key", "val=ue"); }