LoadBalancingKMSClientProvider extends KeyProvider implements
    CryptoExtension,
    KeyProviderDelegationTokenExtension.DelegationTokenExtension { @Override public KeyVersion createKey(final String name, final byte[] material, final Options options) throws IOException { return doOp(new ProviderCallable<KeyVersion>() { @Override public KeyVersion call(KMSClientProvider provider) throws IOException { return provider.createKey(name, material, options); } }, nextIdx(), false); } LoadBalancingKMSClientProvider(URI providerUri,
      KMSClientProvider[] providers, Configuration conf); @VisibleForTesting  LoadBalancingKMSClientProvider(KMSClientProvider[] providers, long seed,
      Configuration conf); private  LoadBalancingKMSClientProvider(URI uri,
      KMSClientProvider[] providers, long seed, Configuration conf); @VisibleForTesting KMSClientProvider[] getProviders(); @Override org.apache.hadoop.security.token.Token<? extends TokenIdentifier> selectDelegationToken(Credentials creds); @Override String getCanonicalServiceName(); @Override Token<?> getDelegationToken(String renewer); @Override long renewDelegationToken(final Token<?> token); @Override Void cancelDelegationToken(final Token<?> token); @Override void warmUpEncryptedKeys(String... keyNames); @Override void drain(String keyName); @Override void invalidateCache(String keyName); @Override EncryptedKeyVersion generateEncryptedKey(final String encryptionKeyName); @Override KeyVersion decryptEncryptedKey(final EncryptedKeyVersion encryptedKeyVersion); @Override EncryptedKeyVersion reencryptEncryptedKey(
      final EncryptedKeyVersion ekv); @Override void reencryptEncryptedKeys(final List<EncryptedKeyVersion> ekvs); @Override KeyVersion getKeyVersion(final String versionName); @Override List<String> getKeys(); @Override Metadata[] getKeysMetadata(final String... names); @Override List<KeyVersion> getKeyVersions(final String name); @Override KeyVersion getCurrentKey(final String name); @Override Metadata getMetadata(final String name); @Override KeyVersion createKey(final String name, final byte[] material,
      final Options options); @Override KeyVersion createKey(final String name, final Options options); @Override void deleteKey(final String name); @Override KeyVersion rollNewVersion(final String name, final byte[] material); @Override KeyVersion rollNewVersion(final String name); @Override void close(); @Override void flush(); static Logger LOG; }
@Test public void testLoadBalancing() throws Exception { Configuration conf = new Configuration(); KMSClientProvider p1 = mock(KMSClientProvider.class); when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenReturn( new KMSClientProvider.KMSKeyVersion("p1", "v1", new byte[0])); KMSClientProvider p2 = mock(KMSClientProvider.class); when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenReturn( new KMSClientProvider.KMSKeyVersion("p2", "v2", new byte[0])); KMSClientProvider p3 = mock(KMSClientProvider.class); when(p3.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenReturn( new KMSClientProvider.KMSKeyVersion("p3", "v3", new byte[0])); KeyProvider kp = new LoadBalancingKMSClientProvider( new KMSClientProvider[] { p1, p2, p3 }, 0, conf); assertEquals("p1", kp.createKey("test1", new Options(conf)).getName()); assertEquals("p2", kp.createKey("test2", new Options(conf)).getName()); assertEquals("p3", kp.createKey("test3", new Options(conf)).getName()); assertEquals("p1", kp.createKey("test4", new Options(conf)).getName()); }
@Test public void testLoadBalancingWithFailure() throws Exception { Configuration conf = new Configuration(); KMSClientProvider p1 = mock(KMSClientProvider.class); when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenReturn( new KMSClientProvider.KMSKeyVersion("p1", "v1", new byte[0])); when(p1.getKMSUrl()).thenReturn("p1"); KMSClientProvider p2 = mock(KMSClientProvider.class); when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new NoSuchAlgorithmException("p2")); when(p2.getKMSUrl()).thenReturn("p2"); KMSClientProvider p3 = mock(KMSClientProvider.class); when(p3.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenReturn( new KMSClientProvider.KMSKeyVersion("p3", "v3", new byte[0])); when(p3.getKMSUrl()).thenReturn("p3"); KMSClientProvider p4 = mock(KMSClientProvider.class); when(p4.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new IOException("p4")); when(p4.getKMSUrl()).thenReturn("p4"); KeyProvider kp = new LoadBalancingKMSClientProvider( new KMSClientProvider[] { p1, p2, p3, p4 }, 0, conf); assertEquals("p1", kp.createKey("test4", new Options(conf)).getName()); try { kp.createKey("test1", new Options(conf)).getName(); fail("Should fail since its not an IOException"); } catch (Exception e) { assertTrue(e instanceof NoSuchAlgorithmException); } assertEquals("p3", kp.createKey("test2", new Options(conf)).getName()); assertEquals("p1", kp.createKey("test3", new Options(conf)).getName()); }
@Test public void testLoadBalancingWithAllBadNodes() throws Exception { Configuration conf = new Configuration(); KMSClientProvider p1 = mock(KMSClientProvider.class); when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new IOException("p1")); KMSClientProvider p2 = mock(KMSClientProvider.class); when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new IOException("p2")); KMSClientProvider p3 = mock(KMSClientProvider.class); when(p3.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new IOException("p3")); KMSClientProvider p4 = mock(KMSClientProvider.class); when(p4.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new IOException("p4")); when(p1.getKMSUrl()).thenReturn("p1"); when(p2.getKMSUrl()).thenReturn("p2"); when(p3.getKMSUrl()).thenReturn("p3"); when(p4.getKMSUrl()).thenReturn("p4"); KeyProvider kp = new LoadBalancingKMSClientProvider( new KMSClientProvider[] { p1, p2, p3, p4 }, 0, conf); try { kp.createKey("test3", new Options(conf)).getName(); fail("Should fail since all providers threw an IOException"); } catch (Exception e) { assertTrue(e instanceof IOException); } }
@Test public void testClientRetriesWithAccessControlException() throws Exception { Configuration conf = new Configuration(); conf.setInt( CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 3); KMSClientProvider p1 = mock(KMSClientProvider.class); when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new AccessControlException("p1")); KMSClientProvider p2 = mock(KMSClientProvider.class); when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new IOException("p2")); KMSClientProvider p3 = mock(KMSClientProvider.class); when(p3.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new IOException("p3")); when(p1.getKMSUrl()).thenReturn("p1"); when(p2.getKMSUrl()).thenReturn("p2"); when(p3.getKMSUrl()).thenReturn("p3"); LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider( new KMSClientProvider[] {p1, p2, p3}, 0, conf); try { kp.createKey("test3", new Options(conf)); fail("Should fail because provider p1 threw an AccessControlException"); } catch (Exception e) { assertTrue(e instanceof AccessControlException); } verify(p1, Mockito.times(1)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); verify(p2, Mockito.never()).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); verify(p3, Mockito.never()).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); }
@Test public void testClientRetriesWithRuntimeException() throws Exception { Configuration conf = new Configuration(); conf.setInt( CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 3); KMSClientProvider p1 = mock(KMSClientProvider.class); when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new RuntimeException("p1")); KMSClientProvider p2 = mock(KMSClientProvider.class); when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new IOException("p2")); when(p1.getKMSUrl()).thenReturn("p1"); when(p2.getKMSUrl()).thenReturn("p2"); LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider( new KMSClientProvider[] {p1, p2}, 0, conf); try { kp.createKey("test3", new Options(conf)); fail("Should fail since provider p1 threw RuntimeException"); } catch (Exception e) { assertTrue(e instanceof RuntimeException); } verify(p1, Mockito.times(1)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); verify(p2, Mockito.never()).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); }
@Test public void testClientRetriesWithTimeoutsException() throws Exception { Configuration conf = new Configuration(); conf.setInt( CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 4); KMSClientProvider p1 = mock(KMSClientProvider.class); when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new ConnectTimeoutException("p1")); KMSClientProvider p2 = mock(KMSClientProvider.class); when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new UnknownHostException("p2")); KMSClientProvider p3 = mock(KMSClientProvider.class); when(p3.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new NoRouteToHostException("p3")); KMSClientProvider p4 = mock(KMSClientProvider.class); when(p4.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenReturn( new KMSClientProvider.KMSKeyVersion("test3", "v1", new byte[0])); when(p1.getKMSUrl()).thenReturn("p1"); when(p2.getKMSUrl()).thenReturn("p2"); when(p3.getKMSUrl()).thenReturn("p3"); when(p4.getKMSUrl()).thenReturn("p4"); LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider( new KMSClientProvider[] {p1, p2, p3, p4}, 0, conf); try { kp.createKey("test3", new Options(conf)); } catch (Exception e) { fail("Provider p4 should have answered the request."); } verify(p1, Mockito.times(1)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); verify(p2, Mockito.times(1)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); verify(p3, Mockito.times(1)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); verify(p4, Mockito.times(1)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); }
@Test public void testClientRetriesSucceedsSecondTime() throws Exception { Configuration conf = new Configuration(); conf.setInt( CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 3); KMSClientProvider p1 = mock(KMSClientProvider.class); when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new ConnectTimeoutException("p1")) .thenReturn(new KMSClientProvider.KMSKeyVersion("test3", "v1", new byte[0])); KMSClientProvider p2 = mock(KMSClientProvider.class); when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new ConnectTimeoutException("p2")); when(p1.getKMSUrl()).thenReturn("p1"); when(p2.getKMSUrl()).thenReturn("p2"); LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider( new KMSClientProvider[] {p1, p2}, 0, conf); try { kp.createKey("test3", new Options(conf)); } catch (Exception e) { fail("Provider p1 should have answered the request second time."); } verify(p1, Mockito.times(2)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); verify(p2, Mockito.times(1)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); }
@Test public void testClientRetriesSpecifiedNumberOfTimes() throws Exception { Configuration conf = new Configuration(); conf.setInt( CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 10); KMSClientProvider p1 = mock(KMSClientProvider.class); when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new ConnectTimeoutException("p1")); KMSClientProvider p2 = mock(KMSClientProvider.class); when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new ConnectTimeoutException("p2")); when(p1.getKMSUrl()).thenReturn("p1"); when(p2.getKMSUrl()).thenReturn("p2"); LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider( new KMSClientProvider[] {p1, p2}, 0, conf); try { kp.createKey("test3", new Options(conf)); fail("Should fail"); } catch (Exception e) { assert (e instanceof ConnectTimeoutException); } verify(p1, Mockito.times(6)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); verify(p2, Mockito.times(5)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); }
@Test public void testClientRetriesIfMaxAttemptsNotSet() throws Exception { Configuration conf = new Configuration(); KMSClientProvider p1 = mock(KMSClientProvider.class); when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new ConnectTimeoutException("p1")); KMSClientProvider p2 = mock(KMSClientProvider.class); when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new ConnectTimeoutException("p2")); when(p1.getKMSUrl()).thenReturn("p1"); when(p2.getKMSUrl()).thenReturn("p2"); LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider( new KMSClientProvider[] {p1, p2}, 0, conf); try { kp.createKey("test3", new Options(conf)); fail("Should fail"); } catch (Exception e) { assert (e instanceof ConnectTimeoutException); } verify(p1, Mockito.times(2)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); verify(p2, Mockito.times(1)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); }
@Test public void testClientRetriesWithAuthenticationExceptionWrappedinIOException() throws Exception { Configuration conf = new Configuration(); conf.setInt( CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 3); KMSClientProvider p1 = mock(KMSClientProvider.class); when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new IOException(new AuthenticationException("p1"))); KMSClientProvider p2 = mock(KMSClientProvider.class); when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new IOException(new AuthenticationException("p1"))); when(p1.getKMSUrl()).thenReturn("p1"); when(p2.getKMSUrl()).thenReturn("p2"); LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider( new KMSClientProvider[] {p1, p2}, 0, conf); try { kp.createKey("test3", new Options(conf)); fail("Should fail since provider p1 threw AuthenticationException"); } catch (Exception e) { assertTrue(e.getCause() instanceof AuthenticationException); } verify(p1, Mockito.times(1)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); verify(p2, Mockito.times(1)).createKey(Mockito.eq("test3"), Mockito.any(Options.class)); }
@Test public void testClientRetriesWithSSLHandshakeExceptionSucceedsSecondTime() throws Exception { Configuration conf = new Configuration(); conf.setInt( CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 3); final String keyName = "test"; KMSClientProvider p1 = mock(KMSClientProvider.class); when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new SSLHandshakeException("p1")) .thenReturn(new KMSClientProvider.KMSKeyVersion(keyName, "v1", new byte[0])); KMSClientProvider p2 = mock(KMSClientProvider.class); when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new ConnectException("p2")); when(p1.getKMSUrl()).thenReturn("p1"); when(p2.getKMSUrl()).thenReturn("p2"); LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider( new KMSClientProvider[] {p1, p2}, 0, conf); kp.createKey(keyName, new Options(conf)); verify(p1, Mockito.times(2)).createKey(Mockito.eq(keyName), Mockito.any(Options.class)); verify(p2, Mockito.times(1)).createKey(Mockito.eq(keyName), Mockito.any(Options.class)); }
@Test public void testClientRetriesWithSSLHandshakeExceptionFailsAtEveryAttempt() throws Exception { Configuration conf = new Configuration(); conf.setInt( CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 2); final String keyName = "test"; final String exceptionMessage = "p1 exception message"; KMSClientProvider p1 = mock(KMSClientProvider.class); Exception originalSslEx = new SSLHandshakeException(exceptionMessage); when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(originalSslEx); KMSClientProvider p2 = mock(KMSClientProvider.class); when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class))) .thenThrow(new ConnectException("p2 exception message")); when(p1.getKMSUrl()).thenReturn("p1"); when(p2.getKMSUrl()).thenReturn("p2"); LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider( new KMSClientProvider[] {p1, p2}, 0, conf); Exception interceptedEx = intercept(ConnectException.class, "SSLHandshakeException: " + exceptionMessage, ()-> kp.createKey(keyName, new Options(conf))); assertEquals(originalSslEx, interceptedEx.getCause()); verify(p1, Mockito.times(2)).createKey(Mockito.eq(keyName), Mockito.any(Options.class)); verify(p2, Mockito.times(1)).createKey(Mockito.eq(keyName), Mockito.any(Options.class)); }