KMS { @POST @Path(KMSRESTConstants.KEYS_RESOURCE) @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) @SuppressWarnings("unchecked") public Response createKey(Map jsonKey) throws Exception { try{ LOG.trace("Entering createKey Method."); KMSWebApp.getAdminCallsMeter().mark(); UserGroupInformation user = HttpUserGroupInformation.get(); final String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD); checkNotEmpty(name, KMSRESTConstants.NAME_FIELD); assertAccess(KMSACLs.Type.CREATE, user, KMSOp.CREATE_KEY, name); String cipher = (String) jsonKey.get(KMSRESTConstants.CIPHER_FIELD); final String material; material = (String) jsonKey.get(KMSRESTConstants.MATERIAL_FIELD); int length = (jsonKey.containsKey(KMSRESTConstants.LENGTH_FIELD)) ? (Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0; String description = (String) jsonKey.get(KMSRESTConstants.DESCRIPTION_FIELD); LOG.debug("Creating key with name {}, cipher being used{}, " + "length of key {}, description of key {}", name, cipher, length, description); Map<String, String> attributes = (Map<String, String>) jsonKey.get(KMSRESTConstants.ATTRIBUTES_FIELD); if (material != null) { assertAccess(KMSACLs.Type.SET_KEY_MATERIAL, user, KMSOp.CREATE_KEY, name); } final KeyProvider.Options options = new KeyProvider.Options( KMSWebApp.getConfiguration()); if (cipher != null) { options.setCipher(cipher); } if (length != 0) { options.setBitLength(length); } options.setDescription(description); options.setAttributes(attributes); KeyProvider.KeyVersion keyVersion = user.doAs( new PrivilegedExceptionAction<KeyVersion>() { @Override public KeyVersion run() throws Exception { KeyProvider.KeyVersion keyVersion = (material != null) ? provider.createKey(name, Base64.decodeBase64(material), options) : provider.createKey(name, options); provider.flush(); return keyVersion; } } ); kmsAudit.ok(user, KMSOp.CREATE_KEY, name, "UserProvidedMaterial:" + (material != null) + " Description:" + description); if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user)) { keyVersion = removeKeyMaterial(keyVersion); } Map json = KMSUtil.toJSON(keyVersion); String requestURL = KMSMDCFilter.getURL(); int idx = requestURL.lastIndexOf(KMSRESTConstants.KEYS_RESOURCE); requestURL = requestURL.substring(0, idx); LOG.trace("Exiting createKey Method."); return Response.created(getKeyURI(KMSRESTConstants.SERVICE_VERSION, name)) .type(MediaType.APPLICATION_JSON) .header("Location", getKeyURI(requestURL, name)).entity(json).build(); } catch (Exception e) { LOG.debug("Exception in createKey.", e); throw e; } } KMS(); @POST @Path(KMSRESTConstants.KEYS_RESOURCE) @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) @SuppressWarnings("unchecked") Response createKey(Map jsonKey); @DELETE @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}") Response deleteKey(@PathParam("name") final String name); @POST @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) Response rolloverKey(@PathParam("name") final String name,
      Map jsonMaterial); @POST @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + KMSRESTConstants.INVALIDATECACHE_RESOURCE) Response invalidateCache(@PathParam("name") final String name); @GET @Path(KMSRESTConstants.KEYS_METADATA_RESOURCE) @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) Response getKeysMetadata(@QueryParam(KMSRESTConstants.KEY)
      List<String> keyNamesList); @GET @Path(KMSRESTConstants.KEYS_NAMES_RESOURCE) @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) Response getKeyNames(); @GET @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}") Response getKey(@PathParam("name") String name); @GET @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + KMSRESTConstants.METADATA_SUB_RESOURCE) @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) Response getMetadata(@PathParam("name") final String name); @GET @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + KMSRESTConstants.CURRENT_VERSION_SUB_RESOURCE) @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) Response getCurrentVersion(@PathParam("name") final String name); @GET @Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}") @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) Response getKeyVersion(
      @PathParam("versionName") final String versionName); @SuppressWarnings({ "rawtypes", "unchecked" }) @GET @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + KMSRESTConstants.EEK_SUB_RESOURCE) @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) Response generateEncryptedKeys(
          @PathParam("name") final String name,
          @QueryParam(KMSRESTConstants.EEK_OP) String edekOp,
          @DefaultValue("1")
          @QueryParam(KMSRESTConstants.EEK_NUM_KEYS) final int numKeys); @SuppressWarnings("rawtypes") @POST @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + KMSRESTConstants.REENCRYPT_BATCH_SUB_RESOURCE) @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) Response reencryptEncryptedKeys(
      @PathParam("name") final String name,
      final List<Map> jsonPayload); @SuppressWarnings("rawtypes") @POST @Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}/" + KMSRESTConstants.EEK_SUB_RESOURCE) @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) Response handleEncryptedKeyOp(
      @PathParam("versionName") final String versionName,
      @QueryParam(KMSRESTConstants.EEK_OP) String eekOp,
      Map jsonPayload); @GET @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" + KMSRESTConstants.VERSIONS_SUB_RESOURCE) @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) Response getKeyVersions(@PathParam("name") final String name);  }
@Test public void testKMSAuthFailureRetry() throws Exception { Configuration conf = new Configuration(); conf.set("hadoop.security.authentication", "kerberos"); final File testDir = getTestDir(); conf = createBaseKMSConf(testDir, conf); conf.set("hadoop.kms.authentication.kerberos.keytab", keytab.getAbsolutePath()); conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost"); conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT"); conf.set("hadoop.kms.authentication.token.validity", "1"); for (KMSACLs.Type type : KMSACLs.Type.values()) { conf.set(type.getAclConfigKey(), type.toString()); } conf.set(KMSACLs.Type.CREATE.getAclConfigKey(), KMSACLs.Type.CREATE.toString() + ",SET_KEY_MATERIAL"); conf.set(KMSACLs.Type.ROLLOVER.getAclConfigKey(), KMSACLs.Type.ROLLOVER.toString() + ",SET_KEY_MATERIAL"); conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k0.ALL", "*"); conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k1.ALL", "*"); conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k2.ALL", "*"); conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k3.ALL", "*"); conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k4.ALL", "*"); writeConf(testDir, conf); runServer(null, null, testDir, new KMSCallable<Void>() { @Override public Void call() throws Exception { final Configuration conf = new Configuration(); conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128); final URI uri = createKMSUri(getKMSUrl()); doAs("SET_KEY_MATERIAL", new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { KeyProvider kp = createProvider(uri, conf); kp.createKey("k0", new byte[16], new KeyProvider.Options(conf)); kp.createKey("k1", new byte[16], new KeyProvider.Options(conf)); Thread.sleep(3500); kp.createKey("k2", new byte[16], new KeyProvider.Options(conf)); return null; } }); return null; } }); runServer(null, null, testDir, new KMSCallable<Void>() { @Override public Void call() throws Exception { final Configuration conf = new Configuration(); conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128); conf.setInt(KMSClientProvider.AUTH_RETRY, 0); final URI uri = createKMSUri(getKMSUrl()); doAs("SET_KEY_MATERIAL", new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { KeyProvider kp = createProvider(uri, conf); kp.createKey("k3", new byte[16], new KeyProvider.Options(conf)); Thread.sleep(3500); try { kp.createKey("k4", new byte[16], new KeyProvider.Options(conf)); Assert.fail("This should not succeed.."); } catch (IOException e) { Assert.assertTrue( "HTTP exception must be a 401 : " + e.getMessage(), e .getMessage().contains("401")); } return null; } }); return null; } }); }
@Test public void testKMSBlackList() throws Exception { Configuration conf = new Configuration(); conf.set("hadoop.security.authentication", "kerberos"); File testDir = getTestDir(); conf = createBaseKMSConf(testDir, conf); conf.set("hadoop.kms.authentication.type", "kerberos"); conf.set("hadoop.kms.authentication.kerberos.keytab", keytab.getAbsolutePath()); conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost"); conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT"); for (KMSACLs.Type type : KMSACLs.Type.values()) { conf.set(type.getAclConfigKey(), " "); } conf.set(KMSACLs.Type.CREATE.getAclConfigKey(), "client,hdfs,otheradmin"); conf.set(KMSACLs.Type.GENERATE_EEK.getAclConfigKey(), "client,hdfs,otheradmin"); conf.set(KMSACLs.Type.DECRYPT_EEK.getAclConfigKey(), "client,hdfs,otheradmin"); conf.set(KMSACLs.Type.DECRYPT_EEK.getBlacklistConfigKey(), "hdfs,otheradmin"); conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "ck0.ALL", "*"); conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "ck1.ALL", "*"); writeConf(testDir, conf); runServer(null, null, testDir, new KMSCallable<Void>() { @Override public Void call() throws Exception { final Configuration conf = new Configuration(); conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128); final URI uri = createKMSUri(getKMSUrl()); doAs("client", new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { try { KeyProvider kp = createProvider(uri, conf); KeyProvider.KeyVersion kv = kp.createKey("ck0", new KeyProvider.Options(conf)); EncryptedKeyVersion eek = ((CryptoExtension)kp).generateEncryptedKey("ck0"); ((CryptoExtension)kp).decryptEncryptedKey(eek); Assert.assertNull(kv.getMaterial()); } catch (Exception ex) { Assert.fail(ex.getMessage()); } return null; } }); doAs("hdfs", new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { try { KeyProvider kp = createProvider(uri, conf); KeyProvider.KeyVersion kv = kp.createKey("ck1", new KeyProvider.Options(conf)); EncryptedKeyVersion eek = ((CryptoExtension)kp).generateEncryptedKey("ck1"); ((CryptoExtension)kp).decryptEncryptedKey(eek); Assert.fail("admin user must not be allowed to decrypt !!"); } catch (Exception ex) { } return null; } }); doAs("otheradmin", new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { try { KeyProvider kp = createProvider(uri, conf); KeyProvider.KeyVersion kv = kp.createKey("ck2", new KeyProvider.Options(conf)); EncryptedKeyVersion eek = ((CryptoExtension)kp).generateEncryptedKey("ck2"); ((CryptoExtension)kp).decryptEncryptedKey(eek); Assert.fail("admin user must not be allowed to decrypt !!"); } catch (Exception ex) { } return null; } }); return null; } }); }
@Test public void testServicePrincipalACLs() throws Exception { Configuration conf = new Configuration(); conf.set("hadoop.security.authentication", "kerberos"); File testDir = getTestDir(); conf = createBaseKMSConf(testDir, conf); conf.set("hadoop.kms.authentication.type", "kerberos"); conf.set("hadoop.kms.authentication.kerberos.keytab", keytab.getAbsolutePath()); conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost"); conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT"); for (KMSACLs.Type type : KMSACLs.Type.values()) { conf.set(type.getAclConfigKey(), " "); } conf.set(KMSACLs.Type.CREATE.getAclConfigKey(), "client"); conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT", "client,client/host"); writeConf(testDir, conf); runServer(null, null, testDir, new KMSCallable<Void>() { @Override public Void call() throws Exception { final Configuration conf = new Configuration(); conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128); final URI uri = createKMSUri(getKMSUrl()); doAs("client", new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { try { KeyProvider kp = createProvider(uri, conf); KeyProvider.KeyVersion kv = kp.createKey("ck0", new KeyProvider.Options(conf)); Assert.assertNull(kv.getMaterial()); } catch (Exception ex) { Assert.fail(ex.getMessage()); } return null; } }); doAs("client/host", new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { try { KeyProvider kp = createProvider(uri, conf); KeyProvider.KeyVersion kv = kp.createKey("ck1", new KeyProvider.Options(conf)); Assert.assertNull(kv.getMaterial()); } catch (Exception ex) { Assert.fail(ex.getMessage()); } return null; } }); return null; } }); }
@Test public void testDelegationTokenAccess() throws Exception { Configuration conf = new Configuration(); conf.set("hadoop.security.authentication", "kerberos"); final File testDir = getTestDir(); conf = createBaseKMSConf(testDir, conf); conf.set("hadoop.kms.authentication.type", "kerberos"); conf.set("hadoop.kms.authentication.kerberos.keytab", keytab.getAbsolutePath()); conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost"); conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT"); final String keyA = "key_a"; final String keyD = "key_d"; conf.set(KeyAuthorizationKeyProvider.KEY_ACL + keyA + ".ALL", "*"); conf.set(KeyAuthorizationKeyProvider.KEY_ACL + keyD + ".ALL", "*"); writeConf(testDir, conf); runServer(null, null, testDir, new KMSCallable<Void>() { @Override public Void call() throws Exception { final Configuration conf = new Configuration(); conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128); final URI uri = createKMSUri(getKMSUrl()); final Credentials credentials = new Credentials(); final UserGroupInformation nonKerberosUgi = UserGroupInformation.getCurrentUser(); try { KeyProvider kp = createProvider(uri, conf); kp.createKey(keyA, new KeyProvider.Options(conf)); } catch (IOException ex) { System.out.println(ex.getMessage()); } doAs("client", new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { KeyProvider kp = createProvider(uri, conf); KeyProviderDelegationTokenExtension kpdte = KeyProviderDelegationTokenExtension. createKeyProviderDelegationTokenExtension(kp); kpdte.addDelegationTokens("foo", credentials); return null; } }); nonKerberosUgi.addCredentials(credentials); try { KeyProvider kp = createProvider(uri, conf); kp.createKey(keyA, new KeyProvider.Options(conf)); } catch (IOException ex) { System.out.println(ex.getMessage()); } nonKerberosUgi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { KeyProvider kp = createProvider(uri, conf); kp.createKey(keyD, new KeyProvider.Options(conf)); return null; } }); return null; } }); }