HttpFSServer { @GET @Path("{path:.*}") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) public Response get(@PathParam("path") String path, @QueryParam(OperationParam.NAME) OperationParam op, @Context Parameters params, @Context HttpServletRequest request) throws IOException, FileSystemAccessException { UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); MDC.put("hostname", request.getRemoteAddr()); switch (op.value()) { case OPEN: { FSOperations.FSOpen command = new FSOperations.FSOpen(path); FileSystem fs = createFileSystem(user); InputStream is = command.execute(fs); Long offset = params.get(OffsetParam.NAME, OffsetParam.class); Long len = params.get(LenParam.NAME, LenParam.class); AUDIT_LOG.info("[{}] offset [{}] len [{}]", new Object[]{path, offset, len}); InputStreamEntity entity = new InputStreamEntity(is, offset, len); response = Response.ok(entity).type(MediaType.APPLICATION_OCTET_STREAM).build(); break; } case GETFILESTATUS: { FSOperations.FSFileStatus command = new FSOperations.FSFileStatus(path); Map json = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case LISTSTATUS: { String filter = params.get(FilterParam.NAME, FilterParam.class); FSOperations.FSListStatus command = new FSOperations.FSListStatus( path, filter); Map json = fsExecute(user, command); AUDIT_LOG.info("[{}] filter [{}]", path, (filter != null) ? filter : "-"); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case GETHOMEDIRECTORY: { enforceRootPath(op.value(), path); FSOperations.FSHomeDir command = new FSOperations.FSHomeDir(); JSONObject json = fsExecute(user, command); AUDIT_LOG.info(""); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case INSTRUMENTATION: { enforceRootPath(op.value(), path); Groups groups = HttpFSServerWebApp.get().get(Groups.class); List<String> userGroups = groups.getGroups(user.getShortUserName()); if (!userGroups.contains(HttpFSServerWebApp.get().getAdminGroup())) { throw new AccessControlException( "User not in HttpFSServer admin group"); } Instrumentation instrumentation = HttpFSServerWebApp.get().get(Instrumentation.class); Map snapshot = instrumentation.getSnapshot(); response = Response.ok(snapshot).build(); break; } case GETCONTENTSUMMARY: { FSOperations.FSContentSummary command = new FSOperations.FSContentSummary(path); Map json = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case GETFILECHECKSUM: { FSOperations.FSFileChecksum command = new FSOperations.FSFileChecksum(path); Map json = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case GETFILEBLOCKLOCATIONS: { response = Response.status(Response.Status.BAD_REQUEST).build(); break; } case GETACLSTATUS: { FSOperations.FSAclStatus command = new FSOperations.FSAclStatus(path); Map json = fsExecute(user, command); AUDIT_LOG.info("ACL status for [{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case GETXATTRS: { List<String> xattrNames = params.getValues(XAttrNameParam.NAME, XAttrNameParam.class); XAttrCodec encoding = params.get(XAttrEncodingParam.NAME, XAttrEncodingParam.class); FSOperations.FSGetXAttrs command = new FSOperations.FSGetXAttrs(path, xattrNames, encoding); @SuppressWarnings("rawtypes") Map json = fsExecute(user, command); AUDIT_LOG.info("XAttrs for [{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case LISTXATTRS: { FSOperations.FSListXAttrs command = new FSOperations.FSListXAttrs(path); @SuppressWarnings("rawtypes") Map json = fsExecute(user, command); AUDIT_LOG.info("XAttr names for [{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } default: { throw new IOException( MessageFormat.format("Invalid HTTP GET operation [{0}]", op.value())); } } return response; }  @GET @Produces(MediaType.APPLICATION_JSON) Response getRoot(@QueryParam(OperationParam.NAME) OperationParam op,
                          @Context Parameters params,
                          @Context HttpServletRequest request); @GET @Path("{path:.*}") @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) Response get(@PathParam("path") String path,
                      @QueryParam(OperationParam.NAME) OperationParam op,
                      @Context Parameters params,
                      @Context HttpServletRequest request); @DELETE @Path("{path:.*}") @Produces(MediaType.APPLICATION_JSON) Response delete(@PathParam("path") String path,
                         @QueryParam(OperationParam.NAME) OperationParam op,
                         @Context Parameters params,
                         @Context HttpServletRequest request); @POST @Path("{path:.*}") @Consumes({"*/*"}) @Produces({MediaType.APPLICATION_JSON}) Response post(InputStream is,
                       @Context UriInfo uriInfo,
                       @PathParam("path") String path,
                       @QueryParam(OperationParam.NAME) OperationParam op,
                       @Context Parameters params,
                       @Context HttpServletRequest request); @PUT @Path("{path:.*}") @Consumes({"*/*"}) @Produces({MediaType.APPLICATION_JSON}) Response put(InputStream is,
                       @Context UriInfo uriInfo,
                       @PathParam("path") String path,
                       @QueryParam(OperationParam.NAME) OperationParam op,
                       @Context Parameters params,
                       @Context HttpServletRequest request);  }
@Test @TestDir @TestJetty @TestHdfs public void testGlobFilter() throws Exception { createHttpFSServer(false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path("/tmp")); fs.create(new Path("/tmp/foo.txt")).close(); String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format("/webhdfs/v1/tmp?user.name={0}&op=liststatus&filter=f*", user)); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream())); reader.readLine(); reader.close(); }
@Test @TestDir @TestJetty @TestHdfs public void testPerms() throws Exception { createHttpFSServer(false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path("/perm")); createWithHttp("/perm/none", null); String statusJson = getStatus("/perm/none", "GETFILESTATUS"); Assert.assertTrue("755".equals(getPerms(statusJson))); createWithHttp("/perm/p-777", "777"); statusJson = getStatus("/perm/p-777", "GETFILESTATUS"); Assert.assertTrue("777".equals(getPerms(statusJson))); createWithHttp("/perm/p-654", "654"); statusJson = getStatus("/perm/p-654", "GETFILESTATUS"); Assert.assertTrue("654".equals(getPerms(statusJson))); createWithHttp("/perm/p-321", "321"); statusJson = getStatus("/perm/p-321", "GETFILESTATUS"); Assert.assertTrue("321".equals(getPerms(statusJson))); }
@Test @TestDir @TestJetty @TestHdfs public void testXAttrs() throws Exception { final String name1 = "user.a1"; final byte[] value1 = new byte[]{0x31, 0x32, 0x33}; final String name2 = "user.a2"; final byte[] value2 = new byte[]{0x41, 0x42, 0x43}; final String dir = "/xattrTest"; final String path = dir + "/file"; createHttpFSServer(false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path(dir)); createWithHttp(path,null); String statusJson = getStatus(path, "GETXATTRS"); Map<String, byte[]> xAttrs = getXAttrs(statusJson); Assert.assertEquals(0, xAttrs.size()); putCmd(path, "SETXATTR", setXAttrParam(name1, value1)); putCmd(path, "SETXATTR", setXAttrParam(name2, value2)); statusJson = getStatus(path, "GETXATTRS"); xAttrs = getXAttrs(statusJson); Assert.assertEquals(2, xAttrs.size()); Assert.assertArrayEquals(value1, xAttrs.get(name1)); Assert.assertArrayEquals(value2, xAttrs.get(name2)); putCmd(path, "REMOVEXATTR", "xattr.name=" + name1); statusJson = getStatus(path, "GETXATTRS"); xAttrs = getXAttrs(statusJson); Assert.assertEquals(1, xAttrs.size()); Assert.assertArrayEquals(value2, xAttrs.get(name2)); putCmd(path, "REMOVEXATTR", "xattr.name=" + name2); statusJson = getStatus(path, "GETXATTRS"); xAttrs = getXAttrs(statusJson); Assert.assertEquals(0, xAttrs.size()); }
@Test @TestDir @TestJetty @TestHdfs public void testFileAcls() throws Exception { final String aclUser1 = "user:foo:rw-"; final String remAclUser1 = "user:foo:"; final String aclUser2 = "user:bar:r--"; final String aclGroup1 = "group::r--"; final String aclSpec = "aclspec=user::rwx," + aclUser1 + "," + aclGroup1 + ",other::---"; final String modAclSpec = "aclspec=" + aclUser2; final String remAclSpec = "aclspec=" + remAclUser1; final String dir = "/aclFileTest"; final String path = dir + "/test"; String statusJson; List<String> aclEntries; createHttpFSServer(false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path(dir)); createWithHttp(path, null); statusJson = getStatus(path, "GETFILESTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "LISTSTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 0); putCmd(path, "SETACL", aclSpec); statusJson = getStatus(path, "GETFILESTATUS"); Assert.assertNotEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "LISTSTATUS"); Assert.assertNotEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 2); Assert.assertTrue(aclEntries.contains(aclUser1)); Assert.assertTrue(aclEntries.contains(aclGroup1)); putCmd(path, "MODIFYACLENTRIES", modAclSpec); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 3); Assert.assertTrue(aclEntries.contains(aclUser1)); Assert.assertTrue(aclEntries.contains(aclUser2)); Assert.assertTrue(aclEntries.contains(aclGroup1)); putCmd(path, "REMOVEACLENTRIES", remAclSpec); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 2); Assert.assertTrue(aclEntries.contains(aclUser2)); Assert.assertTrue(aclEntries.contains(aclGroup1)); putCmd(path, "REMOVEACL", null); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 0); statusJson = getStatus(path, "GETFILESTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "LISTSTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); }
@Test @TestDir @TestJetty @TestHdfs public void testDirAcls() throws Exception { final String defUser1 = "default:user:glarch:r-x"; final String defSpec1 = "aclspec=" + defUser1; final String dir = "/aclDirTest"; String statusJson; List<String> aclEntries; createHttpFSServer(false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path(dir)); statusJson = getStatus(dir, "GETFILESTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 0); putCmd(dir, "SETACL", defSpec1); statusJson = getStatus(dir, "GETFILESTATUS"); Assert.assertNotEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 5); Assert.assertTrue(aclEntries.contains(defUser1)); putCmd(dir, "REMOVEDEFAULTACL", null); statusJson = getStatus(dir, "GETFILESTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 0); }
@Test @TestDir @TestJetty @TestHdfs public void testOpenOffsetLength() throws Exception { createHttpFSServer(false); byte[] array = new byte[]{0, 1, 2, 3}; FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path("/tmp")); OutputStream os = fs.create(new Path("/tmp/foo")); os.write(array); os.close(); String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format("/webhdfs/v1/tmp/foo?user.name={0}&op=open&offset=1&length=2", user)); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); InputStream is = conn.getInputStream(); Assert.assertEquals(1, is.read()); Assert.assertEquals(2, is.read()); Assert.assertEquals(-1, is.read()); }
@Test @TestDir @TestJetty @TestHdfs public void testDelegationTokenOperations() throws Exception { createHttpFSServer(true); URL url = new URL(TestJettyHelper.getJettyURL(), "/webhdfs/v1/?op=GETHOMEDIRECTORY"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, conn.getResponseCode()); AuthenticationToken token = new AuthenticationToken("u", "p", new KerberosDelegationTokenAuthenticationHandler().getType()); token.setExpires(System.currentTimeMillis() + 100000000); SignerSecretProvider secretProvider = StringSignerSecretProviderCreator.newStringSignerSecretProvider(); Properties secretProviderProps = new Properties(); secretProviderProps.setProperty(AuthenticationFilter.SIGNATURE_SECRET, "secret"); secretProvider.init(secretProviderProps, null, -1); Signer signer = new Signer(secretProvider); String tokenSigned = signer.sign(token.toString()); url = new URL(TestJettyHelper.getJettyURL(), "/webhdfs/v1/?op=GETHOMEDIRECTORY"); conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty("Cookie", AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); url = new URL(TestJettyHelper.getJettyURL(), "/webhdfs/v1/?op=GETDELEGATIONTOKEN"); conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty("Cookie", AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); JSONObject json = (JSONObject) new JSONParser().parse(new InputStreamReader(conn.getInputStream())); json = (JSONObject) json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_JSON); String tokenStr = (String) json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON); url = new URL(TestJettyHelper.getJettyURL(), "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr); conn = (HttpURLConnection) url.openConnection(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); url = new URL(TestJettyHelper.getJettyURL(), "/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("PUT"); Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, conn.getResponseCode()); url = new URL(TestJettyHelper.getJettyURL(), "/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("PUT"); conn.setRequestProperty("Cookie", AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); url = new URL(TestJettyHelper.getJettyURL(), "/webhdfs/v1/?op=CANCELDELEGATIONTOKEN&token=" + tokenStr); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("PUT"); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); url = new URL(TestJettyHelper.getJettyURL(), "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr); conn = (HttpURLConnection) url.openConnection(); Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN, conn.getResponseCode()); }
@Test @TestDir @TestJetty @TestHdfs public void testFileAcls() throws Exception { final String aclUser1 = "user:foo:rw-"; final String aclUser2 = "user:bar:r--"; final String aclGroup1 = "group::r--"; final String aclSpec = "aclspec=user::rwx," + aclUser1 + "," + aclGroup1 + ",other::---"; final String modAclSpec = "aclspec=" + aclUser2; final String remAclSpec = "aclspec=" + aclUser1; final String dir = "/aclFileTest"; final String path = dir + "/test"; String statusJson; List<String> aclEntries; createHttpFSServer(false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path(dir)); createWithHttp(path, null); statusJson = getStatus(path, "GETFILESTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "LISTSTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 0); putCmd(path, "SETACL", aclSpec); statusJson = getStatus(path, "GETFILESTATUS"); Assert.assertNotEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "LISTSTATUS"); Assert.assertNotEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 2); Assert.assertTrue(aclEntries.contains(aclUser1)); Assert.assertTrue(aclEntries.contains(aclGroup1)); putCmd(path, "MODIFYACLENTRIES", modAclSpec); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 3); Assert.assertTrue(aclEntries.contains(aclUser1)); Assert.assertTrue(aclEntries.contains(aclUser2)); Assert.assertTrue(aclEntries.contains(aclGroup1)); putCmd(path, "REMOVEACLENTRIES", remAclSpec); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 2); Assert.assertTrue(aclEntries.contains(aclUser2)); Assert.assertTrue(aclEntries.contains(aclGroup1)); putCmd(path, "REMOVEACL", null); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 0); statusJson = getStatus(path, "GETFILESTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "LISTSTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); }