HttpFSServer { @GET @Path("{path:.*}") @Produces({MediaType.APPLICATION_OCTET_STREAM + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8}) public Response get(@PathParam("path") String path, @Context UriInfo uriInfo, @QueryParam(OperationParam.NAME) OperationParam op, @Context Parameters params, @Context HttpServletRequest request) throws IOException, FileSystemAccessException { if((op.value() != HttpFSFileSystem.Operation.GETFILESTATUS) && (op.value() != HttpFSFileSystem.Operation.LISTSTATUS) && accessMode == AccessMode.WRITEONLY) { return Response.status(Response.Status.FORBIDDEN).build(); } UserGroupInformation user = HttpUserGroupInformation.get(); Response response; path = makeAbsolute(path); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); MDC.put("hostname", request.getRemoteAddr()); switch (op.value()) { case OPEN: { Boolean noRedirect = params.get( NoRedirectParam.NAME, NoRedirectParam.class); if (noRedirect) { URI redirectURL = createOpenRedirectionURL(uriInfo); final String js = JsonUtil.toJsonString("Location", redirectURL); response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } else { final FSOperations.FSOpen command = new FSOperations.FSOpen(path); final FileSystem fs = createFileSystem(user); InputStream is = null; UserGroupInformation ugi = UserGroupInformation .createProxyUser(user.getShortUserName(), UserGroupInformation.getLoginUser()); try { is = ugi.doAs(new PrivilegedExceptionAction<InputStream>() { @Override public InputStream run() throws Exception { return command.execute(fs); } }); } catch (InterruptedException ie) { LOG.info("Open interrupted.", ie); Thread.currentThread().interrupt(); } Long offset = params.get(OffsetParam.NAME, OffsetParam.class); Long len = params.get(LenParam.NAME, LenParam.class); AUDIT_LOG.info("[{}] offset [{}] len [{}]", new Object[] { path, offset, len }); InputStreamEntity entity = new InputStreamEntity(is, offset, len); response = Response.ok(entity).type(MediaType.APPLICATION_OCTET_STREAM) .build(); } break; } case GETFILESTATUS: { FSOperations.FSFileStatus command = new FSOperations.FSFileStatus(path); Map json = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case LISTSTATUS: { String filter = params.get(FilterParam.NAME, FilterParam.class); FSOperations.FSListStatus command = new FSOperations.FSListStatus(path, filter); Map json = fsExecute(user, command); AUDIT_LOG.info("[{}] filter [{}]", path, (filter != null) ? filter : "-"); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case GETHOMEDIRECTORY: { enforceRootPath(op.value(), path); FSOperations.FSHomeDir command = new FSOperations.FSHomeDir(); JSONObject json = fsExecute(user, command); AUDIT_LOG.info(""); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case INSTRUMENTATION: { enforceRootPath(op.value(), path); Groups groups = HttpFSServerWebApp.get().get(Groups.class); Set<String> userGroups = groups.getGroupsSet(user.getShortUserName()); if (!userGroups.contains(HttpFSServerWebApp.get().getAdminGroup())) { throw new AccessControlException( "User not in HttpFSServer admin group"); } Instrumentation instrumentation = HttpFSServerWebApp.get().get(Instrumentation.class); Map snapshot = instrumentation.getSnapshot(); response = Response.ok(snapshot).build(); break; } case GETCONTENTSUMMARY: { FSOperations.FSContentSummary command = new FSOperations.FSContentSummary(path); Map json = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case GETQUOTAUSAGE: { FSOperations.FSQuotaUsage command = new FSOperations.FSQuotaUsage(path); Map json = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case GETFILECHECKSUM: { FSOperations.FSFileChecksum command = new FSOperations.FSFileChecksum(path); Boolean noRedirect = params.get( NoRedirectParam.NAME, NoRedirectParam.class); AUDIT_LOG.info("[{}]", path); if (noRedirect) { URI redirectURL = createOpenRedirectionURL(uriInfo); final String js = JsonUtil.toJsonString("Location", redirectURL); response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } else { Map json = fsExecute(user, command); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); } break; } case GETFILEBLOCKLOCATIONS: { response = Response.status(Response.Status.BAD_REQUEST).build(); break; } case GETACLSTATUS: { FSOperations.FSAclStatus command = new FSOperations.FSAclStatus(path); Map json = fsExecute(user, command); AUDIT_LOG.info("ACL status for [{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case GETXATTRS: { List<String> xattrNames = params.getValues(XAttrNameParam.NAME, XAttrNameParam.class); XAttrCodec encoding = params.get(XAttrEncodingParam.NAME, XAttrEncodingParam.class); FSOperations.FSGetXAttrs command = new FSOperations.FSGetXAttrs(path, xattrNames, encoding); @SuppressWarnings("rawtypes") Map json = fsExecute(user, command); AUDIT_LOG.info("XAttrs for [{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case LISTXATTRS: { FSOperations.FSListXAttrs command = new FSOperations.FSListXAttrs(path); @SuppressWarnings("rawtypes") Map json = fsExecute(user, command); AUDIT_LOG.info("XAttr names for [{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case LISTSTATUS_BATCH: { String startAfter = params.get( HttpFSParametersProvider.StartAfterParam.NAME, HttpFSParametersProvider.StartAfterParam.class); byte[] token = HttpFSUtils.EMPTY_BYTES; if (startAfter != null) { token = startAfter.getBytes(Charsets.UTF_8); } FSOperations.FSListStatusBatch command = new FSOperations .FSListStatusBatch(path, token); @SuppressWarnings("rawtypes") Map json = fsExecute(user, command); AUDIT_LOG.info("[{}] token [{}]", path, token); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case GETTRASHROOT: { FSOperations.FSTrashRoot command = new FSOperations.FSTrashRoot(path); JSONObject json = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case GETALLSTORAGEPOLICY: { FSOperations.FSGetAllStoragePolicies command = new FSOperations.FSGetAllStoragePolicies(); JSONObject json = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case GETSTORAGEPOLICY: { FSOperations.FSGetStoragePolicy command = new FSOperations.FSGetStoragePolicy(path); JSONObject json = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(json).type(MediaType.APPLICATION_JSON).build(); break; } case GETSNAPSHOTDIFF: { String oldSnapshotName = params.get(OldSnapshotNameParam.NAME, OldSnapshotNameParam.class); String snapshotName = params.get(SnapshotNameParam.NAME, SnapshotNameParam.class); FSOperations.FSGetSnapshotDiff command = new FSOperations.FSGetSnapshotDiff(path, oldSnapshotName, snapshotName); String js = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); break; } case GETSNAPSHOTTABLEDIRECTORYLIST: { FSOperations.FSGetSnapshottableDirListing command = new FSOperations.FSGetSnapshottableDirListing(); String js = fsExecute(user, command); AUDIT_LOG.info("[{}]", "/"); response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); break; } case GETSNAPSHOTLIST: { FSOperations.FSGetSnapshotListing command = new FSOperations.FSGetSnapshotListing(path); String js = fsExecute(user, command); AUDIT_LOG.info("[{}]", "/"); response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); break; } case GETSERVERDEFAULTS: { FSOperations.FSGetServerDefaults command = new FSOperations.FSGetServerDefaults(); String js = fsExecute(user, command); AUDIT_LOG.info("[{}]", "/"); response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); break; } case CHECKACCESS: { String mode = params.get(FsActionParam.NAME, FsActionParam.class); FsActionParam fsparam = new FsActionParam(mode); FSOperations.FSAccess command = new FSOperations.FSAccess(path, FsAction.getFsAction(fsparam.value())); fsExecute(user, command); AUDIT_LOG.info("[{}]", "/"); response = Response.ok().build(); break; } case GETECPOLICY: { FSOperations.FSGetErasureCodingPolicy command = new FSOperations.FSGetErasureCodingPolicy(path); String js = fsExecute(user, command); AUDIT_LOG.info("[{}]", path); response = Response.ok(js).type(MediaType.APPLICATION_JSON).build(); break; } default: { throw new IOException( MessageFormat.format("Invalid HTTP GET operation [{0}]", op.value())); } } return response; } HttpFSServer(); @GET @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) Response getRoot(@Context UriInfo uriInfo,
                          @QueryParam(OperationParam.NAME) OperationParam op,
                          @Context Parameters params,
                          @Context HttpServletRequest request); @GET @Path("{path:.*}") @Produces({MediaType.APPLICATION_OCTET_STREAM + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8}) Response get(@PathParam("path") String path,
                      @Context UriInfo uriInfo,
                      @QueryParam(OperationParam.NAME) OperationParam op,
                      @Context Parameters params,
                      @Context HttpServletRequest request); @DELETE @Path("{path:.*}") @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8) Response delete(@PathParam("path") String path,
                         @QueryParam(OperationParam.NAME) OperationParam op,
                         @Context Parameters params,
                         @Context HttpServletRequest request); @POST @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8 }) Response postRoot(InputStream is, @Context UriInfo uriInfo,
      @QueryParam(OperationParam.NAME) OperationParam op,
      @Context Parameters params, @Context HttpServletRequest request); @POST @Path("{path:.*}") @Consumes({"*/*"}) @Produces({MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8}) Response post(InputStream is,
                       @Context UriInfo uriInfo,
                       @PathParam("path") String path,
                       @QueryParam(OperationParam.NAME) OperationParam op,
                       @Context Parameters params,
                       @Context HttpServletRequest request); @PUT @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8 }) Response putRoot(InputStream is, @Context UriInfo uriInfo,
      @QueryParam(OperationParam.NAME) OperationParam op,
      @Context Parameters params, @Context HttpServletRequest request); @PUT @Path("{path:.*}") @Consumes({"*/*"}) @Produces({MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8}) Response put(InputStream is,
                       @Context UriInfo uriInfo,
                       @PathParam("path") String path,
                       @QueryParam(OperationParam.NAME) OperationParam op,
                       @Context Parameters params,
                       @Context HttpServletRequest request);  }
@Test @TestDir @TestJetty @TestHdfs public void testGlobFilter() throws Exception { createHttpFSServer(false, false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path("/tmp")); fs.create(new Path("/tmp/foo.txt")).close(); String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format( "/webhdfs/v1/tmp?user.name={0}&op=liststatus&filter=f*", user)); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); BufferedReader reader = new BufferedReader( new InputStreamReader(conn.getInputStream())); reader.readLine(); reader.close(); }
@Test @TestDir @TestJetty @TestHdfs public void testPerms() throws Exception { createHttpFSServer(false, false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path("/perm")); createWithHttp("/perm/none", null); String statusJson = getStatus("/perm/none", "GETFILESTATUS"); Assert.assertTrue("755".equals(getPerms(statusJson))); createWithHttp("/perm/p-777", "777"); statusJson = getStatus("/perm/p-777", "GETFILESTATUS"); Assert.assertTrue("777".equals(getPerms(statusJson))); createWithHttp("/perm/p-654", "654"); statusJson = getStatus("/perm/p-654", "GETFILESTATUS"); Assert.assertTrue("654".equals(getPerms(statusJson))); createWithHttp("/perm/p-321", "321"); statusJson = getStatus("/perm/p-321", "GETFILESTATUS"); Assert.assertTrue("321".equals(getPerms(statusJson))); }
@Test @TestDir @TestJetty @TestHdfs public void testXAttrs() throws Exception { final String name1 = "user.a1"; final byte[] value1 = new byte[]{0x31, 0x32, 0x33}; final String name2 = "user.a2"; final byte[] value2 = new byte[]{0x41, 0x42, 0x43}; final String dir = "/xattrTest"; final String path = dir + "/file"; createHttpFSServer(false, false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path(dir)); createWithHttp(path, null); String statusJson = getStatus(path, "GETXATTRS"); Map<String, byte[]> xAttrs = getXAttrs(statusJson); Assert.assertEquals(0, xAttrs.size()); putCmd(path, "SETXATTR", setXAttrParam(name1, value1)); putCmd(path, "SETXATTR", setXAttrParam(name2, value2)); statusJson = getStatus(path, "GETXATTRS"); xAttrs = getXAttrs(statusJson); Assert.assertEquals(2, xAttrs.size()); Assert.assertArrayEquals(value1, xAttrs.get(name1)); Assert.assertArrayEquals(value2, xAttrs.get(name2)); putCmd(path, "REMOVEXATTR", "xattr.name=" + name1); statusJson = getStatus(path, "GETXATTRS"); xAttrs = getXAttrs(statusJson); Assert.assertEquals(1, xAttrs.size()); Assert.assertArrayEquals(value2, xAttrs.get(name2)); putCmd(path, "REMOVEXATTR", "xattr.name=" + name2); statusJson = getStatus(path, "GETXATTRS"); xAttrs = getXAttrs(statusJson); Assert.assertEquals(0, xAttrs.size()); }
@Test @TestDir @TestJetty @TestHdfs public void testFileAcls() throws Exception { final String aclUser1 = "user:foo:rw-"; final String remAclUser1 = "user:foo:"; final String aclUser2 = "user:bar:r--"; final String aclGroup1 = "group::r--"; final String aclSpec = "aclspec=user::rwx," + aclUser1 + "," + aclGroup1 + ",other::---"; final String modAclSpec = "aclspec=" + aclUser2; final String remAclSpec = "aclspec=" + remAclUser1; final String dir = "/aclFileTest"; final String path = dir + "/test"; String statusJson; List<String> aclEntries; createHttpFSServer(false, false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path(dir)); createWithHttp(path, null); statusJson = getStatus(path, "GETFILESTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "LISTSTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 0); putCmd(path, "SETACL", aclSpec); statusJson = getStatus(path, "GETFILESTATUS"); Assert.assertNotEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "LISTSTATUS"); Assert.assertNotEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 2); Assert.assertTrue(aclEntries.contains(aclUser1)); Assert.assertTrue(aclEntries.contains(aclGroup1)); putCmd(path, "MODIFYACLENTRIES", modAclSpec); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 3); Assert.assertTrue(aclEntries.contains(aclUser1)); Assert.assertTrue(aclEntries.contains(aclUser2)); Assert.assertTrue(aclEntries.contains(aclGroup1)); putCmd(path, "REMOVEACLENTRIES", remAclSpec); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 2); Assert.assertTrue(aclEntries.contains(aclUser2)); Assert.assertTrue(aclEntries.contains(aclGroup1)); putCmd(path, "REMOVEACL", null); statusJson = getStatus(path, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 0); statusJson = getStatus(path, "GETFILESTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "LISTSTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); }
@Test @TestDir @TestJetty @TestHdfs public void testDirAcls() throws Exception { final String defUser1 = "default:user:glarch:r-x"; final String defSpec1 = "aclspec=" + defUser1; final String dir = "/aclDirTest"; String statusJson; List<String> aclEntries; createHttpFSServer(false, false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path(dir)); statusJson = getStatus(dir, "GETFILESTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 0); putCmd(dir, "SETACL", defSpec1); statusJson = getStatus(dir, "GETFILESTATUS"); Assert.assertNotEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 5); Assert.assertTrue(aclEntries.contains(defUser1)); putCmd(dir, "REMOVEDEFAULTACL", null); statusJson = getStatus(dir, "GETFILESTATUS"); Assert.assertEquals(-1, statusJson.indexOf("aclBit")); statusJson = getStatus(dir, "GETACLSTATUS"); aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.size() == 0); }
@Test @TestDir @TestJetty @TestHdfs public void testCustomizedUserAndGroupNames() throws Exception { Server server = createHttpFSServer(false, false); final Configuration conf = HttpFSServerWebApp.get() .get(FileSystemAccess.class).getFileSystemConfiguration(); conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$"); conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY, "^(default:)?(user|group|mask|other):" + "[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?" + "(user|group|mask|other):[[0-9A-Za-z_][@A-Za-z0-9._-]]*:" + "([rwx-]{3})?)*$"); writeConf(conf, "hdfs-site.xml"); server.stop(); server.start(); final String aclUser = "user:123:rw-"; final String aclGroup = "group:foo@bar:r--"; final String aclSpec = "aclspec=user::rwx," + aclUser + ",group::rwx," + aclGroup + ",other::---"; final String dir = "/aclFileTestCustom"; final String path = dir + "/test"; FileSystem fs = FileSystem.get(conf); fs.mkdirs(new Path(dir)); createWithHttp(path, null); putCmd(path, "SETACL", aclSpec); String statusJson = getStatus(path, "GETACLSTATUS"); List<String> aclEntries = getAclEntries(statusJson); Assert.assertTrue(aclEntries.contains(aclUser)); Assert.assertTrue(aclEntries.contains(aclGroup)); }
@Test @TestDir @TestJetty @TestHdfs public void testOpenOffsetLength() throws Exception { createHttpFSServer(false, false); byte[] array = new byte[]{0, 1, 2, 3}; FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path("/tmp")); OutputStream os = fs.create(new Path("/tmp/foo")); os.write(array); os.close(); String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format( "/webhdfs/v1/tmp/foo?user.name={0}&op=open&offset=1&length=2", user)); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); InputStream is = conn.getInputStream(); Assert.assertEquals(1, is.read()); Assert.assertEquals(2, is.read()); Assert.assertEquals(-1, is.read()); }
@Test @TestDir @TestJetty @TestHdfs public void testCreateFileWithUnmaskedPermissions() throws Exception { createHttpFSServer(false, false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path("/tmp")); AclEntry acl = new org.apache.hadoop.fs.permission.AclEntry.Builder() .setType(AclEntryType.USER) .setScope(AclEntryScope.DEFAULT) .setName("user2") .setPermission(FsAction.READ_WRITE) .build(); fs.setAcl(new Path("/tmp"), new ArrayList<AclEntry>(Arrays.asList(acl))); String notUnmaskedFile = "/tmp/notUnmasked"; String unmaskedFile = "/tmp/unmasked"; createWithHttp(notUnmaskedFile, "700"); AclStatus aclStatus = fs.getAclStatus(new Path(notUnmaskedFile)); AclEntry theAcl = findAclWithName(aclStatus, "user2"); Assert.assertNotNull(theAcl); Assert.assertEquals(FsAction.NONE, aclStatus.getEffectivePermission(theAcl)); createWithHttp(unmaskedFile, "700", "777"); aclStatus = fs.getAclStatus(new Path(unmaskedFile)); theAcl = findAclWithName(aclStatus, "user2"); Assert.assertNotNull(theAcl); Assert.assertEquals(FsAction.READ_WRITE, aclStatus.getEffectivePermission(theAcl)); }
@Test @TestDir @TestJetty @TestHdfs public void testMkdirWithUnmaskedPermissions() throws Exception { createHttpFSServer(false, false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path("/tmp")); AclEntry acl = new org.apache.hadoop.fs.permission.AclEntry.Builder() .setType(AclEntryType.USER) .setScope(AclEntryScope.DEFAULT) .setName("user2") .setPermission(FsAction.READ_WRITE) .build(); fs.setAcl(new Path("/tmp"), new ArrayList<AclEntry>(Arrays.asList(acl))); String notUnmaskedDir = "/tmp/notUnmaskedDir"; String unmaskedDir = "/tmp/unmaskedDir"; createDirWithHttp(notUnmaskedDir, "700", null); AclStatus aclStatus = fs.getAclStatus(new Path(notUnmaskedDir)); AclEntry theAcl = findAclWithName(aclStatus, "user2"); Assert.assertNotNull(theAcl); Assert.assertEquals(FsAction.NONE, aclStatus.getEffectivePermission(theAcl)); createDirWithHttp(unmaskedDir, "700", "777"); aclStatus = fs.getAclStatus(new Path(unmaskedDir)); theAcl = findAclWithName(aclStatus, "user2"); Assert.assertNotNull(theAcl); Assert.assertEquals(FsAction.READ_WRITE, aclStatus.getEffectivePermission(theAcl)); }
@Test @TestDir @TestJetty @TestHdfs public void testGetTrashRoot() throws Exception { String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; createHttpFSServer(false, false); String trashJson = getStatus("/", "GETTRASHROOT"); String trashPath = getPath(trashJson); Path expectedPath = new Path(FileSystem.USER_HOME_PREFIX, new Path(user, FileSystem.TRASH_PREFIX)); Assert.assertEquals(expectedPath.toUri().getPath(), trashPath); byte[] array = new byte[]{0, 1, 2, 3}; FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); fs.mkdirs(new Path("/tmp")); OutputStream os = fs.create(new Path("/tmp/foo")); os.write(array); os.close(); trashJson = getStatus("/tmp/foo", "GETTRASHROOT"); trashPath = getPath(trashJson); Assert.assertEquals(expectedPath.toUri().getPath(), trashPath); final Path ezFile = TestHdfsHelper.ENCRYPTED_FILE; final Path ezPath = TestHdfsHelper.ENCRYPTION_ZONE; trashJson = getStatus(ezFile.toUri().getPath(), "GETTRASHROOT"); trashPath = getPath(trashJson); expectedPath = new Path(ezPath, new Path(FileSystem.TRASH_PREFIX, user)); Assert.assertEquals(expectedPath.toUri().getPath(), trashPath); }
@Test @TestDir @TestJetty @TestHdfs public void testGetSnapshotList() throws Exception { createHttpFSServer(false, false); String pathStr = "/tmp/tmp-snap-list-test-1"; createDirWithHttp(pathStr, "700", null); Path path = new Path(pathStr); DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get( path.toUri(), TestHdfsHelper.getHdfsConf()); dfs.allowSnapshot(path); Assert.assertTrue(dfs.getFileStatus(path).isSnapshotEnabled()); verifyGetSnapshotList(dfs, path); String file1 = pathStr + "/file1"; createWithHttp(file1, null); dfs.createSnapshot(path, "snap1"); String file2 = pathStr + "/file2"; createWithHttp(file2, null); dfs.createSnapshot(path, "snap2"); verifyGetSnapshotList(dfs, path); }
@Test @TestDir @TestJetty @TestHdfs public void testNoRedirect() throws Exception { createHttpFSServer(false, false); final String testContent = "Test content"; final String path = "/testfile.txt"; final String username = HadoopUsersConfTestHelper.getHadoopUsers()[0]; URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format( "/webhdfs/v1{0}?user.name={1}&op=CREATE&noredirect=true", path, username)); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(HttpMethod.PUT); conn.connect(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); JSONObject json = (JSONObject)new JSONParser().parse( new InputStreamReader(conn.getInputStream())); String location = (String)json.get("Location"); Assert.assertTrue(location.contains(DataParam.NAME)); Assert.assertFalse(location.contains(NoRedirectParam.NAME)); Assert.assertTrue(location.contains("CREATE")); Assert.assertTrue("Wrong location: " + location, location.startsWith(TestJettyHelper.getJettyURL().toString())); url = new URL(location); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(HttpMethod.PUT); conn.setRequestProperty( "Content-Type", MediaType.APPLICATION_OCTET_STREAM); conn.setDoOutput(true); conn.connect(); OutputStream os = conn.getOutputStream(); os.write(testContent.getBytes()); os.close(); Assert.assertEquals( HttpURLConnection.HTTP_CREATED, conn.getResponseCode()); json = (JSONObject)new JSONParser().parse( new InputStreamReader(conn.getInputStream())); location = (String)json.get("Location"); Assert.assertEquals( TestJettyHelper.getJettyURL() + "/webhdfs/v1" + path, location); url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format( "/webhdfs/v1{0}?user.name={1}&op=OPEN&noredirect=true", path, username)); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(HttpMethod.GET); conn.connect(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); json = (JSONObject)new JSONParser().parse( new InputStreamReader(conn.getInputStream())); location = (String)json.get("Location"); Assert.assertTrue(!location.contains(NoRedirectParam.NAME)); Assert.assertTrue(location.contains("OPEN")); Assert.assertTrue("Wrong location: " + location, location.startsWith(TestJettyHelper.getJettyURL().toString())); url = new URL(location); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(HttpMethod.GET); conn.connect(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); String content = IOUtils.toString( conn.getInputStream(), Charset.defaultCharset()); Assert.assertEquals(testContent, content); url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format( "/webhdfs/v1{0}?user.name={1}&op=GETFILECHECKSUM&noredirect=true", path, username)); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(HttpMethod.GET); conn.connect(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); json = (JSONObject)new JSONParser().parse( new InputStreamReader(conn.getInputStream())); location = (String)json.get("Location"); Assert.assertTrue(!location.contains(NoRedirectParam.NAME)); Assert.assertTrue(location.contains("GETFILECHECKSUM")); Assert.assertTrue("Wrong location: " + location, location.startsWith(TestJettyHelper.getJettyURL().toString())); url = new URL(location); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(HttpMethod.GET); conn.connect(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); json = (JSONObject)new JSONParser().parse( new InputStreamReader(conn.getInputStream())); JSONObject checksum = (JSONObject)json.get("FileChecksum"); Assert.assertEquals( "0000020000000000000000001b9c0a445fed3c0bf1e1aa7438d96b1500000000", checksum.get("bytes")); Assert.assertEquals(28L, checksum.get("length")); Assert.assertEquals("MD5-of-0MD5-of-512CRC32C", checksum.get("algorithm")); }
@Test @TestDir @TestJetty @TestHdfs public void testGetServerDefaults() throws Exception { createHttpFSServer(false, false); String pathStr1 = "/"; Path path1 = new Path(pathStr1); DistributedFileSystem dfs = (DistributedFileSystem) FileSystem .get(path1.toUri(), TestHdfsHelper.getHdfsConf()); verifyGetServerDefaults(dfs); }
@Test @TestDir @TestJetty @TestHdfs public void testAccess() throws Exception { createHttpFSServer(false, false); final String dir = "/xattrTest"; Path path1 = new Path(dir); DistributedFileSystem dfs = (DistributedFileSystem) FileSystem .get(path1.toUri(), TestHdfsHelper.getHdfsConf()); dfs.mkdirs(new Path(dir)); HttpURLConnection conn = sendRequestToHttpFSServer(dir, "CHECKACCESS", "fsaction=r--"); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); HttpURLConnection conn1 = sendRequestToHttpFSServer(dir, "CHECKACCESS", "fsaction=-w-"); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn1.getResponseCode()); }
@Test @TestDir @TestJetty @TestHdfs public void testECPolicy() throws Exception { createHttpFSServer(false, false); final ErasureCodingPolicy ecPolicy = SystemErasureCodingPolicies .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID); final String ecPolicyName = ecPolicy.getName(); final Path ecDir = new Path("/ec"); DistributedFileSystem dfs = (DistributedFileSystem) FileSystem .get(ecDir.toUri(), TestHdfsHelper.getHdfsConf()); Path ecFile = new Path(ecDir, "ec_file.txt"); dfs.mkdirs(ecDir); dfs.enableErasureCodingPolicy(ecPolicyName); dfs.setErasureCodingPolicy(ecDir, ecPolicyName); DFSTestUtil.createFile(dfs, ecFile, 1024, (short) 1, 0); String getFileStatusResponse = getStatus(ecFile.toString(), "GETFILESTATUS"); JSONParser parser = new JSONParser(); JSONObject jsonObject = (JSONObject) parser.parse(getFileStatusResponse); JSONObject details = (JSONObject) jsonObject.get("FileStatus"); String ecpolicyForECfile = (String) details.get("ecPolicy"); assertEquals("EC policy for ecFile should match the set EC policy", ecpolicyForECfile, ecPolicyName); WebHdfsFileSystem httpfsWebHdfs = (WebHdfsFileSystem) FileSystem.get( new URI("webhdfs: + TestJettyHelper.getJettyURL().toURI().getAuthority()), TestHdfsHelper.getHdfsConf()); HdfsFileStatus httpfsFileStatus = (HdfsFileStatus) httpfsWebHdfs.getFileStatus(ecFile); assertNotNull(httpfsFileStatus.getErasureCodingPolicy()); }
@Test @TestDir @TestJetty @TestHdfs public void testErasureCodingPolicy() throws Exception { createHttpFSServer(false, false); final String dir = "/ecPolicy"; Path path1 = new Path(dir); final ErasureCodingPolicy ecPolicy = SystemErasureCodingPolicies .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID); final String ecPolicyName = ecPolicy.getName(); DistributedFileSystem dfs = (DistributedFileSystem) FileSystem .get(path1.toUri(), TestHdfsHelper.getHdfsConf()); dfs.mkdirs(new Path(dir)); dfs.enableErasureCodingPolicy(ecPolicyName); HttpURLConnection conn = putCmdWithReturn(dir, "SETECPOLICY", "ecpolicy=" + ecPolicyName); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); HttpURLConnection conn1 = sendRequestToHttpFSServer(dir, "GETECPOLICY", ""); Assert.assertEquals(conn1.getResponseCode(), HttpURLConnection.HTTP_OK); BufferedReader reader = new BufferedReader(new InputStreamReader(conn1.getInputStream())); String dirLst = reader.readLine(); ErasureCodingPolicy dfsDirLst = dfs.getErasureCodingPolicy(path1); Assert.assertNotNull(dfsDirLst); Assert.assertEquals(dirLst, JsonUtil.toJsonString(dfsDirLst)); String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format("/webhdfs/v1{0}?user.name={1}&op={2}&{3}", dir, user, "UNSETECPOLICY", "")); HttpURLConnection conn2 = (HttpURLConnection) url.openConnection(); conn2.setRequestMethod("POST"); conn2.connect(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn2.getResponseCode()); dfsDirLst = dfs.getErasureCodingPolicy(path1); Assert.assertNull(dfsDirLst); final String dir1 = "/"; HttpURLConnection conn3 = putCmdWithReturn(dir1, "SETECPOLICY", "ecpolicy=" + ecPolicyName); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn3.getResponseCode()); final String dir2 = "/"; URL url1 = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format("/webhdfs/v1{0}?user.name={1}&op={2}&{3}", dir2, user, "UNSETECPOLICY", "")); HttpURLConnection conn4 = (HttpURLConnection) url1.openConnection(); conn4.setRequestMethod("POST"); conn4.connect(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn4.getResponseCode()); }
@Test @TestDir @TestJetty @TestHdfs public void testStoragePolicySatisfier() throws Exception { createHttpFSServer(false, false); final String dir = "/parent"; Path path1 = new Path(dir); String file = "/parent/file"; Path filePath = new Path(file); DistributedFileSystem dfs = (DistributedFileSystem) FileSystem .get(path1.toUri(), TestHdfsHelper.getHdfsConf()); dfs.mkdirs(path1); dfs.create(filePath).close(); dfs.setStoragePolicy(filePath, HdfsConstants.COLD_STORAGE_POLICY_NAME); BlockStoragePolicy storagePolicy = (BlockStoragePolicy) dfs.getStoragePolicy(filePath); assertEquals(HdfsConstants.COLD_STORAGE_POLICY_NAME, storagePolicy.getName()); HttpURLConnection conn = putCmdWithReturn(dir, "SATISFYSTORAGEPOLICY", ""); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); Map<String, byte[]> xAttrs = dfs.getXAttrs(path1); assertTrue( xAttrs.containsKey(HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY)); }
@Test @TestDir @TestJetty @TestHdfs public void testNoRedirectWithData() throws Exception { createHttpFSServer(false, false); final String path = "/file"; final String username = HadoopUsersConfTestHelper.getHadoopUsers()[0]; URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format( "/webhdfs/v1{0}?user.name={1}&op=CREATE&data=true&noredirect=true", path, username)); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(HttpMethod.PUT); conn.setRequestProperty("Content-Type", MediaType.APPLICATION_OCTET_STREAM); conn.setDoOutput(true); conn.connect(); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); JSONObject json = (JSONObject) new JSONParser() .parse(new InputStreamReader(conn.getInputStream())); String location = (String) json.get("Location"); Assert.assertTrue(location.contains(DataParam.NAME)); Assert.assertTrue(location.contains("CREATE")); url = new URL(location); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(HttpMethod.PUT); conn.setRequestProperty("Content-Type", MediaType.APPLICATION_OCTET_STREAM); conn.setDoOutput(true); conn.connect(); final String writeStr = "write some content"; OutputStream os = conn.getOutputStream(); os.write(writeStr.getBytes()); os.close(); Assert.assertEquals(HttpURLConnection.HTTP_CREATED, conn.getResponseCode()); json = (JSONObject) new JSONParser() .parse(new InputStreamReader(conn.getInputStream())); location = (String) json.get("Location"); Assert.assertEquals(TestJettyHelper.getJettyURL() + "/webhdfs/v1" + path, location); }
@Test @TestDir @TestJetty @TestHdfs public void testContentType() throws Exception { createHttpFSServer(false, false); FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf()); Path dir = new Path("/tmp"); Path file = new Path(dir, "foo"); fs.mkdirs(dir); fs.create(file); String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format( "/webhdfs/v1/tmp/foo?user.name={0}&op=open&offset=1&length=2", user)); final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(Operation.OPEN.getMethod()); conn.connect(); LambdaTestUtils.intercept(IOException.class, "Content-Type \"text/html;charset=iso-8859-1\" " + "is incompatible with \"application/json\"", () -> HttpFSUtils.jsonParse(conn)); conn.disconnect(); }