DFSAdmin extends FsShell { @Override public int run(String[] argv) { if (argv.length < 1) { printUsage(""); return -1; } int exitCode = -1; int i = 0; String cmd = argv[i++]; if ("-safemode".equals(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-allowSnapshot".equalsIgnoreCase(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-disallowSnapshot".equalsIgnoreCase(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-provisionSnapshotTrash".equalsIgnoreCase(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-report".equals(cmd)) { if (argv.length > 6) { printUsage(cmd); return exitCode; } } else if ("-saveNamespace".equals(cmd)) { if (argv.length != 1 && argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-rollEdits".equals(cmd)) { if (argv.length != 1) { printUsage(cmd); return exitCode; } } else if ("-restoreFailedStorage".equals(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-refreshNodes".equals(cmd)) { if (argv.length != 1) { printUsage(cmd); return exitCode; } } else if ("-finalizeUpgrade".equals(cmd)) { if (argv.length != 1) { printUsage(cmd); return exitCode; } } else if (RollingUpgradeCommand.matches(cmd)) { if (argv.length > 2) { printUsage(cmd); return exitCode; } } else if ("-upgrade".equals(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-metasave".equals(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-refreshServiceAcl".equals(cmd)) { if (argv.length != 1) { printUsage(cmd); return exitCode; } } else if ("-refresh".equals(cmd)) { if (argv.length < 3) { printUsage(cmd); return exitCode; } } else if ("-refreshUserToGroupsMappings".equals(cmd)) { if (argv.length != 1) { printUsage(cmd); return exitCode; } } else if ("-printTopology".equals(cmd)) { if(argv.length != 1) { printUsage(cmd); return exitCode; } } else if ("-refreshNamenodes".equals(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-getVolumeReport".equals(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-reconfig".equals(cmd)) { if (argv.length != 4) { printUsage(cmd); return exitCode; } } else if ("-deleteBlockPool".equals(cmd)) { if ((argv.length != 3) && (argv.length != 4)) { printUsage(cmd); return exitCode; } } else if ("-setBalancerBandwidth".equals(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-getBalancerBandwidth".equalsIgnoreCase(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-fetchImage".equals(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-shutdownDatanode".equals(cmd)) { if ((argv.length != 2) && (argv.length != 3)) { printUsage(cmd); return exitCode; } } else if ("-getDatanodeInfo".equals(cmd)) { if (argv.length != 2) { printUsage(cmd); return exitCode; } } else if ("-triggerBlockReport".equals(cmd)) { if ((argv.length < 2) || (argv.length > 5)) { printUsage(cmd); return exitCode; } } else if ("-listOpenFiles".equals(cmd)) { if ((argv.length > 4)) { printUsage(cmd); return exitCode; } } init(); Exception debugException = null; exitCode = 0; try { if ("-report".equals(cmd)) { report(argv, i); } else if ("-safemode".equals(cmd)) { setSafeMode(argv, i); } else if ("-allowSnapshot".equalsIgnoreCase(cmd)) { allowSnapshot(argv); } else if ("-disallowSnapshot".equalsIgnoreCase(cmd)) { disallowSnapshot(argv); } else if ("-provisionSnapshotTrash".equalsIgnoreCase(cmd)) { provisionSnapshotTrash(argv); } else if ("-saveNamespace".equals(cmd)) { exitCode = saveNamespace(argv); } else if ("-rollEdits".equals(cmd)) { exitCode = rollEdits(); } else if ("-restoreFailedStorage".equals(cmd)) { exitCode = restoreFailedStorage(argv[i]); } else if ("-refreshNodes".equals(cmd)) { exitCode = refreshNodes(); } else if ("-finalizeUpgrade".equals(cmd)) { exitCode = finalizeUpgrade(); } else if (RollingUpgradeCommand.matches(cmd)) { exitCode = RollingUpgradeCommand.run(getDFS(), argv, i); } else if ("-upgrade".equals(cmd)) { exitCode = upgrade(argv[i]); } else if ("-metasave".equals(cmd)) { exitCode = metaSave(argv, i); } else if (ClearQuotaCommand.matches(cmd)) { exitCode = new ClearQuotaCommand(argv, i, getConf()).runAll(); } else if (SetQuotaCommand.matches(cmd)) { exitCode = new SetQuotaCommand(argv, i, getConf()).runAll(); } else if (ClearSpaceQuotaCommand.matches(cmd)) { exitCode = new ClearSpaceQuotaCommand(argv, i, getConf()).runAll(); } else if (SetSpaceQuotaCommand.matches(cmd)) { exitCode = new SetSpaceQuotaCommand(argv, i, getConf()).runAll(); } else if ("-refreshServiceAcl".equals(cmd)) { exitCode = refreshServiceAcl(); } else if ("-refreshUserToGroupsMappings".equals(cmd)) { exitCode = refreshUserToGroupsMappings(); } else if ("-refreshSuperUserGroupsConfiguration".equals(cmd)) { exitCode = refreshSuperUserGroupsConfiguration(); } else if ("-refreshCallQueue".equals(cmd)) { exitCode = refreshCallQueue(); } else if ("-refresh".equals(cmd)) { exitCode = genericRefresh(argv, i); } else if ("-printTopology".equals(cmd)) { exitCode = printTopology(); } else if ("-refreshNamenodes".equals(cmd)) { exitCode = refreshNamenodes(argv, i); } else if ("-getVolumeReport".equals(cmd)) { exitCode = getVolumeReport(argv, i); } else if ("-deleteBlockPool".equals(cmd)) { exitCode = deleteBlockPool(argv, i); } else if ("-setBalancerBandwidth".equals(cmd)) { exitCode = setBalancerBandwidth(argv, i); } else if ("-getBalancerBandwidth".equals(cmd)) { exitCode = getBalancerBandwidth(argv, i); } else if ("-fetchImage".equals(cmd)) { exitCode = fetchImage(argv, i); } else if ("-shutdownDatanode".equals(cmd)) { exitCode = shutdownDatanode(argv, i); } else if ("-evictWriters".equals(cmd)) { exitCode = evictWriters(argv, i); } else if ("-getDatanodeInfo".equals(cmd)) { exitCode = getDatanodeInfo(argv, i); } else if ("-reconfig".equals(cmd)) { exitCode = reconfig(argv, i); } else if ("-triggerBlockReport".equals(cmd)) { exitCode = triggerBlockReport(argv); } else if ("-listOpenFiles".equals(cmd)) { exitCode = listOpenFiles(argv); } else if ("-help".equals(cmd)) { if (i < argv.length) { printHelp(argv[i]); } else { printHelp(""); } } else { exitCode = -1; System.err.println(cmd.substring(1) + ": Unknown command"); printUsage(""); } } catch (IllegalArgumentException arge) { debugException = arge; exitCode = -1; System.err.println(cmd.substring(1) + ": " + arge.getLocalizedMessage()); printUsage(cmd); } catch (RemoteException e) { exitCode = -1; debugException = e; try { String[] content; content = e.getLocalizedMessage().split("\n"); System.err.println(cmd.substring(1) + ": " + content[0]); } catch (Exception ex) { System.err.println(cmd.substring(1) + ": " + ex.getLocalizedMessage()); debugException = ex; } } catch (Exception e) { exitCode = -1; debugException = e; System.err.println(cmd.substring(1) + ": " + e.getLocalizedMessage()); } if (LOG.isDebugEnabled() && debugException != null) { LOG.debug("Exception encountered:", debugException); } return exitCode; } DFSAdmin();  DFSAdmin(Configuration conf); void report(String[] argv, int i); void setSafeMode(String[] argv, int idx); int triggerBlockReport(String[] argv); void allowSnapshot(String[] argv); void disallowSnapshot(String[] argv); void provisionSnapshotTrash(String[] argv); int saveNamespace(String[] argv); int rollEdits(); int restoreFailedStorage(String arg); int refreshNodes(); int listOpenFiles(String[] argv); int setBalancerBandwidth(String[] argv, int idx); int getBalancerBandwidth(String[] argv, int idx); int fetchImage(final String[] argv, final int idx); int finalizeUpgrade(); int getUpgradeStatus(); int upgrade(String arg); int metaSave(String[] argv, int idx); int printTopology(); int refreshServiceAcl(); int refreshUserToGroupsMappings(); int refreshSuperUserGroupsConfiguration(); int refreshCallQueue(); int reconfig(String[] argv, int i); int genericRefresh(String[] argv, int i); @Override int run(String[] argv); static void main(String[] argv);  }
@Test(timeout = 60000) public void testDFSAdminUnreachableDatanode() throws Exception { redirectStream(); final DFSAdmin dfsAdmin = new DFSAdmin(conf); for (String command : new String[]{"-getDatanodeInfo", "-evictWriters", "-getBalancerBandwidth"}) { final String dnDataAddr = datanode.getXferAddress().getHostString() + ":" + datanode.getXferPort(); resetStream(); final List<String> outs = Lists.newArrayList(); final int ret = ToolRunner.run(dfsAdmin, new String[]{command, dnDataAddr}); assertEquals(-1, ret); scanIntoList(out, outs); assertTrue("Unexpected " + command + " stdout: " + out, outs.isEmpty()); assertTrue("Unexpected " + command + " stderr: " + err, err.toString().contains("Exception")); } }
@Test(timeout = 180000) public void testReportCommand() throws Exception { tearDown(); redirectStream(); final Configuration dfsConf = new HdfsConfiguration(); ErasureCodingPolicy ecPolicy = SystemErasureCodingPolicies.getByID( SystemErasureCodingPolicies.XOR_2_1_POLICY_ID); dfsConf.setInt( DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500); dfsConf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1); final Path baseDir = new Path( PathUtils.getTestDir(getClass()).getAbsolutePath(), GenericTestUtils.getMethodName()); dfsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.toString()); final int numDn = ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits(); try(MiniDFSCluster miniCluster = new MiniDFSCluster .Builder(dfsConf) .numDataNodes(numDn).build()) { miniCluster.waitActive(); assertEquals(numDn, miniCluster.getDataNodes().size()); final DFSAdmin dfsAdmin = new DFSAdmin(dfsConf); final DFSClient client = miniCluster.getFileSystem().getClient(); resetStream(); assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"})); verifyNodesAndCorruptBlocks(numDn, numDn, 0, 0, client, 0L, 0L); final short replFactor = 1; final long fileLength = 512L; final DistributedFileSystem fs = miniCluster.getFileSystem(); final Path file = new Path(baseDir, "/corrupted"); fs.enableErasureCodingPolicy(ecPolicy.getName()); DFSTestUtil.createFile(fs, file, fileLength, replFactor, 12345L); DFSTestUtil.waitReplication(fs, file, replFactor); final ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file); LocatedBlocks lbs = miniCluster.getFileSystem().getClient(). getNamenode().getBlockLocations( file.toString(), 0, fileLength); assertTrue("Unexpected block type: " + lbs.get(0), lbs.get(0) instanceof LocatedBlock); LocatedBlock locatedBlock = lbs.get(0); DatanodeInfo locatedDataNode = locatedBlock.getLocations()[0]; LOG.info("Replica block located on: " + locatedDataNode); Path ecDir = new Path(baseDir, "ec"); fs.mkdirs(ecDir); fs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy.getName()); Path ecFile = new Path(ecDir, "ec-file"); int stripesPerBlock = 2; int cellSize = ecPolicy.getCellSize(); int blockSize = stripesPerBlock * cellSize; int blockGroupSize = ecPolicy.getNumDataUnits() * blockSize; int totalBlockGroups = 1; DFSTestUtil.createStripedFile(miniCluster, ecFile, ecDir, totalBlockGroups, stripesPerBlock, false, ecPolicy); resetStream(); assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"})); verifyNodesAndCorruptBlocks(numDn, numDn, 0, 0, client, 0L, 0L); final List<DataNode> datanodes = miniCluster.getDataNodes(); DataNode dataNodeToShutdown = null; for (DataNode dn : datanodes) { if (!dn.getDatanodeId().getDatanodeUuid().equals( locatedDataNode.getDatanodeUuid())) { dataNodeToShutdown = dn; break; } } assertTrue("Unable to choose a DataNode to shutdown!", dataNodeToShutdown != null); LOG.info("Shutting down: " + dataNodeToShutdown); dataNodeToShutdown.shutdown(); miniCluster.setDataNodeDead(dataNodeToShutdown.getDatanodeId()); assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"})); verifyNodesAndCorruptBlocks(numDn, numDn - 1, 0, 0, client, 0L, 1L); final int blockFilesCorrupted = miniCluster .corruptBlockOnDataNodes(block); assertEquals("Fail to corrupt all replicas for block " + block, replFactor, blockFilesCorrupted); try { IOUtils.copyBytes(fs.open(file), new IOUtils.NullOutputStream(), conf, true); fail("Should have failed to read the file with corrupted blocks."); } catch (ChecksumException ignored) { } fs.setReplication(file, (short) (replFactor + 1)); BlockManagerTestUtil.updateState( miniCluster.getNameNode().getNamesystem().getBlockManager()); waitForCorruptBlock(miniCluster, client, file); resetStream(); assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"})); verifyNodesAndCorruptBlocks(numDn, numDn - 1, 1, 0, client, 0L, 1L); lbs = miniCluster.getFileSystem().getClient(). getNamenode().getBlockLocations( ecFile.toString(), 0, blockGroupSize); assertTrue("Unexpected block type: " + lbs.get(0), lbs.get(0) instanceof LocatedStripedBlock); LocatedStripedBlock bg = (LocatedStripedBlock)(lbs.get(0)); miniCluster.getNamesystem().writeLock(); try { BlockManager bm = miniCluster.getNamesystem().getBlockManager(); bm.findAndMarkBlockAsCorrupt(bg.getBlock(), bg.getLocations()[0], "STORAGE_ID", "TEST"); BlockManagerTestUtil.updateState(bm); } finally { miniCluster.getNamesystem().writeUnlock(); } waitForCorruptBlock(miniCluster, client, file); resetStream(); assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"})); verifyNodesAndCorruptBlocks(numDn, numDn - 1, 1, 1, client, 0L, 0L); } }
@Test public void testAllowDisallowSnapshot() throws Exception { final Path dirPath = new Path("/ssdir1"); final Path trashRoot = new Path(dirPath, ".Trash"); final DistributedFileSystem dfs = cluster.getFileSystem(); final DFSAdmin dfsAdmin = new DFSAdmin(conf); dfs.mkdirs(dirPath); assertEquals(0, ToolRunner.run(dfsAdmin, new String[]{"-allowSnapshot", dirPath.toString()})); assertTrue(dfs.exists(trashRoot)); assertEquals(TRASH_PERMISSION, dfs.getFileStatus(trashRoot).getPermission()); final Path file1 = new Path(dirPath, "file1"); try (FSDataOutputStream s = dfs.create(file1)) { s.write(0); } FsShell fsShell = new FsShell(dfs.getConf()); assertEquals(0, ToolRunner.run(fsShell, new String[]{"-rm", file1.toString()})); final String username = UserGroupInformation.getLoginUser().getShortUserName(); final Path trashRootUserSubdir = new Path(trashRoot, username); assertTrue(dfs.exists(trashRootUserSubdir)); final FsPermission trashUserdirPermission = new FsPermission( FsAction.ALL, FsAction.NONE, FsAction.NONE, false); assertEquals(trashUserdirPermission, dfs.getFileStatus(trashRootUserSubdir).getPermission()); assertNotEquals(0, ToolRunner.run(dfsAdmin, new String[]{"-disallowSnapshot", dirPath.toString()})); dfs.delete(trashRootUserSubdir, true); assertEquals(0, ToolRunner.run(dfsAdmin, new String[]{"-disallowSnapshot", dirPath.toString()})); dfs.delete(dirPath, true); }
@Test public void testRefreshProxyUser() throws Exception { Path dirPath = new Path("/testdir1"); Path subDirPath = new Path("/testdir1/subdir1"); UserGroupInformation loginUserUgi = UserGroupInformation.getLoginUser(); String proxyUser = "fakeuser"; String realUser = loginUserUgi.getShortUserName(); UserGroupInformation proxyUgi = UserGroupInformation.createProxyUserForTesting(proxyUser, loginUserUgi, loginUserUgi.getGroupNames()); loginUserUgi.doAs(new PrivilegedExceptionAction<Integer>() { @Override public Integer run() throws Exception { cluster.getFileSystem().mkdirs(dirPath); cluster.getFileSystem().setOwner(dirPath, proxyUser, proxyUgi.getPrimaryGroupName()); return 0; } }); try { proxyUgi.doAs(new PrivilegedExceptionAction<Integer>() { @Override public Integer run() throws Exception { cluster.getFileSystem().mkdirs(subDirPath); return 0; } }); } catch (RemoteException re) { Assert.assertTrue(re.unwrapRemoteException() instanceof AccessControlException); Assert.assertTrue(re.unwrapRemoteException().getMessage() .equals("User: " + realUser + " is not allowed to impersonate " + proxyUser)); } String userKeyGroups = DefaultImpersonationProvider.getTestProvider(). getProxySuperuserGroupConfKey(realUser); String userKeyHosts = DefaultImpersonationProvider.getTestProvider(). getProxySuperuserIpConfKey(realUser); String rsrc = "testGroupMappingRefresh_rsrc.xml"; tempResource = TestRefreshUserMappings.addNewConfigResource(rsrc, userKeyGroups, "*", userKeyHosts, "*"); String[] args = new String[]{"-refreshSuperUserGroupsConfiguration"}; admin.run(args); proxyUgi.doAs(new PrivilegedExceptionAction<Integer>() { @Override public Integer run() throws Exception { cluster.getFileSystem().mkdirs(dirPath); return 0; } }); }