OptionsParser { public static DistCpOptions parse(String args[]) throws IllegalArgumentException { CommandLineParser parser = new CustomParser(); CommandLine command; try { command = parser.parse(cliOptions, args, true); } catch (ParseException e) { throw new IllegalArgumentException("Unable to parse arguments. " + Arrays.toString(args), e); } DistCpOptions option = parseSourceAndTargetPaths(command); if (command.hasOption(DistCpOptionSwitch.IGNORE_FAILURES.getSwitch())) { option.setIgnoreFailures(true); } if (command.hasOption(DistCpOptionSwitch.ATOMIC_COMMIT.getSwitch())) { option.setAtomicCommit(true); } if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch()) && option.shouldAtomicCommit()) { String workPath = getVal(command, DistCpOptionSwitch.WORK_PATH.getSwitch()); if (workPath != null && !workPath.isEmpty()) { option.setAtomicWorkPath(new Path(workPath)); } } else if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch())) { throw new IllegalArgumentException("-tmp work-path can only be specified along with -atomic"); } if (command.hasOption(DistCpOptionSwitch.LOG_PATH.getSwitch())) { option.setLogPath(new Path(getVal(command, DistCpOptionSwitch.LOG_PATH.getSwitch()))); } if (command.hasOption(DistCpOptionSwitch.SYNC_FOLDERS.getSwitch())) { option.setSyncFolder(true); } if (command.hasOption(DistCpOptionSwitch.OVERWRITE.getSwitch())) { option.setOverwrite(true); } if (command.hasOption(DistCpOptionSwitch.APPEND.getSwitch())) { option.setAppend(true); } if (command.hasOption(DistCpOptionSwitch.DELETE_MISSING.getSwitch())) { option.setDeleteMissing(true); } if (command.hasOption(DistCpOptionSwitch.SKIP_CRC.getSwitch())) { option.setSkipCRC(true); } if (command.hasOption(DistCpOptionSwitch.BLOCKING.getSwitch())) { option.setBlocking(false); } parseBandwidth(command, option); if (command.hasOption(DistCpOptionSwitch.SSL_CONF.getSwitch())) { option.setSslConfigurationFile(command. getOptionValue(DistCpOptionSwitch.SSL_CONF.getSwitch())); } parseNumListStatusThreads(command, option); parseMaxMaps(command, option); if (command.hasOption(DistCpOptionSwitch.COPY_STRATEGY.getSwitch())) { option.setCopyStrategy( getVal(command, DistCpOptionSwitch.COPY_STRATEGY.getSwitch())); } parsePreserveStatus(command, option); if (command.hasOption(DistCpOptionSwitch.DIFF.getSwitch())) { String[] snapshots = getVals(command, DistCpOptionSwitch.DIFF.getSwitch()); Preconditions.checkArgument(snapshots != null && snapshots.length == 2, "Must provide both the starting and ending snapshot names"); option.setUseDiff(true, snapshots[0], snapshots[1]); } parseFileLimit(command); parseSizeLimit(command); if (command.hasOption(DistCpOptionSwitch.FILTERS.getSwitch())) { option.setFiltersFile(getVal(command, DistCpOptionSwitch.FILTERS.getSwitch())); } return option; }  static DistCpOptions parse(String args[]); static void usage();  }
@Test public void testParseIgnoreFailure() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertFalse(options.shouldIgnoreFailures()); options = OptionsParser.parse(new String[] { "-i", "hdfs: "hdfs: Assert.assertTrue(options.shouldIgnoreFailures()); }
@Test public void testParseOverwrite() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertFalse(options.shouldOverwrite()); options = OptionsParser.parse(new String[] { "-overwrite", "hdfs: "hdfs: Assert.assertTrue(options.shouldOverwrite()); try { OptionsParser.parse(new String[] { "-update", "-overwrite", "hdfs: "hdfs: Assert.fail("Update and overwrite aren't allowed together"); } catch (IllegalArgumentException ignore) { } }
@Test public void testLogPath() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertNull(options.getLogPath()); options = OptionsParser.parse(new String[] { "-log", "hdfs: "hdfs: "hdfs: Assert.assertEquals(options.getLogPath(), new Path("hdfs: }
@Test public void testParseBlokcing() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertTrue(options.shouldBlock()); options = OptionsParser.parse(new String[] { "-async", "hdfs: "hdfs: Assert.assertFalse(options.shouldBlock()); }
@Test(expected=IllegalArgumentException.class) public void testParseNonPositiveBandwidth() { OptionsParser.parse(new String[] { "-bandwidth", "-11", "hdfs: "hdfs: }
@Test(expected=IllegalArgumentException.class) public void testParseZeroBandwidth() { OptionsParser.parse(new String[] { "-bandwidth", "0", "hdfs: "hdfs: }
@Test public void testParseSkipCRC() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertFalse(options.shouldSkipCRC()); options = OptionsParser.parse(new String[] { "-update", "-skipcrccheck", "hdfs: "hdfs: Assert.assertTrue(options.shouldSyncFolder()); Assert.assertTrue(options.shouldSkipCRC()); }
@Test public void testParseAtomicCommit() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertFalse(options.shouldAtomicCommit()); options = OptionsParser.parse(new String[] { "-atomic", "hdfs: "hdfs: Assert.assertTrue(options.shouldAtomicCommit()); try { OptionsParser.parse(new String[] { "-atomic", "-update", "hdfs: "hdfs: Assert.fail("Atomic and sync folders were allowed"); } catch (IllegalArgumentException ignore) { } }
@Test public void testParseWorkPath() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertNull(options.getAtomicWorkPath()); options = OptionsParser.parse(new String[] { "-atomic", "hdfs: "hdfs: Assert.assertNull(options.getAtomicWorkPath()); options = OptionsParser.parse(new String[] { "-atomic", "-tmp", "hdfs: "hdfs: "hdfs: Assert.assertEquals(options.getAtomicWorkPath(), new Path("hdfs: try { OptionsParser.parse(new String[] { "-tmp", "hdfs: "hdfs: "hdfs: Assert.fail("work path was allowed without -atomic switch"); } catch (IllegalArgumentException ignore) {} }
@Test public void testParseSyncFolders() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertFalse(options.shouldSyncFolder()); options = OptionsParser.parse(new String[] { "-update", "hdfs: "hdfs: Assert.assertTrue(options.shouldSyncFolder()); }
@Test public void testParseDeleteMissing() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertFalse(options.shouldDeleteMissing()); options = OptionsParser.parse(new String[] { "-update", "-delete", "hdfs: "hdfs: Assert.assertTrue(options.shouldSyncFolder()); Assert.assertTrue(options.shouldDeleteMissing()); options = OptionsParser.parse(new String[] { "-overwrite", "-delete", "hdfs: "hdfs: Assert.assertTrue(options.shouldOverwrite()); Assert.assertTrue(options.shouldDeleteMissing()); try { OptionsParser.parse(new String[] { "-atomic", "-delete", "hdfs: "hdfs: Assert.fail("Atomic and delete folders were allowed"); } catch (IllegalArgumentException ignore) { } }
@Test public void testParseSSLConf() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertNull(options.getSslConfigurationFile()); options = OptionsParser.parse(new String[] { "-mapredSslConf", "/tmp/ssl-client.xml", "hdfs: "hdfs: Assert.assertEquals(options.getSslConfigurationFile(), "/tmp/ssl-client.xml"); }
@Test public void testParseMaps() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertEquals(options.getMaxMaps(), DistCpConstants.DEFAULT_MAPS); options = OptionsParser.parse(new String[] { "-m", "1", "hdfs: "hdfs: Assert.assertEquals(options.getMaxMaps(), 1); options = OptionsParser.parse(new String[] { "-m", "0", "hdfs: "hdfs: Assert.assertEquals(options.getMaxMaps(), 1); try { OptionsParser.parse(new String[] { "-m", "hello", "hdfs: "hdfs: Assert.fail("Non numberic map parsed"); } catch (IllegalArgumentException ignore) { } try { OptionsParser.parse(new String[] { "-mapredXslConf", "hdfs: "hdfs: Assert.fail("Non numberic map parsed"); } catch (IllegalArgumentException ignore) { } }
@Test public void testSourceListing() { DistCpOptions options = OptionsParser.parse(new String[] { "-f", "hdfs: "hdfs: Assert.assertEquals(options.getSourceFileListing(), new Path("hdfs: }
@Test public void testSourceListingAndSourcePath() { try { OptionsParser.parse(new String[] { "-f", "hdfs: "hdfs: "hdfs: Assert.fail("Both source listing & source paths allowed"); } catch (IllegalArgumentException ignore) {} }
@Test public void testMissingSourceInfo() { try { OptionsParser.parse(new String[] { "hdfs: Assert.fail("Neither source listing not source paths present"); } catch (IllegalArgumentException ignore) {} }
@Test public void testMissingTarget() { try { OptionsParser.parse(new String[] { "-f", "hdfs: Assert.fail("Missing target allowed"); } catch (IllegalArgumentException ignore) {} }
@Test public void testInvalidArgs() { try { OptionsParser.parse(new String[] { "-m", "-f", "hdfs: Assert.fail("Missing map value"); } catch (IllegalArgumentException ignore) {} }
@Test public void testCopyStrategy() { DistCpOptions options = OptionsParser.parse(new String[] { "-strategy", "dynamic", "-f", "hdfs: "hdfs: Assert.assertEquals(options.getCopyStrategy(), "dynamic"); options = OptionsParser.parse(new String[] { "-f", "hdfs: "hdfs: Assert.assertEquals(options.getCopyStrategy(), DistCpConstants.UNIFORMSIZE); }
@Test public void testTargetPath() { DistCpOptions options = OptionsParser.parse(new String[] { "-f", "hdfs: "hdfs: Assert.assertEquals(options.getTargetPath(), new Path("hdfs: }
@Test public void testPreserve() { DistCpOptions options = OptionsParser.parse(new String[] { "-f", "hdfs: "hdfs: Assert.assertFalse(options.shouldPreserve(FileAttribute.BLOCKSIZE)); Assert.assertFalse(options.shouldPreserve(FileAttribute.REPLICATION)); Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); Assert.assertFalse(options.shouldPreserve(FileAttribute.USER)); Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP)); Assert.assertFalse(options.shouldPreserve(FileAttribute.CHECKSUMTYPE)); options = OptionsParser.parse(new String[] { "-p", "-f", "hdfs: "hdfs: Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.USER)); Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP)); Assert.assertTrue(options.shouldPreserve(FileAttribute.CHECKSUMTYPE)); Assert.assertFalse(options.shouldPreserve(FileAttribute.ACL)); Assert.assertFalse(options.shouldPreserve(FileAttribute.XATTR)); options = OptionsParser.parse(new String[] { "-p", "hdfs: "hdfs: Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.USER)); Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP)); Assert.assertTrue(options.shouldPreserve(FileAttribute.CHECKSUMTYPE)); Assert.assertFalse(options.shouldPreserve(FileAttribute.ACL)); Assert.assertFalse(options.shouldPreserve(FileAttribute.XATTR)); options = OptionsParser.parse(new String[] { "-pbr", "-f", "hdfs: "hdfs: Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); Assert.assertFalse(options.shouldPreserve(FileAttribute.USER)); Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP)); Assert.assertFalse(options.shouldPreserve(FileAttribute.CHECKSUMTYPE)); Assert.assertFalse(options.shouldPreserve(FileAttribute.ACL)); Assert.assertFalse(options.shouldPreserve(FileAttribute.XATTR)); options = OptionsParser.parse(new String[] { "-pbrgup", "-f", "hdfs: "hdfs: Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.USER)); Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP)); Assert.assertFalse(options.shouldPreserve(FileAttribute.CHECKSUMTYPE)); Assert.assertFalse(options.shouldPreserve(FileAttribute.ACL)); Assert.assertFalse(options.shouldPreserve(FileAttribute.XATTR)); options = OptionsParser.parse(new String[] { "-pbrgupcax", "-f", "hdfs: "hdfs: Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.USER)); Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP)); Assert.assertTrue(options.shouldPreserve(FileAttribute.CHECKSUMTYPE)); Assert.assertTrue(options.shouldPreserve(FileAttribute.ACL)); Assert.assertTrue(options.shouldPreserve(FileAttribute.XATTR)); options = OptionsParser.parse(new String[] { "-pc", "-f", "hdfs: "hdfs: Assert.assertFalse(options.shouldPreserve(FileAttribute.BLOCKSIZE)); Assert.assertFalse(options.shouldPreserve(FileAttribute.REPLICATION)); Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); Assert.assertFalse(options.shouldPreserve(FileAttribute.USER)); Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP)); Assert.assertTrue(options.shouldPreserve(FileAttribute.CHECKSUMTYPE)); Assert.assertFalse(options.shouldPreserve(FileAttribute.ACL)); Assert.assertFalse(options.shouldPreserve(FileAttribute.XATTR)); options = OptionsParser.parse(new String[] { "-p", "-f", "hdfs: "hdfs: int i = 0; Iterator<FileAttribute> attribIterator = options.preserveAttributes(); while (attribIterator.hasNext()) { attribIterator.next(); i++; } Assert.assertEquals(i, DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.length() - 2); try { OptionsParser.parse(new String[] { "-pabcd", "-f", "hdfs: "hdfs: Assert.fail("Invalid preserve attribute"); } catch (IllegalArgumentException ignore) {} catch (NoSuchElementException ignore) {} options = OptionsParser.parse(new String[] { "-f", "hdfs: "hdfs: Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); options.preserve(FileAttribute.PERMISSION); Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); options.preserve(FileAttribute.PERMISSION); Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); }
@Test public void testOptionsAppendToConf() { Configuration conf = new Configuration(); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false)); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false)); DistCpOptions options = OptionsParser.parse(new String[] { "-atomic", "-i", "hdfs: "hdfs: options.appendToConf(conf); Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false)); Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false)); Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA); conf = new Configuration(); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.DELETE_MISSING.getConfigLabel(), false)); Assert.assertEquals(conf.get(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel()), null); options = OptionsParser.parse(new String[] { "-update", "-delete", "-pu", "-bandwidth", "11.2", "hdfs: "hdfs: options.appendToConf(conf); Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.DELETE_MISSING.getConfigLabel(), false)); Assert.assertEquals(conf.get(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel()), "U"); Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 11.2, DELTA); }
@Test public void testAppendOption() { Configuration conf = new Configuration(); Assert.assertFalse(conf.getBoolean( DistCpOptionSwitch.APPEND.getConfigLabel(), false)); Assert.assertFalse(conf.getBoolean( DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); DistCpOptions options = OptionsParser.parse(new String[] { "-update", "-append", "hdfs: "hdfs: options.appendToConf(conf); Assert.assertTrue(conf.getBoolean( DistCpOptionSwitch.APPEND.getConfigLabel(), false)); Assert.assertTrue(conf.getBoolean( DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); try { OptionsParser.parse(new String[] { "-append", "hdfs: "hdfs: fail("Append should fail if update option is not specified"); } catch (IllegalArgumentException e) { GenericTestUtils.assertExceptionContains( "Append is valid only with update options", e); } try { OptionsParser.parse(new String[] { "-append", "-update", "-skipcrccheck", "hdfs: "hdfs: fail("Append should fail if skipCrc option is specified"); } catch (IllegalArgumentException e) { GenericTestUtils.assertExceptionContains( "Append is disallowed when skipping CRC", e); } }
@Test public void testDiffOption() { Configuration conf = new Configuration(); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.DIFF.getConfigLabel(), false)); DistCpOptions options = OptionsParser.parse(new String[] { "-update", "-diff", "s1", "s2", "hdfs: "hdfs: options.appendToConf(conf); Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.DIFF.getConfigLabel(), false)); Assert.assertTrue(options.shouldUseDiff()); Assert.assertEquals("s1", options.getFromSnapshot()); Assert.assertEquals("s2", options.getToSnapshot()); options = OptionsParser.parse(new String[] { "-diff", "s1", ".", "-update", "hdfs: "hdfs: options.appendToConf(conf); Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.DIFF.getConfigLabel(), false)); Assert.assertTrue(options.shouldUseDiff()); Assert.assertEquals("s1", options.getFromSnapshot()); Assert.assertEquals(".", options.getToSnapshot()); try { OptionsParser.parse(new String[] {"-diff", "s1", "-update", "hdfs: "hdfs: fail("-diff should fail with only one snapshot name"); } catch (IllegalArgumentException e) { GenericTestUtils.assertExceptionContains( "Must provide both the starting and ending snapshot names", e); } try { OptionsParser.parse(new String[] { "-diff", "s1", "s2", "hdfs: "hdfs: fail("-diff should fail if -update option is not specified"); } catch (IllegalArgumentException e) { GenericTestUtils.assertExceptionContains( "Diff is valid only with update options", e); } try { OptionsParser.parse(new String[] { "-diff", "s1", "s2", "-update", "-delete", "hdfs: "hdfs: fail("-diff should fail if -delete option is specified"); } catch (IllegalArgumentException e) { GenericTestUtils.assertExceptionContains( "Diff is valid only with update options", e); } try { OptionsParser.parse(new String[] { "-diff", "s1", "s2", "-delete", "-overwrite", "hdfs: "hdfs: fail("-diff should fail if -update option is not specified"); } catch (IllegalArgumentException e) { GenericTestUtils.assertExceptionContains( "Diff is valid only with update options", e); } }
@Test public void testExclusionsOption() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertNull(options.getFiltersFile()); options = OptionsParser.parse(new String[] { "-filters", "/tmp/filters.txt", "hdfs: "hdfs: Assert.assertEquals(options.getFiltersFile(), "/tmp/filters.txt"); }