OptionsParser { public static DistCpOptions parse(String args[]) throws IllegalArgumentException { CommandLineParser parser = new CustomParser(); CommandLine command; try { command = parser.parse(cliOptions, args, true); } catch (ParseException e) { throw new IllegalArgumentException("Unable to parse arguments. " + Arrays.toString(args), e); } DistCpOptions option; Path targetPath; List<Path> sourcePaths = new ArrayList<Path>(); String leftOverArgs[] = command.getArgs(); if (leftOverArgs == null || leftOverArgs.length < 1) { throw new IllegalArgumentException("Target path not specified"); } targetPath = new Path(leftOverArgs[leftOverArgs.length -1].trim()); for (int index = 0; index < leftOverArgs.length - 1; index++) { sourcePaths.add(new Path(leftOverArgs[index].trim())); } if (command.hasOption(DistCpOptionSwitch.SOURCE_FILE_LISTING.getSwitch())) { if (!sourcePaths.isEmpty()) { throw new IllegalArgumentException("Both source file listing and source paths present"); } option = new DistCpOptions(new Path(getVal(command, DistCpOptionSwitch. SOURCE_FILE_LISTING.getSwitch())), targetPath); } else { if (sourcePaths.isEmpty()) { throw new IllegalArgumentException("Neither source file listing nor source paths present"); } option = new DistCpOptions(sourcePaths, targetPath); } if (command.hasOption(DistCpOptionSwitch.IGNORE_FAILURES.getSwitch())) { option.setIgnoreFailures(true); } if (command.hasOption(DistCpOptionSwitch.PRESERVE_SRC_PATH.getSwitch())) { option.setPreserveSrcPath(true); } if (command.hasOption(DistCpOptionSwitch.ATOMIC_COMMIT.getSwitch())) { option.setAtomicCommit(true); } if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch()) && option.shouldAtomicCommit()) { String workPath = getVal(command, DistCpOptionSwitch.WORK_PATH.getSwitch()); if (workPath != null && !workPath.isEmpty()) { option.setAtomicWorkPath(new Path(workPath)); } } else if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch())) { throw new IllegalArgumentException("-tmp work-path can only be specified along with -atomic"); } if (command.hasOption(DistCpOptionSwitch.LOG_PATH.getSwitch())) { option.setLogPath(new Path(getVal(command, DistCpOptionSwitch.LOG_PATH.getSwitch()))); } if (command.hasOption(DistCpOptionSwitch.SYNC_FOLDERS.getSwitch())) { option.setSyncFolder(true); } if (command.hasOption(DistCpOptionSwitch.OVERWRITE.getSwitch())) { option.setOverwrite(true); } if (command.hasOption(DistCpOptionSwitch.DELETE_MISSING.getSwitch())) { option.setDeleteMissing(true); } if (command.hasOption(DistCpOptionSwitch.SKIP_CRC.getSwitch())) { option.setSkipCRC(true); } if (command.hasOption(DistCpOptionSwitch.BLOCKING.getSwitch())) { option.setBlocking(false); } if (command.hasOption(DistCpOptionSwitch.BANDWIDTH.getSwitch())) { try { Integer mapBandwidth = Integer.parseInt( getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()).trim()); option.setMapBandwidth(mapBandwidth); } catch (NumberFormatException e) { throw new IllegalArgumentException("Bandwidth specified is invalid: " + getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()), e); } } if (command.hasOption(DistCpOptionSwitch.BANDWIDTH_KB.getSwitch())) { try { Integer mapBandwidth = Integer.parseInt( getVal(command, DistCpOptionSwitch.BANDWIDTH_KB.getSwitch()).trim()); option.setMapBandwidthKB(mapBandwidth); } catch (NumberFormatException e) { throw new IllegalArgumentException("Bandwidth specified is invalid: " + getVal(command, DistCpOptionSwitch.BANDWIDTH_KB.getSwitch()), e); } } if (command.hasOption(DistCpOptionSwitch.SSL_CONF.getSwitch())) { option.setSslConfigurationFile(command. getOptionValue(DistCpOptionSwitch.SSL_CONF.getSwitch())); } if (command.hasOption(DistCpOptionSwitch.MAX_MAPS.getSwitch())) { try { Integer maps = Integer.parseInt( getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch()).trim()); option.setMaxMaps(maps); } catch (NumberFormatException e) { throw new IllegalArgumentException("Number of maps is invalid: " + getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch()), e); } } if (command.hasOption(DistCpOptionSwitch.COPY_STRATEGY.getSwitch())) { option.setCopyStrategy( getVal(command, DistCpOptionSwitch.COPY_STRATEGY.getSwitch())); } if (command.hasOption(DistCpOptionSwitch.PRESERVE_STATUS.getSwitch())) { String attributes = getVal(command, DistCpOptionSwitch.PRESERVE_STATUS.getSwitch()); if (attributes == null || attributes.isEmpty()) { for (FileAttribute attribute : FileAttribute.values()) { option.preserve(attribute); } } else { for (int index = 0; index < attributes.length(); index++) { option.preserve(FileAttribute. getAttribute(attributes.charAt(index))); } } } if (command.hasOption(DistCpOptionSwitch.FILE_LIMIT.getSwitch())) { String fileLimitString = getVal(command, DistCpOptionSwitch.FILE_LIMIT.getSwitch().trim()); try { Integer.parseInt(fileLimitString); } catch (NumberFormatException e) { throw new IllegalArgumentException("File-limit is invalid: " + fileLimitString, e); } LOG.warn(DistCpOptionSwitch.FILE_LIMIT.getSwitch() + " is a deprecated" + " option. Ignoring."); } if (command.hasOption(DistCpOptionSwitch.SIZE_LIMIT.getSwitch())) { String sizeLimitString = getVal(command, DistCpOptionSwitch.SIZE_LIMIT.getSwitch().trim()); try { Long.parseLong(sizeLimitString); } catch (NumberFormatException e) { throw new IllegalArgumentException("Size-limit is invalid: " + sizeLimitString, e); } LOG.warn(DistCpOptionSwitch.SIZE_LIMIT.getSwitch() + " is a deprecated" + " option. Ignoring."); } return option; }  static DistCpOptions parse(String args[]); static void usage();  }
@Test public void testParseIgnoreFailure() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertFalse(options.shouldIgnoreFailures()); options = OptionsParser.parse(new String[] { "-i", "hdfs: "hdfs: Assert.assertTrue(options.shouldIgnoreFailures()); }
@Test public void testParseOverwrite() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertFalse(options.shouldOverwrite()); options = OptionsParser.parse(new String[] { "-overwrite", "hdfs: "hdfs: Assert.assertTrue(options.shouldOverwrite()); try { OptionsParser.parse(new String[] { "-update", "-overwrite", "hdfs: "hdfs: Assert.fail("Update and overwrite aren't allowed together"); } catch (IllegalArgumentException ignore) { } }
@Test public void testLogPath() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertNull(options.getLogPath()); options = OptionsParser.parse(new String[] { "-log", "hdfs: "hdfs: "hdfs: Assert.assertEquals(options.getLogPath(), new Path("hdfs: }
@Test public void testParseBlokcing() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertTrue(options.shouldBlock()); options = OptionsParser.parse(new String[] { "-async", "hdfs: "hdfs: Assert.assertFalse(options.shouldBlock()); }
@Test public void testParsebandwidth() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertEquals(options.getMapBandwidth(), DistCpConstants.DEFAULT_BANDWIDTH_KB); options = OptionsParser.parse(new String[] { "-bandwidthKB", "11", "hdfs: "hdfs: Assert.assertEquals(options.getMapBandwidth(), 11); options = OptionsParser.parse(new String[] { "-bandwidth", "12", "hdfs: "hdfs: Assert.assertEquals(options.getMapBandwidth(), 12 * 1024); }
@Test public void testParseSkipCRC() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertFalse(options.shouldSkipCRC()); options = OptionsParser.parse(new String[] { "-update", "-skipcrccheck", "hdfs: "hdfs: Assert.assertTrue(options.shouldSyncFolder()); Assert.assertTrue(options.shouldSkipCRC()); }
@Test public void testParseAtomicCommit() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertFalse(options.shouldAtomicCommit()); options = OptionsParser.parse(new String[] { "-atomic", "hdfs: "hdfs: Assert.assertTrue(options.shouldAtomicCommit()); try { OptionsParser.parse(new String[] { "-atomic", "-update", "hdfs: "hdfs: Assert.fail("Atomic and sync folders were allowed"); } catch (IllegalArgumentException ignore) { } }
@Test public void testParseWorkPath() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertNull(options.getAtomicWorkPath()); options = OptionsParser.parse(new String[] { "-atomic", "hdfs: "hdfs: Assert.assertNull(options.getAtomicWorkPath()); options = OptionsParser.parse(new String[] { "-atomic", "-tmp", "hdfs: "hdfs: "hdfs: Assert.assertEquals(options.getAtomicWorkPath(), new Path("hdfs: try { OptionsParser.parse(new String[] { "-tmp", "hdfs: "hdfs: "hdfs: Assert.fail("work path was allowed without -atomic switch"); } catch (IllegalArgumentException ignore) {} }
@Test public void testParseSyncFolders() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertFalse(options.shouldSyncFolder()); options = OptionsParser.parse(new String[] { "-update", "hdfs: "hdfs: Assert.assertTrue(options.shouldSyncFolder()); }
@Test public void testParseDeleteMissing() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertFalse(options.shouldDeleteMissing()); options = OptionsParser.parse(new String[] { "-update", "-delete", "hdfs: "hdfs: Assert.assertTrue(options.shouldSyncFolder()); Assert.assertTrue(options.shouldDeleteMissing()); options = OptionsParser.parse(new String[] { "-overwrite", "-delete", "hdfs: "hdfs: Assert.assertTrue(options.shouldOverwrite()); Assert.assertTrue(options.shouldDeleteMissing()); try { OptionsParser.parse(new String[] { "-atomic", "-delete", "hdfs: "hdfs: Assert.fail("Atomic and delete folders were allowed"); } catch (IllegalArgumentException ignore) { } }
@Test public void testParseSSLConf() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertNull(options.getSslConfigurationFile()); options = OptionsParser.parse(new String[] { "-mapredSslConf", "/tmp/ssl-client.xml", "hdfs: "hdfs: Assert.assertEquals(options.getSslConfigurationFile(), "/tmp/ssl-client.xml"); }
@Test public void testParseMaps() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs: "hdfs: Assert.assertEquals(options.getMaxMaps(), DistCpConstants.DEFAULT_MAPS); options = OptionsParser.parse(new String[] { "-m", "1", "hdfs: "hdfs: Assert.assertEquals(options.getMaxMaps(), 1); try { OptionsParser.parse(new String[] { "-m", "hello", "hdfs: "hdfs: Assert.fail("Non numberic map parsed"); } catch (IllegalArgumentException ignore) { } try { OptionsParser.parse(new String[] { "-mapredXslConf", "hdfs: "hdfs: Assert.fail("Non numberic map parsed"); } catch (IllegalArgumentException ignore) { } }
@Test public void testSourceListing() { DistCpOptions options = OptionsParser.parse(new String[] { "-f", "hdfs: "hdfs: Assert.assertEquals(options.getSourceFileListing(), new Path("hdfs: }
@Test public void testSourceListingAndSourcePath() { try { OptionsParser.parse(new String[] { "-f", "hdfs: "hdfs: "hdfs: Assert.fail("Both source listing & source paths allowed"); } catch (IllegalArgumentException ignore) {} }
@Test public void testMissingSourceInfo() { try { OptionsParser.parse(new String[] { "hdfs: Assert.fail("Neither source listing not source paths present"); } catch (IllegalArgumentException ignore) {} }
@Test public void testMissingTarget() { try { OptionsParser.parse(new String[] { "-f", "hdfs: Assert.fail("Missing target allowed"); } catch (IllegalArgumentException ignore) {} }
@Test public void testInvalidArgs() { try { OptionsParser.parse(new String[] { "-m", "-f", "hdfs: Assert.fail("Missing map value"); } catch (IllegalArgumentException ignore) {} }
@Test public void testCopyStrategy() { DistCpOptions options = OptionsParser.parse(new String[] { "-strategy", "dynamic", "-f", "hdfs: "hdfs: Assert.assertEquals(options.getCopyStrategy(), "dynamic"); options = OptionsParser.parse(new String[] { "-f", "hdfs: "hdfs: Assert.assertEquals(options.getCopyStrategy(), DistCpConstants.DEFAULT_COPY_STRATEGY); options = OptionsParser.parse(new String[] { "-strategy", "dynamic", "-f", "hdfs: "hdfs: Assert.assertEquals(options.getCopyStrategy(), "dynamic"); }
@Test public void testTargetPath() { DistCpOptions options = OptionsParser.parse(new String[] { "-f", "hdfs: "hdfs: Assert.assertEquals(options.getTargetPath(), new Path("hdfs: }
@Test public void testPreserve() { DistCpOptions options = OptionsParser.parse(new String[] { "-f", "hdfs: "hdfs: Assert.assertFalse(options.shouldPreserve(FileAttribute.BLOCKSIZE)); Assert.assertFalse(options.shouldPreserve(FileAttribute.REPLICATION)); Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); Assert.assertFalse(options.shouldPreserve(FileAttribute.USER)); Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP)); options = OptionsParser.parse(new String[] { "-p", "-f", "hdfs: "hdfs: Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.USER)); Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP)); options = OptionsParser.parse(new String[] { "-p", "hdfs: "hdfs: Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.USER)); Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP)); options = OptionsParser.parse(new String[] { "-pbr", "-f", "hdfs: "hdfs: Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); Assert.assertFalse(options.shouldPreserve(FileAttribute.USER)); Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP)); options = OptionsParser.parse(new String[] { "-pbrgup", "-f", "hdfs: "hdfs: Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); Assert.assertTrue(options.shouldPreserve(FileAttribute.USER)); Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP)); options = OptionsParser.parse(new String[] { "-p", "-f", "hdfs: "hdfs: int i = 0; Iterator<FileAttribute> attribIterator = options.preserveAttributes(); while (attribIterator.hasNext()) { attribIterator.next(); i++; } Assert.assertEquals(i, 5); try { OptionsParser.parse(new String[] { "-pabc", "-f", "hdfs: "hdfs: Assert.fail("Invalid preserve attribute"); } catch (IllegalArgumentException ignore) {} catch (NoSuchElementException ignore) {} options = OptionsParser.parse(new String[] { "-f", "hdfs: "hdfs: Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); options.preserve(FileAttribute.PERMISSION); Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); options.preserve(FileAttribute.PERMISSION); Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); }
@Test public void testOptionsAppendToConf() { Configuration conf = new Configuration(); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false)); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false)); DistCpOptions options = OptionsParser.parse(new String[] { "-atomic", "-i", "hdfs: "hdfs: options.appendToConf(conf); Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false)); Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false)); Assert.assertEquals(conf.getInt(DistCpOptionSwitch.BANDWIDTH_KB.getConfigLabel(), -1), DistCpConstants.DEFAULT_BANDWIDTH_KB); conf = new Configuration(); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.DELETE_MISSING.getConfigLabel(), false)); Assert.assertEquals(conf.get(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel()), null); options = OptionsParser.parse(new String[] { "-update", "-delete", "-pu", "-bandwidth", "11", "hdfs: "hdfs: options.appendToConf(conf); Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.DELETE_MISSING.getConfigLabel(), false)); Assert.assertEquals(conf.get(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel()), "U"); Assert.assertEquals(conf.getInt(DistCpOptionSwitch.BANDWIDTH_KB.getConfigLabel(), -1), 11 * 1024); }