@Test public void testFragmenterErrorsWhenPathDoesNotExist() throws Exception { expectedException.expect(InvalidInputException.class); expectedException.expectMessage("Input path does not exist:"); String path = this.getClass().getClassLoader().getResource("csv/").getPath(); context.setDataSource(path + "non-existent"); fragmenter.initialize(context); fragmenter.getFragments(); }
@Override public List<Fragment> getFragments() throws Exception { String fileName = hcfsType.getDataUri(jobConf, context); Path path = new Path(fileName); PxfInputFormat pxfInputFormat = new PxfInputFormat(); PxfInputFormat.setInputPaths(jobConf, path); FileStatus[] fileStatusArray; try { fileStatusArray = pxfInputFormat.listStatus(jobConf); } catch (InvalidInputException e) { if (StringUtils.equalsIgnoreCase("true", context.getOption(IGNORE_MISSING_PATH_OPTION))) { LOG.debug("Ignoring InvalidInputException", e); return fragments; } throw e; } fragments = Arrays.stream(fileStatusArray) .map(fileStatus -> new Fragment(fileStatus.getPath().toUri().toString())) .collect(Collectors.toList()); LOG.debug("Total number of fragments = {}", fragments.size()); return fragments; }
HdfsFileFragmenter extends HdfsDataFragmenter { @Override public List<Fragment> getFragments() throws Exception { String fileName = hcfsType.getDataUri(jobConf, context); Path path = new Path(fileName); PxfInputFormat pxfInputFormat = new PxfInputFormat(); PxfInputFormat.setInputPaths(jobConf, path); FileStatus[] fileStatusArray; try { fileStatusArray = pxfInputFormat.listStatus(jobConf); } catch (InvalidInputException e) { if (StringUtils.equalsIgnoreCase("true", context.getOption(IGNORE_MISSING_PATH_OPTION))) { LOG.debug("Ignoring InvalidInputException", e); return fragments; } throw e; } fragments = Arrays.stream(fileStatusArray) .map(fileStatus -> new Fragment(fileStatus.getPath().toUri().toString())) .collect(Collectors.toList()); LOG.debug("Total number of fragments = {}", fragments.size()); return fragments; } }
HdfsFileFragmenter extends HdfsDataFragmenter { @Override public List<Fragment> getFragments() throws Exception { String fileName = hcfsType.getDataUri(jobConf, context); Path path = new Path(fileName); PxfInputFormat pxfInputFormat = new PxfInputFormat(); PxfInputFormat.setInputPaths(jobConf, path); FileStatus[] fileStatusArray; try { fileStatusArray = pxfInputFormat.listStatus(jobConf); } catch (InvalidInputException e) { if (StringUtils.equalsIgnoreCase("true", context.getOption(IGNORE_MISSING_PATH_OPTION))) { LOG.debug("Ignoring InvalidInputException", e); return fragments; } throw e; } fragments = Arrays.stream(fileStatusArray) .map(fileStatus -> new Fragment(fileStatus.getPath().toUri().toString())) .collect(Collectors.toList()); LOG.debug("Total number of fragments = {}", fragments.size()); return fragments; } HdfsFileFragmenter();   HdfsFileFragmenter(ConfigurationFactory configurationFactory); }
HdfsFileFragmenter extends HdfsDataFragmenter { @Override public List<Fragment> getFragments() throws Exception { String fileName = hcfsType.getDataUri(jobConf, context); Path path = new Path(fileName); PxfInputFormat pxfInputFormat = new PxfInputFormat(); PxfInputFormat.setInputPaths(jobConf, path); FileStatus[] fileStatusArray; try { fileStatusArray = pxfInputFormat.listStatus(jobConf); } catch (InvalidInputException e) { if (StringUtils.equalsIgnoreCase("true", context.getOption(IGNORE_MISSING_PATH_OPTION))) { LOG.debug("Ignoring InvalidInputException", e); return fragments; } throw e; } fragments = Arrays.stream(fileStatusArray) .map(fileStatus -> new Fragment(fileStatus.getPath().toUri().toString())) .collect(Collectors.toList()); LOG.debug("Total number of fragments = {}", fragments.size()); return fragments; } HdfsFileFragmenter();   HdfsFileFragmenter(ConfigurationFactory configurationFactory); @Override List<Fragment> getFragments(); }
HdfsFileFragmenter extends HdfsDataFragmenter { @Override public List<Fragment> getFragments() throws Exception { String fileName = hcfsType.getDataUri(jobConf, context); Path path = new Path(fileName); PxfInputFormat pxfInputFormat = new PxfInputFormat(); PxfInputFormat.setInputPaths(jobConf, path); FileStatus[] fileStatusArray; try { fileStatusArray = pxfInputFormat.listStatus(jobConf); } catch (InvalidInputException e) { if (StringUtils.equalsIgnoreCase("true", context.getOption(IGNORE_MISSING_PATH_OPTION))) { LOG.debug("Ignoring InvalidInputException", e); return fragments; } throw e; } fragments = Arrays.stream(fileStatusArray) .map(fileStatus -> new Fragment(fileStatus.getPath().toUri().toString())) .collect(Collectors.toList()); LOG.debug("Total number of fragments = {}", fragments.size()); return fragments; } HdfsFileFragmenter();   HdfsFileFragmenter(ConfigurationFactory configurationFactory); @Override List<Fragment> getFragments();  }