@Test public void testOldSplitsGen() throws Exception { InputSplitInfo info = generateOldSplits(oldSplitsDir); Assert.assertEquals(new Path(oldSplitsDir, MRHelpers.JOB_SPLIT_RESOURCE_NAME), info.getSplitsFile()); Assert.assertEquals(new Path(oldSplitsDir, MRHelpers.JOB_SPLIT_METAINFO_RESOURCE_NAME), info.getSplitsMetaInfoFile()); RemoteIterator<LocatedFileStatus> files = remoteFs.listFiles(oldSplitsDir, false); boolean foundSplitsFile = false; boolean foundMetaFile = false; int totalFilesFound = 0; while (files.hasNext()) { LocatedFileStatus status = files.next(); String fName = status.getPath().getName(); totalFilesFound++; if (fName.equals(MRHelpers.JOB_SPLIT_RESOURCE_NAME)) { foundSplitsFile = true; } else if (fName.equals(MRHelpers.JOB_SPLIT_METAINFO_RESOURCE_NAME)) { foundMetaFile = true; } else { Assert.fail("Found invalid file in splits dir, filename=" + fName); } Assert.assertTrue(status.getLen() > 0); } Assert.assertEquals(2, totalFilesFound); Assert.assertTrue(foundSplitsFile); Assert.assertTrue(foundMetaFile); verifyLocationHints(oldSplitsDir, info.getTaskLocationHints()); }
@SuppressWarnings({ "rawtypes", "unchecked" }) @Private public static org.apache.hadoop.mapred.InputSplit[] generateOldSplits( JobConf jobConf, String inputFormatName, int numTasks) throws IOException { org.apache.hadoop.mapred.InputFormat inputFormat = jobConf.getInputFormat(); org.apache.hadoop.mapred.InputFormat finalInputFormat = inputFormat; if (inputFormatName != null && !inputFormatName.isEmpty()) { if (!inputFormat.getClass().equals( org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.class)){ throw new TezUncheckedException( "Expected " + org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.class.getName() + " in conf but got: " + inputFormat.getClass().getName()); } try { inputFormat = (org.apache.hadoop.mapred.InputFormat) ReflectionUtils.newInstance(Class.forName(inputFormatName), jobConf); } catch (ClassNotFoundException e) { throw new TezUncheckedException(e); } org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat groupedFormat = new org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat(); groupedFormat.setConf(jobConf); groupedFormat.setInputFormat(inputFormat); groupedFormat.setDesiredNumberOfSplits(numTasks); finalInputFormat = groupedFormat; } org.apache.hadoop.mapred.InputSplit[] splits = finalInputFormat .getSplits(jobConf, jobConf.getNumMapTasks()); Arrays.sort(splits, new OldInputSplitComparator()); return splits; }
MRHelpers { @SuppressWarnings({ "rawtypes", "unchecked" }) @Private public static org.apache.hadoop.mapred.InputSplit[] generateOldSplits( JobConf jobConf, String inputFormatName, int numTasks) throws IOException { org.apache.hadoop.mapred.InputFormat inputFormat = jobConf.getInputFormat(); org.apache.hadoop.mapred.InputFormat finalInputFormat = inputFormat; if (inputFormatName != null && !inputFormatName.isEmpty()) { if (!inputFormat.getClass().equals( org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.class)){ throw new TezUncheckedException( "Expected " + org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.class.getName() + " in conf but got: " + inputFormat.getClass().getName()); } try { inputFormat = (org.apache.hadoop.mapred.InputFormat) ReflectionUtils.newInstance(Class.forName(inputFormatName), jobConf); } catch (ClassNotFoundException e) { throw new TezUncheckedException(e); } org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat groupedFormat = new org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat(); groupedFormat.setConf(jobConf); groupedFormat.setInputFormat(inputFormat); groupedFormat.setDesiredNumberOfSplits(numTasks); finalInputFormat = groupedFormat; } org.apache.hadoop.mapred.InputSplit[] splits = finalInputFormat .getSplits(jobConf, jobConf.getNumMapTasks()); Arrays.sort(splits, new OldInputSplitComparator()); return splits; } }
MRHelpers { @SuppressWarnings({ "rawtypes", "unchecked" }) @Private public static org.apache.hadoop.mapred.InputSplit[] generateOldSplits( JobConf jobConf, String inputFormatName, int numTasks) throws IOException { org.apache.hadoop.mapred.InputFormat inputFormat = jobConf.getInputFormat(); org.apache.hadoop.mapred.InputFormat finalInputFormat = inputFormat; if (inputFormatName != null && !inputFormatName.isEmpty()) { if (!inputFormat.getClass().equals( org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.class)){ throw new TezUncheckedException( "Expected " + org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.class.getName() + " in conf but got: " + inputFormat.getClass().getName()); } try { inputFormat = (org.apache.hadoop.mapred.InputFormat) ReflectionUtils.newInstance(Class.forName(inputFormatName), jobConf); } catch (ClassNotFoundException e) { throw new TezUncheckedException(e); } org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat groupedFormat = new org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat(); groupedFormat.setConf(jobConf); groupedFormat.setInputFormat(inputFormat); groupedFormat.setDesiredNumberOfSplits(numTasks); finalInputFormat = groupedFormat; } org.apache.hadoop.mapred.InputSplit[] splits = finalInputFormat .getSplits(jobConf, jobConf.getNumMapTasks()); Arrays.sort(splits, new OldInputSplitComparator()); return splits; }  }
MRHelpers { @SuppressWarnings({ "rawtypes", "unchecked" }) @Private public static org.apache.hadoop.mapred.InputSplit[] generateOldSplits( JobConf jobConf, String inputFormatName, int numTasks) throws IOException { org.apache.hadoop.mapred.InputFormat inputFormat = jobConf.getInputFormat(); org.apache.hadoop.mapred.InputFormat finalInputFormat = inputFormat; if (inputFormatName != null && !inputFormatName.isEmpty()) { if (!inputFormat.getClass().equals( org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.class)){ throw new TezUncheckedException( "Expected " + org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.class.getName() + " in conf but got: " + inputFormat.getClass().getName()); } try { inputFormat = (org.apache.hadoop.mapred.InputFormat) ReflectionUtils.newInstance(Class.forName(inputFormatName), jobConf); } catch (ClassNotFoundException e) { throw new TezUncheckedException(e); } org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat groupedFormat = new org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat(); groupedFormat.setConf(jobConf); groupedFormat.setInputFormat(inputFormat); groupedFormat.setDesiredNumberOfSplits(numTasks); finalInputFormat = groupedFormat; } org.apache.hadoop.mapred.InputSplit[] splits = finalInputFormat .getSplits(jobConf, jobConf.getNumMapTasks()); Arrays.sort(splits, new OldInputSplitComparator()); return splits; }  @LimitedPrivate("Hive, Pig") @Unstable static void translateVertexConfToTez(Configuration conf); @SuppressWarnings({ "rawtypes", "unchecked" }) @Private static org.apache.hadoop.mapreduce.InputSplit[] generateNewSplits(
      JobContext jobContext, String inputFormatName, int numTasks); @SuppressWarnings({ "rawtypes", "unchecked" }) @Private static org.apache.hadoop.mapred.InputSplit[] generateOldSplits(
      JobConf jobConf, String inputFormatName, int numTasks); static InputSplitInfoDisk generateInputSplits(Configuration conf,
      Path inputSplitsDir); static InputSplitInfoMem generateInputSplitsToMem(Configuration conf); @Private static MRSplitProto createSplitProto(
      T newSplit, SerializationFactory serializationFactory); @Private static MRSplitProto createSplitProto(
      org.apache.hadoop.mapred.InputSplit oldSplit); static void addLog4jSystemProperties(String logLevel,
      List<String> vargs); @SuppressWarnings("deprecation") static String getMapJavaOpts(Configuration conf); @SuppressWarnings("deprecation") static String getReduceJavaOpts(Configuration conf); static void doJobClientMagic(Configuration conf); @LimitedPrivate("Hive, Pig") @Unstable static byte[] createUserPayloadFromConf(Configuration conf); @LimitedPrivate("Hive, Pig") static ByteString createByteStringFromConf(Configuration conf); @LimitedPrivate("Hive, Pig") @Unstable static Configuration createConfFromUserPayload(byte[] bb); @LimitedPrivate("Hive, Pig") static Configuration createConfFromByteString(ByteString bs); static byte[] createMRInputPayload(byte[] configurationBytes,
      MRSplitsProto mrSplitsProto); static byte[] createMRInputPayload(Configuration conf,
      MRSplitsProto mrSplitsProto); static byte[] createMRInputPayloadWithGrouping(byte[] configurationBytes,
      String inputFormatName); static byte[] createMRInputPayloadWithGrouping(Configuration conf,
      String inputFormatName); static MRInputUserPayloadProto parseMRInputPayload(byte[] bytes); static void updateLocalResourcesForInputSplits(
      FileSystem fs,
      InputSplitInfo inputSplitInfo,
      Map<String, LocalResource> localResources); static Resource getMapResource(Configuration conf); static Resource getReduceResource(Configuration conf); static void updateEnvironmentForMRTasks(Configuration conf,
      Map<String, String> environment, boolean isMap); static Configuration getBaseMRConfiguration(Configuration conf); static Configuration getBaseMRConfiguration(); static void updateEnvironmentForMRAM(Configuration conf, Map<String, String> environment); static String getMRAMJavaOpts(Configuration conf); static void addMRInput(Vertex vertex, byte[] userPayload,
      Class<? extends TezRootInputInitializer> initClazz); static void addMROutput(Vertex vertex, byte[] userPayload); @Private static void addMROutputLegacy(Vertex vertex, byte[] userPayload); @SuppressWarnings("unchecked") static InputSplit createOldFormatSplitFromUserPayload(
      MRSplitProto splitProto, SerializationFactory serializationFactory); @SuppressWarnings("unchecked") static org.apache.hadoop.mapreduce.InputSplit createNewFormatSplitFromUserPayload(
      MRSplitProto splitProto, SerializationFactory serializationFactory); static void mergeMRBinaryTokens(Credentials creds,
      Configuration conf); }
MRHelpers { @SuppressWarnings({ "rawtypes", "unchecked" }) @Private public static org.apache.hadoop.mapred.InputSplit[] generateOldSplits( JobConf jobConf, String inputFormatName, int numTasks) throws IOException { org.apache.hadoop.mapred.InputFormat inputFormat = jobConf.getInputFormat(); org.apache.hadoop.mapred.InputFormat finalInputFormat = inputFormat; if (inputFormatName != null && !inputFormatName.isEmpty()) { if (!inputFormat.getClass().equals( org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.class)){ throw new TezUncheckedException( "Expected " + org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.class.getName() + " in conf but got: " + inputFormat.getClass().getName()); } try { inputFormat = (org.apache.hadoop.mapred.InputFormat) ReflectionUtils.newInstance(Class.forName(inputFormatName), jobConf); } catch (ClassNotFoundException e) { throw new TezUncheckedException(e); } org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat groupedFormat = new org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat(); groupedFormat.setConf(jobConf); groupedFormat.setInputFormat(inputFormat); groupedFormat.setDesiredNumberOfSplits(numTasks); finalInputFormat = groupedFormat; } org.apache.hadoop.mapred.InputSplit[] splits = finalInputFormat .getSplits(jobConf, jobConf.getNumMapTasks()); Arrays.sort(splits, new OldInputSplitComparator()); return splits; }  @LimitedPrivate("Hive, Pig") @Unstable static void translateVertexConfToTez(Configuration conf); @SuppressWarnings({ "rawtypes", "unchecked" }) @Private static org.apache.hadoop.mapreduce.InputSplit[] generateNewSplits(
      JobContext jobContext, String inputFormatName, int numTasks); @SuppressWarnings({ "rawtypes", "unchecked" }) @Private static org.apache.hadoop.mapred.InputSplit[] generateOldSplits(
      JobConf jobConf, String inputFormatName, int numTasks); static InputSplitInfoDisk generateInputSplits(Configuration conf,
      Path inputSplitsDir); static InputSplitInfoMem generateInputSplitsToMem(Configuration conf); @Private static MRSplitProto createSplitProto(
      T newSplit, SerializationFactory serializationFactory); @Private static MRSplitProto createSplitProto(
      org.apache.hadoop.mapred.InputSplit oldSplit); static void addLog4jSystemProperties(String logLevel,
      List<String> vargs); @SuppressWarnings("deprecation") static String getMapJavaOpts(Configuration conf); @SuppressWarnings("deprecation") static String getReduceJavaOpts(Configuration conf); static void doJobClientMagic(Configuration conf); @LimitedPrivate("Hive, Pig") @Unstable static byte[] createUserPayloadFromConf(Configuration conf); @LimitedPrivate("Hive, Pig") static ByteString createByteStringFromConf(Configuration conf); @LimitedPrivate("Hive, Pig") @Unstable static Configuration createConfFromUserPayload(byte[] bb); @LimitedPrivate("Hive, Pig") static Configuration createConfFromByteString(ByteString bs); static byte[] createMRInputPayload(byte[] configurationBytes,
      MRSplitsProto mrSplitsProto); static byte[] createMRInputPayload(Configuration conf,
      MRSplitsProto mrSplitsProto); static byte[] createMRInputPayloadWithGrouping(byte[] configurationBytes,
      String inputFormatName); static byte[] createMRInputPayloadWithGrouping(Configuration conf,
      String inputFormatName); static MRInputUserPayloadProto parseMRInputPayload(byte[] bytes); static void updateLocalResourcesForInputSplits(
      FileSystem fs,
      InputSplitInfo inputSplitInfo,
      Map<String, LocalResource> localResources); static Resource getMapResource(Configuration conf); static Resource getReduceResource(Configuration conf); static void updateEnvironmentForMRTasks(Configuration conf,
      Map<String, String> environment, boolean isMap); static Configuration getBaseMRConfiguration(Configuration conf); static Configuration getBaseMRConfiguration(); static void updateEnvironmentForMRAM(Configuration conf, Map<String, String> environment); static String getMRAMJavaOpts(Configuration conf); static void addMRInput(Vertex vertex, byte[] userPayload,
      Class<? extends TezRootInputInitializer> initClazz); static void addMROutput(Vertex vertex, byte[] userPayload); @Private static void addMROutputLegacy(Vertex vertex, byte[] userPayload); @SuppressWarnings("unchecked") static InputSplit createOldFormatSplitFromUserPayload(
      MRSplitProto splitProto, SerializationFactory serializationFactory); @SuppressWarnings("unchecked") static org.apache.hadoop.mapreduce.InputSplit createNewFormatSplitFromUserPayload(
      MRSplitProto splitProto, SerializationFactory serializationFactory); static void mergeMRBinaryTokens(Credentials creds,
      Configuration conf);  }