TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> { List<ImmutablePair<String, Integer>> parse(String value) { List<ImmutablePair<String, Integer>> result = newArrayList(); StringTokenizer tokenizer = new StringTokenizer(value, SEPARATORS); while (tokenizer.hasMoreElements()) { String line = (String) tokenizer.nextElement(); String[] parsed = line.split(",", -1); String origin = parsed[0]; String carrier = parsed[1]; String carrierDelay = parsed[2]; String canceled = parsed[3]; if (isNotCanceled(canceled)) { result.add(of(merge(origin, carrier), onTime(carrierDelay))); } } return result; }   static final String SEPARATORS; }
@Test public void givenString_whenCodes_whenTokenized() { TokenizerMapper tokenizerMapper = new TokenizerMapper(); List<ImmutablePair<String, Integer>> pairs = tokenizerMapper.parse("LAS,WN,23.00,0.00\n" + "LAS,WN,0.00,1.00\n" + "WRO,AA,0.00,0.00\n" + "WRO,BB,-10.00,0.00\n" + "WRO,CC,,0.00\n"); assertTrue(pairs.size() == 4); assertTrue(pairs.get(0).left.equals("LAS-WN")); assertTrue(pairs.get(0).right == 0); assertTrue(pairs.get(1).left.equals("WRO-AA")); assertTrue(pairs.get(1).right == 1); assertTrue(pairs.get(2).left.equals("WRO-BB")); assertTrue(pairs.get(2).right == 1); assertTrue(pairs.get(3).left.equals("WRO-CC")); assertTrue(pairs.get(3).right == 1); }