TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> { List<ImmutablePair<String, Integer>> parse(String value) { List<ImmutablePair<String, Integer>> result = newArrayList(); StringTokenizer tokenizer = new StringTokenizer(value, SEPARATORS); while (tokenizer.hasMoreElements()) { String line = (String) tokenizer.nextElement(); String[] parsed = line.split(",", -1); String origin = parsed[0]; String dest = parsed[1]; String arrDelay = parsed[2]; if (isNotBlank(arrDelay)) { result.add(of(merge(origin, dest), Double.valueOf(arrDelay).intValue())); } } return result; }   static final String SEPARATORS; }
@Test public void givenString_whenCodes_whenTokenized() { TokenizerMapper tokenizerMapper = new TokenizerMapper(); List<ImmutablePair<String, Integer>> pairs = tokenizerMapper.parse("HOU,LIT,16.00\n" + "HOU,NYC,10.00\n"); assertTrue(pairs.size() == 2); assertTrue(pairs.get(0).left.equals("HOU-LIT")); assertTrue(pairs.get(0).right == 16); assertTrue(pairs.get(1).left.equals("HOU-NYC")); assertTrue(pairs.get(1).right == 10); }