TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> { List<ImmutablePair<String, Integer>> parse(String value) { List<ImmutablePair<String, Integer>> result = newArrayList(); StringTokenizer tokenizer = new StringTokenizer(value, SEPARATORS); while (tokenizer.hasMoreElements()) { String line = (String) tokenizer.nextElement(); String[] parsed = line.split(",", -1); String origin = parsed[0]; String delay = parsed[1]; String dest = parsed[2]; result.add(of(merge(origin, dest), onTime(delay))); } return result; }   static final String SEPARATORS; }
@Test public void givenString_whenCodes_whenTokenized() { TokenizerMapper tokenizerMapper = new TokenizerMapper(); List<ImmutablePair<String, Integer>> pairs = tokenizerMapper.parse("HOU,-1.00,LIT\n" + "HOU,,SAT\n" + "HOU,1.00,AAA"); assertTrue(pairs.size() == 3); assertTrue(pairs.get(0).left.equals("HOU-LIT")); assertTrue(pairs.get(0).right == 1); assertTrue(pairs.get(1).left.equals("HOU-SAT")); assertTrue(pairs.get(1).right == 0); assertTrue(pairs.get(2).left.equals("HOU-AAA")); assertTrue(pairs.get(2).right == 0); }