StringSplitTokenizer implements ITokenizer { @Override public String[] tokenize(String input) { String[] result = input.split(columnDelimiter); if(trimValues) { result = StringUtils.trimAllElements(result); } return result; } StringSplitTokenizer(String columnDelimiter, boolean trimValues); @Override String[] tokenize(String input); @Override String toString();  }
@Test public void testStringSplitTokenizer_TrimLine() { String input = " Value0,Value1 ,Value2 "; String[] result = new StringSplitTokenizer(",", true).tokenize(input); Assert.assertNotNull(result); Assert.assertEquals(3, result.length); Assert.assertEquals("Value0", result[0]); Assert.assertEquals("Value1", result[1]); Assert.assertEquals("Value2", result[2]); }
@Test public void testStringSplitTokenizer_NoTrimLine() { String input = " Value0,Value1 ,Value2 "; String[] result = new StringSplitTokenizer(",", false).tokenize(input); Assert.assertNotNull(result); Assert.assertEquals(3, result.length); Assert.assertEquals(" Value0", result[0]); Assert.assertEquals("Value1 ", result[1]); Assert.assertEquals("Value2 ", result[2]); }