RFC4180Tokenizer implements ITokenizer { @Override public String[] tokenize(String input) { return internalTokenize(input); } RFC4180Tokenizer(Options options); @Override String[] tokenize(String input);  }
@Test public void Rfc4180_QuotedString_Double_Quoted_Data_Test() { Options options = new Options('"', '\\', ','); ITokenizer tokenizer = new RFC4180Tokenizer(options); String line = "\"Michael, Chester\",24,\"Also goes by \"\"Mike\"\", among friends that is\""; String[] tokens = tokenizer.tokenize(line); Assert.assertNotNull(tokens); Assert.assertEquals(3, tokens.length); Assert.assertEquals("Michael, Chester", tokens[0]); Assert.assertEquals("24", tokens[1]); Assert.assertEquals("Also goes by \"Mike\", among friends that is", tokens[2]); }
@Test public void Rfc4180_Issue3_Empty_Column_Test() { Options options = new Options('"', '\\', ','); ITokenizer tokenizer = new RFC4180Tokenizer(options); String line = "\"Robert, Willliamson\", , \"All-around nice guy who always says hi\""; String[] tokens = tokenizer.tokenize(line); Assert.assertNotNull(tokens); Assert.assertEquals(3, tokens.length); Assert.assertEquals("Robert, Willliamson", tokens[0]); Assert.assertEquals("", tokens[1]); Assert.assertEquals("All-around nice guy who always says hi", tokens[2]); }
@Test public void Rfc4180_Issue3_Empty_First_Column_Test() { Options options = new Options('"', '\\', ','); ITokenizer tokenizer = new RFC4180Tokenizer(options); String line = " , 24 ,\"Great Guy\""; String[] tokens = tokenizer.tokenize(line); Assert.assertNotNull(tokens); Assert.assertEquals(3, tokens.length); Assert.assertEquals("", tokens[0]); Assert.assertEquals("24", tokens[1]); Assert.assertEquals("Great Guy", tokens[2]); }
@Test public void Rfc4180_Issue3_Empty_Last_Columns_Test() { Options options = new Options('"', '\\', ','); ITokenizer tokenizer = new RFC4180Tokenizer(options); String line = "\"Robert, Willliamson\",27,"; String[] tokens = tokenizer.tokenize(line); Assert.assertNotNull(tokens); Assert.assertEquals(3, tokens.length); Assert.assertEquals("Robert, Willliamson", tokens[0]); Assert.assertEquals("27", tokens[1]); Assert.assertEquals("", tokens[2]); }
@Test public void All_Empty_Last_Columns_Test() { Options options = new Options('"', '\\', ','); ITokenizer tokenizer = new RFC4180Tokenizer(options); String line = ",,"; String[] tokens = tokenizer.tokenize(line); Assert.assertNotNull(tokens); Assert.assertEquals(3, tokens.length); Assert.assertEquals("", tokens[0]); Assert.assertEquals("", tokens[1]); Assert.assertEquals("", tokens[2]); }
@Test public void All_Empty_Last_Column_Not_Empty_Test() { Options options = new Options('"', '\\', ','); ITokenizer tokenizer = new RFC4180Tokenizer(options); String line = ",,a"; String[] tokens = tokenizer.tokenize(line); Assert.assertNotNull(tokens); Assert.assertEquals(3, tokens.length); Assert.assertEquals("", tokens[0]); Assert.assertEquals("", tokens[1]); Assert.assertEquals("a", tokens[2]); }