CsvParser { public Tuple2<Boolean, Row> parse(String line) { Row output = new Row(this.parsers.length); for (int i = 0; i < output.getArity(); i++) { output.setField(i, null); } if (line == null || line.isEmpty()) { return Tuple2.of(false, output); } int startPos = 0; boolean succ = true; final int limit = line.length(); for (int i = 0; i < output.getArity(); i++) { if (startPos > limit) { succ = false; break; } boolean isStringCol = isString[i]; int delimPos = findNextDelimPos(line, startPos, limit, isStringCol); if (delimPos < 0) { delimPos = limit; } String token = line.substring(startPos, delimPos); if (!token.isEmpty()) { Tuple2<Boolean, Object> parsed = parseField(parsers[i], token, isStringCol); if (!parsed.f0) { succ = false; } output.setField(i, parsed.f1); } startPos = delimPos + this.lenFieldDelim; } return Tuple2.of(succ, output); } CsvParser(TypeInformation[] types, String fieldDelim, @Nullable Character quoteChar); Tuple2<Boolean, Row> parse(String line);  }
@Test public void testParser() throws Exception { CsvParser parser = new CsvParser(new TypeInformation[]{Types.STRING}, ",", '"'); Assert.assertEquals(parser.parse("\"hello, world\"").f1.getField(0), "hello, world"); Assert.assertEquals(parser.parse("").f1.getField(0), null); Assert.assertEquals(parser.parse("\"\"").f1.getField(0), ""); Assert.assertEquals(parser.parse("\"\"\"\"\"\"").f1.getField(0), "\"\""); }
@Test public void testLongFieldSeparator() throws Exception { CsvParser parser = new CsvParser(new TypeInformation[]{Types.STRING, Types.STRING, Types.STRING}, "____", '"'); Assert.assertEquals(parser.parse("hello_____world____").f1.getField(0), "hello"); Assert.assertEquals(parser.parse("hello_____world____").f1.getField(1), "_world"); Assert.assertEquals(parser.parse("hello_____world____").f1.getField(2), null); Assert.assertEquals(parser.parse("\"hello_____world____\"").f1.getField(0), "hello_____world____"); Assert.assertEquals(parser.parse("\"hello_____world____\"").f1.getField(1), null); Assert.assertEquals(parser.parse("\"hello_____world____\"").f1.getField(2), null); }
@Test public void testMalFormatString1() throws Exception { CsvParser parser = new CsvParser(new TypeInformation[]{Types.STRING, Types.LONG}, ",", '"'); Assert.assertTrue(parser.parse("\"hello\" world,1").f0); }
@Test public void testMalFormatString2() throws Exception { CsvParser parser = new CsvParser(new TypeInformation[]{Types.STRING, Types.LONG}, ",", '"'); Assert.assertFalse(parser.parse("\"hello world,1").f0); }