@Test public void testQuotedFieldWithUnexpectedNewlineNoNextLineRead() throws Exception { final String input = "col1,col2\n" + "\"foo\",\"bar\n" + "\"baz\",\"zoo\"\n" + "\"aaa\",\"bbb\""; CsvPreference pref = new CsvPreference.Builder(EXCEL_PREFERENCE) .maxLinesPerRow(1).build(); tokenizer = createTokenizer(input, pref); try { final boolean first = tokenizer.readColumns(columns); assertEquals(true , first); assertEquals("[col1, col2]" , columns.toString()); tokenizer.readColumns(columns); fail("should have thrown SuperCsvException"); } catch(SuperCsvException e) { assertEquals("unexpected end of line while reading quoted column on line 2", e.getMessage()); } final boolean third = tokenizer.readColumns(columns); assertEquals(true , third); assertEquals("[baz, zoo]" , columns.toString()); final boolean fourth = tokenizer.readColumns(columns); assertEquals(true , fourth); assertEquals("[aaa, bbb]" , columns.toString()); final boolean fifth = tokenizer.readColumns(columns); assertEquals(false , fifth); }
public boolean readColumns(final List<String> columns) throws IOException { if( columns == null ) { throw new NullPointerException("columns should not be null"); } columns.clear(); currentColumn.setLength(0); currentRow.setLength(0); String line; do { line = readLine(); if( line == null ) { return false; } } while( ignoreEmptyLines && line.length() == 0 || (commentMatcher != null && commentMatcher.isComment(line)) ); currentRow.append(line); TokenizerState state = TokenizerState.NORMAL; int quoteScopeStartingLine = -1; int potentialSpaces = 0; int charIndex = 0; while( true ) { boolean endOfLineReached = charIndex == line.length(); if( endOfLineReached ) { if( TokenizerState.NORMAL.equals(state) ) { if( !surroundingSpacesNeedQuotes ) { appendSpaces(currentColumn, potentialSpaces); } addColumn(columns, line, charIndex); return true; } else { currentColumn.append(NEWLINE); currentRow.append(NEWLINE); charIndex = 0; if (maxLinesPerRow > 0 && getLineNumber() - quoteScopeStartingLine + 1 >= maxLinesPerRow) { String msg = maxLinesPerRow == 1 ? String.format("unexpected end of line while reading quoted column on line %d", getLineNumber()) : String.format("max number of lines to read exceeded while reading quoted column" + " beginning on line %d and ending on line %d", quoteScopeStartingLine, getLineNumber()); throw new SuperCsvException(msg); } else if( (line = readLine()) == null ) { throw new SuperCsvException( String .format( "unexpected end of file while reading quoted column beginning on line %d and ending on line %d", quoteScopeStartingLine, getLineNumber())); } currentRow.append(line); if (line.length() == 0){ continue; } } } final char c = line.charAt(charIndex); if( TokenizerState.NORMAL.equals(state) ) { if( c == delimiterChar) { if( !surroundingSpacesNeedQuotes ) { appendSpaces(currentColumn, potentialSpaces); } addColumn(columns, line, charIndex); potentialSpaces = 0; currentColumn.setLength(0); } else if( c == SPACE ) { potentialSpaces++; } else if( c == quoteChar ) { state = TokenizerState.QUOTE_MODE; quoteScopeStartingLine = getLineNumber(); if( !surroundingSpacesNeedQuotes || currentColumn.length() > 0 ) { appendSpaces(currentColumn, potentialSpaces); } potentialSpaces = 0; } else { if( !surroundingSpacesNeedQuotes || currentColumn.length() > 0 ) { appendSpaces(currentColumn, potentialSpaces); } potentialSpaces = 0; currentColumn.append(c); } } else { if( c == quoteEscapeChar ) { int nextCharIndex = charIndex + 1; boolean availableCharacters = nextCharIndex < line.length(); boolean nextCharIsQuote = availableCharacters && line.charAt(nextCharIndex) == quoteChar; boolean nextCharIsEscapeQuoteChar = availableCharacters && line.charAt(nextCharIndex) == quoteEscapeChar; if( nextCharIsQuote ) { charIndex++; currentColumn.append(quoteChar); } else if( nextCharIsEscapeQuoteChar ) { currentColumn.append(c); charIndex++; } else if( quoteEscapeChar == quoteChar ) { state = TokenizerState.NORMAL; quoteScopeStartingLine = -1; } else { currentColumn.append(c); } } else if( c == quoteChar ) { state = TokenizerState.NORMAL; quoteScopeStartingLine = -1; int nextCharIndex = charIndex + 1; boolean availableCharacters = nextCharIndex < line.length(); boolean nextCharIsQuote = availableCharacters && line.charAt(nextCharIndex) == quoteChar; if( quoteEscapeChar != quoteChar && nextCharIsQuote ) { throw new SuperCsvException("Encountered repeat quote char (" + quoteChar + ") when quoteEscapeChar was (" + quoteEscapeChar + ")" + ". Cannot process data where quotes are escaped both with " + quoteChar + " and with " + quoteEscapeChar); } } else { currentColumn.append(c); } } charIndex++; } }
Tokenizer extends AbstractTokenizer { public boolean readColumns(final List<String> columns) throws IOException { if( columns == null ) { throw new NullPointerException("columns should not be null"); } columns.clear(); currentColumn.setLength(0); currentRow.setLength(0); String line; do { line = readLine(); if( line == null ) { return false; } } while( ignoreEmptyLines && line.length() == 0 || (commentMatcher != null && commentMatcher.isComment(line)) ); currentRow.append(line); TokenizerState state = TokenizerState.NORMAL; int quoteScopeStartingLine = -1; int potentialSpaces = 0; int charIndex = 0; while( true ) { boolean endOfLineReached = charIndex == line.length(); if( endOfLineReached ) { if( TokenizerState.NORMAL.equals(state) ) { if( !surroundingSpacesNeedQuotes ) { appendSpaces(currentColumn, potentialSpaces); } addColumn(columns, line, charIndex); return true; } else { currentColumn.append(NEWLINE); currentRow.append(NEWLINE); charIndex = 0; if (maxLinesPerRow > 0 && getLineNumber() - quoteScopeStartingLine + 1 >= maxLinesPerRow) { String msg = maxLinesPerRow == 1 ? String.format("unexpected end of line while reading quoted column on line %d", getLineNumber()) : String.format("max number of lines to read exceeded while reading quoted column" + " beginning on line %d and ending on line %d", quoteScopeStartingLine, getLineNumber()); throw new SuperCsvException(msg); } else if( (line = readLine()) == null ) { throw new SuperCsvException( String .format( "unexpected end of file while reading quoted column beginning on line %d and ending on line %d", quoteScopeStartingLine, getLineNumber())); } currentRow.append(line); if (line.length() == 0){ continue; } } } final char c = line.charAt(charIndex); if( TokenizerState.NORMAL.equals(state) ) { if( c == delimiterChar) { if( !surroundingSpacesNeedQuotes ) { appendSpaces(currentColumn, potentialSpaces); } addColumn(columns, line, charIndex); potentialSpaces = 0; currentColumn.setLength(0); } else if( c == SPACE ) { potentialSpaces++; } else if( c == quoteChar ) { state = TokenizerState.QUOTE_MODE; quoteScopeStartingLine = getLineNumber(); if( !surroundingSpacesNeedQuotes || currentColumn.length() > 0 ) { appendSpaces(currentColumn, potentialSpaces); } potentialSpaces = 0; } else { if( !surroundingSpacesNeedQuotes || currentColumn.length() > 0 ) { appendSpaces(currentColumn, potentialSpaces); } potentialSpaces = 0; currentColumn.append(c); } } else { if( c == quoteEscapeChar ) { int nextCharIndex = charIndex + 1; boolean availableCharacters = nextCharIndex < line.length(); boolean nextCharIsQuote = availableCharacters && line.charAt(nextCharIndex) == quoteChar; boolean nextCharIsEscapeQuoteChar = availableCharacters && line.charAt(nextCharIndex) == quoteEscapeChar; if( nextCharIsQuote ) { charIndex++; currentColumn.append(quoteChar); } else if( nextCharIsEscapeQuoteChar ) { currentColumn.append(c); charIndex++; } else if( quoteEscapeChar == quoteChar ) { state = TokenizerState.NORMAL; quoteScopeStartingLine = -1; } else { currentColumn.append(c); } } else if( c == quoteChar ) { state = TokenizerState.NORMAL; quoteScopeStartingLine = -1; int nextCharIndex = charIndex + 1; boolean availableCharacters = nextCharIndex < line.length(); boolean nextCharIsQuote = availableCharacters && line.charAt(nextCharIndex) == quoteChar; if( quoteEscapeChar != quoteChar && nextCharIsQuote ) { throw new SuperCsvException("Encountered repeat quote char (" + quoteChar + ") when quoteEscapeChar was (" + quoteEscapeChar + ")" + ". Cannot process data where quotes are escaped both with " + quoteChar + " and with " + quoteEscapeChar); } } else { currentColumn.append(c); } } charIndex++; } } }
Tokenizer extends AbstractTokenizer { public boolean readColumns(final List<String> columns) throws IOException { if( columns == null ) { throw new NullPointerException("columns should not be null"); } columns.clear(); currentColumn.setLength(0); currentRow.setLength(0); String line; do { line = readLine(); if( line == null ) { return false; } } while( ignoreEmptyLines && line.length() == 0 || (commentMatcher != null && commentMatcher.isComment(line)) ); currentRow.append(line); TokenizerState state = TokenizerState.NORMAL; int quoteScopeStartingLine = -1; int potentialSpaces = 0; int charIndex = 0; while( true ) { boolean endOfLineReached = charIndex == line.length(); if( endOfLineReached ) { if( TokenizerState.NORMAL.equals(state) ) { if( !surroundingSpacesNeedQuotes ) { appendSpaces(currentColumn, potentialSpaces); } addColumn(columns, line, charIndex); return true; } else { currentColumn.append(NEWLINE); currentRow.append(NEWLINE); charIndex = 0; if (maxLinesPerRow > 0 && getLineNumber() - quoteScopeStartingLine + 1 >= maxLinesPerRow) { String msg = maxLinesPerRow == 1 ? String.format("unexpected end of line while reading quoted column on line %d", getLineNumber()) : String.format("max number of lines to read exceeded while reading quoted column" + " beginning on line %d and ending on line %d", quoteScopeStartingLine, getLineNumber()); throw new SuperCsvException(msg); } else if( (line = readLine()) == null ) { throw new SuperCsvException( String .format( "unexpected end of file while reading quoted column beginning on line %d and ending on line %d", quoteScopeStartingLine, getLineNumber())); } currentRow.append(line); if (line.length() == 0){ continue; } } } final char c = line.charAt(charIndex); if( TokenizerState.NORMAL.equals(state) ) { if( c == delimiterChar) { if( !surroundingSpacesNeedQuotes ) { appendSpaces(currentColumn, potentialSpaces); } addColumn(columns, line, charIndex); potentialSpaces = 0; currentColumn.setLength(0); } else if( c == SPACE ) { potentialSpaces++; } else if( c == quoteChar ) { state = TokenizerState.QUOTE_MODE; quoteScopeStartingLine = getLineNumber(); if( !surroundingSpacesNeedQuotes || currentColumn.length() > 0 ) { appendSpaces(currentColumn, potentialSpaces); } potentialSpaces = 0; } else { if( !surroundingSpacesNeedQuotes || currentColumn.length() > 0 ) { appendSpaces(currentColumn, potentialSpaces); } potentialSpaces = 0; currentColumn.append(c); } } else { if( c == quoteEscapeChar ) { int nextCharIndex = charIndex + 1; boolean availableCharacters = nextCharIndex < line.length(); boolean nextCharIsQuote = availableCharacters && line.charAt(nextCharIndex) == quoteChar; boolean nextCharIsEscapeQuoteChar = availableCharacters && line.charAt(nextCharIndex) == quoteEscapeChar; if( nextCharIsQuote ) { charIndex++; currentColumn.append(quoteChar); } else if( nextCharIsEscapeQuoteChar ) { currentColumn.append(c); charIndex++; } else if( quoteEscapeChar == quoteChar ) { state = TokenizerState.NORMAL; quoteScopeStartingLine = -1; } else { currentColumn.append(c); } } else if( c == quoteChar ) { state = TokenizerState.NORMAL; quoteScopeStartingLine = -1; int nextCharIndex = charIndex + 1; boolean availableCharacters = nextCharIndex < line.length(); boolean nextCharIsQuote = availableCharacters && line.charAt(nextCharIndex) == quoteChar; if( quoteEscapeChar != quoteChar && nextCharIsQuote ) { throw new SuperCsvException("Encountered repeat quote char (" + quoteChar + ") when quoteEscapeChar was (" + quoteEscapeChar + ")" + ". Cannot process data where quotes are escaped both with " + quoteChar + " and with " + quoteEscapeChar); } } else { currentColumn.append(c); } } charIndex++; } } Tokenizer(final Reader reader, final CsvPreference preferences); }
Tokenizer extends AbstractTokenizer { public boolean readColumns(final List<String> columns) throws IOException { if( columns == null ) { throw new NullPointerException("columns should not be null"); } columns.clear(); currentColumn.setLength(0); currentRow.setLength(0); String line; do { line = readLine(); if( line == null ) { return false; } } while( ignoreEmptyLines && line.length() == 0 || (commentMatcher != null && commentMatcher.isComment(line)) ); currentRow.append(line); TokenizerState state = TokenizerState.NORMAL; int quoteScopeStartingLine = -1; int potentialSpaces = 0; int charIndex = 0; while( true ) { boolean endOfLineReached = charIndex == line.length(); if( endOfLineReached ) { if( TokenizerState.NORMAL.equals(state) ) { if( !surroundingSpacesNeedQuotes ) { appendSpaces(currentColumn, potentialSpaces); } addColumn(columns, line, charIndex); return true; } else { currentColumn.append(NEWLINE); currentRow.append(NEWLINE); charIndex = 0; if (maxLinesPerRow > 0 && getLineNumber() - quoteScopeStartingLine + 1 >= maxLinesPerRow) { String msg = maxLinesPerRow == 1 ? String.format("unexpected end of line while reading quoted column on line %d", getLineNumber()) : String.format("max number of lines to read exceeded while reading quoted column" + " beginning on line %d and ending on line %d", quoteScopeStartingLine, getLineNumber()); throw new SuperCsvException(msg); } else if( (line = readLine()) == null ) { throw new SuperCsvException( String .format( "unexpected end of file while reading quoted column beginning on line %d and ending on line %d", quoteScopeStartingLine, getLineNumber())); } currentRow.append(line); if (line.length() == 0){ continue; } } } final char c = line.charAt(charIndex); if( TokenizerState.NORMAL.equals(state) ) { if( c == delimiterChar) { if( !surroundingSpacesNeedQuotes ) { appendSpaces(currentColumn, potentialSpaces); } addColumn(columns, line, charIndex); potentialSpaces = 0; currentColumn.setLength(0); } else if( c == SPACE ) { potentialSpaces++; } else if( c == quoteChar ) { state = TokenizerState.QUOTE_MODE; quoteScopeStartingLine = getLineNumber(); if( !surroundingSpacesNeedQuotes || currentColumn.length() > 0 ) { appendSpaces(currentColumn, potentialSpaces); } potentialSpaces = 0; } else { if( !surroundingSpacesNeedQuotes || currentColumn.length() > 0 ) { appendSpaces(currentColumn, potentialSpaces); } potentialSpaces = 0; currentColumn.append(c); } } else { if( c == quoteEscapeChar ) { int nextCharIndex = charIndex + 1; boolean availableCharacters = nextCharIndex < line.length(); boolean nextCharIsQuote = availableCharacters && line.charAt(nextCharIndex) == quoteChar; boolean nextCharIsEscapeQuoteChar = availableCharacters && line.charAt(nextCharIndex) == quoteEscapeChar; if( nextCharIsQuote ) { charIndex++; currentColumn.append(quoteChar); } else if( nextCharIsEscapeQuoteChar ) { currentColumn.append(c); charIndex++; } else if( quoteEscapeChar == quoteChar ) { state = TokenizerState.NORMAL; quoteScopeStartingLine = -1; } else { currentColumn.append(c); } } else if( c == quoteChar ) { state = TokenizerState.NORMAL; quoteScopeStartingLine = -1; int nextCharIndex = charIndex + 1; boolean availableCharacters = nextCharIndex < line.length(); boolean nextCharIsQuote = availableCharacters && line.charAt(nextCharIndex) == quoteChar; if( quoteEscapeChar != quoteChar && nextCharIsQuote ) { throw new SuperCsvException("Encountered repeat quote char (" + quoteChar + ") when quoteEscapeChar was (" + quoteEscapeChar + ")" + ". Cannot process data where quotes are escaped both with " + quoteChar + " and with " + quoteEscapeChar); } } else { currentColumn.append(c); } } charIndex++; } } Tokenizer(final Reader reader, final CsvPreference preferences); boolean readColumns(final List<String> columns); String getUntokenizedRow(); }
Tokenizer extends AbstractTokenizer { public boolean readColumns(final List<String> columns) throws IOException { if( columns == null ) { throw new NullPointerException("columns should not be null"); } columns.clear(); currentColumn.setLength(0); currentRow.setLength(0); String line; do { line = readLine(); if( line == null ) { return false; } } while( ignoreEmptyLines && line.length() == 0 || (commentMatcher != null && commentMatcher.isComment(line)) ); currentRow.append(line); TokenizerState state = TokenizerState.NORMAL; int quoteScopeStartingLine = -1; int potentialSpaces = 0; int charIndex = 0; while( true ) { boolean endOfLineReached = charIndex == line.length(); if( endOfLineReached ) { if( TokenizerState.NORMAL.equals(state) ) { if( !surroundingSpacesNeedQuotes ) { appendSpaces(currentColumn, potentialSpaces); } addColumn(columns, line, charIndex); return true; } else { currentColumn.append(NEWLINE); currentRow.append(NEWLINE); charIndex = 0; if (maxLinesPerRow > 0 && getLineNumber() - quoteScopeStartingLine + 1 >= maxLinesPerRow) { String msg = maxLinesPerRow == 1 ? String.format("unexpected end of line while reading quoted column on line %d", getLineNumber()) : String.format("max number of lines to read exceeded while reading quoted column" + " beginning on line %d and ending on line %d", quoteScopeStartingLine, getLineNumber()); throw new SuperCsvException(msg); } else if( (line = readLine()) == null ) { throw new SuperCsvException( String .format( "unexpected end of file while reading quoted column beginning on line %d and ending on line %d", quoteScopeStartingLine, getLineNumber())); } currentRow.append(line); if (line.length() == 0){ continue; } } } final char c = line.charAt(charIndex); if( TokenizerState.NORMAL.equals(state) ) { if( c == delimiterChar) { if( !surroundingSpacesNeedQuotes ) { appendSpaces(currentColumn, potentialSpaces); } addColumn(columns, line, charIndex); potentialSpaces = 0; currentColumn.setLength(0); } else if( c == SPACE ) { potentialSpaces++; } else if( c == quoteChar ) { state = TokenizerState.QUOTE_MODE; quoteScopeStartingLine = getLineNumber(); if( !surroundingSpacesNeedQuotes || currentColumn.length() > 0 ) { appendSpaces(currentColumn, potentialSpaces); } potentialSpaces = 0; } else { if( !surroundingSpacesNeedQuotes || currentColumn.length() > 0 ) { appendSpaces(currentColumn, potentialSpaces); } potentialSpaces = 0; currentColumn.append(c); } } else { if( c == quoteEscapeChar ) { int nextCharIndex = charIndex + 1; boolean availableCharacters = nextCharIndex < line.length(); boolean nextCharIsQuote = availableCharacters && line.charAt(nextCharIndex) == quoteChar; boolean nextCharIsEscapeQuoteChar = availableCharacters && line.charAt(nextCharIndex) == quoteEscapeChar; if( nextCharIsQuote ) { charIndex++; currentColumn.append(quoteChar); } else if( nextCharIsEscapeQuoteChar ) { currentColumn.append(c); charIndex++; } else if( quoteEscapeChar == quoteChar ) { state = TokenizerState.NORMAL; quoteScopeStartingLine = -1; } else { currentColumn.append(c); } } else if( c == quoteChar ) { state = TokenizerState.NORMAL; quoteScopeStartingLine = -1; int nextCharIndex = charIndex + 1; boolean availableCharacters = nextCharIndex < line.length(); boolean nextCharIsQuote = availableCharacters && line.charAt(nextCharIndex) == quoteChar; if( quoteEscapeChar != quoteChar && nextCharIsQuote ) { throw new SuperCsvException("Encountered repeat quote char (" + quoteChar + ") when quoteEscapeChar was (" + quoteEscapeChar + ")" + ". Cannot process data where quotes are escaped both with " + quoteChar + " and with " + quoteEscapeChar); } } else { currentColumn.append(c); } } charIndex++; } } Tokenizer(final Reader reader, final CsvPreference preferences); boolean readColumns(final List<String> columns); String getUntokenizedRow();  }