fix csv find_quote
@@ -20,7 +20,6 @@ use common_datavalues::DataType;
use common_datavalues::TypeDeserializer;
use common_exception::ErrorCode;
use common_exception::Result;
-use common_io::prelude::position2;
use common_io::prelude::position4;
use common_io::prelude::BufferReadExt;
use common_io::prelude::FormatSettings;
@@ -119,14 +118,13 @@ impl CsvInputFormat {
}))
}
- fn find_quotes(buf: &[u8], pos: usize, state: &mut CsvInputState) -> usize {
- let index = pos + position2::<true, b'"', b'\''>(&buf[pos..]);
-
- if index != buf.len() {
- state.quotes = 0;
+ fn find_quote(buf: &[u8], pos: usize, state: &mut CsvInputState, quote: u8) -> usize {
+ for (index, item) in buf.iter().enumerate().skip(pos) {
+ if *item == quote {
+ state.quotes = *item;
return index + 1;
}
-
+ }
buf.len()
}
@@ -150,11 +148,10 @@ impl CsvInputFormat {
b'"' | b'\'' => {
state.quotes = buf[position];
position + 1
- },
- b'\n' => {
- self.accept_row::<0>(buf, pos, state, position)
- },
- _ => { // b'\r'
+ }
+ b'\n' => self.accept_row::<0>(buf, pos, state, position),
+ _ => {
+ // b'\r'
self.accept_row::<b'\n'>(buf, pos, state, position)
}
};
@@ -288,7 +285,7 @@ impl InputFormat for CsvInputFormat {
state.need_more_data = true;
while index < buf.len() && state.need_more_data {
index = match state.quotes != 0 {
- true => Self::find_quotes(buf, index, state),
+ true => Self::find_quote(buf, index, state, state.quotes),
false => self.find_delimiter(buf, index, state),
}
}
@@ -304,7 +301,7 @@ impl InputFormat for CsvInputFormat {
while index < buf.len() {
index = match state.quotes != 0 {
- true => Self::find_quotes(buf, index, state),
+ true => Self::find_quote(buf, index, state, state.quotes),
false => self.find_delimiter(buf, index, state),
};
