chore(format): do not need size to init parquet file reader.
@@ -86,7 +86,7 @@ impl InputFormat for InputFormatParquet {
for path in files {
let obj = op.object(path);
let size = obj.metadata().await?.content_length() as usize;
- let mut reader = obj.seekable_reader(..(size as u64));
+ let mut reader = obj.seekable_reader(..);
let mut file_meta = read_metadata_async(&mut reader).await?;
let row_groups = mem::take(&mut file_meta.row_groups);
let infer_schema = infer_schema(&file_meta)?;
@@ -131,8 +131,7 @@ impl InputFormat for InputFormatParquet {
async fn infer_schema(&self, path: &str, op: &Operator) -> Result<DataSchema> {
let obj = op.object(path);
- let size = obj.content_length().await?;
- let mut reader = obj.seekable_reader(..size);
+ let mut reader = obj.seekable_reader(..);
let file_meta = read_metadata_async(&mut reader).await?;
let arrow_schema = infer_schema(&file_meta)?;
Ok(DataSchema::from(arrow_schema))
@@ -164,7 +163,7 @@ impl InputFormatPipe for ParquetFormatPipe {
let meta = Self::get_split_meta(split_info).expect("must success");
let op = ctx.source.get_operator()?;
let obj = op.object(&split_info.file.path);
- let mut reader = obj.seekable_reader(..(split_info.file.size as u64));
+ let mut reader = obj.seekable_reader(..);
let input_fields = Arc::new(get_used_fields(&meta.file.fields, &ctx.schema)?);
RowGroupInMemory::read_async(&mut reader, meta.meta.clone(), input_fields).await
}
