pub fn finish(self, df: &DataFrame) -> Result<()> {
        let df = to_arrow_compatible_df(df);
        let mut fields = df.schema().to_arrow().fields().clone();

        // date64 is not supported by parquet and will be be truncated to date32
        // We coerce these to timestamp(ms)
        let date64_columns = df
            .get_columns()
            .iter()
            .enumerate()
            .filter_map(|(i, s)| match s.dtype() {
                DataType::Date64 => {
                    fields[i] = ArrowField::new(
                        s.name(),
                        ArrowDataType::Timestamp(TimeUnit::Millisecond, None),
                        s.null_count() > 0,
                    );
                    Some(i)
                }
                _ => None,
            })
            .collect::<Vec<_>>();
        let column_names = df
            .get_columns()
            .iter()
            .map(|s| s.name().to_string())
            .collect::<Vec<_>>();

        let iter = df.iter_record_batches().map(|rb| {
            if !date64_columns.is_empty() {
                let mut columns = rb.columns().to_vec();
                for i in &date64_columns {
                    let array = cast(&columns[*i], &ArrowDataType::Int64).unwrap();
                    let array = cast(
                        &array,
                        &ArrowDataType::Timestamp(TimeUnit::Millisecond, None),
                    )
                    .unwrap();
                    columns[*i] = array;
                }
                RecordBatch::try_from_iter(column_names.iter().zip(columns)).unwrap()
            } else {
                rb
            }
        });

        let mut parquet_writer =
            ParquetArrowWriter::try_new(self.writer, Arc::new(ArrowSchema::new(fields)), None)?;

        for batch in iter {
            parquet_writer.write(&batch)?
        }
        let _ = parquet_writer.close()?;
        Ok(())
    }
/// Write the given DataFrame in the the writer `W`.
Write the given DataFrame in the the writer `W`.
Write the given DataFrame in the the writer ` W ` .
pub fn finish(self, df: &DataFrame) -> Result<()> {
        let df = to_arrow_compatible_df(df);
        let mut fields = df.schema().to_arrow().fields().clone();
        // date64 is not supported by parquet and will be be truncated to date32
        // We coerce these to timestamp(ms)
        let date64_columns = df
            .get_columns()
            .iter()
            .enumerate()
            .filter_map(|(i, s)| match s.dtype() {
                DataType::Date64 => {
                    fields[i] = ArrowField::new(
                        s.name(),
                        ArrowDataType::Timestamp(TimeUnit::Millisecond, None),
                        s.null_count() > 0,
                    );
                    Some(i)
                }
                _ => None,
            })
            .collect::<Vec<_>>();
        let column_names = df
            .get_columns()
            .iter()
            .map(|s| s.name().to_string())
            .collect::<Vec<_>>();
        let iter = df.iter_record_batches().map(|rb| {
            if !date64_columns.is_empty() {
                let mut columns = rb.columns().to_vec();
                for i in &date64_columns {
                    let array = cast(&columns[*i], &ArrowDataType::Int64).unwrap();
                    let array = cast(
                        &array,
                        &ArrowDataType::Timestamp(TimeUnit::Millisecond, None),
                    )
                    .unwrap();
                    columns[*i] = array;
                }
                RecordBatch::try_from_iter(column_names.iter().zip(columns)).unwrap()
            } else {
                rb
            }
        });
        let mut parquet_writer =
            ParquetArrowWriter::try_new(self.writer, Arc::new(ArrowSchema::new(fields)), None)?;
        for batch in iter {
            parquet_writer.write(&batch)?
        }
        let _ = parquet_writer.close()?;
        Ok(())
    }
pub fn finish ( self , df : & DataFrame ) -> Result < ( ) > { let df = to_arrow_compatible_df ( df ) ; let mut fields = df . schema ( ) . to_arrow ( ) . fields ( ) . clone ( ) ; // date64 is not supported by parquet and will be be truncated to date32 // We coerce these to timestamp(ms) let date64_columns = df . get_columns ( ) . iter ( ) . enumerate ( ) . filter_map ( | ( i , s ) | match s . dtype ( ) { DataType :: Date64 => { fields [ i ] = ArrowField :: new ( s . name ( ) , ArrowDataType :: Timestamp ( TimeUnit :: Millisecond , None ) , s . null_count ( ) > 0 , ) ; Some ( i ) } _ => None , } ) . collect :: < Vec < _ > > ( ) ; let column_names = df . get_columns ( ) . iter ( ) . map ( | s | s . name ( ) . to_string ( ) ) . collect :: < Vec < _ > > ( ) ; let iter = df . iter_record_batches ( ) . map ( | rb | { if ! date64_columns . is_empty ( ) { let mut columns = rb . columns ( ) . to_vec ( ) ; for i in & date64_columns { let array = cast ( & columns [ * i ] , & ArrowDataType :: Int64 ) . unwrap ( ) ; let array = cast ( & array , & ArrowDataType :: Timestamp ( TimeUnit :: Millisecond , None ) , ) . unwrap ( ) ; columns [ * i ] = array ; } RecordBatch :: try_from_iter ( column_names . iter ( ) . zip ( columns ) ) . unwrap ( ) } else { rb } } ) ; let mut parquet_writer = ParquetArrowWriter :: try_new ( self . writer , Arc :: new ( ArrowSchema :: new ( fields ) ) , None ) ? ; for batch in iter { parquet_writer . write ( & batch ) ? } let _ = parquet_writer . close ( ) ? ; Ok ( ( ) ) }
Write the given DataFrame in the the writer `W`.
Write the given DataFrame in the the writer ` W ` .
self
df
&DataFrame
self
df
&DataFrame