pub(crate) fn infer_file_schema<R: Read + Seek>(
    reader: &mut R,
    delimiter: u8,
    max_read_records: Option<usize>,
    has_header: bool,
) -> Result<(Schema, usize)> {
    // We use lossy utf8 here because we don't want the schema inference to fail on utf8.
    // It may later.
    let encoding = CsvEncoding::LossyUtf8;
    // set headers to false otherwise the csv crate, skips them.
    let csv_reader = init_csv_reader(reader, false, delimiter);

    let mut records = csv_reader.into_byte_records();
    let header_length;

    // get or create header names
    // when has_header is false, creates default column names with column_ prefix
    let headers: Vec<String> = if let Some(byterecord) = records.next() {
        let byterecord = byterecord.map_err(anyhow::Error::from)?;
        header_length = byterecord.len();
        if has_header {
            byterecord
                .iter()
                .map(|slice| {
                    let s = parse_bytes_with_encoding(slice, encoding)?;
                    Ok(s.into())
                })
                .collect::<Result<_>>()?
        } else {
            (0..header_length)
                .map(|i| format!("column_{}", i + 1))
                .collect()
        }
    } else {
        return Err(PolarsError::NoData("empty csv".into()));
    };

    // keep track of inferred field types
    let mut column_types: Vec<HashSet<ArrowDataType, RandomState>> =
        vec![HashSet::with_hasher(RandomState::new()); header_length];
    // keep track of columns with nulls
    let mut nulls: Vec<bool> = vec![false; header_length];

    let mut records_count = 0;
    let mut fields = Vec::with_capacity(header_length);

    // needed to prevent ownership going into the iterator loop
    let records_ref = &mut records;

    for result in records_ref.take(max_read_records.unwrap_or(std::usize::MAX)) {
        let record = result.map_err(anyhow::Error::from)?;
        records_count += 1;

        for i in 0..header_length {
            if let Some(slice) = record.get(i) {
                if slice.is_empty() {
                    nulls[i] = true;
                } else {
                    let s = parse_bytes_with_encoding(slice, encoding)?;
                    column_types[i].insert(infer_field_schema(&s));
                }
            }
        }
    }

    // build schema from inference results
    for i in 0..header_length {
        let possibilities = &column_types[i];
        let has_nulls = nulls[i];
        let field_name = &headers[i];

        // determine data type based on possible types
        // if there are incompatible types, use DataType::Utf8
        match possibilities.len() {
            1 => {
                for dtype in possibilities.iter() {
                    fields.push(Field::new(&field_name, dtype.clone(), has_nulls));
                }
            }
            2 => {
                if possibilities.contains(&ArrowDataType::Int64)
                    && possibilities.contains(&ArrowDataType::Float64)
                {
                    // we have an integer and double, fall down to double
                    fields.push(Field::new(&field_name, ArrowDataType::Float64, has_nulls));
                } else {
                    // default to Utf8 for conflicting datatypes (e.g bool and int)
                    fields.push(Field::new(&field_name, ArrowDataType::Utf8, has_nulls));
                }
            }
            _ => fields.push(Field::new(&field_name, ArrowDataType::Utf8, has_nulls)),
        }
    }
    let csv_reader = records.into_reader();

    // return the reader seek back to the start
    csv_reader.into_inner().seek(SeekFrom::Start(0))?;

    Ok((Schema::new(fields), records_count))
}
/// Infer the schema of a CSV file by reading through the first n records of the file,
/// with `max_read_records` controlling the maximum number of records to read.
///
/// If `max_read_records` is not set, the whole file is read to infer its schema.
///
/// Return infered schema and number of records used for inference.
Infer the schema of a CSV file by reading through the first n records of the file,
with `max_read_records` controlling the maximum number of records to read.

If `max_read_records` is not set, the whole file is read to infer its schema.

Return infered schema and number of records used for inference.
Infer the schema of a CSV file by reading through the first n records of the file with ` max_read_records ` controlling the maximum number of records to read . If ` max_read_records ` is not set the whole file is read to infer its schema . Return infered schema and number of records used for inference .
pub(crate) fn infer_file_schema<R: Read + Seek>(
    reader: &mut R,
    delimiter: u8,
    max_read_records: Option<usize>,
    has_header: bool,
) -> Result<(Schema, usize)> {
    // We use lossy utf8 here because we don't want the schema inference to fail on utf8.
    // It may later.
    let encoding = CsvEncoding::LossyUtf8;
    // set headers to false otherwise the csv crate, skips them.
    let csv_reader = init_csv_reader(reader, false, delimiter);
    let mut records = csv_reader.into_byte_records();
    let header_length;
    // get or create header names
    // when has_header is false, creates default column names with column_ prefix
    let headers: Vec<String> = if let Some(byterecord) = records.next() {
        let byterecord = byterecord.map_err(anyhow::Error::from)?;
        header_length = byterecord.len();
        if has_header {
            byterecord
                .iter()
                .map(|slice| {
                    let s = parse_bytes_with_encoding(slice, encoding)?;
                    Ok(s.into())
                })
                .collect::<Result<_>>()?
        } else {
            (0..header_length)
                .map(|i| format!("column_{}", i + 1))
                .collect()
        }
    } else {
        return Err(PolarsError::NoData("empty csv".into()));
    };
    // keep track of inferred field types
    let mut column_types: Vec<HashSet<ArrowDataType, RandomState>> =
        vec![HashSet::with_hasher(RandomState::new()); header_length];
    // keep track of columns with nulls
    let mut nulls: Vec<bool> = vec![false; header_length];
    let mut records_count = 0;
    let mut fields = Vec::with_capacity(header_length);
    // needed to prevent ownership going into the iterator loop
    let records_ref = &mut records;
    for result in records_ref.take(max_read_records.unwrap_or(std::usize::MAX)) {
        let record = result.map_err(anyhow::Error::from)?;
        records_count += 1;
        for i in 0..header_length {
            if let Some(slice) = record.get(i) {
                if slice.is_empty() {
                    nulls[i] = true;
                } else {
                    let s = parse_bytes_with_encoding(slice, encoding)?;
                    column_types[i].insert(infer_field_schema(&s));
                }
            }
        }
    }
    // build schema from inference results
    for i in 0..header_length {
        let possibilities = &column_types[i];
        let has_nulls = nulls[i];
        let field_name = &headers[i];
        // determine data type based on possible types
        // if there are incompatible types, use DataType::Utf8
        match possibilities.len() {
            1 => {
                for dtype in possibilities.iter() {
                    fields.push(Field::new(&field_name, dtype.clone(), has_nulls));
                }
            }
            2 => {
                if possibilities.contains(&ArrowDataType::Int64)
                    && possibilities.contains(&ArrowDataType::Float64)
                {
                    // we have an integer and double, fall down to double
                    fields.push(Field::new(&field_name, ArrowDataType::Float64, has_nulls));
                } else {
                    // default to Utf8 for conflicting datatypes (e.g bool and int)
                    fields.push(Field::new(&field_name, ArrowDataType::Utf8, has_nulls));
                }
            }
            _ => fields.push(Field::new(&field_name, ArrowDataType::Utf8, has_nulls)),
        }
    }
    let csv_reader = records.into_reader();
    // return the reader seek back to the start
    csv_reader.into_inner().seek(SeekFrom::Start(0))?;
    Ok((Schema::new(fields), records_count))
}
pub ( crate ) fn infer_file_schema < R : Read + Seek > ( reader : & mut R , delimiter : u8 , max_read_records : Option < usize > , has_header : bool , ) -> Result < ( Schema , usize ) > { // We use lossy utf8 here because we don't want the schema inference to fail on utf8. // It may later. let encoding = CsvEncoding :: LossyUtf8 ; // set headers to false otherwise the csv crate, skips them. let csv_reader = init_csv_reader ( reader , false , delimiter ) ; let mut records = csv_reader . into_byte_records ( ) ; let header_length ; // get or create header names // when has_header is false, creates default column names with column_ prefix let headers : Vec < String > = if let Some ( byterecord ) = records . next ( ) { let byterecord = byterecord . map_err ( anyhow :: Error :: from ) ? ; header_length = byterecord . len ( ) ; if has_header { byterecord . iter ( ) . map ( | slice | { let s = parse_bytes_with_encoding ( slice , encoding ) ? ; Ok ( s . into ( ) ) } ) . collect :: < Result < _ > > ( ) ? } else { ( 0 .. header_length ) . map ( | i | format ! ( " " i 1 ) ) . collect ( ) } } else { return Err ( PolarsError :: NoData ( " " . into ( ) ) ) ; } ; // keep track of inferred field types let mut column_types : Vec < HashSet < ArrowDataType , RandomState > > = vec ! [ HashSet with_hasher ( RandomState new ( ) ) header_length ] ; // keep track of columns with nulls let mut nulls : Vec < bool > = vec ! [ false header_length ] ; let mut records_count = 0 ; let mut fields = Vec :: with_capacity ( header_length ) ; // needed to prevent ownership going into the iterator loop let records_ref = & mut records ; for result in records_ref . take ( max_read_records . unwrap_or ( std :: usize :: MAX ) ) { let record = result . map_err ( anyhow :: Error :: from ) ? ; records_count += 1 ; for i in 0 .. header_length { if let Some ( slice ) = record . get ( i ) { if slice . is_empty ( ) { nulls [ i ] = true ; } else { let s = parse_bytes_with_encoding ( slice , encoding ) ? ; column_types [ i ] . insert ( infer_field_schema ( & s ) ) ; } } } } // build schema from inference results for i in 0 .. header_length { let possibilities = & column_types [ i ] ; let has_nulls = nulls [ i ] ; let field_name = & headers [ i ] ; // determine data type based on possible types // if there are incompatible types, use DataType::Utf8 match possibilities . len ( ) { 1 => { for dtype in possibilities . iter ( ) { fields . push ( Field :: new ( & field_name , dtype . clone ( ) , has_nulls ) ) ; } } 2 => { if possibilities . contains ( & ArrowDataType :: Int64 ) && possibilities . contains ( & ArrowDataType :: Float64 ) { // we have an integer and double, fall down to double fields . push ( Field :: new ( & field_name , ArrowDataType :: Float64 , has_nulls ) ) ; } else { // default to Utf8 for conflicting datatypes (e.g bool and int) fields . push ( Field :: new ( & field_name , ArrowDataType :: Utf8 , has_nulls ) ) ; } } _ => fields . push ( Field :: new ( & field_name , ArrowDataType :: Utf8 , has_nulls ) ) , } } let csv_reader = records . into_reader ( ) ; // return the reader seek back to the start csv_reader . into_inner ( ) . seek ( SeekFrom :: Start ( 0 ) ) ? ; Ok ( ( Schema :: new ( fields ) , records_count ) ) }
Infer the schema of a CSV file by reading through the first n records of the file,
with `max_read_records` controlling the maximum number of records to read.
Infer the schema of a CSV file by reading through the first n records of the file with ` max_read_records ` controlling the maximum number of records to read .
reader
&mut R
delimiter
u8
max_read_records
Option<usize>
has_header
bool
reader
&mut R
delimiter
u8
max_read_records
Option<usize>
has_header
bool