pub fn new(
        paths: Vec<PathBuf>,
        nthreads: usize,
        delimiter: u8,
    ) -> Result<SvmScanner, Box<dyn Error>> {
        // TODO: panic if repeat path

        // Blocks determine the number of atomic chunks to divide inputs into.
        // To minimize the number of merges, this should not be significantly
        // more than nthreads. To minimize wasted time from work imbalance,
        // this should be a little larger than `nthreads`. Note that the user
        let nblocks = nthreads * 4;

        // Consider reading whole thing into memory with ByteSlice::from_path.
        let mmaps = paths
            .iter()
            .map(|path: &PathBuf| -> Result<_, Box<dyn Error>> {
                let file = File::open(path)?;
                // SAFETY: caller must guarantee this file is not
                // mutated during use, per method documentation.
                // https://users.rust-lang.org/t/how-unsafe-is-mmap/19635/24
                Ok(unsafe { MmapOptions::new().map(&file)? })
            })
            .collect::<Result<Vec<_>, Box<dyn Error>>>()?;
        let file_lengths: Vec<_> = mmaps.iter().map(|mmap| mmap.len()).collect();
        let tot_bytes = file_lengths.iter().sum();
        let nblocks = nblocks.min(tot_bytes / 1024 / 1024).max(1);

        // Compute block start offsets by starting with evenly spaced offsets,
        // referring to a global offset of bytes along the entire list of files.
        let mut offsets: Vec<_> = (0..nblocks).map(|b| b * tot_bytes / nblocks).collect();

        // To make sure we respect file boundaries, throw in all the file offsets.
        let file_offsets: Vec<_> = iter::once(0)
            .chain(file_lengths.iter().scan(0, |state, &flen| {
                *state += flen;
                Some(*state)
            }))
            .collect();
        offsets.extend(file_offsets.iter());
        offsets.sort_unstable();

        // drop the ending global offset (multiple if last few files are empty)
        if let Some(pos) = offsets.iter().position(|&o| o == tot_bytes) {
            offsets.truncate(pos);
        }

        // Compute global offsets, local file offsets, and the file (mmap) index
        // for each proposed offset.
        //
        // Shifts offsets so they point to the character directly after the
        // preceding newline or beginning-of-file, whichever is closer.
        let (global, local_file): (Vec<usize>, Vec<(usize, usize)>) = offsets
            .into_iter()
            .scan(0, |mmap_index, global_offset| {
                if global_offset >= file_offsets[*mmap_index + 1] {
                    *mmap_index += 1;
                };
                let local_offset = global_offset - file_offsets[*mmap_index];
                let shifted_local = start_of_line(local_offset, &mmaps[*mmap_index]);
                let global_offset = global_offset + shifted_local - local_offset;
                Some((global_offset, (shifted_local, *mmap_index)))
            })
            .unzip();

        // Figure out block lengths based on global offsets.
        let block_lengths_rev: Vec<_> = global
            .into_iter()
            .rev()
            .scan(tot_bytes, |state: &mut usize, offset| {
                let blen = *state - offset;
                *state -= blen;
                Some(blen)
            })
            .collect();

        let blocks: Vec<_> = block_lengths_rev
            .into_iter()
            .rev()
            .zip(local_file.into_iter())
            .flat_map(|(blen, (local_offset, mmap_index))| {
                if blen == 0 {
                    None
                } else {
                    Some(ByteBlock {
                        mmap_index,
                        mmap_offset: local_offset,
                        length: blen,
                    })
                }
            })
            .collect();

        assert!(nthreads > 1); // mpsc strategy doesn't work otherwise
        Ok(SvmScanner {
            paths,
            mmaps,
            blocks,
            delimiter,
            nthreads,
        })
    }
/// `nthreads` is the desired parallelism level for the processing code
/// (and should align with the default `rayon` pool used during processing).
///
/// Internally, this parameter affects the number of blocks used.
`nthreads` is the desired parallelism level for the processing code
(and should align with the default `rayon` pool used during processing).

Internally, this parameter affects the number of blocks used.
` nthreads ` is the desired parallelism level for the processing code ( and should align with the default ` rayon ` pool used during processing ) . Internally this parameter affects the number of blocks used .
pub fn new(
        paths: Vec<PathBuf>,
        nthreads: usize,
        delimiter: u8,
    ) -> Result<SvmScanner, Box<dyn Error>> {
        // TODO: panic if repeat path
        // Blocks determine the number of atomic chunks to divide inputs into.
        // To minimize the number of merges, this should not be significantly
        // more than nthreads. To minimize wasted time from work imbalance,
        // this should be a little larger than `nthreads`. Note that the user
        let nblocks = nthreads * 4;
        // Consider reading whole thing into memory with ByteSlice::from_path.
        let mmaps = paths
            .iter()
            .map(|path: &PathBuf| -> Result<_, Box<dyn Error>> {
                let file = File::open(path)?;
                // SAFETY: caller must guarantee this file is not
                // mutated during use, per method documentation.
                // https://users.rust-lang.org/t/how-unsafe-is-mmap/19635/24
                Ok(unsafe { MmapOptions::new().map(&file)? })
            })
            .collect::<Result<Vec<_>, Box<dyn Error>>>()?;
        let file_lengths: Vec<_> = mmaps.iter().map(|mmap| mmap.len()).collect();
        let tot_bytes = file_lengths.iter().sum();
        let nblocks = nblocks.min(tot_bytes / 1024 / 1024).max(1);
        // Compute block start offsets by starting with evenly spaced offsets,
        // referring to a global offset of bytes along the entire list of files.
        let mut offsets: Vec<_> = (0..nblocks).map(|b| b * tot_bytes / nblocks).collect();
        // To make sure we respect file boundaries, throw in all the file offsets.
        let file_offsets: Vec<_> = iter::once(0)
            .chain(file_lengths.iter().scan(0, |state, &flen| {
                *state += flen;
                Some(*state)
            }))
            .collect();
        offsets.extend(file_offsets.iter());
        offsets.sort_unstable();
        // drop the ending global offset (multiple if last few files are empty)
        if let Some(pos) = offsets.iter().position(|&o| o == tot_bytes) {
            offsets.truncate(pos);
        }
        // Compute global offsets, local file offsets, and the file (mmap) index
        // for each proposed offset.
        //
        // Shifts offsets so they point to the character directly after the
        // preceding newline or beginning-of-file, whichever is closer.
        let (global, local_file): (Vec<usize>, Vec<(usize, usize)>) = offsets
            .into_iter()
            .scan(0, |mmap_index, global_offset| {
                if global_offset >= file_offsets[*mmap_index + 1] {
                    *mmap_index += 1;
                };
                let local_offset = global_offset - file_offsets[*mmap_index];
                let shifted_local = start_of_line(local_offset, &mmaps[*mmap_index]);
                let global_offset = global_offset + shifted_local - local_offset;
                Some((global_offset, (shifted_local, *mmap_index)))
            })
            .unzip();
        // Figure out block lengths based on global offsets.
        let block_lengths_rev: Vec<_> = global
            .into_iter()
            .rev()
            .scan(tot_bytes, |state: &mut usize, offset| {
                let blen = *state - offset;
                *state -= blen;
                Some(blen)
            })
            .collect();
        let blocks: Vec<_> = block_lengths_rev
            .into_iter()
            .rev()
            .zip(local_file.into_iter())
            .flat_map(|(blen, (local_offset, mmap_index))| {
                if blen == 0 {
                    None
                } else {
                    Some(ByteBlock {
                        mmap_index,
                        mmap_offset: local_offset,
                        length: blen,
                    })
                }
            })
            .collect();
        assert!(nthreads > 1); // mpsc strategy doesn't work otherwise
        Ok(SvmScanner {
            paths,
            mmaps,
            blocks,
            delimiter,
            nthreads,
        })
    }
pub fn new ( paths : Vec < PathBuf > , nthreads : usize , delimiter : u8 , ) -> Result < SvmScanner , Box < dyn Error > > { // TODO: panic if repeat path // Blocks determine the number of atomic chunks to divide inputs into. // To minimize the number of merges, this should not be significantly // more than nthreads. To minimize wasted time from work imbalance, // this should be a little larger than `nthreads`. Note that the user let nblocks = nthreads * 4 ; // Consider reading whole thing into memory with ByteSlice::from_path. let mmaps = paths . iter ( ) . map ( | path : & PathBuf | -> Result < _ , Box < dyn Error > > { let file = File :: open ( path ) ? ; // SAFETY: caller must guarantee this file is not // mutated during use, per method documentation. // https://users.rust-lang.org/t/how-unsafe-is-mmap/19635/24 Ok ( unsafe { MmapOptions :: new ( ) . map ( & file ) ? } ) } ) . collect :: < Result < Vec < _ > , Box < dyn Error > > > ( ) ? ; let file_lengths : Vec < _ > = mmaps . iter ( ) . map ( | mmap | mmap . len ( ) ) . collect ( ) ; let tot_bytes = file_lengths . iter ( ) . sum ( ) ; let nblocks = nblocks . min ( tot_bytes / 1024 / 1024 ) . max ( 1 ) ; // Compute block start offsets by starting with evenly spaced offsets, // referring to a global offset of bytes along the entire list of files. let mut offsets : Vec < _ > = ( 0 .. nblocks ) . map ( | b | b * tot_bytes / nblocks ) . collect ( ) ; // To make sure we respect file boundaries, throw in all the file offsets. let file_offsets : Vec < _ > = iter :: once ( 0 ) . chain ( file_lengths . iter ( ) . scan ( 0 , | state , & flen | { * state += flen ; Some ( * state ) } ) ) . collect ( ) ; offsets . extend ( file_offsets . iter ( ) ) ; offsets . sort_unstable ( ) ; // drop the ending global offset (multiple if last few files are empty) if let Some ( pos ) = offsets . iter ( ) . position ( | & o | o == tot_bytes ) { offsets . truncate ( pos ) ; } // Compute global offsets, local file offsets, and the file (mmap) index // for each proposed offset. // // Shifts offsets so they point to the character directly after the // preceding newline or beginning-of-file, whichever is closer. let ( global , local_file ) : ( Vec < usize > , Vec < ( usize , usize ) > ) = offsets . into_iter ( ) . scan ( 0 , | mmap_index , global_offset | { if global_offset >= file_offsets [ * mmap_index + 1 ] { * mmap_index += 1 ; } ; let local_offset = global_offset - file_offsets [ * mmap_index ] ; let shifted_local = start_of_line ( local_offset , & mmaps [ * mmap_index ] ) ; let global_offset = global_offset + shifted_local - local_offset ; Some ( ( global_offset , ( shifted_local , * mmap_index ) ) ) } ) . unzip ( ) ; // Figure out block lengths based on global offsets. let block_lengths_rev : Vec < _ > = global . into_iter ( ) . rev ( ) . scan ( tot_bytes , | state : & mut usize , offset | { let blen = * state - offset ; * state -= blen ; Some ( blen ) } ) . collect ( ) ; let blocks : Vec < _ > = block_lengths_rev . into_iter ( ) . rev ( ) . zip ( local_file . into_iter ( ) ) . flat_map ( | ( blen , ( local_offset , mmap_index ) ) | { if blen == 0 { None } else { Some ( ByteBlock { mmap_index , mmap_offset : local_offset , length : blen , } ) } } ) . collect ( ) ; assert ! ( nthreads 1 ) ; // mpsc strategy doesn't work otherwise Ok ( SvmScanner { paths , mmaps , blocks , delimiter , nthreads , } ) }
`nthreads` is the desired parallelism level for the processing code
(and should align with the default `rayon` pool used during processing).
` nthreads ` is the desired parallelism level for the processing code ( and should align with the default ` rayon ` pool used during processing ) .
paths
Vec<PathBuf>
nthreads
usize
delimiter
u8
paths
Vec<PathBuf>
nthreads
usize
delimiter
u8