fn build_chunk_table_sequential(
    metrics: &ManifestMetrics,
    log: &ReplicaLogger,
    root: &Path,
    files: Vec<FileWithSize>,
    max_chunk_size: u32,
    chunk_actions: Vec<ChunkAction>,
) -> (Vec<FileInfo>, Vec<ChunkInfo>) {
    let mut chunk_table = Vec::new();
    let mut file_table = Vec::new();
    let mut chunk_index: usize = 0;

    for (file_index, FileWithSize(relative_path, size_bytes)) in files.into_iter().enumerate() {
        let mut file_hash = if relative_path.ends_with("state_file") {
            cow_file_hasher()
        } else {
            file_hasher()
        };

        let mut bytes_left = size_bytes;

        let num_chunks = count_chunks(size_bytes, max_chunk_size);

        (num_chunks as u32).update_hash(&mut file_hash);

        let compute_file_chunk_hashes = |data: &[u8]| {
            // It's OK to not have any chunks for 0-sized files (though it's unlikely that
            // we have any).
            while bytes_left > 0 {
                let chunk_size = bytes_left.min(max_chunk_size as u64);
                let offset = size_bytes - bytes_left;

                let recompute_chunk_hash = || {
                    let mut hasher = if relative_path.ends_with("state_file") {
                        cow_chunk_hasher()
                    } else {
                        chunk_hasher()
                    };
                    hasher.write(&data[offset as usize..(offset + chunk_size) as usize]);
                    hasher.finish()
                };

                assert!(chunk_index < chunk_actions.len());

                let chunk_hash = match chunk_actions[chunk_index] {
                    ChunkAction::RecomputeAndCompare(reused_chunk_hash) => {
                        metrics
                            .chunk_bytes
                            .with_label_values(&[LABEL_VALUE_HASHED_AND_COMPARED])
                            .inc_by(chunk_size);

                        // We have both a reused and a recomputed hash, so we can compare them to
                        // monitor for issues
                        let recomputed_chunk_hash = recompute_chunk_hash();
                        debug_assert_eq!(recomputed_chunk_hash, reused_chunk_hash);
                        if recomputed_chunk_hash != reused_chunk_hash {
                            metrics.reused_chunk_hash_error_count.inc();
                            error!(
                                log,
                                "{}: Hash mismatch in chunk with index {} in file {}, recomputed hash {:?}, reused hash {:?}",
                                CRITICAL_ERROR_REUSED_CHUNK_HASH,
                                chunk_index,
                                relative_path.display(),
                                recomputed_chunk_hash,
                                reused_chunk_hash
                            );
                        }
                        recomputed_chunk_hash
                    }
                    ChunkAction::UseHash(reused_chunk_hash) => {
                        metrics
                            .chunk_bytes
                            .with_label_values(&[LABEL_VALUE_REUSED])
                            .inc_by(chunk_size);
                        reused_chunk_hash
                    }
                    ChunkAction::Recompute => {
                        metrics
                            .chunk_bytes
                            .with_label_values(&[LABEL_VALUE_REUSED])
                            .inc_by(chunk_size);
                        recompute_chunk_hash()
                    }
                };

                let chunk_info = ChunkInfo {
                    file_index: file_index as u32,
                    size_bytes: chunk_size as u32,
                    offset: offset as u64,
                    hash: chunk_hash,
                };

                write_chunk_hash(&mut file_hash, &chunk_info);

                chunk_table.push(chunk_info);

                bytes_left -= chunk_size;
                chunk_index += 1;
            }

            file_table.push(FileInfo {
                relative_path: relative_path.clone(),
                size_bytes,
                hash: file_hash.finish(),
            });
        };

        if relative_path.ends_with("state_file") {
            let absolute_path = root.join(&relative_path);
            let cow_base_dir = absolute_path.parent().unwrap();
            let cow_mgr = CowMemoryManagerImpl::open_readonly(cow_base_dir.to_path_buf());
            let mapped_state = cow_mgr.get_map();
            mapped_state.make_heap_accessible();
            let data = unsafe {
                std::slice::from_raw_parts(mapped_state.get_heap_base(), size_bytes as usize)
            };
            compute_file_chunk_hashes(data);
        } else {
            let mmap =
                ScopedMmap::from_path(root.join(&relative_path)).expect("failed to open file");
            let data = mmap.as_slice();
            compute_file_chunk_hashes(data);
        };
    }

    assert_eq!(chunk_table.len(), chunk_actions.len());

    (file_table, chunk_table)
}
/// Build a chunk table from the file table.
Build a chunk table from the file table.
Build a chunk table from the file table .
fn build_chunk_table_sequential(
    metrics: &ManifestMetrics,
    log: &ReplicaLogger,
    root: &Path,
    files: Vec<FileWithSize>,
    max_chunk_size: u32,
    chunk_actions: Vec<ChunkAction>,
) -> (Vec<FileInfo>, Vec<ChunkInfo>) {
    let mut chunk_table = Vec::new();
    let mut file_table = Vec::new();
    let mut chunk_index: usize = 0;
    for (file_index, FileWithSize(relative_path, size_bytes)) in files.into_iter().enumerate() {
        let mut file_hash = if relative_path.ends_with("state_file") {
            cow_file_hasher()
        } else {
            file_hasher()
        };
        let mut bytes_left = size_bytes;
        let num_chunks = count_chunks(size_bytes, max_chunk_size);
        (num_chunks as u32).update_hash(&mut file_hash);
        let compute_file_chunk_hashes = |data: &[u8]| {
            // It's OK to not have any chunks for 0-sized files (though it's unlikely that
            // we have any).
            while bytes_left > 0 {
                let chunk_size = bytes_left.min(max_chunk_size as u64);
                let offset = size_bytes - bytes_left;
                let recompute_chunk_hash = || {
                    let mut hasher = if relative_path.ends_with("state_file") {
                        cow_chunk_hasher()
                    } else {
                        chunk_hasher()
                    };
                    hasher.write(&data[offset as usize..(offset + chunk_size) as usize]);
                    hasher.finish()
                };
                assert!(chunk_index < chunk_actions.len());
                let chunk_hash = match chunk_actions[chunk_index] {
                    ChunkAction::RecomputeAndCompare(reused_chunk_hash) => {
                        metrics
                            .chunk_bytes
                            .with_label_values(&[LABEL_VALUE_HASHED_AND_COMPARED])
                            .inc_by(chunk_size);
                        // We have both a reused and a recomputed hash, so we can compare them to
                        // monitor for issues
                        let recomputed_chunk_hash = recompute_chunk_hash();
                        debug_assert_eq!(recomputed_chunk_hash, reused_chunk_hash);
                        if recomputed_chunk_hash != reused_chunk_hash {
                            metrics.reused_chunk_hash_error_count.inc();
                            error!(
                                log,
                                "{}: Hash mismatch in chunk with index {} in file {}, recomputed hash {:?}, reused hash {:?}",
                                CRITICAL_ERROR_REUSED_CHUNK_HASH,
                                chunk_index,
                                relative_path.display(),
                                recomputed_chunk_hash,
                                reused_chunk_hash
                            );
                        }
                        recomputed_chunk_hash
                    }
                    ChunkAction::UseHash(reused_chunk_hash) => {
                        metrics
                            .chunk_bytes
                            .with_label_values(&[LABEL_VALUE_REUSED])
                            .inc_by(chunk_size);
                        reused_chunk_hash
                    }
                    ChunkAction::Recompute => {
                        metrics
                            .chunk_bytes
                            .with_label_values(&[LABEL_VALUE_REUSED])
                            .inc_by(chunk_size);
                        recompute_chunk_hash()
                    }
                };
                let chunk_info = ChunkInfo {
                    file_index: file_index as u32,
                    size_bytes: chunk_size as u32,
                    offset: offset as u64,
                    hash: chunk_hash,
                };
                write_chunk_hash(&mut file_hash, &chunk_info);
                chunk_table.push(chunk_info);
                bytes_left -= chunk_size;
                chunk_index += 1;
            }
            file_table.push(FileInfo {
                relative_path: relative_path.clone(),
                size_bytes,
                hash: file_hash.finish(),
            });
        };
        if relative_path.ends_with("state_file") {
            let absolute_path = root.join(&relative_path);
            let cow_base_dir = absolute_path.parent().unwrap();
            let cow_mgr = CowMemoryManagerImpl::open_readonly(cow_base_dir.to_path_buf());
            let mapped_state = cow_mgr.get_map();
            mapped_state.make_heap_accessible();
            let data = unsafe {
                std::slice::from_raw_parts(mapped_state.get_heap_base(), size_bytes as usize)
            };
            compute_file_chunk_hashes(data);
        } else {
            let mmap =
                ScopedMmap::from_path(root.join(&relative_path)).expect("failed to open file");
            let data = mmap.as_slice();
            compute_file_chunk_hashes(data);
        };
    }
    assert_eq!(chunk_table.len(), chunk_actions.len());
    (file_table, chunk_table)
}
fn build_chunk_table_sequential ( metrics : & ManifestMetrics , log : & ReplicaLogger , root : & Path , files : Vec < FileWithSize > , max_chunk_size : u32 , chunk_actions : Vec < ChunkAction > , ) -> ( Vec < FileInfo > , Vec < ChunkInfo > ) { let mut chunk_table = Vec :: new ( ) ; let mut file_table = Vec :: new ( ) ; let mut chunk_index : usize = 0 ; for ( file_index , FileWithSize ( relative_path , size_bytes ) ) in files . into_iter ( ) . enumerate ( ) { let mut file_hash = if relative_path . ends_with ( " " ) { cow_file_hasher ( ) } else { file_hasher ( ) } ; let mut bytes_left = size_bytes ; let num_chunks = count_chunks ( size_bytes , max_chunk_size ) ; ( num_chunks as u32 ) . update_hash ( & mut file_hash ) ; let compute_file_chunk_hashes = | data : & [ u8 ] | { // It's OK to not have any chunks for 0-sized files (though it's unlikely that // we have any). while bytes_left > 0 { let chunk_size = bytes_left . min ( max_chunk_size as u64 ) ; let offset = size_bytes - bytes_left ; let recompute_chunk_hash = | | { let mut hasher = if relative_path . ends_with ( " " ) { cow_chunk_hasher ( ) } else { chunk_hasher ( ) } ; hasher . write ( & data [ offset as usize .. ( offset + chunk_size ) as usize ] ) ; hasher . finish ( ) } ; assert ! ( chunk_index chunk_actions len ( ) ) ; let chunk_hash = match chunk_actions [ chunk_index ] { ChunkAction :: RecomputeAndCompare ( reused_chunk_hash ) => { metrics . chunk_bytes . with_label_values ( & [ LABEL_VALUE_HASHED_AND_COMPARED ] ) . inc_by ( chunk_size ) ; // We have both a reused and a recomputed hash, so we can compare them to // monitor for issues let recomputed_chunk_hash = recompute_chunk_hash ( ) ; debug_assert_eq ! ( recomputed_chunk_hash reused_chunk_hash ) ; if recomputed_chunk_hash != reused_chunk_hash { metrics . reused_chunk_hash_error_count . inc ( ) ; error ! ( log " " CRITICAL_ERROR_REUSED_CHUNK_HASH chunk_index relative_path display ( ) recomputed_chunk_hash reused_chunk_hash ) ; } recomputed_chunk_hash } ChunkAction :: UseHash ( reused_chunk_hash ) => { metrics . chunk_bytes . with_label_values ( & [ LABEL_VALUE_REUSED ] ) . inc_by ( chunk_size ) ; reused_chunk_hash } ChunkAction :: Recompute => { metrics . chunk_bytes . with_label_values ( & [ LABEL_VALUE_REUSED ] ) . inc_by ( chunk_size ) ; recompute_chunk_hash ( ) } } ; let chunk_info = ChunkInfo { file_index : file_index as u32 , size_bytes : chunk_size as u32 , offset : offset as u64 , hash : chunk_hash , } ; write_chunk_hash ( & mut file_hash , & chunk_info ) ; chunk_table . push ( chunk_info ) ; bytes_left -= chunk_size ; chunk_index += 1 ; } file_table . push ( FileInfo { relative_path : relative_path . clone ( ) , size_bytes , hash : file_hash . finish ( ) , } ) ; } ; if relative_path . ends_with ( " " ) { let absolute_path = root . join ( & relative_path ) ; let cow_base_dir = absolute_path . parent ( ) . unwrap ( ) ; let cow_mgr = CowMemoryManagerImpl :: open_readonly ( cow_base_dir . to_path_buf ( ) ) ; let mapped_state = cow_mgr . get_map ( ) ; mapped_state . make_heap_accessible ( ) ; let data = unsafe { std :: slice :: from_raw_parts ( mapped_state . get_heap_base ( ) , size_bytes as usize ) } ; compute_file_chunk_hashes ( data ) ; } else { let mmap = ScopedMmap :: from_path ( root . join ( & relative_path ) ) . expect ( " " ) ; let data = mmap . as_slice ( ) ; compute_file_chunk_hashes ( data ) ; } ; } assert_eq ! ( chunk_table len ( ) chunk_actions len ( ) ) ; ( file_table , chunk_table ) }
Build a chunk table from the file table.
Build a chunk table from the file table .
metrics
&ManifestMetrics
log
&ReplicaLogger
root
&Path
files
Vec<FileWithSize>
max_chunk_size
u32
chunk_actions
Vec<ChunkAction>
metrics
&ManifestMetrics
log
&ReplicaLogger
root
&Path
files
Vec<FileWithSize>
max_chunk_size
u32
chunk_actions
Vec<ChunkAction>