fn hash_plan(
    base_manifest: &Manifest,
    files: &[FileWithSize],
    dirty_file_chunks: BTreeMap<PathBuf, BitVec>,
    max_chunk_size: u32,
    seed: u64,
    rehash_every_nth: u64,
) -> Vec<ChunkAction> {
    // Even if we could reuse all chunks, we want to ensure that we sometimes still
    // recompute them anyway to not propagate errors indefinitely. We choose a
    // uniformly random offset in [0, rehash_every_nth - 1] and recompute any chunks
    // with ((chunk_index + offset) % rehash_every_nth) == 0. The sampling is done
    // using an rng so that it's not always the same chunks but seeded
    // deterministically. We want to ensure that all replicas have the same hash
    // plan, as otherwise a replica that detects an error might not be able to
    // sway consensus. At the same time, we do not require unpredictability
    // here, as long as we can guarantee that we find faulty chunks within
    // rehash_every_nth checkpoints in expectation.
    let mut rng = ChaChaRng::seed_from_u64(seed);
    let rehash_every_nth = rehash_every_nth.max(1); // 0 will behave like 1
    let offset = rng.gen_range(0, rehash_every_nth);

    debug_assert!(uses_chunk_size(base_manifest, max_chunk_size));

    let mut chunk_actions: Vec<ChunkAction> = Vec::new();

    for FileWithSize(relative_path, size_bytes) in files.iter() {
        let num_chunks = count_chunks(*size_bytes, max_chunk_size);

        let compute_dirty_chunk_bitmap = || -> Option<(&BitVec, usize)> {
            let dirty_chunk_bitmap = dirty_file_chunks.get(relative_path)?;

            let base_file_index = base_manifest
                .file_table
                .binary_search_by_key(&relative_path, |file_info| &file_info.relative_path)
                .ok()?;

            // The chunk table contains chunks from all files and hence `base_index` is
            // needed to know the absolute index. The chunk table is sorted by
            // `file_index` and then `offset`. Therefore binary search can be used to find
            // the first chunk index of a file.
            let base_index = base_manifest
                .chunk_table
                .binary_search_by(|chunk_info| {
                    chunk_info
                        .file_index
                        .cmp(&(base_file_index as u32))
                        .then_with(|| chunk_info.offset.cmp(&0u64))
                })
                .ok()?;
            Some((dirty_chunk_bitmap, base_index))
        };

        if let Some((dirty_chunk_bitmap, base_index)) = compute_dirty_chunk_bitmap() {
            debug_assert_eq!(num_chunks, dirty_chunk_bitmap.len());

            for i in 0..num_chunks {
                let action = if dirty_chunk_bitmap[i] {
                    ChunkAction::Recompute
                } else {
                    let chunk = &base_manifest.chunk_table[base_index + i];

                    debug_assert_eq!(
                        &base_manifest.file_table[chunk.file_index as usize].relative_path,
                        relative_path
                    );
                    debug_assert_eq!(chunk.offset, i as u64 * max_chunk_size as u64);
                    debug_assert_eq!(
                        chunk.size_bytes as u64,
                        (size_bytes - chunk.offset).min(max_chunk_size as u64)
                    );

                    // We are using chunk_actions.len() as shorthand for the chunk_index.
                    let offset_index = (chunk_actions.len() as u64).wrapping_add(offset);

                    if (offset_index % rehash_every_nth) == 0 {
                        ChunkAction::RecomputeAndCompare(chunk.hash)
                    } else {
                        ChunkAction::UseHash(chunk.hash)
                    }
                };
                chunk_actions.push(action);
            }
        } else {
            for _ in 0..num_chunks {
                chunk_actions.push(ChunkAction::Recompute);
            }
        }
    }
    chunk_actions
}
/// Makes a "hash plan": an instruction how to compute the hash of each chunk of
/// the new manifest.
Makes a "hash plan": an instruction how to compute the hash of each chunk of
the new manifest.
Makes a " hash plan " : an instruction how to compute the hash of each chunk of the new manifest .
fn hash_plan(
    base_manifest: &Manifest,
    files: &[FileWithSize],
    dirty_file_chunks: BTreeMap<PathBuf, BitVec>,
    max_chunk_size: u32,
    seed: u64,
    rehash_every_nth: u64,
) -> Vec<ChunkAction> {
    // Even if we could reuse all chunks, we want to ensure that we sometimes still
    // recompute them anyway to not propagate errors indefinitely. We choose a
    // uniformly random offset in [0, rehash_every_nth - 1] and recompute any chunks
    // with ((chunk_index + offset) % rehash_every_nth) == 0. The sampling is done
    // using an rng so that it's not always the same chunks but seeded
    // deterministically. We want to ensure that all replicas have the same hash
    // plan, as otherwise a replica that detects an error might not be able to
    // sway consensus. At the same time, we do not require unpredictability
    // here, as long as we can guarantee that we find faulty chunks within
    // rehash_every_nth checkpoints in expectation.
    let mut rng = ChaChaRng::seed_from_u64(seed);
    let rehash_every_nth = rehash_every_nth.max(1); // 0 will behave like 1
    let offset = rng.gen_range(0, rehash_every_nth);
    debug_assert!(uses_chunk_size(base_manifest, max_chunk_size));
    let mut chunk_actions: Vec<ChunkAction> = Vec::new();
    for FileWithSize(relative_path, size_bytes) in files.iter() {
        let num_chunks = count_chunks(*size_bytes, max_chunk_size);
        let compute_dirty_chunk_bitmap = || -> Option<(&BitVec, usize)> {
            let dirty_chunk_bitmap = dirty_file_chunks.get(relative_path)?;
            let base_file_index = base_manifest
                .file_table
                .binary_search_by_key(&relative_path, |file_info| &file_info.relative_path)
                .ok()?;
            // The chunk table contains chunks from all files and hence `base_index` is
            // needed to know the absolute index. The chunk table is sorted by
            // `file_index` and then `offset`. Therefore binary search can be used to find
            // the first chunk index of a file.
            let base_index = base_manifest
                .chunk_table
                .binary_search_by(|chunk_info| {
                    chunk_info
                        .file_index
                        .cmp(&(base_file_index as u32))
                        .then_with(|| chunk_info.offset.cmp(&0u64))
                })
                .ok()?;
            Some((dirty_chunk_bitmap, base_index))
        };
        if let Some((dirty_chunk_bitmap, base_index)) = compute_dirty_chunk_bitmap() {
            debug_assert_eq!(num_chunks, dirty_chunk_bitmap.len());
            for i in 0..num_chunks {
                let action = if dirty_chunk_bitmap[i] {
                    ChunkAction::Recompute
                } else {
                    let chunk = &base_manifest.chunk_table[base_index + i];
                    debug_assert_eq!(
                        &base_manifest.file_table[chunk.file_index as usize].relative_path,
                        relative_path
                    );
                    debug_assert_eq!(chunk.offset, i as u64 * max_chunk_size as u64);
                    debug_assert_eq!(
                        chunk.size_bytes as u64,
                        (size_bytes - chunk.offset).min(max_chunk_size as u64)
                    );
                    // We are using chunk_actions.len() as shorthand for the chunk_index.
                    let offset_index = (chunk_actions.len() as u64).wrapping_add(offset);
                    if (offset_index % rehash_every_nth) == 0 {
                        ChunkAction::RecomputeAndCompare(chunk.hash)
                    } else {
                        ChunkAction::UseHash(chunk.hash)
                    }
                };
                chunk_actions.push(action);
            }
        } else {
            for _ in 0..num_chunks {
                chunk_actions.push(ChunkAction::Recompute);
            }
        }
    }
    chunk_actions
}
fn hash_plan ( base_manifest : & Manifest , files : & [ FileWithSize ] , dirty_file_chunks : BTreeMap < PathBuf , BitVec > , max_chunk_size : u32 , seed : u64 , rehash_every_nth : u64 , ) -> Vec < ChunkAction > { // Even if we could reuse all chunks, we want to ensure that we sometimes still // recompute them anyway to not propagate errors indefinitely. We choose a // uniformly random offset in [0, rehash_every_nth - 1] and recompute any chunks // with ((chunk_index + offset) % rehash_every_nth) == 0. The sampling is done // using an rng so that it's not always the same chunks but seeded // deterministically. We want to ensure that all replicas have the same hash // plan, as otherwise a replica that detects an error might not be able to // sway consensus. At the same time, we do not require unpredictability // here, as long as we can guarantee that we find faulty chunks within // rehash_every_nth checkpoints in expectation. let mut rng = ChaChaRng :: seed_from_u64 ( seed ) ; let rehash_every_nth = rehash_every_nth . max ( 1 ) ; // 0 will behave like 1 let offset = rng . gen_range ( 0 , rehash_every_nth ) ; debug_assert ! ( uses_chunk_size ( base_manifest max_chunk_size ) ) ; let mut chunk_actions : Vec < ChunkAction > = Vec :: new ( ) ; for FileWithSize ( relative_path , size_bytes ) in files . iter ( ) { let num_chunks = count_chunks ( * size_bytes , max_chunk_size ) ; let compute_dirty_chunk_bitmap = | | -> Option < ( & BitVec , usize ) > { let dirty_chunk_bitmap = dirty_file_chunks . get ( relative_path ) ? ; let base_file_index = base_manifest . file_table . binary_search_by_key ( & relative_path , | file_info | & file_info . relative_path ) . ok ( ) ? ; // The chunk table contains chunks from all files and hence `base_index` is // needed to know the absolute index. The chunk table is sorted by // `file_index` and then `offset`. Therefore binary search can be used to find // the first chunk index of a file. let base_index = base_manifest . chunk_table . binary_search_by ( | chunk_info | { chunk_info . file_index . cmp ( & ( base_file_index as u32 ) ) . then_with ( | | chunk_info . offset . cmp ( & 0u64 ) ) } ) . ok ( ) ? ; Some ( ( dirty_chunk_bitmap , base_index ) ) } ; if let Some ( ( dirty_chunk_bitmap , base_index ) ) = compute_dirty_chunk_bitmap ( ) { debug_assert_eq ! ( num_chunks dirty_chunk_bitmap len ( ) ) ; for i in 0 .. num_chunks { let action = if dirty_chunk_bitmap [ i ] { ChunkAction :: Recompute } else { let chunk = & base_manifest . chunk_table [ base_index + i ] ; debug_assert_eq ! ( base_manifest file_table [ chunk file_index as usize ] relative_path relative_path ) ; debug_assert_eq ! ( chunk offset i as u64 max_chunk_size as u64 ) ; debug_assert_eq ! ( chunk size_bytes as u64 ( size_bytes chunk offset ) min ( max_chunk_size as u64 ) ) ; // We are using chunk_actions.len() as shorthand for the chunk_index. let offset_index = ( chunk_actions . len ( ) as u64 ) . wrapping_add ( offset ) ; if ( offset_index % rehash_every_nth ) == 0 { ChunkAction :: RecomputeAndCompare ( chunk . hash ) } else { ChunkAction :: UseHash ( chunk . hash ) } } ; chunk_actions . push ( action ) ; } } else { for _ in 0 .. num_chunks { chunk_actions . push ( ChunkAction :: Recompute ) ; } } } chunk_actions }
Makes a "hash plan": an instruction how to compute the hash of each chunk of
the new manifest.
Makes a " hash plan " : an instruction how to compute the hash of each chunk of the new manifest .
base_manifest
&Manifest
files
&[FileWithSize]
dirty_file_chunks
BTreeMap<PathBuf, BitVec>
max_chunk_size
u32
seed
u64
rehash_every_nth
u64
base_manifest
&Manifest
files
&[FileWithSize]
dirty_file_chunks
BTreeMap<PathBuf, BitVec>
max_chunk_size
u32
seed
u64
rehash_every_nth
u64