pub fn archive_snapshot_package(
    snapshot_package: &SnapshotPackage,
    maximum_full_snapshot_archives_to_retain: usize,
    maximum_incremental_snapshot_archives_to_retain: usize,
) -> Result<()> {
    info!(
        "Generating snapshot archive for slot {}",
        snapshot_package.slot()
    );

    serialize_status_cache(
        snapshot_package.slot(),
        &snapshot_package.slot_deltas,
        &snapshot_package
            .snapshot_links
            .path()
            .join(SNAPSHOT_STATUS_CACHE_FILE_NAME),
    )?;

    let mut timer = Measure::start("snapshot_package-package_snapshots");
    let tar_dir = snapshot_package
        .path()
        .parent()
        .expect("Tar output path is invalid");

    fs::create_dir_all(tar_dir)
        .map_err(|e| SnapshotError::IoWithSource(e, "create archive path"))?;

    // Create the staging directories
    let staging_dir_prefix = TMP_SNAPSHOT_ARCHIVE_PREFIX;
    let staging_dir = tempfile::Builder::new()
        .prefix(&format!(
            "{}{}-",
            staging_dir_prefix,
            snapshot_package.slot()
        ))
        .tempdir_in(tar_dir)
        .map_err(|e| SnapshotError::IoWithSource(e, "create archive tempdir"))?;

    let staging_accounts_dir = staging_dir.path().join("accounts");
    let staging_snapshots_dir = staging_dir.path().join("snapshots");
    let staging_version_file = staging_dir.path().join("version");
    fs::create_dir_all(&staging_accounts_dir)
        .map_err(|e| SnapshotError::IoWithSource(e, "create staging path"))?;

    // Add the snapshots to the staging directory
    symlink::symlink_dir(
        snapshot_package.snapshot_links.path(),
        &staging_snapshots_dir,
    )
    .map_err(|e| SnapshotError::IoWithSource(e, "create staging symlinks"))?;

    // Add the AppendVecs into the compressible list
    for storage in snapshot_package.snapshot_storages.iter().flatten() {
        storage.flush()?;
        let storage_path = storage.get_path();
        let output_path = staging_accounts_dir.join(crate::append_vec::AppendVec::file_name(
            storage.slot(),
            storage.append_vec_id(),
        ));

        // `storage_path` - The file path where the AppendVec itself is located
        // `output_path` - The file path where the AppendVec will be placed in the staging directory.
        let storage_path =
            fs::canonicalize(storage_path).expect("Could not get absolute path for accounts");
        symlink::symlink_file(storage_path, &output_path)
            .map_err(|e| SnapshotError::IoWithSource(e, "create storage symlink"))?;
        if !output_path.is_file() {
            return Err(SnapshotError::StoragePathSymlinkInvalid);
        }
    }

    // Write version file
    {
        let mut f = fs::File::create(staging_version_file)
            .map_err(|e| SnapshotError::IoWithSource(e, "create version file"))?;
        f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())
            .map_err(|e| SnapshotError::IoWithSource(e, "write version file"))?;
    }

    let file_ext = get_archive_ext(snapshot_package.archive_format());

    // Tar the staging directory into the archive at `archive_path`
    let archive_path = tar_dir.join(format!(
        "{}{}.{}",
        staging_dir_prefix,
        snapshot_package.slot(),
        file_ext
    ));

    {
        let mut archive_file = fs::File::create(&archive_path)?;

        let do_archive_files = |encoder: &mut dyn Write| -> Result<()> {
            let mut archive = tar::Builder::new(encoder);
            for dir in ["accounts", "snapshots"] {
                archive.append_dir_all(dir, staging_dir.as_ref().join(dir))?;
            }
            archive.append_path_with_name(staging_dir.as_ref().join("version"), "version")?;
            archive.into_inner()?;
            Ok(())
        };

        match snapshot_package.archive_format() {
            ArchiveFormat::TarBzip2 => {
                let mut encoder =
                    bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::best());
                do_archive_files(&mut encoder)?;
                encoder.finish()?;
            }
            ArchiveFormat::TarGzip => {
                let mut encoder =
                    flate2::write::GzEncoder::new(archive_file, flate2::Compression::default());
                do_archive_files(&mut encoder)?;
                encoder.finish()?;
            }
            ArchiveFormat::TarZstd => {
                let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?;
                do_archive_files(&mut encoder)?;
                encoder.finish()?;
            }
            ArchiveFormat::Tar => {
                do_archive_files(&mut archive_file)?;
            }
        };
    }

    // Atomically move the archive into position for other validators to find
    let metadata = fs::metadata(&archive_path)
        .map_err(|e| SnapshotError::IoWithSource(e, "archive path stat"))?;
    fs::rename(&archive_path, &snapshot_package.path())
        .map_err(|e| SnapshotError::IoWithSource(e, "archive path rename"))?;

    purge_old_snapshot_archives(
        tar_dir,
        maximum_full_snapshot_archives_to_retain,
        maximum_incremental_snapshot_archives_to_retain,
    );

    timer.stop();
    info!(
        "Successfully created {:?}. slot: {}, elapsed ms: {}, size={}",
        snapshot_package.path(),
        snapshot_package.slot(),
        timer.as_ms(),
        metadata.len()
    );
    datapoint_info!(
        "snapshot-package",
        ("slot", snapshot_package.slot(), i64),
        ("duration_ms", timer.as_ms(), i64),
        ("size", metadata.len(), i64)
    );
    Ok(())
}
/// Make a snapshot archive out of the snapshot package
Make a snapshot archive out of the snapshot package
Make a snapshot archive out of the snapshot package
pub fn archive_snapshot_package(
    snapshot_package: &SnapshotPackage,
    maximum_full_snapshot_archives_to_retain: usize,
    maximum_incremental_snapshot_archives_to_retain: usize,
) -> Result<()> {
    info!(
        "Generating snapshot archive for slot {}",
        snapshot_package.slot()
    );
    serialize_status_cache(
        snapshot_package.slot(),
        &snapshot_package.slot_deltas,
        &snapshot_package
            .snapshot_links
            .path()
            .join(SNAPSHOT_STATUS_CACHE_FILE_NAME),
    )?;
    let mut timer = Measure::start("snapshot_package-package_snapshots");
    let tar_dir = snapshot_package
        .path()
        .parent()
        .expect("Tar output path is invalid");
    fs::create_dir_all(tar_dir)
        .map_err(|e| SnapshotError::IoWithSource(e, "create archive path"))?;
    // Create the staging directories
    let staging_dir_prefix = TMP_SNAPSHOT_ARCHIVE_PREFIX;
    let staging_dir = tempfile::Builder::new()
        .prefix(&format!(
            "{}{}-",
            staging_dir_prefix,
            snapshot_package.slot()
        ))
        .tempdir_in(tar_dir)
        .map_err(|e| SnapshotError::IoWithSource(e, "create archive tempdir"))?;
    let staging_accounts_dir = staging_dir.path().join("accounts");
    let staging_snapshots_dir = staging_dir.path().join("snapshots");
    let staging_version_file = staging_dir.path().join("version");
    fs::create_dir_all(&staging_accounts_dir)
        .map_err(|e| SnapshotError::IoWithSource(e, "create staging path"))?;
    // Add the snapshots to the staging directory
    symlink::symlink_dir(
        snapshot_package.snapshot_links.path(),
        &staging_snapshots_dir,
    )
    .map_err(|e| SnapshotError::IoWithSource(e, "create staging symlinks"))?;
    // Add the AppendVecs into the compressible list
    for storage in snapshot_package.snapshot_storages.iter().flatten() {
        storage.flush()?;
        let storage_path = storage.get_path();
        let output_path = staging_accounts_dir.join(crate::append_vec::AppendVec::file_name(
            storage.slot(),
            storage.append_vec_id(),
        ));
        // `storage_path` - The file path where the AppendVec itself is located
        // `output_path` - The file path where the AppendVec will be placed in the staging directory.
        let storage_path =
            fs::canonicalize(storage_path).expect("Could not get absolute path for accounts");
        symlink::symlink_file(storage_path, &output_path)
            .map_err(|e| SnapshotError::IoWithSource(e, "create storage symlink"))?;
        if !output_path.is_file() {
            return Err(SnapshotError::StoragePathSymlinkInvalid);
        }
    }
    // Write version file
    {
        let mut f = fs::File::create(staging_version_file)
            .map_err(|e| SnapshotError::IoWithSource(e, "create version file"))?;
        f.write_all(snapshot_package.snapshot_version.as_str().as_bytes())
            .map_err(|e| SnapshotError::IoWithSource(e, "write version file"))?;
    }
    let file_ext = get_archive_ext(snapshot_package.archive_format());
    // Tar the staging directory into the archive at `archive_path`
    let archive_path = tar_dir.join(format!(
        "{}{}.{}",
        staging_dir_prefix,
        snapshot_package.slot(),
        file_ext
    ));
    {
        let mut archive_file = fs::File::create(&archive_path)?;
        let do_archive_files = |encoder: &mut dyn Write| -> Result<()> {
            let mut archive = tar::Builder::new(encoder);
            for dir in ["accounts", "snapshots"] {
                archive.append_dir_all(dir, staging_dir.as_ref().join(dir))?;
            }
            archive.append_path_with_name(staging_dir.as_ref().join("version"), "version")?;
            archive.into_inner()?;
            Ok(())
        };
        match snapshot_package.archive_format() {
            ArchiveFormat::TarBzip2 => {
                let mut encoder =
                    bzip2::write::BzEncoder::new(archive_file, bzip2::Compression::best());
                do_archive_files(&mut encoder)?;
                encoder.finish()?;
            }
            ArchiveFormat::TarGzip => {
                let mut encoder =
                    flate2::write::GzEncoder::new(archive_file, flate2::Compression::default());
                do_archive_files(&mut encoder)?;
                encoder.finish()?;
            }
            ArchiveFormat::TarZstd => {
                let mut encoder = zstd::stream::Encoder::new(archive_file, 0)?;
                do_archive_files(&mut encoder)?;
                encoder.finish()?;
            }
            ArchiveFormat::Tar => {
                do_archive_files(&mut archive_file)?;
            }
        };
    }
    // Atomically move the archive into position for other validators to find
    let metadata = fs::metadata(&archive_path)
        .map_err(|e| SnapshotError::IoWithSource(e, "archive path stat"))?;
    fs::rename(&archive_path, &snapshot_package.path())
        .map_err(|e| SnapshotError::IoWithSource(e, "archive path rename"))?;
    purge_old_snapshot_archives(
        tar_dir,
        maximum_full_snapshot_archives_to_retain,
        maximum_incremental_snapshot_archives_to_retain,
    );
    timer.stop();
    info!(
        "Successfully created {:?}. slot: {}, elapsed ms: {}, size={}",
        snapshot_package.path(),
        snapshot_package.slot(),
        timer.as_ms(),
        metadata.len()
    );
    datapoint_info!(
        "snapshot-package",
        ("slot", snapshot_package.slot(), i64),
        ("duration_ms", timer.as_ms(), i64),
        ("size", metadata.len(), i64)
    );
    Ok(())
}
pub fn archive_snapshot_package ( snapshot_package : & SnapshotPackage , maximum_full_snapshot_archives_to_retain : usize , maximum_incremental_snapshot_archives_to_retain : usize , ) -> Result < ( ) > { info ! ( " " snapshot_package slot ( ) ) ; serialize_status_cache ( snapshot_package . slot ( ) , & snapshot_package . slot_deltas , & snapshot_package . snapshot_links . path ( ) . join ( SNAPSHOT_STATUS_CACHE_FILE_NAME ) , ) ? ; let mut timer = Measure :: start ( " " ) ; let tar_dir = snapshot_package . path ( ) . parent ( ) . expect ( " " ) ; fs :: create_dir_all ( tar_dir ) . map_err ( | e | SnapshotError :: IoWithSource ( e , " " ) ) ? ; // Create the staging directories let staging_dir_prefix = TMP_SNAPSHOT_ARCHIVE_PREFIX ; let staging_dir = tempfile :: Builder :: new ( ) . prefix ( & format ! ( " " staging_dir_prefix snapshot_package slot ( ) ) ) . tempdir_in ( tar_dir ) . map_err ( | e | SnapshotError :: IoWithSource ( e , " " ) ) ? ; let staging_accounts_dir = staging_dir . path ( ) . join ( " " ) ; let staging_snapshots_dir = staging_dir . path ( ) . join ( " " ) ; let staging_version_file = staging_dir . path ( ) . join ( " " ) ; fs :: create_dir_all ( & staging_accounts_dir ) . map_err ( | e | SnapshotError :: IoWithSource ( e , " " ) ) ? ; // Add the snapshots to the staging directory symlink :: symlink_dir ( snapshot_package . snapshot_links . path ( ) , & staging_snapshots_dir , ) . map_err ( | e | SnapshotError :: IoWithSource ( e , " " ) ) ? ; // Add the AppendVecs into the compressible list for storage in snapshot_package . snapshot_storages . iter ( ) . flatten ( ) { storage . flush ( ) ? ; let storage_path = storage . get_path ( ) ; let output_path = staging_accounts_dir . join ( crate :: append_vec :: AppendVec :: file_name ( storage . slot ( ) , storage . append_vec_id ( ) , ) ) ; // `storage_path` - The file path where the AppendVec itself is located // `output_path` - The file path where the AppendVec will be placed in the staging directory. let storage_path = fs :: canonicalize ( storage_path ) . expect ( " " ) ; symlink :: symlink_file ( storage_path , & output_path ) . map_err ( | e | SnapshotError :: IoWithSource ( e , " " ) ) ? ; if ! output_path . is_file ( ) { return Err ( SnapshotError :: StoragePathSymlinkInvalid ) ; } } // Write version file { let mut f = fs :: File :: create ( staging_version_file ) . map_err ( | e | SnapshotError :: IoWithSource ( e , " " ) ) ? ; f . write_all ( snapshot_package . snapshot_version . as_str ( ) . as_bytes ( ) ) . map_err ( | e | SnapshotError :: IoWithSource ( e , " " ) ) ? ; } let file_ext = get_archive_ext ( snapshot_package . archive_format ( ) ) ; // Tar the staging directory into the archive at `archive_path` let archive_path = tar_dir . join ( format ! ( " " staging_dir_prefix snapshot_package slot ( ) file_ext ) ) ; { let mut archive_file = fs :: File :: create ( & archive_path ) ? ; let do_archive_files = | encoder : & mut dyn Write | -> Result < ( ) > { let mut archive = tar :: Builder :: new ( encoder ) ; for dir in [ " " , " " ] { archive . append_dir_all ( dir , staging_dir . as_ref ( ) . join ( dir ) ) ? ; } archive . append_path_with_name ( staging_dir . as_ref ( ) . join ( " " ) , " " ) ? ; archive . into_inner ( ) ? ; Ok ( ( ) ) } ; match snapshot_package . archive_format ( ) { ArchiveFormat :: TarBzip2 => { let mut encoder = bzip2 :: write :: BzEncoder :: new ( archive_file , bzip2 :: Compression :: best ( ) ) ; do_archive_files ( & mut encoder ) ? ; encoder . finish ( ) ? ; } ArchiveFormat :: TarGzip => { let mut encoder = flate2 :: write :: GzEncoder :: new ( archive_file , flate2 :: Compression :: default ( ) ) ; do_archive_files ( & mut encoder ) ? ; encoder . finish ( ) ? ; } ArchiveFormat :: TarZstd => { let mut encoder = zstd :: stream :: Encoder :: new ( archive_file , 0 ) ? ; do_archive_files ( & mut encoder ) ? ; encoder . finish ( ) ? ; } ArchiveFormat :: Tar => { do_archive_files ( & mut archive_file ) ? ; } } ; } // Atomically move the archive into position for other validators to find let metadata = fs :: metadata ( & archive_path ) . map_err ( | e | SnapshotError :: IoWithSource ( e , " " ) ) ? ; fs :: rename ( & archive_path , & snapshot_package . path ( ) ) . map_err ( | e | SnapshotError :: IoWithSource ( e , " " ) ) ? ; purge_old_snapshot_archives ( tar_dir , maximum_full_snapshot_archives_to_retain , maximum_incremental_snapshot_archives_to_retain , ) ; timer . stop ( ) ; info ! ( " " snapshot_package path ( ) snapshot_package slot ( ) timer as_ms ( ) metadata len ( ) ) ; datapoint_info ! ( " " ( " " snapshot_package slot ( ) i64 ) ( " " timer as_ms ( ) i64 ) ( " " metadata len ( ) i64 ) ) ; Ok ( ( ) ) }
Make a snapshot archive out of the snapshot package
Make a snapshot archive out of the snapshot package
snapshot_package
&SnapshotPackage
maximum_full_snapshot_archives_to_retain
usize
maximum_incremental_snapshot_archives_to_retain
usize