[vcpkg] Extract binary caching functionality into separate interface
ADD
include/vcpkg/binarycaching.h
+#pragma once
+
+#include <vcpkg/base/files.h>
+#include <vcpkg/packagespec.h>
+#include <vcpkg/vcpkgpaths.h>
+
+namespace vcpkg::Dependencies
+{
+ struct InstallPlanAction;
+}
+namespace vcpkg::Build
+{
+ struct AbiTagAndFile;
+ struct BuildPackageOptions;
+}
+
+namespace vcpkg
+{
+ enum RestoreResult
+ {
+ MISSING,
+ SUCCESS,
+ BUILD_FAILED,
+ };
+
+ struct IBinaryProvider
+ {
+ virtual ~IBinaryProvider() = default;
+ virtual void prefetch() = 0;
+ virtual RestoreResult try_restore(const VcpkgPaths& paths,
+ const PackageSpec& spec,
+ const Build::AbiTagAndFile& abi_tag_and_file,
+ const Build::BuildPackageOptions& build_options) = 0;
+ virtual void push_success(const VcpkgPaths& paths,
+ const Build::AbiTagAndFile& abi_tag_and_file,
+ const Dependencies::InstallPlanAction& action) = 0;
+ virtual void push_failure(const VcpkgPaths& paths,
+ const Build::AbiTagAndFile& abi_tag_and_file,
+ const PackageSpec& spec) = 0;
+ };
+
+ std::unique_ptr<IBinaryProvider> create_archives_provider();
+}

MODIFY
include/vcpkg/build.h
include/vcpkg/build.h
#include <set>
#include <vector>
+namespace vcpkg
+{
+ struct IBinaryProvider;
+}
+
namespace vcpkg::Dependencies
{
struct InstallPlanAction;
@@ -28,7 +33,6 @@ namespace vcpkg::Build
void perform_and_exit_ex(const FullPackageSpec& full_spec,
const SourceControlFileLocation& scfl,
const PortFileProvider::PathsPortFileProvider& provider,
- const ParsedArguments& options,
const VcpkgPaths& paths);
void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet default_triplet);
@@ -202,6 +206,7 @@ namespace vcpkg::Build
ExtendedBuildResult build_package(const VcpkgPaths& paths,
const Dependencies::InstallPlanAction& config,
const CMakeVars::CMakeVarProvider& var_provider,
+ IBinaryProvider* binaries_provider,
const StatusParagraphs& status_db);
enum class BuildPolicy

ADD
src/vcpkg/binarycaching.cpp
+#include <vcpkg/base/checks.h>
+#include <vcpkg/base/files.h>
+#include <vcpkg/base/system.print.h>
+#include <vcpkg/base/system.process.h>
+#include <vcpkg/binarycaching.h>
+#include <vcpkg/build.h>
+#include <vcpkg/dependencies.h>
+
+using namespace vcpkg;
+
+namespace
+{
+ static System::ExitCodeAndOutput decompress_archive(const VcpkgPaths& paths,
+ const PackageSpec& spec,
+ const fs::path& archive_path)
+ {
+ auto& fs = paths.get_filesystem();
+
+ auto pkg_path = paths.package_dir(spec);
+ fs.remove_all(pkg_path, VCPKG_LINE_INFO);
+ std::error_code ec;
+ fs.create_directories(pkg_path, ec);
+ auto files = fs.get_files_non_recursive(pkg_path);
+ Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string());
+
+#if defined(_WIN32)
+ auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);
+ auto cmd = Strings::format(
+ R"("%s" x "%s" -o"%s" -y)", seven_zip_exe.u8string(), archive_path.u8string(), pkg_path.u8string());
+#else
+ auto cmd = Strings::format(R"(unzip -qq "%s" "-d%s")", archive_path.u8string(), pkg_path.u8string());
+#endif
+ return System::cmd_execute_and_capture_output(cmd, System::get_clean_environment());
+ }
+
+ // Compress the source directory into the destination file.
+ static void compress_directory(const VcpkgPaths& paths, const fs::path& source, const fs::path& destination)
+ {
+ auto& fs = paths.get_filesystem();
+
+ std::error_code ec;
+
+ fs.remove(destination, ec);
+ Checks::check_exit(
+ VCPKG_LINE_INFO, !fs.exists(destination), "Could not remove file: %s", destination.u8string());
+#if defined(_WIN32)
+ auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);
+
+ System::cmd_execute_and_capture_output(
+ Strings::format(
+ R"("%s" a "%s" "%s\*")", seven_zip_exe.u8string(), destination.u8string(), source.u8string()),
+ System::get_clean_environment());
+#else
+ System::cmd_execute_clean(
+ Strings::format(R"(cd '%s' && zip --quiet -r '%s' *)", source.u8string(), destination.u8string()));
+#endif
+ }
+
+ static void compress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& destination)
+ {
+ compress_directory(paths, paths.package_dir(spec), destination);
+ }
+
+ struct ArchivesBinaryProvider : IBinaryProvider
+ {
+ ~ArchivesBinaryProvider() = default;
+ void prefetch() override {}
+ RestoreResult try_restore(const VcpkgPaths& paths,
+ const PackageSpec& spec,
+ const Build::AbiTagAndFile& abi_tag_and_file,
+ const Build::BuildPackageOptions& build_options) override
+ {
+ auto& fs = paths.get_filesystem();
+ std::error_code ec;
+ const fs::path archives_root_dir = paths.root / "archives";
+ const std::string archive_name = abi_tag_and_file.tag + ".zip";
+ const fs::path archive_subpath = fs::u8path(abi_tag_and_file.tag.substr(0, 2)) / archive_name;
+ const fs::path archive_path = archives_root_dir / archive_subpath;
+ const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;
+ const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();
+ const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";
+ if (fs.exists(archive_path))
+ {
+ System::print2("Using cached binary package: ", archive_path.u8string(), "\n");
+
+ int archive_result = decompress_archive(paths, spec, archive_path).exit_code;
+
+ if (archive_result != 0)
+ {
+ System::print2("Failed to decompress archive package\n");
+ if (build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO)
+ {
+ return RestoreResult::BUILD_FAILED;
+ }
+ else
+ {
+ System::print2("Purging bad archive\n");
+ fs.remove(archive_path, ec);
+ }
+ }
+ else
+ {
+ return RestoreResult::SUCCESS;
+ }
+ }
+
+ if (fs.exists(archive_tombstone_path))
+ {
+ if (build_options.fail_on_tombstone == Build::FailOnTombstone::YES)
+ {
+ System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");
+ return RestoreResult::BUILD_FAILED;
+ }
+ else
+ {
+ System::print2(
+ System::Color::warning, "Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");
+ }
+ }
+ else
+ {
+ System::printf("Could not locate cached archive: %s\n", archive_path.u8string());
+ }
+
+ return RestoreResult::MISSING;
+ }
+ void push_success(const VcpkgPaths& paths,
+ const Build::AbiTagAndFile& abi_tag_and_file,
+ const Dependencies::InstallPlanAction& action) override
+ {
+ auto& spec = action.spec;
+ auto& fs = paths.get_filesystem();
+ std::error_code ec;
+ const fs::path archives_root_dir = paths.root / "archives";
+ const std::string archive_name = abi_tag_and_file.tag + ".zip";
+ const fs::path archive_subpath = fs::u8path(abi_tag_and_file.tag.substr(0, 2)) / archive_name;
+ const fs::path archive_path = archives_root_dir / archive_subpath;
+ const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;
+ const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();
+ const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";
+
+ const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip");
+
+ compress_archive(paths, spec, tmp_archive_path);
+
+ fs.create_directories(archive_path.parent_path(), ec);
+ fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec);
+ if (ec)
+ {
+ System::printf(System::Color::warning,
+ "Failed to store binary cache %s: %s\n",
+ archive_path.u8string(),
+ ec.message());
+ }
+ else
+ System::printf("Stored binary cache: %s\n", archive_path.u8string());
+ }
+ void push_failure(const VcpkgPaths& paths,
+ const Build::AbiTagAndFile& abi_tag_and_file,
+ const PackageSpec& spec) override
+ {
+ auto& fs = paths.get_filesystem();
+ std::error_code ec;
+ const fs::path archives_root_dir = paths.root / "archives";
+ const std::string archive_name = abi_tag_and_file.tag + ".zip";
+ const fs::path archive_subpath = fs::u8path(abi_tag_and_file.tag.substr(0, 2)) / archive_name;
+ const fs::path archive_path = archives_root_dir / archive_subpath;
+ const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;
+ const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();
+ const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";
+
+ if (!fs.exists(archive_tombstone_path))
+ {
+ // Build failed, store all failure logs in the tombstone.
+ const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs";
+ const auto tmp_log_path_destination = tmp_log_path / spec.name();
+ const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip";
+ fs.create_directories(tmp_log_path_destination, ec);
+
+ for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name()))
+ {
+ if (log_file.path().extension() == ".log")
+ {
+ fs.copy_file(log_file.path(),
+ tmp_log_path_destination / log_file.path().filename(),
+ fs::stdfs::copy_options::none,
+ ec);
+ }
+ }
+
+ compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip");
+
+ fs.create_directories(archive_tombstone_path.parent_path(), ec);
+ fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec);
+
+ // clean up temporary directory
+ fs.remove_all(tmp_log_path, VCPKG_LINE_INFO);
+ }
+ }
+ };
+}
+
+std::unique_ptr<vcpkg::IBinaryProvider> vcpkg::create_archives_provider()
+{
+ return std::make_unique<ArchivesBinaryProvider>();
+}

MODIFY
src/vcpkg/build.cpp
src/vcpkg/build.cpp
#include <vcpkg/base/system.process.h>
#include <vcpkg/base/util.h>
+#include <vcpkg/binarycaching.h>
#include <vcpkg/build.h>
#include <vcpkg/commands.h>
#include <vcpkg/dependencies.h>
@@ -38,11 +39,8 @@ namespace vcpkg::Build::Command
void perform_and_exit_ex(const FullPackageSpec& full_spec,
const SourceControlFileLocation& scfl,
const PathsPortFileProvider& provider,
- const ParsedArguments& options,
const VcpkgPaths& paths)
{
- vcpkg::Util::unused(options);
-
CMakeVars::TripletCMakeVarProvider var_provider(paths);
var_provider.load_dep_info_vars(std::array<PackageSpec, 1>{full_spec.package_spec});
var_provider.load_tag_vars(std::array<FullPackageSpec, 1>{full_spec}, provider);
@@ -94,7 +92,8 @@ namespace vcpkg::Build::Command
action->build_options = build_package_options;
const auto build_timer = Chrono::ElapsedTimer::create_started();
- const auto result = Build::build_package(paths, *action, var_provider, status_db);
+ const auto result =
+ Build::build_package(paths, *action, var_provider, create_archives_provider().get(), status_db);
System::print2("Elapsed time for package ", spec, ": ", build_timer, '\n');
if (result.code == BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES)
@@ -147,7 +146,7 @@ namespace vcpkg::Build::Command
Checks::check_exit(VCPKG_LINE_INFO, scfl != nullptr, "Error: Couldn't find port '%s'", port_name);
- perform_and_exit_ex(spec, *scfl, provider, options, paths);
+ perform_and_exit_ex(spec, *scfl, provider, paths);
}
}
@@ -737,62 +736,14 @@ namespace vcpkg::Build
return nullopt;
}
- static System::ExitCodeAndOutput decompress_archive(const VcpkgPaths& paths,
- const PackageSpec& spec,
- const fs::path& archive_path)
- {
- auto& fs = paths.get_filesystem();
-
- auto pkg_path = paths.package_dir(spec);
- fs.remove_all(pkg_path, VCPKG_LINE_INFO);
- std::error_code ec;
- fs.create_directories(pkg_path, ec);
- auto files = fs.get_files_non_recursive(pkg_path);
- Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string());
-
-#if defined(_WIN32)
- auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);
- auto cmd = Strings::format(
- R"("%s" x "%s" -o"%s" -y)", seven_zip_exe.u8string(), archive_path.u8string(), pkg_path.u8string());
-#else
- auto cmd = Strings::format(R"(unzip -qq "%s" "-d%s")", archive_path.u8string(), pkg_path.u8string());
-#endif
- return System::cmd_execute_and_capture_output(cmd, System::get_clean_environment());
- }
-
- // Compress the source directory into the destination file.
- static void compress_directory(const VcpkgPaths& paths, const fs::path& source, const fs::path& destination)
- {
- auto& fs = paths.get_filesystem();
-
- std::error_code ec;
-
- fs.remove(destination, ec);
- Checks::check_exit(
- VCPKG_LINE_INFO, !fs.exists(destination), "Could not remove file: %s", destination.u8string());
-#if defined(_WIN32)
- auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);
-
- System::cmd_execute_and_capture_output(
- Strings::format(
- R"("%s" a "%s" "%s\*")", seven_zip_exe.u8string(), destination.u8string(), source.u8string()),
- System::get_clean_environment());
-#else
- System::cmd_execute_clean(
- Strings::format(R"(cd '%s' && zip --quiet -r '%s' *)", source.u8string(), destination.u8string()));
-#endif
- }
-
- static void compress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& destination)
- {
- compress_directory(paths, paths.package_dir(spec), destination);
- }
-
ExtendedBuildResult build_package(const VcpkgPaths& paths,
const Dependencies::InstallPlanAction& action,
const CMakeVars::CMakeVarProvider& var_provider,
+ IBinaryProvider* binaries_provider,
const StatusParagraphs& status_db)
{
+ auto binary_caching_enabled = binaries_provider && action.build_options.binary_caching == BinaryCaching::YES;
+
auto& fs = paths.get_filesystem();
Triplet triplet = action.spec.triplet();
const std::string& name = action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO)
@@ -843,116 +794,40 @@ namespace vcpkg::Build
std::error_code ec;
const auto abi_tag_and_file = maybe_abi_tag_and_file.get();
- const fs::path archives_root_dir = paths.root / "archives";
- const std::string archive_name = abi_tag_and_file->tag + ".zip";
- const fs::path archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name;
- const fs::path archive_path = archives_root_dir / archive_subpath;
- const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;
const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();
const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";
-
- if (action.build_options.binary_caching == BinaryCaching::YES)
- {
- if (fs.exists(archive_path))
- {
- System::print2("Using cached binary package: ", archive_path.u8string(), "\n");
-
- int archive_result = decompress_archive(paths, spec, archive_path).exit_code;
-
- if (archive_result != 0)
- {
- System::print2("Failed to decompress archive package\n");
- if (action.build_options.purge_decompress_failure == PurgeDecompressFailure::NO)
+ if (binary_caching_enabled)
{
+ auto restore = binaries_provider->try_restore(paths, spec, *abi_tag_and_file, action.build_options);
+ if (restore == RestoreResult::BUILD_FAILED)
return BuildResult::BUILD_FAILED;
- }
- else
- {
- System::print2("Purging bad archive\n");
- fs.remove(archive_path, ec);
- }
- }
- else
+ else if (restore == RestoreResult::SUCCESS)
{
auto maybe_bcf = Paragraphs::try_load_cached_package(paths, spec);
auto bcf = std::make_unique<BinaryControlFile>(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO));
return {BuildResult::SUCCEEDED, std::move(bcf)};
}
- }
-
- if (fs.exists(archive_tombstone_path))
- {
- if (action.build_options.fail_on_tombstone == FailOnTombstone::YES)
- {
- System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");
- return BuildResult::BUILD_FAILED;
- }
else
{
- System::print2(
- System::Color::warning, "Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");
+ // missing package, proceed to build.
}
}
- System::printf("Could not locate cached archive: %s\n", archive_path.u8string());
- }
-
ExtendedBuildResult result = do_build_package_and_clean_buildtrees(
paths, pre_build_info, pre_build_info.public_abi_override.value_or(abi_tag_and_file->tag), action);
fs.create_directories(abi_package_dir, ec);
- Checks::check_exit(VCPKG_LINE_INFO, !ec, "Coud not create directory %s", abi_package_dir.u8string());
fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec);
Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string());
- if (action.build_options.binary_caching == BinaryCaching::YES && result.code == BuildResult::SUCCEEDED)
+ if (binary_caching_enabled && result.code == BuildResult::SUCCEEDED)
{
- const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip");
-
- compress_archive(paths, spec, tmp_archive_path);
-
- fs.create_directories(archive_path.parent_path(), ec);
- fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec);
- if (ec)
- {
- System::printf(System::Color::warning,
- "Failed to store binary cache %s: %s\n",
- archive_path.u8string(),
- ec.message());
- }
- else
- System::printf("Stored binary cache: %s\n", archive_path.u8string());
+ binaries_provider->push_success(paths, *abi_tag_and_file, action);
}
- else if (action.build_options.binary_caching == BinaryCaching::YES &&
+ else if (binary_caching_enabled &&
(result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED))
{
- if (!fs.exists(archive_tombstone_path))
- {
- // Build failed, store all failure logs in the tombstone.
- const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs";
- const auto tmp_log_path_destination = tmp_log_path / spec.name();
- const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip";
- fs.create_directories(tmp_log_path_destination, ec);
-
- for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name()))
- {
- if (log_file.path().extension() == ".log")
- {
- fs.copy_file(log_file.path(),
- tmp_log_path_destination / log_file.path().filename(),
- fs::stdfs::copy_options::none,
- ec);
- }
- }
-
- compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip");
-
- fs.create_directories(archive_tombstone_path.parent_path(), ec);
- fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec);
-
- // clean up temporary directory
- fs.remove_all(tmp_log_path, VCPKG_LINE_INFO);
- }
+ binaries_provider->push_failure(paths, *abi_tag_and_file, spec);
}
return result;

MODIFY
src/vcpkg/commands.buildexternal.cpp
src/vcpkg/commands.buildexternal.cpp
@@ -33,6 +33,6 @@ namespace vcpkg::Commands::BuildExternal
Checks::check_exit(
VCPKG_LINE_INFO, maybe_scfl.has_value(), "could not load control file for %s", spec.package_spec.name());
- Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, options, paths);
+ Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, paths);
}
}

MODIFY
src/vcpkg/install.cpp
src/vcpkg/install.cpp
#include <vcpkg/base/hash.h>
#include <vcpkg/base/system.print.h>
#include <vcpkg/base/util.h>
+#include <vcpkg/binarycaching.h>
#include <vcpkg/build.h>
#include <vcpkg/cmakevars.h>
#include <vcpkg/commands.h>
@@ -299,6 +300,7 @@ namespace vcpkg::Install
ExtendedBuildResult perform_install_plan_action(const VcpkgPaths& paths,
InstallPlanAction& action,
StatusParagraphs& status_db,
+ IBinaryProvider* binaries_provider,
const CMakeVars::CMakeVarProvider& var_provider)
{
const InstallPlanType& plan_type = action.plan_type;
@@ -339,7 +341,7 @@ namespace vcpkg::Install
else
System::printf("Building package %s...\n", display_name_with_features);
- auto result = Build::build_package(paths, action, var_provider, status_db);
+ auto result = Build::build_package(paths, action, var_provider, binaries_provider, status_db);
if (BuildResult::DOWNLOADED == result.code)
{
@@ -456,13 +458,15 @@ namespace vcpkg::Install
for (auto&& action : action_plan.already_installed)
{
results.emplace_back(action.spec, &action);
- results.back().build_result = perform_install_plan_action(paths, action, status_db, var_provider);
+ results.back().build_result = perform_install_plan_action(paths, action, status_db, nullptr, var_provider);
}
+ auto binary_provider = create_archives_provider();
for (auto&& action : action_plan.install_actions)
{
with_tracking(action.spec, [&]() {
- auto result = perform_install_plan_action(paths, action, status_db, var_provider);
+ auto result =
+ perform_install_plan_action(paths, action, status_db, binary_provider.get(), var_provider);
if (result.code != BuildResult::SUCCEEDED && keep_going == KeepGoing::NO)
{

[vcpkg] Extract binary caching functionality into separate interface