[vcpkg] improve xunit xml output used in CI tests
MODIFY
include/vcpkg/install.h
include/vcpkg/install.h
@@ -37,7 +37,6 @@ namespace vcpkg::Install
std::string total_elapsed_time;
void print() const;
- static std::string xunit_result(const PackageSpec& spec, Chrono::ElapsedTime time, Build::BuildResult code);
std::string xunit_results() const;
};

MODIFY
src/vcpkg/commands.ci.cpp
src/vcpkg/commands.ci.cpp
@@ -52,24 +52,176 @@ namespace vcpkg::Commands::CI
nullptr,
};
+ struct XunitTestResults
+ {
+ public:
+
+ XunitTestResults()
+ {
+ m_assembly_run_datetime = Chrono::CTime::get_current_date_time();
+ }
+
+ void add_test_results(const std::string& spec, const Build::BuildResult& build_result, const Chrono::ElapsedTime& elapsed_time, const std::string& abi_tag)
+ {
+ m_collections.back().tests.push_back({ spec, build_result, elapsed_time, abi_tag });
+ }
+
+ // Starting a new test collection
+ void push_collection( const std::string& name)
+ {
+ m_collections.push_back({name});
+ }
+
+ void collection_time(const vcpkg::Chrono::ElapsedTime& time)
+ {
+ m_collections.back().time = time;
+ }
+
+ const std::string& build_xml()
+ {
+ m_xml.clear();
+ xml_start_assembly();
+
+ for (const auto& collection : m_collections)
+ {
+ xml_start_collection(collection);
+ for (const auto& test : collection.tests)
+ {
+ xml_test(test);
+ }
+ xml_finish_collection();
+ }
+
+ xml_finish_assembly();
+ return m_xml;
+ }
+
+ void assembly_time(const vcpkg::Chrono::ElapsedTime& assembly_time)
+ {
+ m_assembly_time = assembly_time;
+ }
+
+ private:
+
+ struct XunitTest
+ {
+ std::string name;
+ vcpkg::Build::BuildResult result;
+ vcpkg::Chrono::ElapsedTime time;
+ std::string abi_tag;
+ };
+
+ struct XunitCollection
+ {
+ std::string name;
+ vcpkg::Chrono::ElapsedTime time;
+ std::vector<XunitTest> tests;
+ };
+
+ void xml_start_assembly()
+ {
+ std::string datetime;
+ if (m_assembly_run_datetime)
+ {
+ auto rawDateTime = m_assembly_run_datetime.get()->to_string();
+ // The expected format is "yyyy-mm-ddThh:mm:ss.0Z"
+ // 0123456789012345678901
+ datetime = Strings::format(R"(run-date="%s" run-time="%s")",
+ rawDateTime.substr(0, 10), rawDateTime.substr(11, 8));
+ }
+
+ std::string time = Strings::format(R"(time="%lld")", m_assembly_time.as<std::chrono::seconds>().count());
+
+ m_xml += Strings::format(
+ R"(<assemblies>)" "\n"
+ R"( <assembly name="vcpkg" %s %s>)" "\n"
+ , datetime, time);
+ }
+ void xml_finish_assembly()
+ {
+ m_xml += " </assembly>\n"
+ "</assemblies>\n";
+ }
+
+ void xml_start_collection(const XunitCollection& collection)
+ {
+ m_xml += Strings::format(R"( <collection name="%s" time="%lld">)"
+ "\n",
+ collection.name,
+ collection.time.as<std::chrono::seconds>().count());
+ }
+ void xml_finish_collection()
+ {
+ m_xml += " </collection>\n";
+ }
+
+ void xml_test(const XunitTest& test)
+ {
+ std::string message_block;
+ const char* result_string = "";
+ switch (test.result)
+ {
+ case BuildResult::POST_BUILD_CHECKS_FAILED:
+ case BuildResult::FILE_CONFLICTS:
+ case BuildResult::BUILD_FAILED:
+ result_string = "Fail";
+ message_block = Strings::format("<failure><message><![CDATA[%s]]></message></failure>", to_string(test.result));
+ break;
+ case BuildResult::EXCLUDED:
+ case BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES:
+ result_string = "Skip";
+ message_block = Strings::format("<reason><![CDATA[%s]]></reason>", to_string(test.result));
+ break;
+ case BuildResult::SUCCEEDED:
+ result_string = "Pass";
+ break;
+ default:
+ Checks::exit_fail(VCPKG_LINE_INFO);
+ break;
+ }
+
+ std::string traits_block;
+ if (test.abi_tag != "") // only adding if there is a known abi tag
+ {
+ traits_block = Strings::format(R"(<traits><trait name="abi_tag" value="%s" /></traits>)", test.abi_tag);
+ }
+
+ m_xml += Strings::format(R"( <test name="%s" method="%s" time="%lld" result="%s">%s%s</test>)"
+ "\n",
+ test.name,
+ test.name,
+ test.time.as<std::chrono::seconds>().count(),
+ result_string,
+ traits_block,
+ message_block);
+ }
+
+ Optional<vcpkg::Chrono::CTime> m_assembly_run_datetime;
+ vcpkg::Chrono::ElapsedTime m_assembly_time;
+ std::vector<XunitCollection> m_collections;
+
+ std::string m_xml;
+ };
+
+
struct UnknownCIPortsResults
{
std::vector<FullPackageSpec> unknown;
std::map<PackageSpec, Build::BuildResult> known;
std::map<PackageSpec, std::vector<std::string>> features;
+ std::map<PackageSpec, std::string> abi_tag_map;
};
- static UnknownCIPortsResults find_unknown_ports_for_ci(const VcpkgPaths& paths,
+ static std::unique_ptr<UnknownCIPortsResults> find_unknown_ports_for_ci(const VcpkgPaths& paths,
const std::set<std::string>& exclusions,
const Dependencies::PortFileProvider& provider,
const std::vector<FeatureSpec>& fspecs,
const bool purge_tombstones)
{
- UnknownCIPortsResults ret;
+ auto ret = std::make_unique<UnknownCIPortsResults>();
auto& fs = paths.get_filesystem();
- std::map<PackageSpec, std::string> abi_tag_map;
std::set<PackageSpec> will_fail;
const Build::BuildPackageOptions build_options = {
@@ -103,9 +255,9 @@ namespace vcpkg::Commands::CI
auto dependency_abis =
Util::fmap(p->computed_dependencies, [&](const PackageSpec& spec) -> Build::AbiEntry {
- auto it = abi_tag_map.find(spec);
+ auto it = ret->abi_tag_map.find(spec);
- if (it == abi_tag_map.end())
+ if (it == ret->abi_tag_map.end())
return {spec.name(), ""};
else
return {spec.name(), it->second};
@@ -118,13 +270,13 @@ namespace vcpkg::Commands::CI
if (auto tag_and_file = maybe_tag_and_file.get())
{
abi = tag_and_file->tag;
- abi_tag_map.emplace(p->spec, abi);
+ ret->abi_tag_map.emplace(p->spec, abi);
}
}
else if (auto ipv = p->installed_package.get())
{
abi = ipv->core->package.abi;
- if (!abi.empty()) abi_tag_map.emplace(p->spec, abi);
+ if (!abi.empty()) ret->abi_tag_map.emplace(p->spec, abi);
}
std::string state;
@@ -143,35 +295,35 @@ namespace vcpkg::Commands::CI
bool b_will_build = false;
- ret.features.emplace(p->spec,
+ ret->features.emplace(p->spec,
std::vector<std::string> {p->feature_list.begin(), p->feature_list.end()});
if (Util::Sets::contains(exclusions, p->spec.name()))
{
- ret.known.emplace(p->spec, BuildResult::EXCLUDED);
+ ret->known.emplace(p->spec, BuildResult::EXCLUDED);
will_fail.emplace(p->spec);
}
else if (std::any_of(p->computed_dependencies.begin(),
p->computed_dependencies.end(),
[&](const PackageSpec& spec) { return Util::Sets::contains(will_fail, spec); }))
{
- ret.known.emplace(p->spec, BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES);
+ ret->known.emplace(p->spec, BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES);
will_fail.emplace(p->spec);
}
else if (fs.exists(archive_path))
{
state += "pass";
- ret.known.emplace(p->spec, BuildResult::SUCCEEDED);
+ ret->known.emplace(p->spec, BuildResult::SUCCEEDED);
}
else if (fs.exists(archive_tombstone_path))
{
state += "fail";
- ret.known.emplace(p->spec, BuildResult::BUILD_FAILED);
+ ret->known.emplace(p->spec, BuildResult::BUILD_FAILED);
will_fail.emplace(p->spec);
}
else
{
- ret.unknown.push_back({p->spec, {p->feature_list.begin(), p->feature_list.end()}});
+ ret->unknown.push_back({p->spec, {p->feature_list.begin(), p->feature_list.end()}});
b_will_build = true;
}
@@ -229,23 +381,29 @@ namespace vcpkg::Commands::CI
};
std::vector<std::map<PackageSpec, BuildResult>> all_known_results;
+ std::map<PackageSpec, std::string> abi_tag_map;
+
+ XunitTestResults xunitTestResults;
std::vector<std::string> all_ports = Install::get_all_port_names(paths);
std::vector<TripletAndSummary> results;
+ auto timer = Chrono::ElapsedTimer::create_started();
for (const Triplet& triplet : triplets)
{
Input::check_triplet(triplet, paths);
+ xunitTestResults.push_collection(triplet.canonical_name());
+
Dependencies::PackageGraph pgraph(paths_port_file, status_db);
std::vector<PackageSpec> specs = PackageSpec::to_package_specs(all_ports, triplet);
// Install the default features for every package
- auto all_fspecs = Util::fmap(specs, [](auto& spec) { return FeatureSpec(spec, ""); });
+ auto all_feature_specs = Util::fmap(specs, [](auto& spec) { return FeatureSpec(spec, ""); });
auto split_specs =
- find_unknown_ports_for_ci(paths, exclusions_set, paths_port_file, all_fspecs, purge_tombstones);
- auto fspecs = FullPackageSpec::to_feature_specs(split_specs.unknown);
+ find_unknown_ports_for_ci(paths, exclusions_set, paths_port_file, all_feature_specs, purge_tombstones);
+ auto feature_specs = FullPackageSpec::to_feature_specs(split_specs->unknown);
- for (auto&& fspec : fspecs)
+ for (auto&& fspec : feature_specs)
pgraph.install(fspec);
Dependencies::CreateInstallPlanOptions serialize_options;
@@ -284,7 +442,7 @@ namespace vcpkg::Commands::CI
p->plan_type = InstallPlanType::EXCLUDED;
}
- for (auto&& feature : split_specs.features[p->spec])
+ for (auto&& feature : split_specs->features[p->spec])
if (p->feature_list.find(feature) == p->feature_list.end())
{
pgraph.install({p->spec, feature});
@@ -304,13 +462,32 @@ namespace vcpkg::Commands::CI
}
else
{
+ auto collection_timer = Chrono::ElapsedTimer::create_started();
auto summary = Install::perform(action_plan, Install::KeepGoing::YES, paths, status_db);
+ auto collection_time_elapsed = collection_timer.elapsed();
+
+ // Adding results for ports that were built or pulled from an archive
for (auto&& result : summary.results)
- split_specs.known.erase(result.spec);
+ {
+ split_specs->known.erase(result.spec);
+ xunitTestResults.add_test_results(result.spec.to_string(), result.build_result.code, result.timing, split_specs->abi_tag_map.at(result.spec));
+ }
+
+ // Adding results for ports that were not built because they have known states
+ for (auto&& port : split_specs->known)
+ {
+ xunitTestResults.add_test_results(port.first.to_string(), port.second, Chrono::ElapsedTime{}, split_specs->abi_tag_map.at(port.first));
+ }
+
+ all_known_results.emplace_back(std::move(split_specs->known));
+ abi_tag_map.insert(split_specs->abi_tag_map.begin(), split_specs->abi_tag_map.end());
+
results.push_back({ triplet, std::move(summary)});
- all_known_results.emplace_back(std::move(split_specs.known));
+
+ xunitTestResults.collection_time( collection_time_elapsed );
}
}
+ xunitTestResults.assembly_time(timer.elapsed());
for (auto&& result : results)
{
@@ -322,21 +499,7 @@ namespace vcpkg::Commands::CI
auto it_xunit = options.settings.find(OPTION_XUNIT);
if (it_xunit != options.settings.end())
{
- std::string xunit_doc = "<assemblies><assembly><collection>\n";
-
- for (auto&& result : results)
- xunit_doc += result.summary.xunit_results();
- for (auto&& known_result : all_known_results)
- {
- for (auto&& result : known_result)
- {
- xunit_doc +=
- Install::InstallSummary::xunit_result(result.first, Chrono::ElapsedTime {}, result.second);
- }
- }
-
- xunit_doc += "</collection></assembly></assemblies>\n";
- paths.get_filesystem().write_contents(fs::u8path(it_xunit->second), xunit_doc);
+ paths.get_filesystem().write_contents(fs::u8path(it_xunit->second), xunitTestResults.build_xml());
}
Checks::exit_success(VCPKG_LINE_INFO);

MODIFY
src/vcpkg/install.cpp
src/vcpkg/install.cpp
@@ -719,9 +719,9 @@ namespace vcpkg::Install
return nullptr;
}
- std::string InstallSummary::xunit_result(const PackageSpec& spec, Chrono::ElapsedTime time, BuildResult code)
+ static std::string xunit_result(const PackageSpec& spec, Chrono::ElapsedTime time, BuildResult code)
{
- std::string inner_block;
+ std::string message_block;
const char* result_string = "";
switch (code)
{
@@ -729,12 +729,12 @@ namespace vcpkg::Install
case BuildResult::FILE_CONFLICTS:
case BuildResult::BUILD_FAILED:
result_string = "Fail";
- inner_block = Strings::format("<failure><message><![CDATA[%s]]></message></failure>", to_string(code));
+ message_block = Strings::format("<failure><message><![CDATA[%s]]></message></failure>", to_string(code));
break;
case BuildResult::EXCLUDED:
case BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES:
result_string = "Skip";
- inner_block = Strings::format("<reason><![CDATA[%s]]></reason>", to_string(code));
+ message_block = Strings::format("<reason><![CDATA[%s]]></reason>", to_string(code));
break;
case BuildResult::SUCCEEDED: result_string = "Pass"; break;
default: Checks::exit_fail(VCPKG_LINE_INFO);
@@ -746,7 +746,7 @@ namespace vcpkg::Install
spec,
time.as<std::chrono::seconds>().count(),
result_string,
- inner_block);
+ message_block);
}
std::string InstallSummary::xunit_results() const

[vcpkg] improve xunit xml output used in CI tests