Enforce cross platform paths in registries.
* Enforce cross platform paths in registries.
This also drops the dependency on path::iterator which means Bill doesn't have to implement it in project depath
* Also fix array_elements to actually be diengaged on failure.
MODIFY
include/vcpkg/base/jsonreader.h
include/vcpkg/base/jsonreader.h
@@ -73,6 +73,20 @@ namespace vcpkg::Json
int64_t index = -1;
StringView field;
};
+
+ struct PathGuard
+ {
+ PathGuard(std::vector<Path>& path) : m_path{path} { m_path.emplace_back(); }
+ PathGuard(std::vector<Path>& path, int64_t i) : m_path{path} { m_path.emplace_back(i); }
+ PathGuard(std::vector<Path>& path, StringView f) : m_path{path} { m_path.emplace_back(f); }
+ PathGuard(const PathGuard&) = delete;
+ PathGuard& operator=(const PathGuard&) = delete;
+ ~PathGuard() { m_path.pop_back(); }
+
+ private:
+ std::vector<Path>& m_path;
+ };
+
std::vector<Path> m_path;
public:
@@ -100,9 +114,8 @@ namespace vcpkg::Json
template<class Type>
void visit_in_key(const Value& value, StringView key, Type& place, IDeserializer<Type>& visitor)
{
- m_path.push_back(key);
+ PathGuard guard{m_path, key};
auto opt = visitor.visit(*this, value);
-
if (auto p_opt = opt.get())
{
place = std::move(*p_opt);
@@ -111,16 +124,14 @@ namespace vcpkg::Json
{
add_expected_type_error(visitor.type_name());
}
- m_path.pop_back();
}
// value should be the value at key of the currently visited object
template<class Type>
void visit_at_index(const Value& value, int64_t index, Type& place, IDeserializer<Type>& visitor)
{
- m_path.push_back(index);
+ PathGuard guard{m_path, index};
auto opt = visitor.visit(*this, value);
-
if (auto p_opt = opt.get())
{
place = std::move(*p_opt);
@@ -129,7 +140,6 @@ namespace vcpkg::Json
{
add_expected_type_error(visitor.type_name());
}
- m_path.pop_back();
}
// returns whether key \in obj
@@ -161,29 +171,27 @@ namespace vcpkg::Json
template<class Type>
Optional<std::vector<Type>> array_elements(const Array& arr, IDeserializer<Type>& visitor)
{
- std::vector<Type> result;
- m_path.emplace_back();
+ Optional<std::vector<Type>> result{std::vector<Type>()};
+ PathGuard guard{m_path};
for (size_t i = 0; i < arr.size(); ++i)
{
m_path.back().index = static_cast<int64_t>(i);
auto opt = visitor.visit(*this, arr[i]);
- if (auto p = opt.get())
+ if (auto parsed = opt.get())
+ {
+ if (auto result_vec = result.get())
{
- result.push_back(std::move(*p));
+ result_vec->push_back(std::move(*parsed));
+ }
}
else
{
this->add_expected_type_error(visitor.type_name());
- for (++i; i < arr.size(); ++i)
- {
- m_path.back().index = static_cast<int64_t>(i);
- auto opt2 = visitor.visit(*this, arr[i]);
- if (!opt2) this->add_expected_type_error(visitor.type_name());
- }
+ result.clear();
}
}
- m_path.pop_back();
- return std::move(result);
+
+ return result;
}
};

MODIFY
include/vcpkg/base/optional.h
include/vcpkg/base/optional.h
@@ -82,7 +82,7 @@ namespace vcpkg
}
else if (m_is_present && !o.m_is_present)
{
- clear();
+ destroy();
}
return *this;
}
@@ -100,7 +100,7 @@ namespace vcpkg
}
else if (m_is_present && !o.m_is_present)
{
- clear();
+ destroy();
}
return *this;
}
@@ -110,14 +110,14 @@ namespace vcpkg
const T& value() const { return this->m_t; }
T& value() { return this->m_t; }
- private:
- void clear()
+ void destroy()
{
m_is_present = false;
m_t.~T();
m_inactive = '\0';
}
+ private:
bool m_is_present;
union
{
@@ -160,7 +160,7 @@ namespace vcpkg
}
else if (m_is_present && !o.m_is_present)
{
- clear();
+ destroy();
}
return *this;
}
@@ -170,14 +170,14 @@ namespace vcpkg
const T& value() const { return this->m_t; }
T& value() { return this->m_t; }
- private:
- void clear()
+ void destroy()
{
m_is_present = false;
m_t.~T();
m_inactive = '\0';
}
+ private:
bool m_is_present;
union
{
@@ -290,7 +290,7 @@ namespace vcpkg
template<class F, class U = map_t<F>>
Optional<U> map(F f) const&
{
- if (has_value())
+ if (this->m_base.has_value())
{
return f(this->m_base.value());
}
@@ -300,7 +300,7 @@ namespace vcpkg
template<class F, class U = map_t<F>>
U then(F f) const&
{
- if (has_value())
+ if (this->m_base.has_value())
{
return f(this->m_base.value());
}
@@ -313,7 +313,7 @@ namespace vcpkg
template<class F, class U = move_map_t<F>>
Optional<U> map(F f) &&
{
- if (has_value())
+ if (this->m_base.has_value())
{
return f(std::move(this->m_base.value()));
}
@@ -323,13 +323,21 @@ namespace vcpkg
template<class F, class U = move_map_t<F>>
U then(F f) &&
{
- if (has_value())
+ if (this->m_base.has_value())
{
return f(std::move(this->m_base.value()));
}
return nullopt;
}
+ void clear()
+ {
+ if (this->m_base.has_value())
+ {
+ this->m_base.destroy();
+ }
+ }
+
friend bool operator==(const Optional& lhs, const Optional& rhs)
{
if (lhs.m_base.has_value())

MODIFY
include/vcpkg/base/strings.h
include/vcpkg/base/strings.h
@@ -306,6 +306,7 @@ namespace vcpkg::Strings
const char* search(StringView haystack, StringView needle);
bool contains(StringView haystack, StringView needle);
+ bool contains(StringView haystack, char needle);
// base 32 encoding, following IETC RFC 4648
std::string b32_encode(std::uint64_t x) noexcept;

MODIFY
include/vcpkg/registries.h
include/vcpkg/registries.h
@@ -136,4 +136,47 @@ namespace vcpkg
ExpectedS<std::map<std::string, VersionT, std::less<>>> get_builtin_baseline(const VcpkgPaths& paths);
bool is_git_commit_sha(StringView sv);
+
+ struct VersionDbEntry
+ {
+ VersionT version;
+ Versions::Scheme scheme = Versions::Scheme::String;
+
+ // only one of these may be non-empty
+ std::string git_tree;
+ path p;
+ };
+
+ // VersionDbType::Git => VersionDbEntry.git_tree is filled
+ // VersionDbType::Filesystem => VersionDbEntry.path is filled
+ enum class VersionDbType
+ {
+ Git,
+ Filesystem,
+ };
+
+ struct VersionDbEntryDeserializer final : Json::IDeserializer<VersionDbEntry>
+ {
+ static constexpr StringLiteral GIT_TREE = "git-tree";
+ static constexpr StringLiteral PATH = "path";
+
+ StringView type_name() const override;
+ View<StringView> valid_fields() const override;
+ Optional<VersionDbEntry> visit_object(Json::Reader& r, const Json::Object& obj) override;
+ VersionDbEntryDeserializer(VersionDbType type, const path& root) : type(type), registry_root(root) { }
+
+ private:
+ VersionDbType type;
+ path registry_root;
+ };
+
+ struct VersionDbEntryArrayDeserializer final : Json::IDeserializer<std::vector<VersionDbEntry>>
+ {
+ virtual StringView type_name() const override;
+ virtual Optional<std::vector<VersionDbEntry>> visit_array(Json::Reader& r, const Json::Array& arr) override;
+ VersionDbEntryArrayDeserializer(VersionDbType type, const path& root) : underlying{type, root} { }
+
+ private:
+ VersionDbEntryDeserializer underlying;
+ };
}

MODIFY
src/vcpkg-test/registries.cpp
src/vcpkg-test/registries.cpp
@@ -196,3 +196,210 @@ TEST_CASE ("registry_parsing", "[registries]")
INFO(Strings::join("\n", r.errors()));
CHECK(r.errors().empty());
}
+
+TEST_CASE ("git_version_db_parsing", "[registries]")
+{
+ VersionDbEntryArrayDeserializer filesystem_version_db{VersionDbType::Git, "a/b"};
+ Json::Reader r;
+ auto test_json = parse_json(R"json(
+[
+ {
+ "git-tree": "9b07f8a38bbc4d13f8411921e6734753e15f8d50",
+ "version-date": "2021-06-26",
+ "port-version": 0
+ },
+ {
+ "git-tree": "12b84a31469a78dd4b42dcf58a27d4600f6b2d48",
+ "version-date": "2021-01-14",
+ "port-version": 0
+ },
+ {
+ "git-tree": "bd4565e8ab55bc5e098a1750fa5ff0bc4406ca9b",
+ "version-string": "2020-04-12",
+ "port-version": 0
+ }
+]
+)json");
+
+ auto results_opt = r.visit(test_json, filesystem_version_db);
+ auto& results = results_opt.value_or_exit(VCPKG_LINE_INFO);
+ CHECK(results[0].version == VersionT{"2021-06-26", 0});
+ CHECK(results[0].git_tree == "9b07f8a38bbc4d13f8411921e6734753e15f8d50");
+ CHECK(results[1].version == VersionT{"2021-01-14", 0});
+ CHECK(results[1].git_tree == "12b84a31469a78dd4b42dcf58a27d4600f6b2d48");
+ CHECK(results[2].version == VersionT{"2020-04-12", 0});
+ CHECK(results[2].git_tree == "bd4565e8ab55bc5e098a1750fa5ff0bc4406ca9b");
+ CHECK(r.errors().empty());
+}
+
+TEST_CASE ("filesystem_version_db_parsing", "[registries]")
+{
+ VersionDbEntryArrayDeserializer filesystem_version_db{VersionDbType::Filesystem, "a/b"};
+
+ {
+ Json::Reader r;
+ auto test_json = parse_json(R"json(
+[
+ {
+ "version-string": "puppies",
+ "port-version": 0,
+ "path": "$/c/d"
+ },
+ {
+ "version-string": "doggies",
+ "port-version": 0,
+ "path": "$/e/d"
+ },
+ {
+ "version-semver": "1.2.3",
+ "port-version": 0,
+ "path": "$/semvers/here"
+ }
+]
+ )json");
+ auto results_opt = r.visit(test_json, filesystem_version_db);
+ auto& results = results_opt.value_or_exit(VCPKG_LINE_INFO);
+ CHECK(results[0].version == VersionT{"puppies", 0});
+ CHECK(results[0].p == vcpkg::u8path("a/b/c/d"));
+ CHECK(results[1].version == VersionT{"doggies", 0});
+ CHECK(results[1].p == vcpkg::u8path("a/b/e/d"));
+ CHECK(results[2].version == VersionT{"1.2.3", 0});
+ CHECK(results[2].p == vcpkg::u8path("a/b/semvers/here"));
+ CHECK(r.errors().empty());
+ }
+
+ { // missing $/
+ Json::Reader r;
+ auto test_json = parse_json(R"json(
+[
+ {
+ "version-string": "puppies",
+ "port-version": 0,
+ "path": "c/d"
+ }
+]
+ )json");
+ CHECK(!r.visit(test_json, filesystem_version_db).has_value());
+ CHECK(!r.errors().empty());
+ }
+
+ { // uses backslash
+ Json::Reader r;
+ auto test_json = parse_json(R"json(
+[
+ {
+ "version-string": "puppies",
+ "port-version": 0,
+ "path": "$\\c\\d"
+ }
+]
+ )json");
+ CHECK(!r.visit(test_json, filesystem_version_db).has_value());
+ CHECK(!r.errors().empty());
+ }
+
+ { // doubled slash
+ Json::Reader r;
+ auto test_json = parse_json(R"json(
+[
+ {
+ "version-string": "puppies",
+ "port-version": 0,
+ "path": "$/c//d"
+ }
+]
+ )json");
+ CHECK(!r.visit(test_json, filesystem_version_db).has_value());
+ CHECK(!r.errors().empty());
+ }
+
+ { // dot path (first)
+ Json::Reader r;
+ auto test_json = parse_json(R"json(
+[
+ {
+ "version-string": "puppies",
+ "port-version": 0,
+ "path": "$/./d/a/a"
+ }
+]
+ )json");
+ CHECK(!r.visit(test_json, filesystem_version_db).has_value());
+ CHECK(!r.errors().empty());
+ }
+
+ { // dot path (mid)
+ Json::Reader r;
+ auto test_json = parse_json(R"json(
+[
+ {
+ "version-string": "puppies",
+ "port-version": 0,
+ "path": "$/c/d/./a"
+ }
+]
+ )json");
+ CHECK(!r.visit(test_json, filesystem_version_db).has_value());
+ CHECK(!r.errors().empty());
+ }
+
+ { // dot path (last)
+ Json::Reader r;
+ auto test_json = parse_json(R"json(
+[
+ {
+ "version-string": "puppies",
+ "port-version": 0,
+ "path": "$/c/d/."
+ }
+]
+ )json");
+ CHECK(!r.visit(test_json, filesystem_version_db).has_value());
+ CHECK(!r.errors().empty());
+ }
+
+ { // dot dot path (first)
+ Json::Reader r;
+ auto test_json = parse_json(R"json(
+[
+ {
+ "version-string": "puppies",
+ "port-version": 0,
+ "path": "$/../d/a/a"
+ }
+]
+ )json");
+ CHECK(!r.visit(test_json, filesystem_version_db).has_value());
+ CHECK(!r.errors().empty());
+ }
+
+ { // dot dot path (mid)
+ Json::Reader r;
+ auto test_json = parse_json(R"json(
+[
+ {
+ "version-string": "puppies",
+ "port-version": 0,
+ "path": "$/c/d/../a"
+ }
+]
+ )json");
+ CHECK(!r.visit(test_json, filesystem_version_db).has_value());
+ CHECK(!r.errors().empty());
+ }
+
+ { // dot dot path (last)
+ Json::Reader r;
+ auto test_json = parse_json(R"json(
+[
+ {
+ "version-string": "puppies",
+ "port-version": 0,
+ "path": "$/c/d/.."
+ }
+]
+ )json");
+ CHECK(!r.visit(test_json, filesystem_version_db).has_value());
+ CHECK(!r.errors().empty());
+ }
+}

MODIFY
src/vcpkg/base/strings.cpp
src/vcpkg/base/strings.cpp
@@ -311,6 +311,11 @@ bool Strings::contains(StringView haystack, StringView needle)
return Strings::search(haystack, needle) != haystack.end();
}
+bool Strings::contains(StringView haystack, char needle)
+{
+ return std::find(haystack.begin(), haystack.end(), needle) != haystack.end();
+}
+
size_t Strings::byte_edit_distance(StringView a, StringView b)
{
static constexpr size_t max_string_size = 100;

MODIFY
src/vcpkg/registries.cpp
src/vcpkg/registries.cpp
@@ -243,24 +243,6 @@ namespace
DelayedInit<Baseline> m_baseline;
};
- struct VersionDbEntry
- {
- VersionT version;
- Versions::Scheme scheme = Versions::Scheme::String;
-
- // only one of these may be non-empty
- std::string git_tree;
- path p;
- };
-
- // VersionDbType::Git => VersionDbEntry.git_tree is filled
- // VersionDbType::Filesystem => VersionDbEntry.path is filled
- enum class VersionDbType
- {
- Git,
- Filesystem,
- };
-
path relative_path_to_versions(StringView port_name);
ExpectedS<std::vector<VersionDbEntry>> load_versions_file(const Filesystem& fs,
VersionDbType vdb,
@@ -732,106 +714,6 @@ namespace
};
BaselineDeserializer BaselineDeserializer::instance;
- struct VersionDbEntryDeserializer final : Json::IDeserializer<VersionDbEntry>
- {
- static constexpr StringLiteral GIT_TREE = "git-tree";
- static constexpr StringLiteral PATH = "path";
-
- StringView type_name() const override { return "a version database entry"; }
- View<StringView> valid_fields() const override
- {
- static const StringView u_git[] = {GIT_TREE};
- static const StringView u_path[] = {PATH};
- static const auto t_git = vcpkg::Util::Vectors::concat<StringView>(schemed_deserializer_fields(), u_git);
- static const auto t_path = vcpkg::Util::Vectors::concat<StringView>(schemed_deserializer_fields(), u_path);
-
- return type == VersionDbType::Git ? t_git : t_path;
- }
-
- Optional<VersionDbEntry> visit_object(Json::Reader& r, const Json::Object& obj) override
- {
- VersionDbEntry ret;
-
- auto schemed_version = visit_required_schemed_deserializer(type_name(), r, obj);
- ret.scheme = schemed_version.scheme;
- ret.version = std::move(schemed_version.versiont);
-
- static Json::StringDeserializer git_tree_deserializer("a git object SHA");
- static Json::StringDeserializer path_deserializer("a registry path");
-
- switch (type)
- {
- case VersionDbType::Git:
- {
- r.required_object_field(type_name(), obj, GIT_TREE, ret.git_tree, git_tree_deserializer);
- break;
- }
- case VersionDbType::Filesystem:
- {
- std::string path_res;
- r.required_object_field(type_name(), obj, PATH, path_res, path_deserializer);
- path p = vcpkg::u8path(path_res);
- if (p.is_absolute())
- {
- r.add_generic_error("a registry path",
- "A registry path may not be absolute, and must start with a `$` to mean "
- "the registry root; e.g., `$/foo/bar`.");
- return ret;
- }
- else if (p.empty())
- {
- r.add_generic_error("a registry path", "A registry path must not be empty.");
- return ret;
- }
-
- auto it = p.begin();
- if (*it != "$")
- {
- r.add_generic_error(
- "a registry path",
- "A registry path must start with `$` to mean the registry root; e.g., `$/foo/bar`");
- }
-
- ret.p = registry_root;
- ++it;
- std::for_each(it, p.end(), [&r, &ret](const path& p) {
- if (p == "..")
- {
- r.add_generic_error("a registry path", "A registry path must not contain `..`.");
- }
- else
- {
- ret.p /= p;
- }
- });
-
- break;
- }
- }
-
- return ret;
- }
-
- VersionDbEntryDeserializer(VersionDbType type, const path& root) : type(type), registry_root(root) { }
-
- VersionDbType type;
- path registry_root;
- };
-
- struct VersionDbEntryArrayDeserializer final : Json::IDeserializer<std::vector<VersionDbEntry>>
- {
- virtual StringView type_name() const override { return "an array of versions"; }
-
- virtual Optional<std::vector<VersionDbEntry>> visit_array(Json::Reader& r, const Json::Array& arr) override
- {
- return r.array_elements(arr, underlying);
- }
-
- VersionDbEntryArrayDeserializer(VersionDbType type, const path& root) : underlying{type, root} { }
-
- VersionDbEntryDeserializer underlying;
- };
-
struct RegistryImplDeserializer : Json::IDeserializer<std::unique_ptr<RegistryImplementation>>
{
constexpr static StringLiteral KIND = "kind";
@@ -1138,6 +1020,119 @@ namespace
namespace vcpkg
{
+ StringView VersionDbEntryDeserializer::type_name() const { return "a version database entry"; }
+ View<StringView> VersionDbEntryDeserializer::valid_fields() const
+ {
+ static const StringView u_git[] = {GIT_TREE};
+ static const StringView u_path[] = {PATH};
+ static const auto t_git = vcpkg::Util::Vectors::concat<StringView>(schemed_deserializer_fields(), u_git);
+ static const auto t_path = vcpkg::Util::Vectors::concat<StringView>(schemed_deserializer_fields(), u_path);
+
+ return type == VersionDbType::Git ? t_git : t_path;
+ }
+
+ Optional<VersionDbEntry> VersionDbEntryDeserializer::visit_object(Json::Reader& r, const Json::Object& obj)
+ {
+ VersionDbEntry ret;
+
+ auto schemed_version = visit_required_schemed_deserializer(type_name(), r, obj);
+ ret.scheme = schemed_version.scheme;
+ ret.version = std::move(schemed_version.versiont);
+
+ static Json::StringDeserializer git_tree_deserializer("a git object SHA");
+ static Json::StringDeserializer path_deserializer("a registry path");
+
+ switch (type)
+ {
+ case VersionDbType::Git:
+ {
+ r.required_object_field(type_name(), obj, GIT_TREE, ret.git_tree, git_tree_deserializer);
+ break;
+ }
+ case VersionDbType::Filesystem:
+ {
+ std::string path_res;
+ r.required_object_field(type_name(), obj, PATH, path_res, path_deserializer);
+ if (!Strings::starts_with(path_res, "$/"))
+ {
+ r.add_generic_error(
+ "a registry path",
+ "A registry path must start with `$` to mean the registry root; e.g., `$/foo/bar`.");
+ return nullopt;
+ }
+
+ if (Strings::contains(path_res, '\\'))
+ {
+ r.add_generic_error("a registry path",
+ "A registry path must use forward slashes as path separators.");
+ return nullopt;
+ }
+
+ if (Strings::contains(path_res, "//"))
+ {
+ r.add_generic_error("a registry path", "A registry path must not have multiple slashes.");
+ return nullopt;
+ }
+
+ auto first = path_res.begin();
+ const auto last = path_res.end();
+ for (std::string::iterator candidate;; first = candidate)
+ {
+ candidate = std::find(first, last, '/');
+ if (candidate == last)
+ {
+ break;
+ }
+
+ ++candidate;
+ if (candidate == last)
+ {
+ break;
+ }
+
+ if (*candidate != '.')
+ {
+ continue;
+ }
+
+ ++candidate;
+ if (candidate == last || *candidate == '/')
+ {
+ r.add_generic_error("a registry path", "A registry path must not have 'dot' path elements.");
+ return nullopt;
+ }
+
+ if (*candidate != '.')
+ {
+ first = candidate;
+ continue;
+ }
+
+ ++candidate;
+ if (candidate == last || *candidate == '/')
+ {
+ r.add_generic_error("a registry path",
+ "A registry path must not have 'dot dot' path elements.");
+ return nullopt;
+ }
+ }
+
+ ret.p = registry_root / vcpkg::u8path(StringView{path_res}.substr(2));
+ break;
+ }
+ }
+
+ return ret;
+ }
+
+ StringView VersionDbEntryArrayDeserializer::type_name() const { return "an array of versions"; }
+
+ Optional<std::vector<VersionDbEntry>> VersionDbEntryArrayDeserializer::visit_array(Json::Reader& r,
+ const Json::Array& arr)
+ {
+ return r.array_elements(arr, underlying);
+ }
+
LockFile::Entry LockFile::get_or_fetch(const VcpkgPaths& paths, StringView key)
{
auto it = lockdata.find(key);

Enforce cross platform paths in registries. (#137)

* Enforce cross platform paths in registries.

This also drops the dependency on path::iterator which means Bill doesn't have to implement it in project depath

* Also fix array_elements to actually be diengaged on failure.