gpcloud: parse ETag case-insensitively + add NextMarker parsing logic
Some cloud providers form ETAG in a different format, for instance,
Yandex Cloud Object Storage sends it in format `Etag: "......"`.
According to the RFC (https://www.w3.org/Protocols/rfc2616/rfc2616.html)
ETag can be in any case. So we should parse the header case-insensitively.
Also add `NextMarker` parsing logic.
@@ -192,6 +192,7 @@ bool S3InterfaceService::parseBucketXML(ListBucketResult *result, xmlParserCtxtP
char *content = NULL;
char *key = NULL;
char *key_size = NULL;
+ char *next_marker = NULL;
cur = rootElement->xmlChildrenNode;
while (cur != NULL) {
@@ -200,6 +201,14 @@ bool S3InterfaceService::parseBucketXML(ListBucketResult *result, xmlParserCtxtP
key = NULL;
}
+ if (!xmlStrcmp(cur->name, (const xmlChar *)"NextMarker")) {
+ content = (char *)xmlNodeGetContent(cur);
+ if (content) {
+ next_marker = strdup(content);
+ xmlFree(content);
+ }
+ }
+
if (!xmlStrcmp(cur->name, (const xmlChar *)"IsTruncated")) {
content = (char *)xmlNodeGetContent(cur);
if (content) {
@@ -262,7 +271,11 @@ bool S3InterfaceService::parseBucketXML(ListBucketResult *result, xmlParserCtxtP
cur = cur->next;
}
+ if (is_truncated && next_marker) {
+ marker = next_marker;
+ } else {
marker = (is_truncated && key) ? key : "";
+ }
if (key) {
xmlFree(key);
@@ -299,6 +312,7 @@ ListBucketResult S3InterfaceService::listBucket(S3Url &s3Url) {
if (!encodedPrefix.empty()) {
querySs << (marker.empty() ? "prefix=" : "&prefix=") << encodedPrefix;
}
+
s3Url.setPrefix("");
string queryStr = querySs.str();
@@ -470,8 +484,17 @@ string S3InterfaceService::uploadPartOfData(S3VectorUInt8 &data, const S3Url &s3
Response resp = this->putResponseWithRetries(urlWithQuery.str(), headers, data);
if (resp.getStatus() == RESPONSE_OK) {
string headers(resp.getRawHeaders().begin(), resp.getRawHeaders().end());
+ string toSearch = "etag: ";
+
+ auto res =
+ std::search(headers.begin(), headers.end(), toSearch.begin(), toSearch.end(),
+ [](char ch1, char ch2) { return std::tolower(ch1) == std::tolower(ch2); });
+
+ if (res == headers.end()) {
+ S3_DIE(S3RuntimeError, "Response does not contain etag in the header");
+ }
- uint64_t etagStartPos = headers.find("ETag: ") + 6;
+ uint64_t etagStartPos = res - headers.begin() + toSearch.length();
string etagToEnd = headers.substr(etagStartPos);
// RFC 2616 states "HTTP/1.1 defines the sequence CR LF as the end-of-line
// marker for all protocol elements except the entity-body"
