Fix logging to correctly log downloads (#2057)

* Only unset some of the headers

* Remove pointless code

* Unset all of our x-amz-* headers

* Correctly send download events when we get one

# Note: It is VERY important to ensure that any changes to VCL will work
#       properly with both the current version of ``master`` and the version in
#       the pull request that adds any new changes. This is because the
#       configuration will be applied automatically as part of the deployment
#       process, but while the previous version of the code is still up and
#       running. Thus backwards incompatible changes must be broken up over
#       multiple pull requests in order to phase them in over multiple deploys.

sub vcl_recv {

    # I'm not 100% sure on what this is exactly for, it was taken from the
    # Fastly documentation, however, what I *believe* it does is just ensure
    # that we don't serve a stale copy of the page from the shield node when
    # an edge node is requesting content.
    if (req.http.Fastly-FF) {
        set req.max_stale_while_revalidate = 0s;
    }

    # Some (Older) clients will send a hash fragment as part of the URL even
    # though that is a local only modification. This breaks this badly for the
    # files in S3, and in general it's just not needed.
    set req.url = regsub(req.url, "#.*$", "");

    # Fastly does some normalization of the Accept-Encoding header so that it
    # reduces the number of cached copies (when served with the common,
    # Vary: Accept-Encoding) that are cached for any one URL. This makes a lot
    # of sense, except for the fact that we want to enable brotli compression
    # for our static files. Thus we need to work around the normalized encoding
    # in a way that still minimizes cached copies, but which will allow our
    # static files to be served using brotli.
    if (req.url ~ "^/static/" && req.http.Fastly-Orig-Accept-Encoding) {
        if (req.http.User-Agent ~ "MSIE 6") {
            # For that 0.3% of stubborn users out there
            unset req.http.Accept-Encoding;
        } elsif (req.http.Fastly-Orig-Accept-Encoding ~ "br") {
            set req.http.Accept-Encoding = "br";
        } elsif (req.http.Fastly-Orig-Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } else {
            unset req.http.Accept-Encoding;
        }
    }

    # Most of the URLs in Warehouse do not support or require any sort of query
    # parameter. If we strip these at the edge then we'll increase our cache
    # efficiency when they won't otherwise change the output of the pages.
    #
    # This will match any URL except those that start with:
    #
    #   * /admin/
    #   * /search/
    #   * /account/login/
    #   * /account/logout/
    #   * /account/register/
    #   * /pypi
    if (req.url.path !~ "^/(admin/|search(/|$)|account/(login|logout|register)/|pypi)") {
        set req.url = req.url.path;
    }

    # Sort all of our query parameters, this will ensure that the same query
    # parameters in a different order will end up being represented as the same
    # thing, reducing cache misses due to ordering differences.
    set req.url = boltsort.sort(req.url);

    # We have a number of items that we'll pass back to the origin.
    # Set a header to tell the backend if we're using https or http.
    if (req.http.Fastly-SSL) {
        set req.http.Warehouse-Proto = "https";
    } else {
        set req.http.Warehouse-Proto = "http";
    }

    # Pass the client IP address back to the backend.
    if (req.http.Fastly-Client-IP) {
        set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
    }

    # Pass the real host value back to the backend.
    if (req.http.Host) {
        set req.http.Warehouse-Host = req.http.host;
    }

#FASTLY recv

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Redirect pypi.io, www.pypi.io, and warehouse.python.org to pypi.org, this
    # is purposely done *after* the HTTPS checks.
    if (std.tolower(req.http.host) ~ "^(www.pypi.org|(www.)?pypi.io|warehouse.python.org)$") {
        set req.http.Location = "https://pypi.org" req.url;
        error 750 "Redirect to Primary Domain";
    }
    # Redirect warehouse-staging.python.org to test.pypi.io.
    if (std.tolower(req.http.host) ~ "^(test.pypi.io|warehouse-staging.python.org)$") {
        set req.http.Location = "https://test.pypi.org" req.url;
        error 750 "Redirect to Primary Domain";
    }

    # Requests to /packages/ get dispatched to Amazon instead of to our typical
    # Origin. This requires a a bit of setup to make it work.
    if (req.http.host ~ "^(test-)?files.pythonhosted.org$"
            && req.url ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/") {
        # Setup our environment to better match what S3 expects/needs
        set req.http.Host = req.http.AWS-Bucket-Name ".s3.amazonaws.com";
        set req.http.Date = now;
        set req.url = regsuball(req.url, "\+", urlencode("+"));

        # Compute the Authorization header that S3 requires to be able to
        # access the files stored there.
        set req.http.Authorization = "AWS " req.http.AWS-Access-Key-ID ":" digest.hmac_sha1_base64(req.http.AWS-Secret-Access-Key, "GET" LF LF LF req.http.Date LF "/" req.http.AWS-Bucket-Name req.url.path);

        # We don't want to send our Warehouse-Token to S3, so we'll go ahead
        # and remove it.
        unset req.http.Warehouse-Token;
    }

    # We no longer need any of these variables, which would exist only to
    # shuffle configuration from the Fastly UI into our VCL.
    unset req.http.AWS-Access-Key-ID;
    unset req.http.AWS-Secret-Access-Key;
    unset req.http.AWS-Bucket-Name;

    # On a POST, we want to skip the shielding and hit backends directly.
    if (req.request == "POST") {
        set req.backend = F_Heroku;
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # We don't ever want to cache our health URL. Outside systems should be
    # able to use it to reach past Fastly and get an end to end health check.
    if (req.url == "/_health/") {
        return(pass);
    }

    # We never want to cache our admin URLs, while this should be "safe" due to
    # the architecure of Warehouse, it'll just be easier to debug issues if
    # these always are uncached.
    if (req.url ~ "^/admin/") {
        return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {

    # These are newer kinds of redirects which should be able to be cached by
    # default, even though Fastly doesn't currently have them in their default
    # list of cacheable status codes.
    if (http_status_matches(beresp.status, "303,307,308")) {
        set beresp.cacheable = true;
    }

    # For any 5xx status code we want to see if a stale object exists for it,
    # if so we'll go ahead and serve it.
    if (beresp.status >= 500 && beresp.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

#FASTLY fetch

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 502 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 502 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (http_status_matches(beresp.status, "500,502,503")) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_hit {
#FASTLY hit

    # If the object we have isn't cacheable, then just serve it directly
    # without going through any of the caching mechanisms.
    if (!obj.cacheable) {
        return(pass);
    }

    return(deliver);
}

sub vcl_deliver {
    # If this is an error and we have a stale response available, restart so
    # that we can pick it up and serve it.
    if (resp.status >= 500 && resp.status < 600) {
        if (stale.exists) {
            restart;
        }
    }

#FASTLY deliver

    # Unset headers that we don't need/want to send on to the client because
    # they are not generally useful.
    unset resp.http.Via;

    # Set our standard security headers, we do this in VCL rather than in
    # Warehouse itself so that we always get these headers, regardless of the
    # origin server being used.
    set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload";
    set resp.http.X-Frame-Options = "deny";
    set resp.http.X-XSS-Protection = "1; mode=block";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-Permitted-Cross-Domain-Policies = "none";

    # If we're not executing a shielding request, and the URL is one of our file
    # URLs, and it's a GET request, and the response is either a 200 or a 304
    # then we want to log an event stating that a download has taken place.
    if (!req.http.Fastly-FF
            && std.tolower(req.http.Warehouse-Host) == "files.pythonhosted.org"
            && req.url.path ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/"
            && req.request == "GET"
            && http_status_matches(resp.status, "200,304")) {
        log {"syslog "} req.service_id {" linehaul :: "} "2@" now "|" geoip.country_code "|" req.url.path "|" tls.client.protocol "|" tls.client.cipher "|" resp.http.x-amz-meta-project "|" resp.http.x-amz-meta-version "|" resp.http.x-amz-meta-package-type "|" req.http.user-agent;
        log {"syslog "} req.service_id {" downloads :: "} "2@" now "|" geoip.country_code "|" req.url.path "|" tls.client.protocol "|" tls.client.cipher "|" resp.http.x-amz-meta-project "|" resp.http.x-amz-meta-version "|" resp.http.x-amz-meta-package-type "|" req.http.user-agent;
    }

    # Unset a few headers set by Amazon that we don't really have a need/desire
    # to send to clients.
    unset resp.http.x-amz-replication-status;
    unset resp.http.x-amz-meta-python-version;
    unset resp.http.x-amz-meta-version;
    unset resp.http.x-amz-meta-package-type;
    unset resp.http.x-amz-meta-project;

    return(deliver);
}

sub vcl_error {
#FASTLY error

    # If we have a 5xx error and there is a stale object available, then we
    # will deliver that stale object.
    if (obj.status >= 500 && obj.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    } else if (obj.status == 750) {
        set obj.status = 301;
        set obj.http.Location = req.http.Location;
        set obj.http.Content-Type = "text/html; charset=UTF-8";
        synthetic {"<title>301 Moved Permanently</title>
301 Moved Permanently
"};
        return(deliver);
    }

}

# Note: It is VERY important to ensure that any changes to VCL will work
#       properly with both the current version of ``master`` and the version in
#       the pull request that adds any new changes. This is because the
#       configuration will be applied automatically as part of the deployment
#       process, but while the previous version of the code is still up and
#       running. Thus backwards incompatible changes must be broken up over
#       multiple pull requests in order to phase them in over multiple deploys.

sub vcl_recv {

    # I'm not 100% sure on what this is exactly for, it was taken from the
    # Fastly documentation, however, what I *believe* it does is just ensure
    # that we don't serve a stale copy of the page from the shield node when
    # an edge node is requesting content.
    if (req.http.Fastly-FF) {
        set req.max_stale_while_revalidate = 0s;
    }

    # Some (Older) clients will send a hash fragment as part of the URL even
    # though that is a local only modification. This breaks this badly for the
    # files in S3, and in general it's just not needed.
    set req.url = regsub(req.url, "#.*$", "");

    # Fastly does some normalization of the Accept-Encoding header so that it
    # reduces the number of cached copies (when served with the common,
    # Vary: Accept-Encoding) that are cached for any one URL. This makes a lot
    # of sense, except for the fact that we want to enable brotli compression
    # for our static files. Thus we need to work around the normalized encoding
    # in a way that still minimizes cached copies, but which will allow our
    # static files to be served using brotli.
    if (req.url ~ "^/static/" && req.http.Fastly-Orig-Accept-Encoding) {
        if (req.http.User-Agent ~ "MSIE 6") {
            # For that 0.3% of stubborn users out there
            unset req.http.Accept-Encoding;
        } elsif (req.http.Fastly-Orig-Accept-Encoding ~ "br") {
            set req.http.Accept-Encoding = "br";
        } elsif (req.http.Fastly-Orig-Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } else {
            unset req.http.Accept-Encoding;
        }
    }

    # Most of the URLs in Warehouse do not support or require any sort of query
    # parameter. If we strip these at the edge then we'll increase our cache
    # efficiency when they won't otherwise change the output of the pages.
    #
    # This will match any URL except those that start with:
    #
    #   * /admin/
    #   * /search/
    #   * /account/login/
    #   * /account/logout/
    #   * /account/register/
    #   * /pypi
    if (req.url.path !~ "^/(admin/|search(/|$)|account/(login|logout|register)/|pypi)") {
        set req.url = req.url.path;
    }

    # Sort all of our query parameters, this will ensure that the same query
    # parameters in a different order will end up being represented as the same
    # thing, reducing cache misses due to ordering differences.
    set req.url = boltsort.sort(req.url);

#FASTLY recv

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Redirect pypi.io, www.pypi.io, and warehouse.python.org to pypi.org, this
    # is purposely done *after* the HTTPS checks.
    if (std.tolower(req.http.host) ~ "^(www.pypi.org|(www.)?pypi.io|warehouse.python.org)$") {
        set req.http.Location = "https://pypi.org" req.url;
        error 750 "Redirect to Primary Domain";
    }
    # Redirect warehouse-staging.python.org to test.pypi.io.
    if (std.tolower(req.http.host) ~ "^(test.pypi.io|warehouse-staging.python.org)$") {
        set req.http.Location = "https://test.pypi.org" req.url;
        error 750 "Redirect to Primary Domain";
    }

    # Requests to /packages/ get dispatched to Amazon instead of to our typical
    # Origin. This requires a a bit of setup to make it work.
    if (req.http.host ~ "^(test-)?files.pythonhosted.org$"
            && req.url ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/") {
        # Setup our environment to better match what S3 expects/needs
        set req.http.Host = req.http.AWS-Bucket-Name ".s3.amazonaws.com";
        set req.http.Date = now;
        set req.url = regsuball(req.url, "\+", urlencode("+"));

        # Compute the Authorization header that S3 requires to be able to
        # access the files stored there.
        set req.http.Authorization = "AWS " req.http.AWS-Access-Key-ID ":" digest.hmac_sha1_base64(req.http.AWS-Secret-Access-Key, "GET" LF LF LF req.http.Date LF "/" req.http.AWS-Bucket-Name req.url.path);

        # We don't want to send our Warehouse-Token to S3, so we'll go ahead
        # and remove it.
        unset req.http.Warehouse-Token;
    }

    # We no longer need any of these variables, which would exist only to
    # shuffle configuration from the Fastly UI into our VCL.
    unset req.http.AWS-Access-Key-ID;
    unset req.http.AWS-Secret-Access-Key;
    unset req.http.AWS-Bucket-Name;

    # We have a number of items that we'll pass back to the origin, but only
    # if we have a Warehouse-Token that will allow them to be accepted.
    if (req.http.Warehouse-Token) {
        # Set a header to tell the backend if we're using https or http.
        if (req.http.Fastly-SSL) {
            set req.http.Warehouse-Proto = "https";
        } else {
            set req.http.Warehouse-Proto = "http";
        }

        # Pass the client IP address back to the backend.
        if (req.http.Fastly-Client-IP) {
            set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
        }

        # Pass the real host value back to the backend.
        if (req.http.Host) {
            set req.http.Warehouse-Host = req.http.host;
        }
    }

    # On a POST, we want to skip the shielding and hit backends directly.
    if (req.request == "POST") {
        set req.backend = F_Heroku;
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # We don't ever want to cache our health URL. Outside systems should be
    # able to use it to reach past Fastly and get an end to end health check.
    if (req.url == "/_health/") {
        return(pass);
    }

    # We never want to cache our admin URLs, while this should be "safe" due to
    # the architecure of Warehouse, it'll just be easier to debug issues if
    # these always are uncached.
    if (req.url ~ "^/admin/") {
        return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {

    # These are newer kinds of redirects which should be able to be cached by
    # default, even though Fastly doesn't currently have them in their default
    # list of cacheable status codes.
    if (http_status_matches(beresp.status, "303,307,308")) {
        set beresp.cacheable = true;
    }

    # For any 5xx status code we want to see if a stale object exists for it,
    # if so we'll go ahead and serve it.
    if (beresp.status >= 500 && beresp.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    # When delivering a 304 response, we don't always have access to all the
    # headers in the resp because a 304 response is supposed to remove most of
    # the headers. So we'll instead stash these headers on the request so that
    # we can log this data from there instead of from the response.
    if (beresp.http.x-amz-meta-project
            || beresp.http.x-amz-meta-version
            || beresp.http.x-amz-meta-package-type) {
        set req.http.Fastly-amz-meta-project = beresp.http.x-amz-meta-project;
        set req.http.Fastly-amz-meta-version = beresp.http.x-amz-meta-version;
        set req.http.Fastly-amz-meta-package-type = beresp.http.x-amz-meta-package-type;
    }

#FASTLY fetch

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 502 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 502 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (http_status_matches(beresp.status, "500,502,503")) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_hit {
#FASTLY hit

    # If the object we have isn't cacheable, then just serve it directly
    # without going through any of the caching mechanisms.
    if (!obj.cacheable) {
        return(pass);
    }

    return(deliver);
}

sub vcl_deliver {
    # If this is an error and we have a stale response available, restart so
    # that we can pick it up and serve it.
    if (resp.status >= 500 && resp.status < 600) {
        if (stale.exists) {
            restart;
        }
    }

#FASTLY deliver

    # Unset headers that we don't need/want to send on to the client because
    # they are not generally useful.
    unset resp.http.Via;

    # Unset a few headers set by Amazon that we don't really have a need/desire
    # to send to clients.
    unset resp.http.X-AMZ-Replication-Status;
    unset resp.http.X-AMZ-Meta-Python-Version;
    unset resp.http.X-AMZ-Meta-Version;
    unset resp.http.X-AMZ-Meta-Package-Type;
    unset resp.http.X-AMZ-Meta-Project;

    # Set our standard security headers, we do this in VCL rather than in
    # Warehouse itself so that we always get these headers, regardless of the
    # origin server being used.
    set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload";
    set resp.http.X-Frame-Options = "deny";
    set resp.http.X-XSS-Protection = "1; mode=block";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-Permitted-Cross-Domain-Policies = "none";

    # Unstash our information about what project/version/package-type a
    # particular file download was for.
    if (req.http.Fastly-amz-meta-project
            || req.http.Fastly-amz-meta-version
            || req.http.Fastly-amz-meta-package-type) {
        set resp.http.x-amz-meta-project = req.http.Fastly-amz-meta-project;
        set resp.http.x-amz-meta-version = req.http.Fastly-amz-meta-version;
        set resp.http.x-amz-meta-package-type = req.http.Fastly-amz-meta-package-type;
    }

    # If we're not executing a shielding request, and the URL is one of our file
    # URLs, and it's a GET request, and the response is either a 200 or a 304
    # then we want to log an event stating that a download has taken place.
    if (!req.http.Fastly-FF
            && std.tolower(req.http.host) == "files.pythonhosted.org"
            && req.url.path ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/"
            && req.request == "GET"
            && http_status_matches(resp.status, "200,304")) {
        log {"syslog "} req.service_id {" linehaul :: "} "2@" now "|" geoip.country_code "|" req.url.path "|" tls.client.protocol "|" tls.client.cipher "|" resp.http.x-amz-meta-project "|" resp.http.x-amz-meta-version "|" resp.http.x-amz-meta-package-type "|" req.http.user-agent;
        log {"syslog "} req.service_id {" downloads :: "} "2@" now "|" geoip.country_code "|" req.url.path "|" tls.client.protocol "|" tls.client.cipher "|" resp.http.x-amz-meta-project "|" resp.http.x-amz-meta-version "|" resp.http.x-amz-meta-package-type "|" req.http.user-agent;
    }

    return(deliver);
}

sub vcl_error {
#FASTLY error

    # If we have a 5xx error and there is a stale object available, then we
    # will deliver that stale object.
    if (obj.status >= 500 && obj.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    } else if (obj.status == 750) {
        set obj.status = 301;
        set obj.http.Location = req.http.Location;
        set obj.http.Content-Type = "text/html; charset=UTF-8";
        synthetic {"<title>301 Moved Permanently</title>
301 Moved Permanently
"};
        return(deliver);
    }

}


Updated VCL with further refinements to regexp, and case normalization.

vcl 4.0;

import directors;
import std;

###
# Varnish Documentation
#   https://www.varnish-cache.org/docs/trunk/users-guide/vcl-built-in-subs.html
#   https://www.varnish-cache.org/docs/trunk/reference/vcl.html
##

{% for backend in varnish_backend_hosts %}
backend application_{{ loop.index }} {
  .host = "{{ backend }}";
  .port = "{{ varnish_backend_port }}";
{% if varnish_health_checks_enabled %}
  .probe = {
    .url = "{{ varnish_health_check_url }}";
    .interval = {{ varnish_health_check_interval }};
    .timeout  = {{ varnish_health_check_timeout }};
    .window = {{ varnish_health_check_window }};
    .threshold = {{ varnish_health_check_threshold }};
  }
{% endif %}
}
{% endfor %}

###
# vcl_init
#   Called when VCL is loaded, before any requests pass through it. Typically used to initialize VMODs.
###
# The vcl_init subroutine may terminate with calling return() with one of the following keywords:
#
# ok
#   Normal return, VCL continues loading.
##

sub vcl_init {
  new application = directors.round_robin();
{% for backend in varnish_backend_hosts %}
  application.add_backend(application_{{ loop.index }});
{% endfor %}
}

###
# vcl_recv
#   Called at the beginning of a request, after the complete request has been received and parsed.
#   Its purpose is to decide whether or not to serve the request, how to do it, and, if applicable, which backend to use.
#   It is also used to modify the request
###
# The vcl_recv subroutine may terminate with calling return() on one of the following keywords:
#
# synth(status code, reason)
#   Return a synthetic object with the specified status code to the client and abandon the request.
# pass
#   Switch to pass mode. Control will eventually pass to vcl_pass.
# pipe
#   Switch to pipe mode. Control will eventually pass to vcl_pipe.
# hash
#   Continue processing the object as a potential candidate for caching. Passes the control over to vcl_hash.
# purge
#   Purge the object and it's variants. Control passes through vcl_hash to vcl_purge.
#

sub vcl_recv {
  set req.backend_hint = application.backend();

  ###
  # Avoid processing uncacheable requests
  ###

  if (req.http.Authorization) {
    return (pass);
  }

  ###
  # Inject X-Forwarded-For headers, but only once.
  ###

  if (req.restarts == 0) {
    if (req.http.X-Forwarded-For) {
      set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip;
    } else {
      set req.http.X-Forwarded-For = client.ip;
    }
  }

{% if varnish_header_sanitization_enabled %}
  ###
  # Header Sanitization
  ###

{% if varnish_discards_port_from_host_header %}
  set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
{% endif %}

  if (req.http.Accept-Encoding) {
    if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
      unset req.http.Accept-Encoding;
    } elsif (req.http.Accept-Encoding ~ "gzip") {
      set req.http.Accept-Encoding = "gzip";
    } elsif (req.http.Accept-Encoding ~ "deflate") {
      set req.http.Accept-Encoding = "deflate";
    } else {
      unset req.http.Accept-Encoding;
    }
  }
{% endif %}

{% if varnish_cookie_sanitization_enabled %}
  ###
  # Cookie Sanitization
  ###

  ### Google Analytics
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");

  ### Quantcast
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");

  ### AddThis
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");

  ### General Cleanup
  set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");
  if (req.http.cookie ~ "^\s*$") {
      unset req.http.cookie;
  }
{% endif %}

{% if varnish_uri_sanitization_enabled %}
  ###
  # URI Sanitization
  ###

  if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=") {
    set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
    set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
    set req.url = regsub(req.url, "\?&", "?");
    set req.url = regsub(req.url, "\?$", "");
  }
  if (req.url ~ "\#") {
    set req.url = regsub(req.url, "\#.*$", "");
  }

{% endif %}

{% if varnish_workaround_telusdotcom_browser_profile %}
  ###
  # Workaround for BrowserProfile and Language/Region Detection on TELUS.com
  ###

  if (req.http.Cookie ~ "BrowserProfile") {
    set req.http.X-Language = std.tolower(regsuball(req.http.Cookie, "(.*)language..:\\.([^\\]*)(.*)", "\2"));
    set req.http.X-Province = std.tolower(regsuball(req.http.Cookie, "(.*)region..:\\.([^\\]*)(.*)", "\2"));
  } else {
    if (req.http.Cookie ~ "lang=") {
      set req.http.X-Language = std.tolower(regsuball(req.http.Cookie, "(.*)lang=([^;]*)(.*)", "\2"));
    } else {
      set req.http.X-Language = "en";
    }

    if (req.http.Cookie ~ "prov=") {
      set req.http.X-Province = std.tolower(regsuball(req.http.Cookie, "(.*)prov=([^;]*)(.*)", "\2"));
    } else {
      set req.http.X-Province = "BC";
    }
  }

  if (req.url ~ "/(en|fr)/") {
    if (req.url ~ "/en/" && req.http.X-Language != "en") {
      set req.http.X-Passthrough-Reason = "Unaligned Path and Cookie (Language)";
      return(pass);
    }
    if (req.url ~ "/fr/" && req.http.X-Language != "fr") {
      set req.http.X-Passthrough-Reason = "Unaligned Path and Cookie (Language)";
      return(pass);
    }
  } else {
      set req.http.X-Passthrough-Reason = "Not Set in Path (Language)";
    return(pass);
  }

  if (req.url ~ "/(ab|bc|mb|nb|nl|ns|nt|nu|on|pe|qc|sk|yt)/") {
    {% for province in ['ab', 'bc', 'mb', 'nb', 'nl', 'ns', 'nt', 'nu', 'on', 'pe', 'qc', 'sk', 'yt'] %}
    if (req.url ~ "{{ province }}" && req.http.X-Province != "{{ province }}") {
      set req.http.X-Passthrough-Reason = "Unaligned Path and Cookie (Province)";
      return(pass);
    }
    {% endfor %}
  } else {
      set req.http.X-Passthrough-Reason = "Not Set in Path (Province)";
    return(pass);
  }
{% endif %}

{% if varnish_blacklist_enabled %}
  if ((req.url ~ "{{ varnish_blacklist_regexp }}")) {
    set req.http.X-Passthrough-Reason = "Path in Blacklist";
    return(pass);
  } else {
{% if varnish_discards_client_cookies %}
    unset req.http.Cookie;
{% endif %}
  }
{% endif %}

}

###
# vcl_hit
#   Called when a cache lookup is successful.
###
# The vcl_hit subroutine may terminate with calling return() with one of the following keywords:
#
# restart
#   Restart the transaction. Increases the restart counter. If the number of restarts is higher than max_restarts Varnish emits a guru meditation error.
# deliver
#   Deliver the object. Control passes to vcl_deliver.
# synth(status code, reason)
#   Return the specified status code to the client and abandon the request.
###

sub vcl_hit {
  if (obj.ttl > 0s) {
    return (deliver);
  }
{% if varnish_health_checks_enabled %}
  if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) {
    return (deliver);
  } else {
    return (fetch);
  }
{% else %}
  if (obj.ttl + obj.grace > 0s) {
     return (deliver);
  }

  return (fetch);
{% endif %}
}

###
# vcl_backend_response
#   Called after the response headers has been successfully retrieved from the backend.
###
# The vcl_backend_response subroutine may terminate with calling return() with one of the following keywords:
#
# deliver
#   Possibly insert the object into the cache, then deliver it to the Control will eventually pass to vcl_deliver.
# abandon
#   Abandon the backend request and generates an error.
# retry
#   Retry the backend transaction. Increases the retries counter. If the number of retries is higher than max_retries Varnish emits a guru meditation error.
###

sub vcl_backend_response {
{% if varnish_workaround_telusdotcom_browser_profile %}
  if (bereq.http.X-Language) {
    set beresp.http.X-Language = bereq.http.X-Language;
  }
  if (bereq.http.X-Province) {
    set beresp.http.X-Province = bereq.http.X-Province;
  }
{% endif %}

{% if varnish_blacklist_enabled %}
  if (bereq.url ~ "{{ varnish_blacklist_regexp }}") {
    set beresp.http.X-Cacheable = "NO:Path in Blacklist";
    set beresp.uncacheable = true;
    set beresp.ttl = {{ varnish_backend_response_ttl }}s;
    return(deliver);
  } else {
    if (bereq.http.X-Passthrough-Reason) {
      set beresp.http.X-Passthrough-Reason = bereq.http.X-Passthrough-Reason;
      return(deliver);
    } else {
{% if varnish_discards_server_cookies %}
      unset beresp.http.Set-Cookie;
{% endif %}
    }
  }
{% else %}
{% if varnish_discards_server_cookies %}
  unset beresp.http.Set-Cookie;
{% endif %}
{% endif %}

{% if varnish_cache_diagnostics_enabled %}
  if (bereq.http.Cookie) {
    set beresp.http.X-Cacheable = "NO:Cookie in Request";
    set beresp.uncacheable = true;
    set beresp.ttl = {{ varnish_backend_response_ttl }}s;
    return(deliver);
  } elsif (beresp.http.Set-Cookie) {
    set beresp.http.X-Cacheable = "NO:Set-Cookie in Response";
    set beresp.uncacheable = true;
    set beresp.ttl = {{ varnish_backend_response_ttl }}s;
    return(deliver);
  } elsif (beresp.http.Cache-Control ~ "private") {
    set beresp.http.X-Cacheable = "NO:Cache-Control=private";
    set beresp.uncacheable = true;
    set beresp.ttl = {{ varnish_backend_response_ttl }}s;
    return(deliver);
  } elsif (beresp.http.X-No-Cache) {
    set beresp.http.X-Cacheable = "NO:X-No-Cache";
    set beresp.uncacheable = true;
    set beresp.ttl = {{ varnish_backend_response_ttl }}s;
    return(deliver);
  } elsif (beresp.ttl <= 0s) {
    set beresp.http.X-Cacheable = "NO:Not Cacheable";
    set beresp.uncacheable = true;
    set beresp.ttl = {{ varnish_backend_response_ttl }}s;
    return(deliver);
  } else {
    set beresp.http.X-Cacheable = "YES";
  }
{% endif %}

  set beresp.grace = {{ varnish_grace_period }};
}

###
# vcl_deliver
#   Called before a cached object is delivered to the client.
###
# The vcl_deliver subroutine may terminate with calling return() with one of the following keywords:
#
# deliver
#   Deliver the object to the client.
# restart
#   Restart the transaction. Increases the restart counter. If the number of restarts is higher than max_restarts Varnish emits a guru meditation error.
###

sub vcl_deliver {
  if (obj.hits > 0) {
    set resp.http.X-Cache = "HIT";
  } else {
    set resp.http.X-Cache = "MISS";
  }
}

vcl 4.0;

import directors;
import std;

###
# Varnish Documentation
#   https://www.varnish-cache.org/docs/trunk/users-guide/vcl-built-in-subs.html
#   https://www.varnish-cache.org/docs/trunk/reference/vcl.html
##

{% for backend in varnish_backend_hosts %}
backend application_{{ loop.index }} {
  .host = "{{ backend }}";
  .port = "{{ varnish_backend_port }}";
{% if varnish_health_checks_enabled %}
  .probe = {
    .url = "{{ varnish_health_check_url }}";
    .interval = {{ varnish_health_check_interval }};
    .timeout  = {{ varnish_health_check_timeout }};
    .window = {{ varnish_health_check_window }};
    .threshold = {{ varnish_health_check_threshold }};
  }
{% endif %}
}
{% endfor %}

###
# vcl_init
#   Called when VCL is loaded, before any requests pass through it. Typically used to initialize VMODs.
###
# The vcl_init subroutine may terminate with calling return() with one of the following keywords:
#
# ok
#   Normal return, VCL continues loading.
##

sub vcl_init {
  new application = directors.round_robin();
{% for backend in varnish_backend_hosts %}
  application.add_backend(application_{{ loop.index }});
{% endfor %}
}

###
# vcl_recv
#   Called at the beginning of a request, after the complete request has been received and parsed.
#   Its purpose is to decide whether or not to serve the request, how to do it, and, if applicable, which backend to use.
#   It is also used to modify the request
###
# The vcl_recv subroutine may terminate with calling return() on one of the following keywords:
#
# synth(status code, reason)
#   Return a synthetic object with the specified status code to the client and abandon the request.
# pass
#   Switch to pass mode. Control will eventually pass to vcl_pass.
# pipe
#   Switch to pipe mode. Control will eventually pass to vcl_pipe.
# hash
#   Continue processing the object as a potential candidate for caching. Passes the control over to vcl_hash.
# purge
#   Purge the object and it's variants. Control passes through vcl_hash to vcl_purge.
#

sub vcl_recv {
  set req.backend_hint = application.backend();

  ###
  # Avoid processing uncacheable requests
  ###

  if (req.http.Authorization) {
    return (pass);
  }

  ###
  # Inject X-Forwarded-For headers, but only once.
  ###

  if (req.restarts == 0) {
    if (req.http.X-Forwarded-For) {
      set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip;
    } else {
      set req.http.X-Forwarded-For = client.ip;
    }
  }

{% if varnish_header_sanitization_enabled %}
  ###
  # Header Sanitization
  ###

{% if varnish_discards_port_from_host_header %}
  set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
{% endif %}

  if (req.http.Accept-Encoding) {
    if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
      unset req.http.Accept-Encoding;
    } elsif (req.http.Accept-Encoding ~ "gzip") {
      set req.http.Accept-Encoding = "gzip";
    } elsif (req.http.Accept-Encoding ~ "deflate") {
      set req.http.Accept-Encoding = "deflate";
    } else {
      unset req.http.Accept-Encoding;
    }
  }
{% endif %}

{% if varnish_cookie_sanitization_enabled %}
  ###
  # Cookie Sanitization
  ###

  ### Google Analytics
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");

  ### Quantcast
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");

  ### AddThis
  set req.http.Cookie = regsuball(req.http.Cookie, "***REMOVED***=[^;]+(; )?", "");

  ### General Cleanup
  set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");
  if (req.http.cookie ~ "^\s*$") {
      unset req.http.cookie;
  }
{% endif %}

{% if varnish_uri_sanitization_enabled %}
  ###
  # URI Sanitization
  ###

  if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=") {
    set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
    set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
    set req.url = regsub(req.url, "\?&", "?");
    set req.url = regsub(req.url, "\?$", "");
  }
  if (req.url ~ "\#") {
    set req.url = regsub(req.url, "\#.*$", "");
  }

{% endif %}

{% if varnish_workaround_telusdotcom_browser_profile %}
  ###
  # Workaround for BrowserProfile and Language/Region Detection on TELUS.com
  ###

  if (req.http.Cookie ~ "BrowserProfile") {
    set req.http.X-Language = regsuball(req.http.Cookie, "language..:\\.([^\\]*)", "\1");
    set req.http.X-Province = regsuball(req.http.Cookie, "region..:\\.([^\\]*)", "\1");
  } else {
    if (req.http.Cookie ~ "lang=") {
      set req.http.X-Language = regsuball(req.http.Cookie, "lang=([^;]*)", "\1");
    } else {
      set req.http.X-Language = "en";
    }

    if (req.http.Cookie ~ "prov=") {
      set req.http.X-Province = regsuball(req.http.Cookie, "prov=([^;]*)", "\1");
    } else {
      set req.http.X-Province = "BC";
    }
  }

  if (req.url ~ "/(en|fr)/") {
    if (req.url ~ "/en/" && req.http.X-Language != "en") {
      set req.http.X-Passthrough-Reason = "Unaligned Path and Cookie (Language)";
      return(pass);
    }
    if (req.url ~ "/fr/" && req.http.X-Language != "fr") {
      set req.http.X-Passthrough-Reason = "Unaligned Path and Cookie (Language)";
      return(pass);
    }
  } else {
      set req.http.X-Passthrough-Reason = "Not Set in Path (Language)";
    return(pass);
  }

  if (req.url ~ "/(ab|bc|mb|nb|nl|ns|nt|nu|on|pe|qc|sk|yt)/") {
    {% for province in ['ab', 'bc', 'mb', 'nb', 'nl', 'ns', 'nt', 'nu', 'on', 'pe', 'qc', 'sk', 'yt'] %}
    if (req.url ~ "{{ province }}" && req.http.X-Province != "{{ province }}") {
      set req.http.X-Passthrough-Reason = "Unaligned Path and Cookie (Province)";
      return(pass);
    }
    {% endfor %}
  } else {
      set req.http.X-Passthrough-Reason = "Not Set in Path (Province)";
    return(pass);
  }
{% endif %}

{% if varnish_blacklist_enabled %}
  if ((req.url ~ "{{ varnish_blacklist_regexp }}")) {
    set req.http.X-Passthrough-Reason = "Path in Blacklist";
    return(pass);
  } else {
{% if varnish_discards_client_cookies %}
    unset req.http.Cookie;
{% endif %}
  }
{% endif %}

}

###
# vcl_hit
#   Called when a cache lookup is successful.
###
# The vcl_hit subroutine may terminate with calling return() with one of the following keywords:
#
# restart
#   Restart the transaction. Increases the restart counter. If the number of restarts is higher than max_restarts Varnish emits a guru meditation error.
# deliver
#   Deliver the object. Control passes to vcl_deliver.
# synth(status code, reason)
#   Return the specified status code to the client and abandon the request.
###

sub vcl_hit {
  if (obj.ttl > 0s) {
    return (deliver);
  }
{% if varnish_health_checks_enabled %}
  if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) {
    return (deliver);
  } else {
    return (fetch);
  }
{% else %}
  if (obj.ttl + obj.grace > 0s) {
     return (deliver);
  }

  return (fetch);
{% endif %}
}

###
# vcl_backend_response
#   Called after the response headers has been successfully retrieved from the backend.
###
# The vcl_backend_response subroutine may terminate with calling return() with one of the following keywords:
#
# deliver
#   Possibly insert the object into the cache, then deliver it to the Control will eventually pass to vcl_deliver.
# abandon
#   Abandon the backend request and generates an error.
# retry
#   Retry the backend transaction. Increases the retries counter. If the number of retries is higher than max_retries Varnish emits a guru meditation error.
###

sub vcl_backend_response {
{% if varnish_workaround_telusdotcom_browser_profile %}
  if (bereq.http.X-Language) {
    set beresp.http.X-Language = bereq.http.X-Language;
  }
  if (bereq.http.X-Province) {
    set beresp.http.X-Province = bereq.http.X-Province;
  }
{% endif %}

{% if varnish_blacklist_enabled %}
  if (bereq.url ~ "{{ varnish_blacklist_regexp }}") {
    set beresp.http.X-Cacheable = "NO:Path in Blacklist";
    set beresp.uncacheable = true;
    set beresp.ttl = {{ varnish_backend_response_ttl }}s;
    return(deliver);
  } else {
    if (bereq.http.X-Passthrough-Reason) {
      set beresp.http.X-Passthrough-Reason = bereq.http.X-Passthrough-Reason;
      return(deliver);
    } else {
{% if varnish_discards_server_cookies %}
      unset beresp.http.Set-Cookie;
{% endif %}
    }
  }
{% else %}
{% if varnish_discards_server_cookies %}
  unset beresp.http.Set-Cookie;
{% endif %}
{% endif %}

{% if varnish_cache_diagnostics_enabled %}
  if (bereq.http.Cookie) {
    set beresp.http.X-Cacheable = "NO:Cookie in Request";
    set beresp.uncacheable = true;
    set beresp.ttl = {{ varnish_backend_response_ttl }}s;
    return(deliver);
  } elsif (beresp.http.Set-Cookie) {
    set beresp.http.X-Cacheable = "NO:Set-Cookie in Response";
    set beresp.uncacheable = true;
    set beresp.ttl = {{ varnish_backend_response_ttl }}s;
    return(deliver);
  } elsif (beresp.http.Cache-Control ~ "private") {
    set beresp.http.X-Cacheable = "NO:Cache-Control=private";
    set beresp.uncacheable = true;
    set beresp.ttl = {{ varnish_backend_response_ttl }}s;
    return(deliver);
  } elsif (beresp.http.X-No-Cache) {
    set beresp.http.X-Cacheable = "NO:X-No-Cache";
    set beresp.uncacheable = true;
    set beresp.ttl = {{ varnish_backend_response_ttl }}s;
    return(deliver);
  } elsif (beresp.ttl <= 0s) {
    set beresp.http.X-Cacheable = "NO:Not Cacheable";
    set beresp.uncacheable = true;
    set beresp.ttl = {{ varnish_backend_response_ttl }}s;
    return(deliver);
  } else {
    set beresp.http.X-Cacheable = "YES";
  }
{% endif %}

  set beresp.grace = {{ varnish_grace_period }};
}

###
# vcl_deliver
#   Called before a cached object is delivered to the client.
###
# The vcl_deliver subroutine may terminate with calling return() with one of the following keywords:
#
# deliver
#   Deliver the object to the client.
# restart
#   Restart the transaction. Increases the restart counter. If the number of restarts is higher than max_restarts Varnish emits a guru meditation error.
###

sub vcl_deliver {
  if (obj.hits > 0) {
    set resp.http.X-Cache = "HIT";
  } else {
    set resp.http.X-Cache = "MISS";
  }
}


Documentation for vcl_pipe and connection: close

Note in the default VCL that connection: close might be wanted.

git-svn-id: 7d4b18ab7d176792635d6a7a77dd8cbbea8e8daa@3852 d4fa192b-c00b-0410-8231-f00ffab90ce4

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/wikimedia/operations-debs-varnish.git/'


Added example of VCL-file to improve integration with Spree

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/fastly/spree_fastly.git/'


Add vcl sample from previous commit.

Add vcl sample from previous commit.

/*
  This is the minimal VCL configuration required for passing the Apache
  mod_pagespeed system tests. To install varnish and start the varnish
  server at the right port, do the following:
  1) sudo apt-get install varnish
  2) sudo vim /etc/default/varnish and put in the following lines at the
     bottom of the file:
  DAEMON_OPTS="-a :8020 \
             -T localhost:6082 \
             -f /etc/varnish/default.vcl \
             -S /etc/varnish/secret \
             -s file,/var/lib/varnish/$INSTANCE/varnish_storage.bin,1G"
  3) sudo cp /path/to/install/sample_conf.vcl /etc/varnish/default.vcl
  4) sudo service varnish restart
*/

backend default {
  # Location of Apache mod_pagespeed server.
  .host = "127.0.0.1";
  .port = "8080";
}

acl purge {
  # Purge requests are only allowed from localhost.
  "localhost";
  "127.0.0.1";
}

sub vcl_recv {
  if (req.request == "PURGE") {
    if (!client.ip ~ purge) {
      error 405 "Not allowed.";
    }
    return (lookup);
  }
}

sub vcl_hit {
  if (req.request == "PURGE") {
    purge;
    error 200 "Purged.";
  }
}

sub vcl_miss {
  if (req.request == "PURGE") {
     purge;
     error 200 "Purged.";
  }
}

sub vcl_fetch {
   # Cache everything for 30s.
   set beresp.ttl   = 30s;
   set beresp.grace = 30s;
   set beresp.http.X-Cacheable = "YES";
   return (deliver);
}

error: pathspec 'src/install/sample_conf.vcl' did not match any file(s) known to git


Race condition fix

Race condition fix

#Add the following to your Varnish 4 vcl file
#In varnish 4 inlice C disabled by default. You must enable it using -p vcc_allow_inline_c=true

C{
  #include <dlfcn.h>
  #include <stdlib.h>
  #include <stdio.h>

static const char* (*md5_hash)(char* str) = NULL;

__attribute__((constructor)) void
load_module()
{
        const char* symbol_name = "md5_hash";
        const char* plugin_name = "/etc/varnish/modules/md5/libmd5varnish.so";
        void* handle = NULL;

        handle = dlopen( plugin_name, RTLD_NOW );
        if (handle != NULL) {
                md5_hash = dlsym( handle, symbol_name );
                if (md5_hash == NULL)
                        fprintf( stderr, "\nError: Could not load MD5 plugin:\n%s\n\n", dlerror() );
                else
                        printf( "MD5 plugin loaded successfully.\n");
        }
        else
                fprintf( stderr, "\nError: Could not load MD5 plugin:\n%s\n\n", dlerror() );
}
}C

## An Example of How to use it in vcl_recv
#
sub vcl_recv {

    C{
        //\6X-MD5: - \6 is length of string in octal encoding
        static const struct gethdr_s VGC_HDR_REQ_VARNISH_X_MD5 = {
            HDR_REQ, "\6X-MD5:"
        };

        static const struct gethdr_s VGC_HDR_REQ_VARNISH_X_DIGEST = {
            HDR_REQ, "\24X-MD5-Digest-String:"
        };

        char* md5DigestString = VRT_GetHdr(ctx, &VGC_HDR_REQ_VARNISH_X_DIGEST);
        const char* calculatedMD5 = (*md5_hash)(md5DigestString);
        VRT_SetHdr(ctx, &VGC_HDR_REQ_VARNISH_X_MD5, calculatedMD5, vrt_magic_string_end);
        free((char*)calculatedMD5);
    }C

}

error: pathspec 'md5_v4.vcl' did not match any file(s) known to git


added default.cl 4.0 vcl file

added default.cl 4.0 vcl file

#
# This is an example VCL file for Varnish.
#
# It does not do anything by default, delegating control to the
# builtin VCL. The builtin VCL is called when there is no explicit
# return statement.
#
# See the VCL chapters in the Users Guide at https://www.varnish-cache.org/docs/
# and http://varnish-cache.org/trac/wiki/VCLExamples for more examples.

# Update for work with Varnish 4

# Marker to tell the VCL compiler that this VCL has been adapted to the
# new 4.0 format.
vcl 4.0;

# Default backend definition. Set this to point to your content server.
backend default {
    .host = "127.0.0.1";
    .port = "8080";
    .connect_timeout = 600s;
    .first_byte_timeout = 600s;
    .between_bytes_timeout = 600s;
    .max_connections = 800;
}

# Only allow purging from specific IPs
acl purge {
    "localhost";
    "127.0.0.1";
}

# This function is used when a request is send by a HTTP client (Browser)
sub vcl_recv {
  # Normalize the header, remove the port (in case you're testing this on various TCP ports)
  set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");

  # Allow purging from ACL
  if (req.method == "PURGE") {
    # If not allowed then a error 405 is returned
    if (!client.ip ~ purge) {
      return(synth(405, "This IP is not allowed to send PURGE requests."));
    }
    # If allowed, do a cache_lookup -> vlc_hit() or vlc_miss()
    return (purge);
  }

  # Post requests will not be cached
  if (req.http.Authorization || req.method == "POST") {
    return (pass);
  }

  # --- Wordpress specific configuration

  # Did not cache the RSS feed
  if (req.url ~ "/feed") {
    return (pass);
  }

  # Blitz hack
        if (req.url ~ "/mu-.*") {
                return (pass);
        }

  # Did not cache the admin and login pages
  if (req.url ~ "/wp-(login|admin)") {
    return (pass);
  }

   # Do not cache the WooCommerce pages
   ### REMOVE IT IF YOU DO NOT USE WOOCOMMERCE ###
  if (req.url ~ "/(cart|my-account|checkout|addons|/?add-to-cart=)") {
          return (pass);
      }

  # Remove the "has_js" cookie
  set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", "");

  # Remove any Google Analytics based cookies
  set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", "");

  # Remove the Quant Capital cookies (added by some plugin, all __qca)
  set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", "");

  # Remove the wp-settings-1 cookie
  set req.http.Cookie = regsuball(req.http.Cookie, "wp-settings-1=[^;]+(; )?", "");

  # Remove the wp-settings-time-1 cookie
  set req.http.Cookie = regsuball(req.http.Cookie, "wp-settings-time-1=[^;]+(; )?", "");

  # Remove the wp test cookie
  set req.http.Cookie = regsuball(req.http.Cookie, "wordpress_test_cookie=[^;]+(; )?", "");

  # Are there cookies left with only spaces or that are empty?
  if (req.http.cookie ~ "^ *$") {
        unset req.http.cookie;
  }

  # Cache the following files extensions
  if (req.url ~ "\.(css|js|png|gif|jp(e)?g|swf|ico)") {
    unset req.http.cookie;
  }

  # Normalize Accept-Encoding header and compression
  # https://www.varnish-cache.org/docs/3.0/tutorial/vary.html
  if (req.http.Accept-Encoding) {
    # Do no compress compressed files...
    if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
          unset req.http.Accept-Encoding;
    } elsif (req.http.Accept-Encoding ~ "gzip") {
          set req.http.Accept-Encoding = "gzip";
    } elsif (req.http.Accept-Encoding ~ "deflate") {
          set req.http.Accept-Encoding = "deflate";
    } else {
      unset req.http.Accept-Encoding;
    }
  }

  # Check the cookies for wordpress-specific items
  if (req.http.Cookie ~ "wordpress_" || req.http.Cookie ~ "comment_") {
    return (pass);
  }
  if (!req.http.cookie) {
    unset req.http.cookie;
  }

  # --- End of Wordpress specific configuration

  # Did not cache HTTP authentication and HTTP Cookie
  if (req.http.Authorization || req.http.Cookie) {
    # Not cacheable by default
    return (pass);
  }

  # Cache all others requests
  return (hash);
}

sub vcl_pipe {
  return (pipe);
}

sub vcl_pass {
  return (fetch);
}

# The data on which the hashing will take place
sub vcl_hash {
  hash_data(req.url);
  if (req.http.host) {
      hash_data(req.http.host);
  } else {
      hash_data(server.ip);
  }

  # If the client supports compression, keep that in a different cache
      if (req.http.Accept-Encoding) {
          hash_data(req.http.Accept-Encoding);
  }

  return (lookup);
}

# This function is used when a request is sent by our backend (Nginx server)
sub vcl_backend_response {
  # Remove some headers we never want to see
  unset beresp.http.Server;
  unset beresp.http.X-Powered-By;

  # For static content strip all backend cookies
  if (bereq.url ~ "\.(css|js|png|gif|jp(e?)g)|swf|ico") {
    unset beresp.http.cookie;
  }

  # Only allow cookies to be set if we're in admin area
  if (beresp.http.Set-Cookie && bereq.url !~ "^/wp-(login|admin)") {
          unset beresp.http.Set-Cookie;
      }

  # don't cache response to posted requests or those with basic auth
  if ( bereq.method == "POST" || bereq.http.Authorization ) {
          set beresp.uncacheable = true;
    set beresp.ttl = 120s;
    return (deliver);
      }

      # don't cache search results
  if ( bereq.url ~ "\?s=" ){
    set beresp.uncacheable = true;
                set beresp.ttl = 120s;
                return (deliver);
  }

  # only cache status ok
  if ( beresp.status != 200 ) {
    set beresp.uncacheable = true;
                set beresp.ttl = 120s;
                return (deliver);
  }

  # A TTL of 24h
  set beresp.ttl = 24h;
  # Define the default grace period to serve cached content
  set beresp.grace = 30s;

  return (deliver);
}

# The routine when we deliver the HTTP request to the user
# Last chance to modify headers that are sent to the client
sub vcl_deliver {
  if (obj.hits > 0) {
    set resp.http.X-Cache = "cached";
  } else {
    set resp.http.x-Cache = "uncached";
  }

  # Remove some headers: PHP version
  unset resp.http.X-Powered-By;

  # Remove some headers: Apache version & OS
  unset resp.http.Server;

  # Remove some heanders: Varnish
  unset resp.http.Via;
  unset resp.http.X-Varnish;

  return (deliver);
}

sub vcl_init {
  return (ok);
}

sub vcl_fini {
  return (ok);
}

error: pathspec 'files/default/default4.0.vcl' did not match any file(s) known to git


Added example vcl to use in front of zope+plone (this could perhaps go in a contrib/ directory instead)

git-svn-id: 7d4b18ab7d176792635d6a7a77dd8cbbea8e8daa@1427 d4fa192b-c00b-0410-8231-f00ffab90ce4


Add a skeleton VCL file to help new users along.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/alarky/varnish-cache-doc-ja.git/'


updating varnish conf

updating varnish conf

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/cristianocasella/cristianocasella.com.git/'


comment reformatting on drupal_base.vcl

comment reformatting on drupal_base.vcl

error: pathspec 'varnish' did not match any file(s) known to git
error: pathspec '3' did not match any file(s) known to git
error: pathspec 'vcls/drupal_base.vcl' did not match any file(s) known to git


added sample config for gzipping content on Varnish


comment and code reformatting on drupal_base.vcl

fatal: unable to access 'https://github.com/NITEMAN/Varnish_VCL_samps-hacks.git/': The requested URL returned error: 403


adding default.vcl file

adding default.vcl file

backend default {
    .host = "127.0.0.1";
    .port = "9000";
}

sub vcl_recv {
	return (pass);
}

sub vcl_backend_response {
}

sub vcl_deliver {
}

error: pathspec 'default.vcl' did not match any file(s) known to git


Add default.vcl Varnish file

Add default.vcl Varnish file

C{
#include <stdlib.h>
#include <stdio.h>
#include <time.h>
#include <pthread.h>
static pthread_mutex_t lrand_mutex = PTHREAD_MUTEX_INITIALIZER;
void generate_uuid(char* buf) {
pthread_mutex_lock(&lrand_mutex);
long a = lrand48();
long b = lrand48();
long c = lrand48();
long d = lrand48();
pthread_mutex_unlock(&lrand_mutex);
// SID must match this regex for Kount compat /^\w{1,32}$/
sprintf(buf, "frontend=%08lx%04lx%04lx%04lx%04lx%08lx",
a,
b & 0xffff,
(b & ((long)0x0fff0000) >> 16) | 0x4000,
(c & 0x0fff) | 0x8000,
(c & (long)0xffff0000) >> 16,
d
);
return;
}
}C
import std;
backend default {
.host = "127.0.0.1";
.port = "8080";
.first_byte_timeout = 300s;
.between_bytes_timeout = 300s;
}
backend admin {
.host = "127.0.0.1";
.port = "8080";
.first_byte_timeout = 21600s;
.between_bytes_timeout = 21600s;
}
acl crawler_acl {
"127.0.0.1";
}
acl debug_acl {
}
sub generate_session {
if (req.url ~ ".*[&?]SID=([^&]+).*") {
set req.http.X-Varnish-Faked-Session = regsub(
req.url, ".*[&?]SID=([^&]+).*", "frontend=\1");
} else {
C{
char uuid_buf [50];
generate_uuid(uuid_buf);
VRT_SetHdr(sp, HDR_REQ,
"\030X-Varnish-Faked-Session:",
uuid_buf,
vrt_magic_string_end
);
}C
}
if (req.http.Cookie) {
std.collect(req.http.Cookie);
set req.http.Cookie = req.http.X-Varnish-Faked-Session +
"; " + req.http.Cookie;
} else {
set req.http.Cookie = req.http.X-Varnish-Faked-Session;
}
}
sub generate_session_expires {
C{
time_t now = time(NULL);
struct tm now_tm = *gmtime(&now);
now_tm.tm_sec += 86400;
mktime(&now_tm);
char date_buf [50];
strftime(date_buf, sizeof(date_buf)-1, "%a, %d-%b-%Y %H:%M:%S %Z", &now_tm);
VRT_SetHdr(sp, HDR_RESP,
"\031X-Varnish-Cookie-Expires:",
date_buf,
vrt_magic_string_end
);
}C
}
sub vcl_recv {
if (req.restarts == 0) {
if (req.http.X-Forwarded-For) {
set req.http.X-Forwarded-For =
req.http.X-Forwarded-For + ", " + client.ip;
} else {
set req.http.X-Forwarded-For = client.ip;
}
}
if (!true || req.http.Authorization ||
req.request !~ "^(GET|HEAD)$" ||
req.http.Cookie ~ "varnish_bypass=1") {
return (pipe);
}
set req.url = regsuball(req.url, "(.*)//+(.*)", "\1/\2");
if (req.http.Accept-Encoding) {
if (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} else if (req.http.Accept-Encoding ~ "deflate") {
set req.http.Accept-Encoding = "deflate";
} else {
unset req.http.Accept-Encoding;
}
}
if (req.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|lightspeed)\.php/)?") {
set req.http.X-Turpentine-Secret-Handshake = "1";
if (req.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|lightspeed)\.php/)?alexpie") {
set req.backend = admin;
return (pipe);
}
if (req.http.Cookie ~ "\bcurrency=") {
set req.http.X-Varnish-Currency = regsub(
req.http.Cookie, ".*\bcurrency=([^;]*).*", "\1");
}
if (req.http.Cookie ~ "\bstore=") {
set req.http.X-Varnish-Store = regsub(
req.http.Cookie, ".*\bstore=([^;]*).*", "\1");
}
if (req.url ~ "/turpentine/esi/get(?:Block|FormKey)/") {
set req.http.X-Varnish-Esi-Method = regsub(
req.url, ".*/method/(\w+)/.*", "\1");
set req.http.X-Varnish-Esi-Access = regsub(
req.url, ".*/access/(\w+)/.*", "\1");
if (req.http.X-Varnish-Esi-Method == "esi" && req.esi_level == 0 &&
!(true || client.ip ~ debug_acl)) {
error 403 "External ESI requests are not allowed";
}
}
if (req.http.Cookie !~ "frontend=") {
if (client.ip ~ crawler_acl ||
req.http.User-Agent ~ "^(?:ApacheBench/.*|.*Googlebot.*|JoeDog/.*Siege.*|magespeedtest\.com|Nexcessnet_Turpentine/.*)$") {
set req.http.Cookie = "frontend=crawler-session";
} else {
call generate_session;
}
}
if (true &&
req.url ~ ".*\.(?:css|js|jpe?g|png|gif|ico|swf)(?=\?|&|$)") {
unset req.http.Cookie;
unset req.http.X-Varnish-Faked-Session;
return (lookup);
}
if (req.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|lightspeed)\.php/)?(?:alexpie|api|cron\.php)" ||
req.url ~ "\?.*__from_store=") {
return (pipe);
}
if (true &&
req.url ~ "(?:[?&](?:__SID|XDEBUG_PROFILE)(?=[&=]|$))") {
return (pass);
}
if (req.url ~ "[?&](utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=") {
set req.url = regsuball(req.url, "(?:(\?)?|&)(?:utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=[^&]+", "\1");
set req.url = regsuball(req.url, "(?:(\?)&|\?$)", "\1");
}
return (lookup);
}
}
sub vcl_pipe {
unset bereq.http.X-Turpentine-Secret-Handshake;
set bereq.http.Connection = "close";
}
sub vcl_hash {
hash_data(req.url);
if (req.http.Host) {
hash_data(req.http.Host);
} else {
hash_data(server.ip);
}
hash_data(req.http.Ssl-Offloaded);
if (req.http.X-Normalized-User-Agent) {
hash_data(req.http.X-Normalized-User-Agent);
}
if (req.http.Accept-Encoding) {
hash_data(req.http.Accept-Encoding);
}
if (req.http.X-Varnish-Store || req.http.X-Varnish-Currency) {
hash_data("s=" + req.http.X-Varnish-Store + "&c=" + req.http.X-Varnish-Currency);
}
if (req.http.X-Varnish-Esi-Access == "private" &&
req.http.Cookie ~ "frontend=") {
hash_data(regsub(req.http.Cookie, "^.*?frontend=([^;]*);*.*$", "\1"));
}
return (hash);
}
sub vcl_hit {
}
sub vcl_fetch {
set req.grace = 15s;
set beresp.http.X-Varnish-Host = req.http.host;
set beresp.http.X-Varnish-URL = req.url;
if (req.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|lightspeed)\.php/)?") {
unset beresp.http.Vary;
set beresp.do_gzip = true;
if (beresp.status != 200 && beresp.status != 404) {
set beresp.ttl = 15s;
return (hit_for_pass);
} else {
if (beresp.http.Set-Cookie) {
set beresp.http.X-Varnish-Set-Cookie = beresp.http.Set-Cookie;
unset beresp.http.Set-Cookie;
}
unset beresp.http.Cache-Control;
unset beresp.http.Expires;
unset beresp.http.Pragma;
unset beresp.http.Cache;
unset beresp.http.Age;
if (beresp.http.X-Turpentine-Esi == "1") {
set beresp.do_esi = true;
}
if (beresp.http.X-Turpentine-Cache == "0") {
set beresp.ttl = 15s;
return (hit_for_pass);
} else {
if (true &&
bereq.url ~ ".*\.(?:css|js|jpe?g|png|gif|ico|swf)(?=\?|&|$)") {
set beresp.ttl = 28800s;
set beresp.http.Cache-Control = "max-age=28800";
} elseif (req.http.X-Varnish-Esi-Method) {
if (req.http.X-Varnish-Esi-Access == "private" &&
req.http.Cookie ~ "frontend=") {
set beresp.http.X-Varnish-Session = regsub(req.http.Cookie,
"^.*?frontend=([^;]*);*.*$", "\1");
}
if (req.http.X-Varnish-Esi-Method == "ajax" &&
req.http.X-Varnish-Esi-Access == "public") {
set beresp.http.Cache-Control = "max-age=" + regsub(
req.url, ".*/ttl/(\d+)/.*", "\1");
}
set beresp.ttl = std.duration(
regsub(
req.url, ".*/ttl/(\d+)/.*", "\1s"),
300s);
if (beresp.ttl == 0s) {
set beresp.ttl = 15s;
return (hit_for_pass);
}
} else {
set beresp.ttl = 3600s;
}
}
}
return (deliver);
}
}
sub vcl_deliver {
if (req.http.X-Varnish-Faked-Session) {
call generate_session_expires;
set resp.http.Set-Cookie = req.http.X-Varnish-Faked-Session +
"; expires=" + resp.http.X-Varnish-Cookie-Expires + "; path=/";
if (req.http.Host) {
set resp.http.Set-Cookie = resp.http.Set-Cookie +
"; domain=" + regsub(req.http.Host, ":\d+$", "");
}
set resp.http.Set-Cookie = resp.http.Set-Cookie + "; httponly";
unset resp.http.X-Varnish-Cookie-Expires;
}
if (req.http.X-Varnish-Esi-Method == "ajax" && req.http.X-Varnish-Esi-Access == "private") {
set resp.http.Cache-Control = "no-cache";
}
if (true || client.ip ~ debug_acl) {
set resp.http.X-Varnish-Hits = obj.hits;
set resp.http.X-Varnish-Esi-Method = req.http.X-Varnish-Esi-Method;
set resp.http.X-Varnish-Esi-Access = req.http.X-Varnish-Esi-Access;
set resp.http.X-Varnish-Currency = req.http.X-Varnish-Currency;
set resp.http.X-Varnish-Store = req.http.X-Varnish-Store;
} else {
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Turpentine-Cache;
unset resp.http.X-Turpentine-Esi;
unset resp.http.X-Turpentine-Flush-Events;
unset resp.http.X-Turpentine-Block;
unset resp.http.X-Varnish-Session;
unset resp.http.X-Varnish-Host;
unset resp.http.X-Varnish-URL;
unset resp.http.X-Varnish-Set-Cookie;
}
}

error: pathspec 'ansible/files/varnish/default.vcl' did not match any file(s) known to git


added the VCL code for the VEBAN method

added the VCL code for the VEBAN method

# 1. VEBAN method
sub vcl_recv {
  if (req.request == "VEBAN") {
    if (!client.ip ~ purge) {
      error 501 "Method Not Implemented";
    }
    if (req.http.X-VE-BanKey != "complex-stuff-here") {
      error 403 "Forbidden";
    }
    if (!req.http.X-VE-BanStr) {
      error 911 "Bad VEBAN request (missing X-VE-BanStr)";
    }
    ban(req.http.X-VE-BanStr);
    std.log("VEBAN added ban [" + req.http.X-VE-BanStr + "]");
    error 910 "VEBAN OK";
  }
}

error: pathspec 'vcl/veban.vcl' did not match any file(s) known to git


Create default.vcl

Create default.vcl

# This is a basic VCL configuration file for varnish.  See the vcl(7)
# man page for details on VCL syntax and semantics.
#
# Default backend definition.  Set this to point to your content
# server.
#
backend default {
.host = "127.0.0.1";
.port = "8080";
.connect_timeout = 600s;
.first_byte_timeout = 600s;
.between_bytes_timeout = 600s;
.max_connections = 800;

}
# Drop any cookies sent to WordPress.
sub vcl_recv {
    if (!(req.url ~ "wp-(login|admin)")) {
        unset req.http.cookie;
    }
}

# Drop any cookies WordPress tries to send back to the client.
sub vcl_fetch {
    if (!(req.url ~ "wp-(login|admin)")) {
        unset beresp.http.set-cookie;
    }
}

acl purge {
      "localhost";
}

sub vcl_recv {
        if (req.request == "PURGE") {
                if (!client.ip ~ purge) {
                        error 405 "Not allowed.";
                }
                return(lookup);
        }
if (req.url ~ "^/$") {
               unset req.http.cookie;
            }
}
sub vcl_hit {
        if (req.request == "PURGE") {
                set obj.ttl = 0s;
                error 200 "Purged.";
        }
}
sub vcl_miss {
        if (req.request == "PURGE") {
                error 404 "Not in cache.";
        }
if (!(req.url ~ "wp-(login|admin)")) {
                        unset req.http.cookie;
                }
    if (req.url ~ "^/[^?]+.(jpeg|jpg|png|gif|ico|js|css|txt|gz|zip|lzma|bz2|tgz|tbz|html|htm)(\?.|)$") {
       unset req.http.cookie;
       set req.url = regsub(req.url, "\?.$", "");
    }
    if (req.url ~ "^/$") {
       unset req.http.cookie;
    }
}
sub vcl_fetch {
        if (req.url ~ "^/$") {
                unset beresp.http.set-cookie;
        }
if (!(req.url ~ "wp-(login|admin)")) {
                        unset beresp.http.set-cookie;
}

set req.grace = 2m;

# Set X-Forwarded-For header for logging in nginx
remove req.http.X-Forwarded-For;
set    req.http.X-Forwarded-For = client.ip;

# Remove has_js and CloudFlare/Google Analytics __* cookies.
set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", "");
# Remove a ";" prefix, if present.
set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");

# Remove the wp-settings-1 cookie
set req.http.Cookie = regsuball(req.http.Cookie, "wp-settings-1=[^;]+(; )?", "");

# Remove the wp-settings-time-1 cookie
set req.http.Cookie = regsuball(req.http.Cookie, "wp-settings-time-1=[^;]+(; )?", "");

# Remove the wp test cookie
set req.http.Cookie = regsuball(req.http.Cookie, "wordpress_test_cookie=[^;]+(; )?", "");

# Static content unique to the theme can be cached (so no user uploaded images)
# The reason I don't take the wp-content/uploads is because of cache size on bigger blogs
# that would fill up with all those files getting pushed into cache
if (req.url ~ "wp-content/themes/" && req.url ~ "\.(css|js|png|gif|jp(e)?g)") {
    unset req.http.cookie;
}
}

sub vcl_fetch {
	#set obj.grace = 5m;
    set beresp.grace = 2m;

}

sub vcl_hit {
        if (req.request == "PURGE") {
                purge;
                error 200 "Purged.";
        }
}

sub vcl_miss {
        if (req.request == "PURGE") {
                purge;
                error 200 "Purged.";
        }
}

error: pathspec 'etc/varnish/default.vcl' did not match any file(s) known to git


Create m2-default.vcl

Create m2-default.vcl

vcl 4.0;

import std;
# The minimal Varnish version is 4.0

backend default {
    .host = "127.0.0.1";
    .port = "8080";
    .first_byte_timeout = 1000s;
}

acl purge {
    "127.0.0.1";
}

sub vcl_recv {
    if (req.method == "PURGE") {
        if (client.ip !~ purge) {
            return (synth(405, "Method not allowed"));
        }
        if (!req.http.X-Magento-Tags-Pattern) {
            return (synth(400, "X-Magento-Tags-Pattern header required"));
        }
        ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
        return (synth(200, "Purged"));
    }

    if (req.method != "GET" &&
        req.method != "HEAD" &&
        req.method != "PUT" &&
        req.method != "POST" &&
        req.method != "TRACE" &&
        req.method != "OPTIONS" &&
        req.method != "DELETE") {
          /* Non-RFC2616 or CONNECT which is weird. */
          return (pipe);
    }

    # We only deal with GET and HEAD by default
    if (req.method != "GET" && req.method != "HEAD") {
        return (pass);
    }

    # Bypass shopping cart and checkout requests
    if (req.url ~ "/checkout") {
        return (pass);
    }

    # normalize url in case of leading HTTP scheme and domain
    set req.url = regsub(req.url, "^http[s]?://", "");

    # collect all cookies
    std.collect(req.http.Cookie);

    # static files are always cacheable. remove SSL flag and cookie
        if (req.url ~ "^/(pub/)?(media|static)/.*\.(ico|css|js|jpg|jpeg|png|gif|tiff|bmp|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)$") {
        unset req.http.Https;
        unset req.http.Cookie;
    }

    return (hash);
}

sub vcl_hash {
    if (req.http.cookie ~ "X-Magento-Vary=") {
        hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1"));
    }

}

sub vcl_backend_response {
    if (beresp.http.content-type ~ "text") {
        set beresp.do_esi = true;
    }

    if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
        set beresp.do_gzip = true;
    }

    # cache only successfully responses and 404s
    if (beresp.status != 200 && beresp.status != 404) {
        set beresp.ttl = 0s;
        set beresp.uncacheable = true;
        return (deliver);
    } elsif (beresp.http.Cache-Control ~ "private") {
        set beresp.uncacheable = true;
        set beresp.ttl = 86400s;
        return (deliver);
    }

    if (beresp.http.X-Magento-Debug) {
        set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
    }

    # validate if we need to cache it and prevent from setting cookie
    # images, css and js are cacheable by default so we have to remove cookie also
    if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
        unset beresp.http.set-cookie;
        if (bereq.url !~ "\.(ico|css|js|jpg|jpeg|png|gif|tiff|bmp|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)(\?|$)") {
            set beresp.http.Pragma = "no-cache";
            set beresp.http.Expires = "-1";
            set beresp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
            set beresp.grace = 1m;
        }
    }
    return (deliver);
}

sub vcl_deliver {
    if (resp.http.X-Magento-Debug) {
        if (resp.http.x-varnish ~ " ") {
            set resp.http.X-Magento-Cache-Debug = "HIT";
        } else {
            set resp.http.X-Magento-Cache-Debug = "MISS";
        }
    } else {
        unset resp.http.Age;
    }

    unset resp.http.X-Magento-Debug;
    unset resp.http.X-Magento-Tags;
    unset resp.http.X-Powered-By;
    unset resp.http.Server;
    unset resp.http.X-Varnish;
    unset resp.http.Via;
    unset resp.http.Link;
}

error: pathspec 'm2-default.vcl' did not match any file(s) known to git


Revert accidental change to backend port in Varnish config example (3000 is misleading as it suggests we recommend using Rack as the default backend)

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/4bic/alaveteli.git/'


fixing ip address for backend

fixing ip address for backend


Don't strip last_seen_* cookies.  Fixes #405


added proper vary header to gzipping example


Add Varnish VCL test-file

Add Varnish VCL test-file

fatal: unable to access 'https://github.com/sol/pygments.git/': The requested URL returned error: 403


more draft

more draft

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/erezny/twitter-import.git/'


tuning cache for static files

tuning cache for static files

fatal: unable to access 'https://github.com/cristianocasella/cristianocasella.com.git/': The requested URL returned error: 403


added gzip content-type

added gzip content-type


Add tarpitting of rate limited requests

Add tarpitting of rate limited requests

# Tarpit Rate limited requests
if ( resp.status == 429 && req.http.Rate-Limit ) {
  resp.tarpit(5, 100000);
}

error: pathspec 'etc/vcl_snippets_rate_limiting/deliver.vcl' did not match any file(s) known to git


Added templates directory for the varnish template.

error: pathspec 'templates/backend.vcl' did not match any file(s) known to git


Create default.vcl

Create default.vcl

backend default {
    .host = "127.0.0.1";
    .port = "20080";
}

sub vcl_recv {
    set req.hash_ignore_busy = true;
    if (req.restarts == 0) {
        if (req.http.x-forwarded-for) {
            set req.http.X-Forwarded-For =
                req.http.X-Forwarded-For + ", " + client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
     }

     if (req.request != "GET" && req.request != "HEAD") {
         /* We only deal with GET and HEAD by default */
         error 403 "Forbidden Method";
     }

     set req.backend = default;
     set req.grace = 1h;

     return (lookup);
}

sub vcl_hash {
/***
    hash_data(req.url);
    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }
***/

///    if (req.url ~ "\?") {
///        set req.http.X-TMP = regsub(req.url, "^([^?]+).*", "\1");
        //// DELETE parameter
        set req.http.X-TMP = regsub(req.url, "\?.*$", "");
        hash_data(req.http.X-TMP);
        remove req.http.X-TMP;
///    }

    return (hash);
}

sub vcl_fetch {
     set beresp.grace = 1h;
    set beresp.ttl = 10s;

    if (beresp.status >= 400 && beresp.status < 500) {
         set beresp.ttl = 20s;
    }

    if (beresp.status >= 500 && beresp.status < 600) {
         set beresp.saintmode = 90s;
         return (restart);
    }

    // 圧縮されていない場合、圧縮する
    set beresp.do_gzip = true;

    // 圧縮されている場合、解凍する
    // set beresp.do_gunzip = true;

    return (deliver);
}

sub vcl_deliver {
    if(obj.hits > 0){
        set resp.http.X-Cache = "Hit";
    } else {
        set resp.http.X-Cache = "Miss";
    }
}

sub vcl_error {
    if((obj.status >= 100 && obj.status < 200)
    || obj.status == 204
    || obj.status == 304){
        return (deliver);
    }

    set obj.http.Content-Type = "text/html; charset=utf-8";
    set obj.http.Retry-After = "5";
    synthetic {"
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

    <title>"} + obj.status + " " + obj.response + {"</title>

Error "} + obj.status + " " + obj.response + {"

"};
    return (deliver);
}

error: pathspec 'varnish3/default.vcl' did not match any file(s) known to git


Updated with varnish-2

Updated with varnish-2

fatal: unable to access 'https://github.com/kiranthomas/sylius-docker.git/': The requested URL returned error: 403