Redirect test.pypi.io to test.pypi.org (#1416)

sub vcl_recv {

    # I'm not 100% sure on what this is exactly for, it was taken from the
    # Fastly documentation, however, what I *believe* it does is just ensure
    # that we don't serve a stale copy of the page from the shield node when
    # an edge node is requesting content.
    if (req.http.Fastly-FF) {
        set req.max_stale_while_revalidate = 0s;
    }

    # Some (Older) clients will send a hash fragment as part of the URL even
    # though that is a local only modification. This breaks this badly for the
    # files in S3, and in general it's just not needed.
    set req.url = regsub(req.url, "#.*$", "");

    # Fastly does some normalization of the Accept-Encoding header so that it
    # reduces the number of cached copies (when served with the common,
    # Vary: Accept-Encoding) that are cached for any one URL. This makes a lot
    # of sense, except for the fact that we want to enable brotli compression
    # for our static files. Thus we need to work around the normalized encoding
    # in a way that still minimizes cached copies, but which will allow our
    # static files to be served using brotli.
    if (req.url ~ "^/static/" && req.http.Fastly-Orig-Accept-Encoding) {
        if (req.http.User-Agent ~ "MSIE 6") {
            # For that 0.3% of stubborn users out there
            unset req.http.Accept-Encoding;
        } elsif (req.http.Fastly-Orig-Accept-Encoding ~ "br") {
            set req.http.Accept-Encoding = "br";
        } elsif (req.http.Fastly-Orig-Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } else {
            unset req.http.Accept-Encoding;
        }
    }

    # Most of the URLs in Warehouse do not support or require any sort of query
    # parameter. If we strip these at the edge then we'll increase our cache
    # efficiency when they won't otherwise change the output of the pages.
    #
    # This will match any URL except those that start with:
    #
    #   * /search/
    #   * /account/login/
    #   * /account/logout/
    #   * /account/register/
    #   * /pypi
    if (req.url !~ "^/(search/|account/(login|logout|register)/|pypi)") {
        set req.url = req.url.path;
    }

    # Sort all of our query parameters, this will ensure that the same query
    # parameters in a different order will end up being represented as the same
    # thing, reducing cache misses due to ordering differences.
    set req.url = boltsort.sort(req.url);

#FASTLY recv

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Redirect www.pypi.io and warehouse.python.org to pypi.io, this is
    # purposely done *after* the HTTPS checks.
    if (std.tolower(req.http.host) ~ "^(www.pypi.io|warehouse.python.org)$") {
        set req.http.Location = "https://pypi.io" req.url;
        error 750 "Redirect to Primary Domain";
    }
    # Redirect warehouse-staging.python.org to test.pypi.io.
    if (std.tolower(req.http.host) ~ "^(test.pypi.io|warehouse-staging.python.org)$") {
        set req.http.Location = "https://test.pypi.org" req.url;
        error 750 "Redirect to Primary Domain";
    }

    # Requests to /packages/ get dispatched to Amazon instead of to our typical
    # Origin. This requires a a bit of setup to make it work.
    if (req.http.host ~ "^(test-)?files.pythonhosted.org$"
            && req.url ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/") {
        # Setup our environment to better match what S3 expects/needs
        set req.http.Host = req.http.AWS-Bucket-Name ".s3.amazonaws.com";
        set req.http.Date = now;
        set req.url = regsuball(req.url, "\+", urlencode("+"));

        # Compute the Authorization header that S3 requires to be able to
        # access the files stored there.
        set req.http.Authorization = "AWS " req.http.AWS-Access-Key-ID ":" digest.hmac_sha1_base64(req.http.AWS-Secret-Access-Key, "GET" LF LF LF req.http.Date LF "/" req.http.AWS-Bucket-Name req.url.path);

        # We don't want to send our Warehouse-Token to S3, so we'll go ahead
        # and remove it.
        unset req.http.Warehouse-Token;
    }

    # We no longer need any of these variables, which would exist only to
    # shuffle configuration from the Fastly UI into our VCL.
    unset req.http.AWS-Access-Key-ID;
    unset req.http.AWS-Secret-Access-Key;
    unset req.http.AWS-Bucket-Name;

    # We have a number of items that we'll pass back to the origin, but only
    # if we have a Warehouse-Token that will allow them to be accepted.
    if (req.http.Warehouse-Token) {
        # Set a header to tell the backend if we're using https or http.
        if (req.http.Fastly-SSL) {
            set req.http.Warehouse-Proto = "https";
        } else {
            set req.http.Warehouse-Proto = "http";
        }

        # Pass the client IP address back to the backend.
        if (req.http.Fastly-Client-IP) {
            set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
        }

        # Pass the real host value back to the backend.
        if (req.http.Host) {
            set req.http.Warehouse-Host = req.http.host;
        }
    }

    # On a POST, we want to skip the shielding and hit backends directly.
    if (req.request == "POST") {
        set req.backend = F_Heroku;
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # We don't ever want to cache our health URL. Outside systems should be
    # able to use it to reach past Fastly and get an end to end health check.
    if (req.url == "/_health/") {
        return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {

    # These are newer kinds of redirects which should be able to be cached by
    # default, even though Fastly doesn't currently have them in their default
    # list of cacheable status codes.
    if (http_status_matches(beresp.status, "303,307,308")) {
        set beresp.cacheable = true;
    }

    # For any 5xx status code we want to see if a stale object exists for it,
    # if so we'll go ahead and serve it.
    if (beresp.status >= 500 && beresp.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    # When delivering a 304 response, we don't always have access to all the
    # headers in the resp because a 304 response is supposed to remove most of
    # the headers. So we'll instead stash these headers on the request so that
    # we can log this data from there instead of from the response.
    if (beresp.http.x-amz-meta-project
            || beresp.http.x-amz-meta-version
            || beresp.http.x-amz-meta-package-type) {
        set req.http.Fastly-amz-meta-project = beresp.http.x-amz-meta-project;
        set req.http.Fastly-amz-meta-version = beresp.http.x-amz-meta-version;
        set req.http.Fastly-amz-meta-package-type = beresp.http.x-amz-meta-package-type;
    }

#FASTLY fetch

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 502 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 502 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (http_status_matches(beresp.status, "500,502,503")) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_hit {
#FASTLY hit

    # If the object we have isn't cacheable, then just serve it directly
    # without going through any of the caching mechanisms.
    if (!obj.cacheable) {
        return(pass);
    }

    return(deliver);
}

sub vcl_deliver {
    # If this is an error and we have a stale response available, restart so
    # that we can pick it up and serve it.
    if (resp.status >= 500 && resp.status < 600) {
        if (stale.exists) {
            restart;
        }
    }

#FASTLY deliver

    # Unset headers that we don't need/want to send on to the client because
    # they are not generally useful.
    unset resp.http.Server;
    unset resp.http.Via;

    # Unset a few headers set by Amazon that we don't really have a need/desire
    # to send to clients.
    unset resp.http.X-AMZ-Replication-Status;
    unset resp.http.X-AMZ-Meta-Python-Version;
    unset resp.http.X-AMZ-Meta-Version;
    unset resp.http.X-AMZ-Meta-Package-Type;
    unset resp.http.X-AMZ-Meta-Project;

    # Set our standard security headers, we do this in VCL rather than in
    # Warehouse itself so that we always get these headers, regardless of the
    # origin server being used.
    set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload";
    set resp.http.X-Frame-Options = "deny";
    set resp.http.X-XSS-Protection = "1; mode=block";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-Permitted-Cross-Domain-Policies = "none";

    # Unstash our information about what project/version/package-type a
    # particular file download was for.
    if (req.http.Fastly-amz-meta-project
            || req.http.Fastly-amz-meta-version
            || req.http.Fastly-amz-meta-package-type) {
        set resp.http.x-amz-meta-project = req.http.Fastly-amz-meta-project;
        set resp.http.x-amz-meta-version = req.http.Fastly-amz-meta-version;
        set resp.http.x-amz-meta-package-type = req.http.Fastly-amz-meta-package-type;
    }

    # If we're not executing a shielding request, and the URL is one of our file
    # URLs, and it's a GET request, and the response is either a 200 or a 304
    # then we want to log an event stating that a download has taken place.
    if (!req.http.Fastly-FF
            && std.tolower(req.http.host) == "files.pythonhosted.org"
            && req.url.path ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/"
            && req.request == "GET"
            && http_status_matches(resp.status, "200,304")) {
        log {"syslog "} req.service_id {" linehaul :: "} "2@" now "|" geoip.country_code "|" req.url.path "|" tls.client.protocol "|" tls.client.cipher "|" resp.http.x-amz-meta-project "|" resp.http.x-amz-meta-version "|" resp.http.x-amz-meta-package-type "|" req.http.user-agent;
        log {"syslog "} req.service_id {" downloads :: "} "2@" now "|" geoip.country_code "|" req.url.path "|" tls.client.protocol "|" tls.client.cipher "|" resp.http.x-amz-meta-project "|" resp.http.x-amz-meta-version "|" resp.http.x-amz-meta-package-type "|" req.http.user-agent;
    }

    return(deliver);
}

sub vcl_error {
#FASTLY error

    # If we have a 5xx error and there is a stale object available, then we
    # will deliver that stale object.
    if (obj.status >= 500 && obj.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    } else if (obj.status == 750) {
        set obj.status = 301;
        set obj.http.Location = req.http.Location;
        set obj.http.Content-Type = "text/html; charset=UTF-8";
        synthetic {"<title>301 Moved Permanently</title>
301 Moved Permanently
"};
        return(deliver);
    }

}

sub vcl_recv {

    # I'm not 100% sure on what this is exactly for, it was taken from the
    # Fastly documentation, however, what I *believe* it does is just ensure
    # that we don't serve a stale copy of the page from the shield node when
    # an edge node is requesting content.
    if (req.http.Fastly-FF) {
        set req.max_stale_while_revalidate = 0s;
    }

    # Some (Older) clients will send a hash fragment as part of the URL even
    # though that is a local only modification. This breaks this badly for the
    # files in S3, and in general it's just not needed.
    set req.url = regsub(req.url, "#.*$", "");

    # Fastly does some normalization of the Accept-Encoding header so that it
    # reduces the number of cached copies (when served with the common,
    # Vary: Accept-Encoding) that are cached for any one URL. This makes a lot
    # of sense, except for the fact that we want to enable brotli compression
    # for our static files. Thus we need to work around the normalized encoding
    # in a way that still minimizes cached copies, but which will allow our
    # static files to be served using brotli.
    if (req.url ~ "^/static/" && req.http.Fastly-Orig-Accept-Encoding) {
        if (req.http.User-Agent ~ "MSIE 6") {
            # For that 0.3% of stubborn users out there
            unset req.http.Accept-Encoding;
        } elsif (req.http.Fastly-Orig-Accept-Encoding ~ "br") {
            set req.http.Accept-Encoding = "br";
        } elsif (req.http.Fastly-Orig-Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } else {
            unset req.http.Accept-Encoding;
        }
    }

    # Most of the URLs in Warehouse do not support or require any sort of query
    # parameter. If we strip these at the edge then we'll increase our cache
    # efficiency when they won't otherwise change the output of the pages.
    #
    # This will match any URL except those that start with:
    #
    #   * /search/
    #   * /account/login/
    #   * /account/logout/
    #   * /account/register/
    #   * /pypi
    if (req.url !~ "^/(search/|account/(login|logout|register)/|pypi)") {
        set req.url = req.url.path;
    }

    # Sort all of our query parameters, this will ensure that the same query
    # parameters in a different order will end up being represented as the same
    # thing, reducing cache misses due to ordering differences.
    set req.url = boltsort.sort(req.url);

#FASTLY recv

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Redirect www.pypi.io and warehouse.python.org to pypi.io, this is
    # purposely done *after* the HTTPS checks.
    if (std.tolower(req.http.host) ~ "^(www.pypi.io|warehouse.python.org)$") {
        set req.http.Location = "https://pypi.io" req.url;
        error 750 "Redirect to Primary Domain";
    }
    # Redirect warehouse-staging.python.org to test.pypi.io.
    if (std.tolower(req.http.host) == "warehouse-staging.python.org") {
        set req.http.Location = "https://test.pypi.io" req.url;
        error 750 "Redirect to Primary Domain";
    }

    # Requests to /packages/ get dispatched to Amazon instead of to our typical
    # Origin. This requires a a bit of setup to make it work.
    if (req.http.host ~ "^(test-)?files.pythonhosted.org$"
            && req.url ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/") {
        # Setup our environment to better match what S3 expects/needs
        set req.http.Host = req.http.AWS-Bucket-Name ".s3.amazonaws.com";
        set req.http.Date = now;
        set req.url = regsuball(req.url, "\+", urlencode("+"));

        # Compute the Authorization header that S3 requires to be able to
        # access the files stored there.
        set req.http.Authorization = "AWS " req.http.AWS-Access-Key-ID ":" digest.hmac_sha1_base64(req.http.AWS-Secret-Access-Key, "GET" LF LF LF req.http.Date LF "/" req.http.AWS-Bucket-Name req.url.path);

        # We don't want to send our Warehouse-Token to S3, so we'll go ahead
        # and remove it.
        unset req.http.Warehouse-Token;
    }

    # We no longer need any of these variables, which would exist only to
    # shuffle configuration from the Fastly UI into our VCL.
    unset req.http.AWS-Access-Key-ID;
    unset req.http.AWS-Secret-Access-Key;
    unset req.http.AWS-Bucket-Name;

    # We have a number of items that we'll pass back to the origin, but only
    # if we have a Warehouse-Token that will allow them to be accepted.
    if (req.http.Warehouse-Token) {
        # Set a header to tell the backend if we're using https or http.
        if (req.http.Fastly-SSL) {
            set req.http.Warehouse-Proto = "https";
        } else {
            set req.http.Warehouse-Proto = "http";
        }

        # Pass the client IP address back to the backend.
        if (req.http.Fastly-Client-IP) {
            set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
        }

        # Pass the real host value back to the backend.
        if (req.http.Host) {
            set req.http.Warehouse-Host = req.http.host;
        }
    }

    # On a POST, we want to skip the shielding and hit backends directly.
    if (req.request == "POST") {
        set req.backend = F_Heroku;
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # We don't ever want to cache our health URL. Outside systems should be
    # able to use it to reach past Fastly and get an end to end health check.
    if (req.url == "/_health/") {
        return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {

    # These are newer kinds of redirects which should be able to be cached by
    # default, even though Fastly doesn't currently have them in their default
    # list of cacheable status codes.
    if (http_status_matches(beresp.status, "303,307,308")) {
        set beresp.cacheable = true;
    }

    # For any 5xx status code we want to see if a stale object exists for it,
    # if so we'll go ahead and serve it.
    if (beresp.status >= 500 && beresp.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    # When delivering a 304 response, we don't always have access to all the
    # headers in the resp because a 304 response is supposed to remove most of
    # the headers. So we'll instead stash these headers on the request so that
    # we can log this data from there instead of from the response.
    if (beresp.http.x-amz-meta-project
            || beresp.http.x-amz-meta-version
            || beresp.http.x-amz-meta-package-type) {
        set req.http.Fastly-amz-meta-project = beresp.http.x-amz-meta-project;
        set req.http.Fastly-amz-meta-version = beresp.http.x-amz-meta-version;
        set req.http.Fastly-amz-meta-package-type = beresp.http.x-amz-meta-package-type;
    }

#FASTLY fetch

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 502 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 502 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (http_status_matches(beresp.status, "500,502,503")) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_hit {
#FASTLY hit

    # If the object we have isn't cacheable, then just serve it directly
    # without going through any of the caching mechanisms.
    if (!obj.cacheable) {
        return(pass);
    }

    return(deliver);
}

sub vcl_deliver {
    # If this is an error and we have a stale response available, restart so
    # that we can pick it up and serve it.
    if (resp.status >= 500 && resp.status < 600) {
        if (stale.exists) {
            restart;
        }
    }

#FASTLY deliver

    # Unset headers that we don't need/want to send on to the client because
    # they are not generally useful.
    unset resp.http.Server;
    unset resp.http.Via;

    # Unset a few headers set by Amazon that we don't really have a need/desire
    # to send to clients.
    unset resp.http.X-AMZ-Replication-Status;
    unset resp.http.X-AMZ-Meta-Python-Version;
    unset resp.http.X-AMZ-Meta-Version;
    unset resp.http.X-AMZ-Meta-Package-Type;
    unset resp.http.X-AMZ-Meta-Project;

    # Set our standard security headers, we do this in VCL rather than in
    # Warehouse itself so that we always get these headers, regardless of the
    # origin server being used.
    set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload";
    set resp.http.X-Frame-Options = "deny";
    set resp.http.X-XSS-Protection = "1; mode=block";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-Permitted-Cross-Domain-Policies = "none";

    # Unstash our information about what project/version/package-type a
    # particular file download was for.
    if (req.http.Fastly-amz-meta-project
            || req.http.Fastly-amz-meta-version
            || req.http.Fastly-amz-meta-package-type) {
        set resp.http.x-amz-meta-project = req.http.Fastly-amz-meta-project;
        set resp.http.x-amz-meta-version = req.http.Fastly-amz-meta-version;
        set resp.http.x-amz-meta-package-type = req.http.Fastly-amz-meta-package-type;
    }

    # If we're not executing a shielding request, and the URL is one of our file
    # URLs, and it's a GET request, and the response is either a 200 or a 304
    # then we want to log an event stating that a download has taken place.
    if (!req.http.Fastly-FF
            && std.tolower(req.http.host) == "files.pythonhosted.org"
            && req.url.path ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/"
            && req.request == "GET"
            && http_status_matches(resp.status, "200,304")) {
        log {"syslog "} req.service_id {" linehaul :: "} "2@" now "|" geoip.country_code "|" req.url.path "|" tls.client.protocol "|" tls.client.cipher "|" resp.http.x-amz-meta-project "|" resp.http.x-amz-meta-version "|" resp.http.x-amz-meta-package-type "|" req.http.user-agent;
        log {"syslog "} req.service_id {" downloads :: "} "2@" now "|" geoip.country_code "|" req.url.path "|" tls.client.protocol "|" tls.client.cipher "|" resp.http.x-amz-meta-project "|" resp.http.x-amz-meta-version "|" resp.http.x-amz-meta-package-type "|" req.http.user-agent;
    }

    return(deliver);
}

sub vcl_error {
#FASTLY error

    # If we have a 5xx error and there is a stale object available, then we
    # will deliver that stale object.
    if (obj.status >= 500 && obj.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    } else if (obj.status == 750) {
        set obj.status = 301;
        set obj.http.Location = req.http.Location;
        set obj.http.Content-Type = "text/html; charset=UTF-8";
        synthetic {"<title>301 Moved Permanently</title>
301 Moved Permanently
"};
        return(deliver);
    }

}


Simplified varnish cookie cleanup rules

Simplified varnish cookie cleanup rules

* Removed cleanup of 'has_js' (remnant of drupal stuff from vcl template)
* Removed specific cookie checks, instead removing any underscore prefixed
* Left in removal of Google Analytics "utm" prefixed cookies (debateable, @ericclemmons might have opinion here)

Fixes #28 but more vcl cleanup to do...

# Default backend definition.  Set this to point to your content server.
# all paths relative to varnish option vcl_dir

include "custom.backend.vcl";
include "custom.acl.vcl";

# Handle the HTTP request received by the client
sub vcl_recv {
    # varnish serves stale (but cacheable) objects while retriving object from backend
    if (req.backend.healthy) {
        set req.grace = 30s;
    } else {
        set req.grace = 1h;
    }

    # shortcut for DFind requests
    if (req.url ~ "^/w00tw00t") {
        error 404 "Not Found";
    }

    if (req.restarts == 0) {
        if (req.http.X-Forwarded-For) {
            set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
    }

    # Normalize the header, remove the port (in case you're testing this on various TCP ports)
    set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");

    # Allow purging
    if (req.request == "PURGE") {
        if (!client.ip ~ purge) {
            # Not from an allowed IP? Then die with an error.
            error 405 "This IP is not allowed to send PURGE requests.";
        }

        # If you got this stage (and didn't error out above), do a cache-lookup
        # That will force entry into vcl_hit() or vcl_miss() below and purge the actual cache
        return (lookup);
    }

    # Only deal with "normal" types
    if (req.request != "GET" &&
            req.request != "HEAD" &&
            req.request != "PUT" &&
            req.request != "POST" &&
            req.request != "TRACE" &&
            req.request != "OPTIONS" &&
            req.request != "PATCH" &&
            req.request != "DELETE") {
        /* Non-RFC2616 or CONNECT which is weird. */
        return (pipe);
    }

    if (req.request != "GET" && req.request != "HEAD") {
        # We only deal with GET and HEAD by default
        return (pass);
    }

    # Some generic URL manipulation, useful for all templates that follow
    # First remove the Google Analytics added parameters, useless for our backend
    if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=") {
        set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
        set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
        set req.url = regsub(req.url, "\?&", "?");
        set req.url = regsub(req.url, "\?$", "");
    }

    # Strip hash, server doesn't need it.
    if (req.url ~ "\#") {
        set req.url = regsub(req.url, "\#.*$", "");
    }

    # Strip a trailing ? if it exists
    if (req.url ~ "\?$") {
        set req.url = regsub(req.url, "\?$", "");
    }

    # This is an example to redirect with a 301/302 HTTP status code from within Varnish
    # if (req.http.Host ~ "secure.mysite.tld") {
    #   # We may want to force our users from the secure site to the HTTPs version?
    #   error 720 "https://secure.mysite.tld";
    #   # If you want to keep the URLs intact, this also works:
    #   error 720 "https://" + req.http.Host + req.url;
    # }
    #
    # Or to force a 302 temporary redirect, use error 721
    # if (req.http.Host ~ "temp.mysite.tld") {
    #   # Temporary redirect
    #   error 721 "http://mysite.tld/temp";
    # }
    #

    # Some generic cookie manipulation, useful for all templates that follow

    # Remove any '_' prefixed cookies
    set req.http.Cookie = regsuball(req.http.Cookie, "_[^=]+=[^;]+(; )?", "");

    # Remove any Google Analytics based cookies
    set req.http.Cookie = regsuball(req.http.Cookie, "utmctr=[^;]+(; )?", "");
    set req.http.Cookie = regsuball(req.http.Cookie, "utmcmd.=[^;]+(; )?", "");
    set req.http.Cookie = regsuball(req.http.Cookie, "utmccn.=[^;]+(; )?", "");

    # Remove a ";" prefix in the cookie if present
    set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");

    # Are there cookies left with only spaces or that are empty?
    if (req.http.cookie ~ "^\s*$") {
        unset req.http.cookie;
    }

    # Normalize Accept-Encoding header
    # straight from the manual: https://www.varnish-cache.org/docs/3.0/tutorial/vary.html
    if (req.http.Accept-Encoding) {
        if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
            # No point in compressing these
            remove req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elsif (req.http.Accept-Encoding ~ "deflate") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            # unkown algorithm
            remove req.http.Accept-Encoding;
        }
    }

    # Include custom vcl_recv logic
    include "conf.d/receive/wordpress.vcl";

    # Remove all cookies for static files
    # A valid discussion could be held on this line: do you really need to cache static files that don't cause load? Only if you have memory left.
    # Sure, there's disk I/O, but chances are your OS will already have these files in their buffers (thus memory).
    # Before you blindly enable this, have a read here: http://mattiasgeniar.be/2012/11/28/stop-caching-static-files/
    if (req.url ~ "^[^?]*\.(bmp|bz2|css|doc|eot|flv|gif|gz|ico|jpeg|jpg|js|less|mp[34]|pdf|png|rar|rtf|swf|tar|tgz|txt|wav|woff|xml|zip)(\?.*)?$") {
        unset req.http.Cookie;
        return (lookup);
    }

    # Send Surrogate-Capability headers to announce ESI support to backend
    set req.http.Surrogate-Capability = "key=ESI/1.0";

    if (req.http.Authorization || req.http.Cookie) {
        # Not cacheable by default
        return (pass);
    }

    return (lookup);
}

sub vcl_pipe {
    # Note that only the first request to the backend will have
    # X-Forwarded-For set.  If you use X-Forwarded-For and want to
    # have it set for all requests, make sure to have:
    # set bereq.http.connection = "close";
    # here.  It is not set by default as it might break some broken web
    # applications, like IIS with NTLM authentication.

    #set bereq.http.Connection = "Close";
    return (pipe);
}

sub vcl_pass {
    return (pass);
}

# The data on which the hashing will take place
sub vcl_hash {
    hash_data(req.url);

    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }

    # hash cookies for object with auth
    if (req.http.Cookie) {
        hash_data(req.http.Cookie);
    }
    if (req.http.Authorization) {
        hash_data(req.http.Authorization);
    }

    return (hash);
}

sub vcl_hit {
    # Allow purges
    if (req.request == "PURGE") {
        purge;
        error 200 "purged";
    }

    return (deliver);
}

sub vcl_miss {
    # Allow purges
    if (req.request == "PURGE") {
        purge;
        error 200 "purged";
    }

    return (fetch);
}

# Handle the HTTP request coming from our backend
sub vcl_fetch {
    set beresp.grace = 1h;

    # Include custom vcl_fetch logic
    include "conf.d/fetch/wordpress.vcl";

    # Parse ESI request and remove Surrogate-Control header
    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
        unset beresp.http.Surrogate-Control;
        set beresp.do_esi = true;
    }

    # If the request to the backend returns a code is 5xx, restart the loop
    # If the number of restarts reaches the value of the parameter max_restarts,
    # the request will be error'ed.  max_restarts defaults to 4.  This prevents
    # an eternal loop in the event that, e.g., the object does not exist at all.
    if (beresp.status >= 500 && beresp.status <= 599){
        return(restart);
    }

    # Enable cache for all static files
    # The same argument as the static caches from above: monitor your cache size, if you get data nuked out of it, consider giving up the static file cache.
    # Before you blindly enable this, have a read here: http://mattiasgeniar.be/2012/11/28/stop-caching-static-files/
    if (req.url ~ "^[^?]*\.(bmp|bz2|css|doc|eot|flv|gif|gz|ico|jpeg|jpg|js|less|mp[34]|pdf|png|rar|rtf|swf|tar|tgz|txt|wav|woff|xml|zip)(\?.*)?$") {
        unset beresp.http.set-cookie;
    }

    # Sometimes, a 301 or 302 redirect formed via Apache's mod_rewrite can mess with the HTTP port that is being passed along.
    # This often happens with simple rewrite rules in a scenario where Varnish runs on :80 and Apache on :8080 on the same box.
    # A redirect can then often redirect the end-user to a URL on :8080, where it should be :80.
    # This may need finetuning on your setup.
    #
    # To prevent accidental replace, we only filter the 301/302 redirects for now.
    if (beresp.status == 301 || beresp.status == 302) {
        set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", "");
    }

    # Don't cache user & server errors
    if (beresp.status >= 400) {
        return (hit_for_pass);
    }

    # Non private responses without cookies and with a ttl of 0 should be artificially extended to 1 hour
    if (beresp.ttl <= 0s && beresp.http.Cache-Control !~ "private" && (!beresp.http.Set-Cookie)) {
        set beresp.ttl = 1h;
        set beresp.http.X-Cache-Extended = "1";
    }

    # Set 2min cache if unset for static files
    if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Vary == "*") {
        set beresp.ttl = 120s;
        return (hit_for_pass);
    }

    return (deliver);
}

# The routine when we deliver the HTTP request to the user
# Last chance to modify headers that are sent to the client
sub vcl_deliver {
    if (obj.hits > 0) {
        set resp.http.X-Cache = "cached";
    } else {
        set resp.http.x-Cache = "uncached";
    }

    # Remove some headers: PHP version
    unset resp.http.X-Powered-By;

    # Remove some headers: Apache version & OS
    unset resp.http.Server;
    unset resp.http.X-Drupal-Cache;
    unset resp.http.X-Varnish;
    unset resp.http.Via;
    unset resp.http.Link;

    return (deliver);
}

sub vcl_error {
    if (obj.status == 720) {
        # We use this special error status 720 to force redirects with 301 (permanent) redirects
        # To use this, call the following from anywhere in vcl_recv: error 720 "http://host/new.html"
        set obj.status = 301;
        set obj.http.Location = obj.response;
        return (deliver);
    } elseif (obj.status == 721) {
        # And we use error status 721 to force redirects with a 302 (temporary) redirect
        # To use this, call the following from anywhere in vcl_recv: error 720 "http://host/new.html"
        set obj.status = 302;
        set obj.http.Location = obj.response;
        return (deliver);
    }

    return (deliver);
}

sub vcl_init {
    return (ok);
}

sub vcl_fini {
    return (ok);
}

# Default backend definition.  Set this to point to your content server.
# all paths relative to varnish option vcl_dir

include "custom.backend.vcl";
include "custom.acl.vcl";

# Handle the HTTP request received by the client
sub vcl_recv {
    # varnish serves stale (but cacheable) objects while retriving object from backend
    if (req.backend.healthy) {
        set req.grace = 30s;
    } else {
        set req.grace = 1h;
    }

    # shortcut for DFind requests
    if (req.url ~ "^/w00tw00t") {
        error 404 "Not Found";
    }

    if (req.restarts == 0) {
        if (req.http.X-Forwarded-For) {
            set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
    }

    # Normalize the header, remove the port (in case you're testing this on various TCP ports)
    set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");

    # Allow purging
    if (req.request == "PURGE") {
        if (!client.ip ~ purge) {
            # Not from an allowed IP? Then die with an error.
            error 405 "This IP is not allowed to send PURGE requests.";
        }

        # If you got this stage (and didn't error out above), do a cache-lookup
        # That will force entry into vcl_hit() or vcl_miss() below and purge the actual cache
        return (lookup);
    }

    # Only deal with "normal" types
    if (req.request != "GET" &&
            req.request != "HEAD" &&
            req.request != "PUT" &&
            req.request != "POST" &&
            req.request != "TRACE" &&
            req.request != "OPTIONS" &&
            req.request != "PATCH" &&
            req.request != "DELETE") {
        /* Non-RFC2616 or CONNECT which is weird. */
        return (pipe);
    }

    if (req.request != "GET" && req.request != "HEAD") {
        # We only deal with GET and HEAD by default
        return (pass);
    }

    # Some generic URL manipulation, useful for all templates that follow
    # First remove the Google Analytics added parameters, useless for our backend
    if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=") {
        set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
        set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
        set req.url = regsub(req.url, "\?&", "?");
        set req.url = regsub(req.url, "\?$", "");
    }

    # Strip hash, server doesn't need it.
    if (req.url ~ "\#") {
        set req.url = regsub(req.url, "\#.*$", "");
    }

    # Strip a trailing ? if it exists
    if (req.url ~ "\?$") {
        set req.url = regsub(req.url, "\?$", "");
    }

    # This is an example to redirect with a 301/302 HTTP status code from within Varnish
    # if (req.http.Host ~ "secure.mysite.tld") {
    #   # We may want to force our users from the secure site to the HTTPs version?
    #   error 720 "https://secure.mysite.tld";
    #   # If you want to keep the URLs intact, this also works:
    #   error 720 "https://" + req.http.Host + req.url;
    # }
    #
    # Or to force a 302 temporary redirect, use error 721
    # if (req.http.Host ~ "temp.mysite.tld") {
    #   # Temporary redirect
    #   error 721 "http://mysite.tld/temp";
    # }
    #

    # Some generic cookie manipulation, useful for all templates that follow
    # Remove the "has_js" cookie
    set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", "");

    # Remove any Satallite cookies
    set req.http.Cookie = regsuball(req.http.Cookie, "__gaid=[^;]+(; )?", "");
    set req.http.Cookie = regsuball(req.http.Cookie, "_sdsat_[^=]+=[^;]+(; )?", "");

    # Remove any Google Analytics based cookies
    set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", "");
    set req.http.Cookie = regsuball(req.http.Cookie, "_ga=[^;]+(; )?", "");
    set req.http.Cookie = regsuball(req.http.Cookie, "utmctr=[^;]+(; )?", "");
    set req.http.Cookie = regsuball(req.http.Cookie, "utmcmd.=[^;]+(; )?", "");
    set req.http.Cookie = regsuball(req.http.Cookie, "utmccn.=[^;]+(; )?", "");

    # Remove any Cloudflare cookies
    set req.http.Cookie = regsuball(req.http.Cookie, "__cfduid=[^;]+(; )?", "");

    # Remove the Quant Capital cookies (added by some plugin, all __qca)
    set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", "");

    # Remove the AddThis cookies
    set req.http.Cookie = regsuball(req.http.Cookie, "__atuvc=[^;]+(; )?", "");

    # Remove a ";" prefix in the cookie if present
    set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");

    # Are there cookies left with only spaces or that are empty?
    if (req.http.cookie ~ "^\s*$") {
        unset req.http.cookie;
    }

    # Normalize Accept-Encoding header
    # straight from the manual: https://www.varnish-cache.org/docs/3.0/tutorial/vary.html
    if (req.http.Accept-Encoding) {
        if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
            # No point in compressing these
            remove req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elsif (req.http.Accept-Encoding ~ "deflate") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            # unkown algorithm
            remove req.http.Accept-Encoding;
        }
    }

    # Include custom vcl_recv logic
    include "conf.d/receive/wordpress.vcl";

    # Remove all cookies for static files
    # A valid discussion could be held on this line: do you really need to cache static files that don't cause load? Only if you have memory left.
    # Sure, there's disk I/O, but chances are your OS will already have these files in their buffers (thus memory).
    # Before you blindly enable this, have a read here: http://mattiasgeniar.be/2012/11/28/stop-caching-static-files/
    if (req.url ~ "^[^?]*\.(bmp|bz2|css|doc|eot|flv|gif|gz|ico|jpeg|jpg|js|less|mp[34]|pdf|png|rar|rtf|swf|tar|tgz|txt|wav|woff|xml|zip)(\?.*)?$") {
        unset req.http.Cookie;
        return (lookup);
    }

    # Send Surrogate-Capability headers to announce ESI support to backend
    set req.http.Surrogate-Capability = "key=ESI/1.0";

    if (req.http.Authorization || req.http.Cookie) {
        # Not cacheable by default
        return (pass);
    }

    return (lookup);
}

sub vcl_pipe {
    # Note that only the first request to the backend will have
    # X-Forwarded-For set.  If you use X-Forwarded-For and want to
    # have it set for all requests, make sure to have:
    # set bereq.http.connection = "close";
    # here.  It is not set by default as it might break some broken web
    # applications, like IIS with NTLM authentication.

    #set bereq.http.Connection = "Close";
    return (pipe);
}

sub vcl_pass {
    return (pass);
}

# The data on which the hashing will take place
sub vcl_hash {
    hash_data(req.url);

    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }

    # hash cookies for object with auth
    if (req.http.Cookie) {
        hash_data(req.http.Cookie);
    }
    if (req.http.Authorization) {
        hash_data(req.http.Authorization);
    }

    return (hash);
}

sub vcl_hit {
    # Allow purges
    if (req.request == "PURGE") {
        purge;
        error 200 "purged";
    }

    return (deliver);
}

sub vcl_miss {
    # Allow purges
    if (req.request == "PURGE") {
        purge;
        error 200 "purged";
    }

    return (fetch);
}

# Handle the HTTP request coming from our backend
sub vcl_fetch {
    set beresp.grace = 1h;

    # Include custom vcl_fetch logic
    include "conf.d/fetch/wordpress.vcl";

    # Parse ESI request and remove Surrogate-Control header
    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
        unset beresp.http.Surrogate-Control;
        set beresp.do_esi = true;
    }

    # If the request to the backend returns a code is 5xx, restart the loop
    # If the number of restarts reaches the value of the parameter max_restarts,
    # the request will be error'ed.  max_restarts defaults to 4.  This prevents
    # an eternal loop in the event that, e.g., the object does not exist at all.
    if (beresp.status >= 500 && beresp.status <= 599){
        return(restart);
    }

    # Enable cache for all static files
    # The same argument as the static caches from above: monitor your cache size, if you get data nuked out of it, consider giving up the static file cache.
    # Before you blindly enable this, have a read here: http://mattiasgeniar.be/2012/11/28/stop-caching-static-files/
    if (req.url ~ "^[^?]*\.(bmp|bz2|css|doc|eot|flv|gif|gz|ico|jpeg|jpg|js|less|mp[34]|pdf|png|rar|rtf|swf|tar|tgz|txt|wav|woff|xml|zip)(\?.*)?$") {
        unset beresp.http.set-cookie;
    }

    # Sometimes, a 301 or 302 redirect formed via Apache's mod_rewrite can mess with the HTTP port that is being passed along.
    # This often happens with simple rewrite rules in a scenario where Varnish runs on :80 and Apache on :8080 on the same box.
    # A redirect can then often redirect the end-user to a URL on :8080, where it should be :80.
    # This may need finetuning on your setup.
    #
    # To prevent accidental replace, we only filter the 301/302 redirects for now.
    if (beresp.status == 301 || beresp.status == 302) {
        set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", "");
    }

    # Don't cache user & server errors
    if (beresp.status >= 400) {
        return (hit_for_pass);
    }

    # Non private responses without cookies and with a ttl of 0 should be artificially extended to 1 hour
    if (beresp.ttl <= 0s && beresp.http.Cache-Control !~ "private" && (!beresp.http.Set-Cookie)) {
        set beresp.ttl = 1h;
        set beresp.http.X-Cache-Extended = "1";
    }

    # Set 2min cache if unset for static files
    if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Vary == "*") {
        set beresp.ttl = 120s;
        return (hit_for_pass);
    }

    return (deliver);
}

# The routine when we deliver the HTTP request to the user
# Last chance to modify headers that are sent to the client
sub vcl_deliver {
    if (obj.hits > 0) {
        set resp.http.X-Cache = "cached";
    } else {
        set resp.http.x-Cache = "uncached";
    }

    # Remove some headers: PHP version
    unset resp.http.X-Powered-By;

    # Remove some headers: Apache version & OS
    unset resp.http.Server;
    unset resp.http.X-Drupal-Cache;
    unset resp.http.X-Varnish;
    unset resp.http.Via;
    unset resp.http.Link;

    return (deliver);
}

sub vcl_error {
    if (obj.status == 720) {
        # We use this special error status 720 to force redirects with 301 (permanent) redirects
        # To use this, call the following from anywhere in vcl_recv: error 720 "http://host/new.html"
        set obj.status = 301;
        set obj.http.Location = obj.response;
        return (deliver);
    } elseif (obj.status == 721) {
        # And we use error status 721 to force redirects with a 302 (temporary) redirect
        # To use this, call the following from anywhere in vcl_recv: error 720 "http://host/new.html"
        set obj.status = 302;
        set obj.http.Location = obj.response;
        return (deliver);
    }

    return (deliver);
}

sub vcl_init {
    return (ok);
}

sub vcl_fini {
    return (ok);
}


update varnish

update varnish

# This is a basic VCL configuration file for varnish.  See the vcl(7)
# man page for details on VCL syntax and semantics.

import std;

# static
backend default {
  .host = "127.0.0.1";
  .port = "4000";
}
backend components {
  .host = "127.0.0.1";
  .port = "3330";
}
backend charts {
  .host = "127.0.0.1";
  .port = "8001";
}
backend chat {
  .host = "127.0.0.1";
  .port = "8002";
}
backend codemirror {
  .host = "127.0.0.1";
  .port = "8003";
}
backend directory {
  .host = "127.0.0.1";
  .port = "8004";
}
backend hello {
  .host = "127.0.0.1";
  .port = "8005";
}
backend sink {
  .host = "127.0.0.1";
  .port = "8006";
}
backend todos {
  .host = "127.0.0.1";
  .port = "8007";
}
backend widgets {
  .host = "127.0.0.1";
  .port = "8008";
}

sub vcl_recv {
  if (!req.http.Host) {
    error 404 "Need a host header";
  }

  set req.http.Host = regsub(req.http.Host, "^www\.", "");
  set req.http.Host = regsub(req.http.Host, ":80$", "");

  if (req.restarts == 0) {
    if (req.http.x-forwarded-for) {
      set req.http.X-Forwarded-For =
        req.http.X-Forwarded-For + ", " + client.ip;
    } else {
      set req.http.X-Forwarded-For = client.ip;
    }
  }

  if (req.http.Host ~ "^components\.") {
    set req.backend = components;
  } else if (req.http.Host ~ "^charts\.") {
    set req.backend = charts;
  } else if (req.http.Host ~ "^chat\.") {
    set req.backend = chat;
  } else if (req.http.Host ~ "^codemirror\.") {
    set req.backend = codemirror;
  } else if (req.http.Host ~ "^directory\.") {
    set req.backend = directory;
  } else if (req.http.Host ~ "^hello\.") {
    set req.backend = hello;
  } else if (req.http.Host ~ "^sink\.") {
    set req.backend = sink;
  } else if (req.http.Host ~ "^todos\.") {
    set req.backend = todos;
  } else if (req.http.Host ~ "^widgets\.") {
    set req.backend = widgets;
  }

  if (req.http.Upgrade ~ "(?i)websocket") {
    return (pipe);
  }
  if (req.request != "GET" &&
      req.request != "HEAD" &&
      req.request != "PUT" &&
      req.request != "POST" &&
      req.request != "TRACE" &&
      req.request != "OPTIONS" &&
      req.request != "DELETE") {
    /* Non-RFC2616 or CONNECT which is weird. */
    return (pipe);
  }
  if (req.request != "GET" && req.request != "HEAD") {
    /* We only deal with GET and HEAD by default */
    return (pass);
  }
  if (req.http.Authorization || req.http.Cookie) {
    /* Not cacheable by default */
    return (pass);
  }
  return (lookup);
}

sub vcl_pipe {
  # Websockets
  if (req.http.upgrade) {
    set bereq.http.upgrade = req.http.upgrade;
  }
  set bereq.http.connection = "close";

  return (pipe);
}

sub vcl_fetch {
  # Compress responses
  if (beresp.http.content-type ~ "text"
      || beresp.http.content-type ~ "json"
      || beresp.http.content-type ~ "javascript") {
    set beresp.do_gzip = true;
  }
}

sub vcl_error {
  if (obj.status == 750) {
    # moved permanently
    set obj.http.Location = req.http.Location;
    set obj.status = 301;
  } else if (obj.status == 752) {
    # moved temporarily
    set obj.http.Location = req.http.Location;
    set obj.status = 302;
  } else {
    set obj.http.Content-Type = "text/html; charset=utf-8";
    set obj.http.Retry-After = "5";
    synthetic std.fileread("/etc/varnish/503.html");
  }
  return (deliver);
}

# This is a basic VCL configuration file for varnish.  See the vcl(7)
# man page for details on VCL syntax and semantics.

import std;

# static
backend default {
  .host = "127.0.0.1";
  .port = "4000";
}
backend examples {
  .host = "127.0.0.1";
  .port = "8080";
}
backend components {
  .host = "127.0.0.1";
  .port = "3330";
}

sub vcl_recv {
  if (!req.http.Host) {
    error 404 "Need a host header";
  }

  set req.http.Host = regsub(req.http.Host, "^www\.", "");
  set req.http.Host = regsub(req.http.Host, ":80$", "");

  if (req.restarts == 0) {
    if (req.http.x-forwarded-for) {
      set req.http.X-Forwarded-For =
        req.http.X-Forwarded-For + ", " + client.ip;
    } else {
      set req.http.X-Forwarded-For = client.ip;
    }
  }

  if (
    req.http.Host ~ "^chat\." ||
    req.http.Host ~ "^charts\." ||
    req.http.Host ~ "^directory\." ||
    req.http.Host ~ "^codemirror\." ||
    req.http.Host ~ "^hello\." ||
    req.http.Host ~ "^sink\." ||
    req.http.Host ~ "^todos\." ||
    req.http.Host ~ "^widgets\."
  ) {
    set req.backend = examples;
  } else if (req.http.Host ~ "^components\.") {
    set req.backend = components;
  }

  if (req.request != "GET" &&
      req.request != "HEAD" &&
      req.request != "PUT" &&
      req.request != "POST" &&
      req.request != "TRACE" &&
      req.request != "OPTIONS" &&
      req.request != "DELETE") {
    /* Non-RFC2616 or CONNECT which is weird. */
    return (pipe);
  }
  if (req.http.Upgrade ~ "(?i)websocket") {
    return (pipe);
  }
  if (req.request != "GET" && req.request != "HEAD") {
    /* We only deal with GET and HEAD by default */
    return (pass);
  }
  if (req.http.Authorization || req.http.Cookie) {
    /* Not cacheable by default */
    return (pass);
  }
  return (lookup);
}

sub vcl_hash {
  hash_data(req.url);
  if (req.http.host) {
    hash_data(req.http.host);
  } else {
    hash_data(server.ip);
  }
  if (req.http.Origin) {
    hash_data(req.http.Origin);
  }
  return (hash);
}

sub vcl_pipe {
  # Note that only the first request to the backend will have
  # X-Forwarded-For set.  If you use X-Forwarded-For and want to
  # have it set for all requests, make sure to have:
  set bereq.http.connection = "close";
  # here.  It is not set by default as it might break some broken web
  # applications, like IIS with NTLM authentication.

  # Websockets
  if (req.http.upgrade) {
    set bereq.http.upgrade = req.http.upgrade;
  }

  return (pipe);
}

sub vcl_fetch {
  # Compress responses
  if (beresp.http.content-type ~ "text"
      || beresp.http.content-type ~ "json"
      || beresp.http.content-type ~ "javascript") {
    set beresp.do_gzip = true;
  }
}

sub vcl_error {
  if (obj.status == 750) {
    # moved permanently
    set obj.http.Location = req.http.Location;
    set obj.status = 301;
  } else if (obj.status == 752) {
    # moved temporarily
    set obj.http.Location = req.http.Location;
    set obj.status = 302;
  } else {
    set obj.http.Content-Type = "text/html; charset=utf-8";
    set obj.http.Retry-After = "5";
    synthetic std.fileread("/etc/varnish/503.html");
  }
  return (deliver);
}


Revert "fix redirects"

Revert "fix redirects"

This reverts commit 2a9026b004e148db83e807411adfde4deece4515.

sub vcl_recv {
	if (req.http.Host ~ "polyfills.io") {
		# Fastly recommend using the 900-999 range of status codes for custom errors within an application's VCL.
		# Do the canonicalise check before the HTTPS check to avoid a double redirect
		error 901 "Canonicalise";
	}
	if (!req.http.Fastly-SSL) {
		# 801 is a special error code that Fastly uses to Force HTTPS on the request
		error 801 "Redirect to HTTPS";
	}
	if (req.http.Orig-URL ~ "^/v1") {
		error 902 "Redirect to V2";
	}

	if (req.url.path == "/" || req.url.path == "/v2" || req.url.path == "/v2/docs/") {
		error 908;
	}
}

sub vcl_error {
  # Redirect to canonical prod/qa origins
	if (obj.status == 901) {
		set obj.status = 301;
		set obj.response = "Moved Permanently";
		set obj.http.Location = "https://polyfill.io" req.url;
		synthetic {""};
		return (deliver);
	}
	# Redirect to v2
	if (obj.status == 902) {
		set obj.status = 301;
		set obj.response = "Moved Permanently";
		# Remove libVersion and gated query parameters if they exist.
		set req.url = querystring.regfilter(req.url, "^\b(libVersion|gated)\b.*");
		set obj.http.Location = if(req.url.path == "/v1", "/v3/", regsub(req.url, "^/v1", "/v2"));
		set obj.http.Deprecation-Notice = "API version 1 has been decommissioned - see the body of this response for more information.";
		synthetic {"API version 1 has been decommissioned. Your request is being redirected to v2.  The `libVersion` and `gated` query string parameters are no longer supported and if present have been removed from your request.\n\nA deprecation period for v1 existed between August and December 2015, during which time v1 requests were honoured but a deprecation warning was added to output."};
		return (deliver);
	}

	# Redirect to v3
	if (obj.status == 908) {
		set obj.status = 301;
		set obj.http.Location = "/v3/";
		return (deliver);
	}
}

sub vcl_recv {
	if (req.http.Host ~ "polyfills.io") {
		# Fastly recommend using the 900-999 range of status codes for custom errors within an application's VCL.
		# Do the canonicalise check before the HTTPS check to avoid a double redirect
		error 901 "Canonicalise";
	}
	if (!req.http.Fastly-SSL) {
		# 801 is a special error code that Fastly uses to Force HTTPS on the request
		error 801 "Redirect to HTTPS";
	}
	if (req.http.Orig-URL ~ "^/v1") {
		error 902 "Redirect to V2";
	}

	if (req.url.path == "/" || req.url.path == "/v2" || req.url.path == "/v2/docs" || req.url.path == "/v2/docs/") {
		error 908;
	}
}

sub vcl_error {
  # Redirect to canonical prod/qa origins
	if (obj.status == 901) {
		set obj.status = 301;
		set obj.response = "Moved Permanently";
		set obj.http.Location = "https://polyfill.io" req.url;
		synthetic {""};
		return (deliver);
	}
	# Redirect to v2
	if (obj.status == 902) {
		set obj.status = 301;
		set obj.response = "Moved Permanently";
		# Remove libVersion and gated query parameters if they exist.
		set req.url = querystring.regfilter(req.url, "^\b(libVersion|gated)\b.*");
		set obj.http.Location = if(req.url.path == "/v1", "/v3/", regsub(req.url, "^/v1", "/v2"));
		set obj.http.Deprecation-Notice = "API version 1 has been decommissioned - see the body of this response for more information.";
		synthetic {"API version 1 has been decommissioned. Your request is being redirected to v2.  The `libVersion` and `gated` query string parameters are no longer supported and if present have been removed from your request.\n\nA deprecation period for v1 existed between August and December 2015, during which time v1 requests were honoured but a deprecation warning was added to output."};
		return (deliver);
	}

	# Redirect to v3
	if (obj.status == 908) {
		set obj.status = 301;
		set obj.http.Location = "/v3/";
		return (deliver);
	}
}


Sort querystring using boltsort for better caching (#812)

* Sort querystring using boltsort for better caching

* Update fastly-config.vcl

import boltsort;

sub vcl_recv {
#FASTLY recv
	if ( req.request == "FASTLYPURGE" ) {
		set req.http.Fastly-Purge-Requires-Auth = "1";
	}

	if (req.request != "HEAD" && req.request != "GET" && req.request != "FASTLYPURGE") {
		return(pass);
	}

	if (!req.http.Fastly-SSL && (req.http.Host == "cdn.polyfill.io" || req.http.Host == "polyfill.io")) {
		error 751 "Force TLS";
	}

	if (req.url ~ "^/v2/polyfill\." && req.url !~ "[\?\&]ua=") {
		set req.http.X-Orig-URL = req.url;
		set req.url = "/v2/normalizeUa?ua=" urlencode(req.http.User-Agent);
	}

	if (req.url ~ "^/v2/recordRumData") {
		error 204 "No Content";
	}

	set req.url = boltsort.sort(req.url);

	return(lookup);
}

sub vcl_deliver {
#FASTLY deliver

	# If the response is to a normalise request and there's a parked "original request", use the normalised UA response to modify the original request and restart it
	if (req.url ~ "^/v\d/normalizeUa" && resp.status == 200 && req.http.X-Orig-URL) {
		set req.http.Fastly-force-Shield = "1";
		if (req.http.X-Orig-URL ~ "\?") {
			set req.url = req.http.X-Orig-URL "&ua=" resp.http.Normalized-User-Agent;
		} else {
			set req.url = req.http.X-Orig-URL "?ua=" resp.http.Normalized-User-Agent;
		}
		restart;

	# If the original request didn't specify a UA override, and we added one when the request was sent to the backend, the backend won't have included a Vary:user-agent header (correctly) but we need to add one
	} else if (req.url ~ "^/v\d/polyfill\..*[\?\&]ua=" && req.http.X-Orig-URL && req.http.X-Orig-URL !~ "[\?\&]ua=") {
		add resp.http.Vary = "User-Agent";
	}
	return(deliver);
}

sub vcl_error {

	# Redirect to SSL
	if (obj.status == 751) {
		set obj.status = 301;
		set obj.response = "Moved Permanently";
		set obj.http.Location = "https://" req.http.host req.url;
		synthetic {""};
		return (deliver);
	}
}

sub vcl_recv {
#FASTLY recv
	if ( req.request == "FASTLYPURGE" ) {
		set req.http.Fastly-Purge-Requires-Auth = "1";
	}

	if (req.request != "HEAD" && req.request != "GET" && req.request != "FASTLYPURGE") {
		return(pass);
	}

	if (!req.http.Fastly-SSL && (req.http.Host == "cdn.polyfill.io" || req.http.Host == "polyfill.io")) {
		error 751 "Force TLS";
	}

	if (req.url ~ "^/v2/polyfill\." && req.url !~ "[\?\&]ua=") {
		set req.http.X-Orig-URL = req.url;
		set req.url = "/v2/normalizeUa?ua=" urlencode(req.http.User-Agent);
	}

	if (req.url ~ "^/v2/recordRumData") {
		error 204 "No Content";
	}

	return(lookup);
}

sub vcl_deliver {
#FASTLY deliver

	# If the response is to a normalise request and there's a parked "original request", use the normalised UA response to modify the original request and restart it
	if (req.url ~ "^/v\d/normalizeUa" && resp.status == 200 && req.http.X-Orig-URL) {
		set req.http.Fastly-force-Shield = "1";
		if (req.http.X-Orig-URL ~ "\?") {
			set req.url = req.http.X-Orig-URL "&ua=" resp.http.Normalized-User-Agent;
		} else {
			set req.url = req.http.X-Orig-URL "?ua=" resp.http.Normalized-User-Agent;
		}
		restart;

	# If the original request didn't specify a UA override, and we added one when the request was sent to the backend, the backend won't have included a Vary:user-agent header (correctly) but we need to add one
	} else if (req.url ~ "^/v\d/polyfill\..*[\?\&]ua=" && req.http.X-Orig-URL && req.http.X-Orig-URL !~ "[\?\&]ua=") {
		add resp.http.Vary = "User-Agent";
	}
	return(deliver);
}

sub vcl_error {

	# Redirect to SSL
	if (obj.status == 751) {
		set obj.status = 301;
		set obj.response = "Moved Permanently";
		set obj.http.Location = "https://" req.http.host req.url;
		synthetic {""};
		return (deliver);
	}
}


Reindent according to Varnish coding standards.

Thanks to Sergey Syrota for pointing this out.

#
# devicedetect-dev.vcl
#
# Enable test URLs and cookie overrides.
#

backend devicetest {
	.host = "127.0.0.1";
	.port = "5911";
}

sub vcl_recv {
	# 701/702 are arbitrary chosen return codes that is only used internally in varnish.
	if (req.url ~ "^/set_ua_device/.+") {
		error 701 regsub(req.url, "^/set_ua_device/", "");
	}
	# set expired cookie if nothing is specified
	if (req.url ~ "^/set_ua_device/") {
		error 702 "OK";
	}
	if (req.url ~ "^/devicetest") {
		set req.backend = devicetest;
	}
}

sub vcl_error {
	if (obj.status == 701 || obj.status == 702) {
		if (obj.status == 702) {
			set obj.status = 200;
			set obj.http.Set-Cookie = "X-UA-Device-force=expiring; Path=/; Expires=Thu, 01 Jan 1970 00:00:01 GMT;";
		} else {
			set obj.status = 200;
			set obj.http.Set-Cookie = "X-UA-Device-force=" + obj.response + "; Path=/;";
		}
		set obj.http.Content-Type = "text/html; charset=utf-8";
		synthetic {"
OK, Cookie updated
/devicetest/"};
		set obj.response = "OK";
		return(deliver);
	}
}


Do not cache readiness probe

Do not cache readiness probe

vcl 4.0;

backend default {
    .host = "localhost";
    .port = "8080";
}

# Set the allowed IPs of purge requests
acl purge {
    "localhost";
    "127.0.0.1";
}

sub vcl_recv {

    # Handle purge requests
    if (req.method == "PURGE") {
        if (client.ip !~ purge) {
            return (synth(405));
        }

        if (req.http.X-Purge-Method == "regex") {
            ban("req.url ~ " + req.url + " && req.http.host ~ " + req.http.host);
            return (synth(200, "Banned."));
        } else {
            return (purge);
        }
    }

    # Do not cache readiness probe
    if (req.url ~ "/ready(/)?") {
	    return ( pass );
    }

    # Do not cache rss feed
    if (req.url ~ "/feed(/)?") {
	    return ( pass );
    }

    # Do not cache cron
    if (req.url ~ "wp-cron\.php.*") {
        return ( pass );
    }

    # Do not cache search result
    if (req.url ~ "/\?s\=") {
	    return ( pass );
    }

    # Exclude admin and login urls
    if (req.url ~ "wp-admin|wp-login") {
        return (pass);
    }

    # Ignore some wp cookies
    set req.http.cookie = regsuball(req.http.cookie, "wp-settings-\d+=[^;]+(; )?", "");
    set req.http.cookie = regsuball(req.http.cookie, "wp-settings-time-\d+=[^;]+(; )?", "");
    set req.http.cookie = regsuball(req.http.cookie, "wordpress_test_cookie=[^;]+(; )?", "");
    if (req.http.cookie == "") {
        unset req.http.cookie;
    }

}

# Extend cache
sub vcl_backend_response {
    if (beresp.ttl == 120s) {
        set beresp.ttl = 1h;
    }
}

vcl 4.0;

backend default {
    .host = "localhost";
    .port = "8080";
}

# Set the allowed IPs of purge requests
acl purge {
    "localhost";
    "127.0.0.1";
}

sub vcl_recv {

    # Handle purge requests
    if (req.method == "PURGE") {
        if (client.ip !~ purge) {
            return (synth(405));
        }

        if (req.http.X-Purge-Method == "regex") {
            ban("req.url ~ " + req.url + " && req.http.host ~ " + req.http.host);
            return (synth(200, "Banned."));
        } else {
            return (purge);
        }
    }

    # Do not cache rss feed
    if (req.url ~ "/feed(/)?") {
	    return ( pass );
    }

    # Do not cache cron
    if (req.url ~ "wp-cron\.php.*") {
        return ( pass );
    }

    # Do not cache search result
    if (req.url ~ "/\?s\=") {
	    return ( pass );
    }

    # Exclude admin and login urls
    if (req.url ~ "wp-admin|wp-login") {
        return (pass);
    }

    # Ignore some wp cookies
    set req.http.cookie = regsuball(req.http.cookie, "wp-settings-\d+=[^;]+(; )?", "");
    set req.http.cookie = regsuball(req.http.cookie, "wp-settings-time-\d+=[^;]+(; )?", "");
    set req.http.cookie = regsuball(req.http.cookie, "wordpress_test_cookie=[^;]+(; )?", "");
    if (req.http.cookie == "") {
        unset req.http.cookie;
    }

}

# Extend cache
sub vcl_backend_response {
    if (beresp.ttl == 120s) {
        set beresp.ttl = 1h;
    }
}


This check is not necessary as Fastly already does this

    # workaround for possible security issue
    if (req.url ~ "^\s") {
        set obj.status = 400;
        set obj.response = "Malformed request";
        synthetic "";
        return (deliver);
    }

    /* handle 503s */
    if (obj.status >= 500 && obj.status < 600) {

        /* deliver stale object if it is available */
        if (stale.exists) {
            return(deliver_stale);
        }
    }


84 - varnish bypass is a direct route to nginx-php now

vcl 4.0;

import std;
import directors;

# set backend backend
backend backend {
  .host = "${VARNISH_HOST:-nginx-php}";
  .port = "8080";
  .first_byte_timeout = 35m;
  .between_bytes_timeout = 10m;
}

sub vcl_init {
  # set director webworker
   new webworker = directors.round_robin();
   webworker.add_backend(backend);
}

# Allow purging from localhost
# @TODO allow from openshift network
acl purge {
      "127.0.0.1";
}

# This configuration is optimized for Drupal hosting:
# Respond to incoming requests.
sub vcl_recv {
  if (req.url ~ "^/varnish_status$")  {
    return (synth(700,"OK"));
  }
  # set the backend, which should be used:
  set req.backend_hint = webworker.backend();

  # Always set the forward ip.
   if (req.restarts == 0) {
     if (req.http.x-forwarded-for) {
         set req.http.X-Forwarded-For =  req.http.X-Forwarded-For + ", " + client.ip;
     } else {
         set req.http.X-Forwarded-For = client.ip;
     }
   }

  # SA-CORE-2014-004 preventing access to /xmlrpc.php
  if (req.url ~ "^/xmlrpc.php$") {
    return (synth(701, "Unauthorized"));
  }

  # Strip out Google Analytics campaign variables. They are only needed
  # by the javascript running on the page
  # utm_source, utm_medium, utm_campaign, gclid
   if(req.url ~ "(\?|&)(gclid|utm_[a-z]+)=") {
     set req.url = regsuball(req.url, "(gclid|utm_[a-z]+)=[^\&]+&?", "");
     set req.url = regsub(req.url, "(\?|&)$", "");
   }

  # We set a header to let a Varnish Chain know that it already has been varnishcached
  set req.http.X-LAGOON-VARNISH = "d3659c5e113e.amazee.io";

  # Bypass a cache hit
  if (req.method == "REFRESH") {
      if (!client.ip ~ purge) { return (synth(405, "Not allowed")); }
      set req.method = "GET";
      set req.hash_always_miss = true;
  }

  # Non-RFC2616 or CONNECT which is weird, we pipe that
  if (req.method != "GET" &&
    req.method != "HEAD" &&
    req.method != "PUT" &&
    req.method != "POST" &&
    req.method != "TRACE" &&
    req.method != "OPTIONS" &&
    req.method != "DELETE") {
     return (pipe);
  }

  # We only try to cache  GET and HEAD, other things are passed
  if (req.method != "GET" && req.method != "HEAD") {
    return (pass);
  }

  # Any requests with Basic Auth are passed
  if (req.http.Authorization || req.http.Authenticate)
  {
    return (pass);
  }

  ## Pass requests which are from blackfire
  if (req.http.X-Blackfire-Query) {
    return (pass);
  }

  # Some URLs should never be cached
  if (req.url ~ "^/status\.php$" ||
      req.url ~ "^/update\.php$" ||
      req.url ~ "^/admin([/?]|$).*$" ||
      req.url ~ "^/info([/?]|$).*$" ||
      req.url ~ "^/flag([/?]|$).*$" ||
      req.url ~ "^.*/system/files([/?]|$).*$" ||
      req.url ~ "^/amazeeadmin" ||
      req.url ~ "^/lithium" ||
      req.url ~ "^/cgi" ||
      req.url ~ "^/cgi-bin"
  ) {
    return (pass);
  }

  # Plupload likes to get piped
  if (req.url ~ "^.*/plupload-handle-uploads.*$"
  ) {
    return (pipe);
  }

  # Handle compression correctly. Different browsers send different
  # "Accept-Encoding" headers, even though they mostly all support the same
  # compression mechanisms. By consolidating these compression headers into
  # a consistent format, we can reduce the size of the cache and get more hits.=
  # @see: http:// varnish.projects.linpro.no/wiki/FAQ/Compression
  if (req.http.Accept-Encoding) {
    if (req.http.Accept-Encoding ~ "gzip") {
      # If the browser supports it, we'll use gzip.
      set req.http.Accept-Encoding = "gzip";
    }
    else if (req.http.Accept-Encoding ~ "deflate") {
      # Next, try deflate if it is supported.
      set req.http.Accept-Encoding = "deflate";
    }
    else {
      # Unknown algorithm. Remove it and send unencoded.
      unset req.http.Accept-Encoding;
    }
  }

  # Always cache the following file types for all users.
  if (req.url ~ "(?i)\.(css|js|jpg|jpeg|gif|ico|png|tiff|tif|img|tga|wmf|swf|html|htm|woff|woff2|mp4|ttf|eot|svg)(\?.*)?$") {
    unset req.http.Cookie;
  }

  # Remove all cookies that Drupal doesn't need to know about. ANY remaining
  # cookie will cause the request to pass-through to a backend. For the most part
  # we always set the NO_CACHE cookie after any POST request, disabling the
  # Varnish cache temporarily. The session cookie allows all authenticated users
  # to pass through as long as they're logged in.
  #
  # 1. Append a semi-colon to the front of the cookie string.
  # 2. Remove all spaces that appear after semi-colons.
  # 3. Match the cookies we want to keep, adding the space we removed
  #    previously, back. (\1) is first matching group in the regsuball.
  # 4. Remove all other cookies, identifying them by the fact that they have
  #    no space after the preceding semi-colon.
  # 5. Remove all spaces and semi-colons from the beginning and end of the
  #    cookie string.
  if (req.http.Cookie) {
    set req.http.CookieCheck = ";" + req.http.Cookie;
    set req.http.CookieCheck = regsuball(req.http.CookieCheck, "; +", ";");
    set req.http.CookieCheck = regsuball(req.http.CookieCheck, ";(SESS[a-z0-9]+|SSESS[a-z0-9]+|NO_CACHE)=", "; \1=");
    set req.http.CookieCheck = regsuball(req.http.CookieCheck, ";[^ ][^;]*", "");
    set req.http.CookieCheck = regsuball(req.http.CookieCheck, "^[; ]+|[; ]+$", "");

    set req.http.Cookie = req.http.Cookie + ";";

    if (req.http.CookieCheck == "") {
      # If there are no remaining cookies, remove the cookie header. If there
      # aren't any cookie headers, Varnish's default behavior will be to cache
      # the page.

      unset req.http.CookieCheck;
      unset req.http.Cookie;
    }
    else {
      # If there is any cookies left (a session or NO_CACHE cookie), do not
      # cache the page. Pass it on to Apache directly.
      unset req.http.CookieCheck;
      return (pass);
    }
  }

  # Cacheable, lookup in cache.
  return (hash);
}

sub vcl_hit {
    if (obj.ttl >= 0s) {
        # normal hit
        return (deliver);
    }
    # We have no fresh fish. Lets look at the stale ones.
    if (std.healthy(req.backend_hint)) {
        # Backend is healthy. If the object is not older then 30secs, deliver it to the client
        # and automatically create a separate backend request to warm the cache for this request.
        if (obj.ttl + 30s > 0s) {
            set req.http.grace = "normal(limited)";
            return (deliver);
        } else {
            # No candidate for grace. Fetch a fresh object.
            return(miss);
        }
    } else {
        # backend is sick - use full grace
        if (obj.ttl + obj.grace > 0s) {
            set req.http.grace = "full";
            return (deliver);
        } else {
            # no graced object.
            return (miss);
        }
    }
}

sub vcl_backend_response {
  # Temporarily disable streaming on files to be tested and removed after Varnish 4.1.x upgrade
  set beresp.do_stream = false;

  # Allow items to be stale if needed.
  set beresp.grace = 6h;

  ## FIXME: rename X-AMAZEEIO-VARNISH-BYPASS to X-AMAZEEIO-VARNISH-BACKEND-BYPASS to avoid confusion
  # If the backend sends a X-AMAZEEIO-VARNISH-BYPASS header we directly deliver
  if(beresp.http.X-AMAZEEIO-VARNISH-BYPASS == "TRUE") {
    return (deliver);
  }

  # Cache 404 and 403 for 10 seconds
  if(beresp.status == 404 || beresp.status == 403) {
    set beresp.ttl = 10s;
    return (deliver);
  }

  # Don't allow static files to set cookies.
  if (bereq.url ~ "(?i)\.(css|js|jpg|jpeg|gif|ico|png|tiff|tif|img|tga|wmf|swf|html|htm|woff|woff2|mp4|ttf|eot|svg)(\?.*)?$") {
    # beresp == Back-end response from the web server.
    unset beresp.http.set-cookie;
    unset beresp.http.Cache-Control;
    set beresp.ttl = 2628001s;
    set beresp.http.Cache-Control = "public, max-age=2628001";
    set beresp.http.Expires = "" + (now + beresp.ttl);
  }

  return (deliver);
}

# Set a header to track a cache HIT/MISS.
sub vcl_deliver {
  if (obj.hits > 0) {
    set resp.http.X-Varnish-Cache = "HIT";
  }
  else {
    set resp.http.X-Varnish-Cache = "MISS";
  }
  unset resp.http.X-Generator;
  unset resp.http.Server;
  # Inject information about grace
  if (req.http.grace) {
    set resp.http.X-Varnish-Grace = req.http.grace;
  }
  set resp.http.X-AMAZEEIO = "varnish>" + resp.http.X-AMAZEEIO;
  return (deliver);
}

sub vcl_hash {
     hash_data(req.url);
     if (req.http.host) {
         hash_data(req.http.host);
     } else {
         hash_data(server.ip);
     }
     if (req.http.X-Forwarded-Proto) {
        hash_data(req.http.X-Forwarded-Proto);
     }
     if (req.http.HTTPS) {
        hash_data(req.http.HTTPS);
     }
  return (lookup);
}

sub vcl_synth {
  if (resp.status == 701) {
    set resp.status = 401;
    set resp.http.Content-Type = "text/plain; charset=utf-8";
    synthetic({"XMLRPC Interface is blocked due to SA-CORE-2014-004 - mail support@amazee.io if you need it."});
    return (deliver);
  }
  if (resp.status == 700) {
    # Set a status the client will understand
    set resp.status = 200;
    # Create our synthetic response
    synthetic("");
    return(deliver);
}
  return (deliver);
}

sub vcl_backend_error {
    # Restart the request, when we have a backend server error, to try another backend.
    # Restart max twice.
    if (bereq.retries < 2) {
      return(retry);
    }

    set beresp.http.Content-Type = "text/html; charset=utf-8";
    set beresp.http.Retry-After = "5";
    synthetic( {"
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

    <title>Server Error</title>

We are sorry...

We encountered a server-side error. This means that the problem is not with your computer or Internet connection, but rather with the website's server.

We are currently working on solving this problem and apologise for the inconvenience.

            Technical Information

            Error "} + beresp.status + " " + beresp.reason + {"

            XID: "} + bereq.xid + {"

"} );
    return (deliver);
}

vcl 4.0;

import std;
import directors;

# set backend backend
backend backend {
  .host = "${VARNISH_HOST:-nginx-php}";
  .port = "8080";
  .first_byte_timeout = 35m;
  .between_bytes_timeout = 10m;
}

sub vcl_init {
  # set director webworker
   new webworker = directors.round_robin();
   webworker.add_backend(backend);
}

# Allow purging from localhost
# @TODO allow from openshift network
acl purge {
      "127.0.0.1";
}

# This configuration is optimized for Drupal hosting:
# Respond to incoming requests.
sub vcl_recv {
  if (req.url ~ "^/varnish_status$")  {
    return (synth(700,"OK"));
  }
  # set the backend, which should be used:
  set req.backend_hint = webworker.backend();

  # Always set the forward ip.
   if (req.restarts == 0) {
     if (req.http.x-forwarded-for) {
         set req.http.X-Forwarded-For =  req.http.X-Forwarded-For + ", " + client.ip;
     } else {
         set req.http.X-Forwarded-For = client.ip;
     }
   }

  # SA-CORE-2014-004 preventing access to /xmlrpc.php
  if (req.url ~ "^/xmlrpc.php$") {
    return (synth(701, "Unauthorized"));
  }

  # Strip out Google Analytics campaign variables. They are only needed
  # by the javascript running on the page
  # utm_source, utm_medium, utm_campaign, gclid
   if(req.url ~ "(\?|&)(gclid|utm_[a-z]+)=") {
     set req.url = regsuball(req.url, "(gclid|utm_[a-z]+)=[^\&]+&?", "");
     set req.url = regsub(req.url, "(\?|&)$", "");
   }

  ## Pass all Requests which are handled via an upstream Varnish
  if (req.http.X-AMAZEEIO-VARNISH) {
    set req.http.X-AMAZEEIO-VARNISH =  "d3659c5e113e.amazee.io, " + req.http.X-AMAZEEIO-VARNISH;
    set req.http.X-AMAZEEIO-VARNISH-BYPASS = "TRUE";
    return (pass);
  } else if (req.http.Fastly-FF) {
    ## Pass all Requests which are handled via Fastly
    set req.http.X-AMAZEEIO-VARNISH = "d3659c5e113e.amazee.io";
    set req.http.X-AMAZEEIO-VARNISH-BYPASS = "TRUE";
    set req.http.X-Forwarded-For = req.http.Fastly-Client-IP;
    return (pass);
  } else if (req.http.CF-RAY) {
    ## Pass all Requests which are handled via CloudFlare
    set req.http.X-AMAZEEIO-VARNISH = "d3659c5e113e.amazee.io";
    set req.http.X-AMAZEEIO-VARNISH-BYPASS = "TRUE";
    set req.http.X-Forwarded-For = req.http.CF-Connecting-IP;
    return (pass);
  } else if (req.http.X-Pull) {
    ## Pass all Requests which are handled via KeyCDN
    set req.http.X-AMAZEEIO-VARNISH = "d3659c5e113e.amazee.io";
    set req.http.X-AMAZEEIO-VARNISH-BYPASS = "TRUE";
    return (pass);
  } else {
    # We set a header to let a Varnish Chain know that it already has been varnishcached
    set req.http.X-AMAZEEIO-VARNISH = "d3659c5e113e.amazee.io";
  }

  # Bypass a cache hit
  if (req.method == "REFRESH") {
      if (!client.ip ~ purge) { return (synth(405, "Not allowed")); }
      set req.method = "GET";
      set req.hash_always_miss = true;
  }

  # Non-RFC2616 or CONNECT which is weird, we pipe that
  if (req.method != "GET" &&
    req.method != "HEAD" &&
    req.method != "PUT" &&
    req.method != "POST" &&
    req.method != "TRACE" &&
    req.method != "OPTIONS" &&
    req.method != "DELETE") {
     return (pipe);
  }

  # We only try to cache  GET and HEAD, other things are passed
  if (req.method != "GET" && req.method != "HEAD") {
    return (pass);
  }

  # Any requests with Basic Auth are passed
  if (req.http.Authorization || req.http.Authenticate)
  {
    return (pass);
  }

  ## Pass requests which are from blackfire
  if (req.http.X-Blackfire-Query) {
    return (pass);
  }

  # Some URLs should never be cached
  if (req.url ~ "^/status\.php$" ||
      req.url ~ "^/update\.php$" ||
      req.url ~ "^/admin([/?]|$).*$" ||
      req.url ~ "^/info([/?]|$).*$" ||
      req.url ~ "^/flag([/?]|$).*$" ||
      req.url ~ "^.*/system/files([/?]|$).*$" ||
      req.url ~ "^/amazeeadmin" ||
      req.url ~ "^/lithium" ||
      req.url ~ "^/cgi" ||
      req.url ~ "^/cgi-bin"
  ) {
    return (pass);
  }

  # Plupload likes to get piped
  if (req.url ~ "^.*/plupload-handle-uploads.*$"
  ) {
    return (pipe);
  }

  # Handle compression correctly. Different browsers send different
  # "Accept-Encoding" headers, even though they mostly all support the same
  # compression mechanisms. By consolidating these compression headers into
  # a consistent format, we can reduce the size of the cache and get more hits.=
  # @see: http:// varnish.projects.linpro.no/wiki/FAQ/Compression
  if (req.http.Accept-Encoding) {
    if (req.http.Accept-Encoding ~ "gzip") {
      # If the browser supports it, we'll use gzip.
      set req.http.Accept-Encoding = "gzip";
    }
    else if (req.http.Accept-Encoding ~ "deflate") {
      # Next, try deflate if it is supported.
      set req.http.Accept-Encoding = "deflate";
    }
    else {
      # Unknown algorithm. Remove it and send unencoded.
      unset req.http.Accept-Encoding;
    }
  }

  # Always cache the following file types for all users.
  if (req.url ~ "(?i)\.(css|js|jpg|jpeg|gif|ico|png|tiff|tif|img|tga|wmf|swf|html|htm|woff|woff2|mp4|ttf|eot|svg)(\?.*)?$") {
    unset req.http.Cookie;
  }

  # Remove all cookies that Drupal doesn't need to know about. ANY remaining
  # cookie will cause the request to pass-through to a backend. For the most part
  # we always set the NO_CACHE cookie after any POST request, disabling the
  # Varnish cache temporarily. The session cookie allows all authenticated users
  # to pass through as long as they're logged in.
  #
  # 1. Append a semi-colon to the front of the cookie string.
  # 2. Remove all spaces that appear after semi-colons.
  # 3. Match the cookies we want to keep, adding the space we removed
  #    previously, back. (\1) is first matching group in the regsuball.
  # 4. Remove all other cookies, identifying them by the fact that they have
  #    no space after the preceding semi-colon.
  # 5. Remove all spaces and semi-colons from the beginning and end of the
  #    cookie string.
  if (req.http.Cookie) {
    set req.http.CookieCheck = ";" + req.http.Cookie;
    set req.http.CookieCheck = regsuball(req.http.CookieCheck, "; +", ";");
    set req.http.CookieCheck = regsuball(req.http.CookieCheck, ";(SESS[a-z0-9]+|SSESS[a-z0-9]+|NO_CACHE)=", "; \1=");
    set req.http.CookieCheck = regsuball(req.http.CookieCheck, ";[^ ][^;]*", "");
    set req.http.CookieCheck = regsuball(req.http.CookieCheck, "^[; ]+|[; ]+$", "");

    set req.http.Cookie = req.http.Cookie + ";";

    if (req.http.CookieCheck == "") {
      # If there are no remaining cookies, remove the cookie header. If there
      # aren't any cookie headers, Varnish's default behavior will be to cache
      # the page.

      unset req.http.CookieCheck;
      unset req.http.Cookie;
    }
    else {
      # If there is any cookies left (a session or NO_CACHE cookie), do not
      # cache the page. Pass it on to Apache directly.
      unset req.http.CookieCheck;
      return (pass);
    }
  }

  # Cacheable, lookup in cache.
  return (hash);
}

sub vcl_hit {
    if (obj.ttl >= 0s) {
        # normal hit
        return (deliver);
    }
    # We have no fresh fish. Lets look at the stale ones.
    if (std.healthy(req.backend_hint)) {
        # Backend is healthy. If the object is not older then 30secs, deliver it to the client
        # and automatically create a separate backend request to warm the cache for this request.
        if (obj.ttl + 30s > 0s) {
            set req.http.grace = "normal(limited)";
            return (deliver);
        } else {
            # No candidate for grace. Fetch a fresh object.
            return(miss);
        }
    } else {
        # backend is sick - use full grace
        if (obj.ttl + obj.grace > 0s) {
            set req.http.grace = "full";
            return (deliver);
        } else {
            # no graced object.
            return (miss);
        }
    }
}

sub vcl_backend_response {
  # Temporarily disable streaming on files to be tested and removed after Varnish 4.1.x upgrade
  set beresp.do_stream = false;

  # Allow items to be stale if needed.
  set beresp.grace = 6h;

  ## FIXME: rename X-AMAZEEIO-VARNISH-BYPASS to X-AMAZEEIO-VARNISH-BACKEND-BYPASS to avoid confusion
  # If the backend sends a X-AMAZEEIO-VARNISH-BYPASS header we directly deliver
  if(beresp.http.X-AMAZEEIO-VARNISH-BYPASS == "TRUE") {
    return (deliver);
  }

  # Cache 404 and 403 for 10 seconds
  if(beresp.status == 404 || beresp.status == 403) {
    set beresp.ttl = 10s;
    return (deliver);
  }

  # Don't allow static files to set cookies.
  if (bereq.url ~ "(?i)\.(css|js|jpg|jpeg|gif|ico|png|tiff|tif|img|tga|wmf|swf|html|htm|woff|woff2|mp4|ttf|eot|svg)(\?.*)?$") {
    # beresp == Back-end response from the web server.
    unset beresp.http.set-cookie;
    unset beresp.http.Cache-Control;
    set beresp.ttl = 2628001s;
    set beresp.http.Cache-Control = "public, max-age=2628001";
    set beresp.http.Expires = "" + (now + beresp.ttl);
  }

  return (deliver);
}

# Set a header to track a cache HIT/MISS.
sub vcl_deliver {
  if (obj.hits > 0) {
    set resp.http.X-Varnish-Cache = "HIT";
  }
  else {
    set resp.http.X-Varnish-Cache = "MISS";
  }
  unset resp.http.X-Generator;
  unset resp.http.Server;
  # Inject information about grace
  if (req.http.grace) {
    set resp.http.X-Varnish-Grace = req.http.grace;
  }
  set resp.http.X-AMAZEEIO = "varnish>" + resp.http.X-AMAZEEIO;
  return (deliver);
}

sub vcl_hash {
     hash_data(req.url);
     if (req.http.host) {
         hash_data(req.http.host);
     } else {
         hash_data(server.ip);
     }
     if (req.http.X-Forwarded-Proto) {
        hash_data(req.http.X-Forwarded-Proto);
     }
     if (req.http.HTTPS) {
        hash_data(req.http.HTTPS);
     }
  return (lookup);
}

sub vcl_synth {
  if (resp.status == 701) {
    set resp.status = 401;
    set resp.http.Content-Type = "text/plain; charset=utf-8";
    synthetic({"XMLRPC Interface is blocked due to SA-CORE-2014-004 - mail support@amazee.io if you need it."});
    return (deliver);
  }
  if (resp.status == 700) {
    # Set a status the client will understand
    set resp.status = 200;
    # Create our synthetic response
    synthetic("");
    return(deliver);
}
  return (deliver);
}

sub vcl_backend_error {
    # Restart the request, when we have a backend server error, to try another backend.
    # Restart max twice.
    if (bereq.retries < 2) {
      return(retry);
    }

    set beresp.http.Content-Type = "text/html; charset=utf-8";
    set beresp.http.Retry-After = "5";
    synthetic( {"
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

    <title>Server Error</title>

We are sorry...

We encountered a server-side error. This means that the problem is not with your computer or Internet connection, but rather with the website's server.

We are currently working on solving this problem and apologise for the inconvenience.

            Technical Information

            Error "} + beresp.status + " " + beresp.reason + {"

            XID: "} + bereq.xid + {"

"} );
    return (deliver);
}


Revert "remove /v2/docs redirects"

Revert "remove /v2/docs redirects"

This reverts commit 6507bf0cfc2d273d31ac32a7b52f9825b578be0c.

sub vcl_recv {
	if (req.http.Host ~ "polyfills.io") {
		# Fastly recommend using the 900-999 range of status codes for custom errors within an application's VCL.
		# Do the canonicalise check before the HTTPS check to avoid a double redirect
		error 901 "Canonicalise";
	}
	if (!req.http.Fastly-SSL) {
		# 801 is a special error code that Fastly uses to Force HTTPS on the request
		error 801 "Redirect to HTTPS";
	}
	if (req.http.Orig-URL ~ "^/v1") {
		error 902 "Redirect to V2";
	}

	if (req.url.path != "/v2/polyfill.js" && req.url.path != "/v2/polyfill.min.js") {
		if (req.url.path ~ "^/v2($|/)") {
			error 908;
		}
	}
	if (req.url.path == "/") {
		error 908;
	}
}

sub vcl_error {
  # Redirect to canonical prod/qa origins
	if (obj.status == 901) {
		set obj.status = 301;
		set obj.response = "Moved Permanently";
		set obj.http.Location = "https://polyfill.io" req.url;
		synthetic {""};
		return (deliver);
	}
	# Redirect to v2
	if (obj.status == 902) {
		set obj.status = 301;
		set obj.response = "Moved Permanently";
		# Remove libVersion and gated query parameters if they exist.
		set req.url = querystring.regfilter(req.url, "^\b(libVersion|gated)\b.*");
		set obj.http.Location = if(req.url.path == "/v1", "/v3/", regsub(req.url, "^/v1", "/v2"));
		set obj.http.Deprecation-Notice = "API version 1 has been decommissioned - see the body of this response for more information.";
		synthetic {"API version 1 has been decommissioned. Your request is being redirected to v2.  The `libVersion` and `gated` query string parameters are no longer supported and if present have been removed from your request.\n\nA deprecation period for v1 existed between August and December 2015, during which time v1 requests were honoured but a deprecation warning was added to output."};
		return (deliver);
	}

	# Redirect to v3
	if (obj.status == 908) {
		set obj.status = 301;
		set obj.http.Location = "/v3/";
		return (deliver);
	}
}

sub vcl_recv {
	if (req.http.Host ~ "polyfills.io") {
		# Fastly recommend using the 900-999 range of status codes for custom errors within an application's VCL.
		# Do the canonicalise check before the HTTPS check to avoid a double redirect
		error 901 "Canonicalise";
	}
	if (!req.http.Fastly-SSL) {
		# 801 is a special error code that Fastly uses to Force HTTPS on the request
		error 801 "Redirect to HTTPS";
	}
	if (req.http.Orig-URL ~ "^/v1") {
		error 902 "Redirect to V2";
	}

	if (req.url.path == "/") {
		error 908;
	}
}

sub vcl_error {
  # Redirect to canonical prod/qa origins
	if (obj.status == 901) {
		set obj.status = 301;
		set obj.response = "Moved Permanently";
		set obj.http.Location = "https://polyfill.io" req.url;
		synthetic {""};
		return (deliver);
	}
	# Redirect to v2
	if (obj.status == 902) {
		set obj.status = 301;
		set obj.response = "Moved Permanently";
		# Remove libVersion and gated query parameters if they exist.
		set req.url = querystring.regfilter(req.url, "^\b(libVersion|gated)\b.*");
		set obj.http.Location = if(req.url.path == "/v1", "/v3/", regsub(req.url, "^/v1", "/v2"));
		set obj.http.Deprecation-Notice = "API version 1 has been decommissioned - see the body of this response for more information.";
		synthetic {"API version 1 has been decommissioned. Your request is being redirected to v2.  The `libVersion` and `gated` query string parameters are no longer supported and if present have been removed from your request.\n\nA deprecation period for v1 existed between August and December 2015, during which time v1 requests were honoured but a deprecation warning was added to output."};
		return (deliver);
	}

	# Redirect to v3
	if (obj.status == 908) {
		set obj.status = 301;
		set obj.http.Location = "/v3/";
		return (deliver);
	}
}