D'oh, fix VCL syntax in an example.

D'oh, fix VCL syntax in an example.

#
# Cookie-based session-affinity: non-cached replies will go to a backend per
# the normal rules, but we'll keep track of the backend that served the
# request and make sure any future request from the same client will go to
# the same backend.
#

import redis;
import std;

backend be1 {
  .host = "192.168.0.1";
  .port = "80";
}

backend be2 {
  .host = "192.168.0.2";
  .port = "80";
}

director default random {
  .retries = 5;
  {
    .backend        = be1;
    .weight         = 7;
  }
  {
    .backend        = be2;
    .weight         = 3;
  }
}

#
# Compute the redis key from a request.
#
# In this example a cookie is used, but it could be anything distinctive enough;
# IP address + User Agent + Accept* headers should work nicely for most purposes.
#
sub redis_key_from_req {
  if (req.http.Cookie) {
    set req.http.redis_key = req.http.Cookie;
  }
}

#
# Compute the redis key from a response.
#
# We use the reposnse cookie if present, as it might be different. We still
# have the request headers in case we want to use a different schema (see the
# comment for redis_key_from_req).
#
# Another option is to have the backend send us an explicit header.
#
sub redis_key_from_beresp {
  if (beresp.http.Set_Cookie) {
    set req.http.redis_key = beresp.http.Set_Cookie;
  } else if (req.http.Cookie) {
    set req.http.redis_key = req.http.Cookie;
  }
}

#
# Compute the backend that will serve further requests for this user.
#
# We just use the backend name, but we may as well use an explicit header
# from the backend. This would enable other scenarios, e.g. a "master" app
# server that creates sessions and then hands them off to other servers.
#
sub backend_from_beresp {
  set beresp.http.x_backend = beresp.backend.name;
}

#
# Set a backend given the name.
#
# This would be a very useful addition to the language, or it could be
# implemented as a vmod.
#
sub set_backend_by_name {
  if (req.http.x_backend) {
    std.log("backend will be: " + req.http.x_backend);

    if (req.http.x_backend == "be1") {
      set req.backend = be1;
    } else if (req.http.x_backend == "be2") {
      set req.backend = be2;
    } else {
      error 503 "No such backend " + req.http.x_backend;
    }
  }
}

#
# This is the main subroutine; it takes care of finding whether we have an
# entry in redis, and sets the corresponding backed.
#
sub select_backend {
  call redis_key_from_req;

  if (req.http.redis_key) {
    set req.http.x_backend = redis.call("HGET sessions " + req.http.redis_key);
    if (req.http.x_backend) {
      std.log("Found in redis");
      call set_backend_by_name;
    } else {
      # we will let the director choose
    }
  } else {
    std.log("Cookie not found");
  }
}

sub vcl_miss {
  call select_backend;
}

sub vcl_pass {
  call select_backend;
}

sub vcl_pipe {
  call select_backend;
}

#
# IF we don't have an entry in redis, we just let the director (or other
# mechanism) choose. When we get the response we need to figure key and
# value and store them.
#
sub vcl_fetch {
  if (!req.http.x_backend) {
    call redis_key_from_beresp;
    call backend_from_beresp;

    if (req.http.redis_key) {
      std.log("Fetched from: " + beresp.http.x_backend + ", key=" + req.http.redis_key);
      redis.send("HSET sessions " + req.http.redis_key + " " + beresp.http.x_backend);
    }
  }
}

#
# Cookie-based session-affinity: non-cached replies will go to a backend per
# the normal rules, but we'll keep track of the backend that served the
# request and make sure any future request from the same client will go to
# the same backend.
#

import redis;
import std;

backend be1 {
  .host = "192.168.0.1";
  .port = "80";
}

backend be2 {
  .host = "192.168.0.2";
  .port = "80";
}

director default random {
  .retries = 5;
  {
    .backend        = be1;
    .weight         = 7;
  }
  {
    .backend        = be2;
    .weight         = 3;
  }
}

#
# Compute the redis key from a request.
#
# In this example a cookie is used, but it could be anything distinctive enough;
# IP address + User Agent + Accept* headers should work nicely for most purposes.
#
sub redis_key_from_req {
  if (req.http.Cookie) {
    set req.http.redis_key = req.http.Cookie;
  }
}

#
# Compute the redis key from a response.
#
# We use the reposnse cookie if present, as it might be different. We still
# have the request headers in case we want to use a different schema (see the
# comment for redis_key_from_req).
#
# Another option is to have the backend send us an explicit header.
#
sub redis_key_from_beresp {
  if (beresp.http.Set_Cookie) {
    set req.http.redis_key = beresp.http.Set_Cookie;
  } else if (req.http.Cookie) {
    set req.http.redis_key = req.http.Cookie;
  }
}

#
# Compute the backend that will serve further requests for this user.
#
# We just use the backend name, but we may as well use an explicit header
# from the backend. This would enable other scenarios, e.g. a "master" app
# server that creates sessions and then hands them off to other servers.
#
sub backend_from_beresp {
  set beresp.http.x_backend = beresp.backend.name;
}

#
# Set a backend given the name.
#
# This would be a very useful addition to the language, or it could be
# implemented as a vmod.
#
sub set_backend_by_name {
  if (req.http.x_backend) {
    std.log("backend will be: " + req.http.x_backend);

    if (req.http.x_backend == "be1") {
      set req.backend = be1;
    } else if (req.http.x_backend == "be2") {
      set req.backend = be2;
    } else {
      return(error 503 "No such backend " + req.http.x_backend);
    }
  }
}

#
# This is the main subroutine; it takes care of finding whether we have an
# entry in redis, and sets the corresponding backed.
#
sub select_backend {
  call redis_key_from_req;

  if (req.http.redis_key) {
    set req.http.x_backend = redis.call("HGET sessions " + req.http.redis_key);
    if (req.http.x_backend) {
      std.log("Found in redis");
      call set_backend_by_name;
    } else {
      # we will let the director choose
    }
  } else {
    std.log("Cookie not found");
  }
}

sub vcl_miss {
  call select_backend;
}

sub vcl_pass {
  call select_backend;
}

sub vcl_pipe {
  call select_backend;
}

#
# IF we don't have an entry in redis, we just let the director (or other
# mechanism) choose. When we get the response we need to figure key and
# value and store them.
#
sub vcl_fetch {
  if (!req.http.x_backend) {
    call redis_key_from_beresp;
    call backend_from_beresp;

    if (req.http.redis_key) {
      std.log("Fetched from: " + beresp.http.x_backend + ", key=" + req.http.redis_key);
      redis.send("HSET sessions " + req.http.redis_key + " " + beresp.http.x_backend);
    }
  }
}


doc + support unsetting the override cookie

#
# devicedetect-dev.vcl
#
# Enable test URLs and cookie overrides.
#

backend devicetest {
    .host = "127.0.0.1";
    .port = "8000";
}

sub vcl_recv {
    # 701/702 are arbitrary chosen return codes that is only used internally in varnish.
    if (req.url ~ "^/set_ua_device/.+") { error 701 regsub(req.url, "^/set_ua_device/", ""); }
    # set expired cookie if nothing is specified
    if (req.url ~ "^/set_ua_device/") { error 702 "OK"; }
    if (req.url ~ "^/devicetest") { set req.backend = devicetest; }
}

sub vcl_error {
    if (obj.status == 701 || obj.status == 702) {
        if (obj.status == 702) {
            set obj.status = 200;
            set obj.http.Set-Cookie = "X-UA-Device-force=expiring; Path=/; Expires=Thu, 01 Jan 1970 00:00:01 GMT;";

	} else {
	    set obj.status = 200;
	    set obj.http.Set-Cookie = "X-UA-Device-force=" + obj.response + "; Path=/;";
	}
        set obj.http.Content-Type = "text/html; charset=utf-8";
        synthetic {"
OK, Cookie updated
/devicetest/"};
        set obj.response = "OK";
        return(deliver);
    }
}

#
# devicedetect-dev.vcl
#
# Enable test URLs and cookie overrides

backend devicetest {
    .host = "127.0.0.1";
    .port = "8000";
}

sub vcl_recv {
    if (req.url ~ "^/set_devicetype/") { error 701 regsub(req.url, "^/set_devicetype/", ""); }
    if (req.url ~ "^/devicetest") { set backend = devicetest; }
}

sub vcl_error {
    if (obj.status == 701 ) {
        # obj.response == mobile-generic
        set obj.http.Set-Cookie = "X-UA-device=" + obj.response;
        set obj.http.Content-Type = "text/plain; charset=utf-8";
        synthetic {" 200 OK, Cookie set "};
	return(deliver)
    }
}
"""


Use a different test for if ESI is being done

sub vcl_recv {
#FASTLY recv

    # Disable ESI processing when doing a shield request.
    if (req.http.Fastly-FF) {
        set req.esi = false;
    }

    # If we're serving an ESI request, and the request in question has NOT
    # opted into cookies, then we'll go ahead and strip any cookies from the
    # request. In addition, we'll strip out any Authorization or Authentication
    # headers.
    if (req.url ~ "^/_esi/") {
        unset req.http.Authenticate;
        unset req.http.Authorization;

        if (req.url !~ "esi-cookies=1") {
            unset req.http.Cookie;
        }
    }

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Set a header to tell the backend if we're using https or http.
    if (req.http.Fastly-SSL) {
        set req.http.Warehouse-Proto = "https";
    } else {
        set req.http.Warehouse-Proto = "http";
    }

    # Pass the client IP address back to the backend.
    if (req.http.Fastly-Client-IP) {
        set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
    }

    # Pass the real host value back to the backend.
    if (req.http.Host) {
        set req.http.Warehouse-Host = req.http.Host;
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {
#FASTLY fetch

    # Only enable ESI on requests that could possibly support it, for now just
    # ones that return HTML, but not the /simple/ API.
    if (beresp.http.Content-type ~ "html" && !(req.url ~ "^/simple/")) {
        # Conditional HTTP requests are not compatible with Varnish's
        # implementation of ESI, in particularl the ETag and the Last-Modified
        # won't be updated when the included content changes, causing Varnish
        # to return a 304 Not Modified.
        unset beresp.http.ETag;
        unset beresp.http.Last-Modified;

        # Enable ESI.
        esi;
    }

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 500 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 500 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (beresp.status == 500 || beresp.status == 503) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_deliver {
#FASTLY deliver

    # If the backend has indicated additional Vary headers to add once the
    # ESI result has been processed, then we'll go ahead and either append them
    # to our existing Vary header or we'll set the Vary header equal to it.
    if (resp.http.Warehouse-ESI-Vary) {
        if (resp.http.Vary) {
            set resp.http.Vary = resp.http.Vary ", " resp.http.Warehouse-ESI-Vary;
        } else {
            set resp.http.Vary = resp.http.Warehouse-ESI-Vary;
        }

        unset resp.http.Warehouse-ESI-Vary;
    }

    return(deliver);
}

sub vcl_error {
#FASTLY error

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    }
}

sub vcl_recv {
#FASTLY recv

    # Disable ESI processing when doing a shield request.
    if (req.http.Fastly-FF) {
        set req.esi = false;
    }

    # If we're serving an ESI request, and the request in question has NOT
    # opted into cookies, then we'll go ahead and strip any cookies from the
    # request. In addition, we'll strip out any Authorization or Authentication
    # headers.
    if (req.esis > 0) {
        unset req.http.Authenticate;
        unset req.http.Authorization;

        if (req.url !~ "esi-cookies=1") {
            unset req.http.Cookie;
        }
    }

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Set a header to tell the backend if we're using https or http.
    if (req.http.Fastly-SSL) {
        set req.http.Warehouse-Proto = "https";
    } else {
        set req.http.Warehouse-Proto = "http";
    }

    # Pass the client IP address back to the backend.
    if (req.http.Fastly-Client-IP) {
        set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
    }

    # Pass the real host value back to the backend.
    if (req.http.Host) {
        set req.http.Warehouse-Host = req.http.Host;
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {
#FASTLY fetch

    # Only enable ESI on requests that could possibly support it, for now just
    # ones that return HTML, but not the /simple/ API.
    if (beresp.http.Content-type ~ "html" && !(req.url ~ "^/simple/")) {
        # Conditional HTTP requests are not compatible with Varnish's
        # implementation of ESI, in particularl the ETag and the Last-Modified
        # won't be updated when the included content changes, causing Varnish
        # to return a 304 Not Modified.
        unset beresp.http.ETag;
        unset beresp.http.Last-Modified;

        # Enable ESI.
        esi;
    }

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 500 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 500 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (beresp.status == 500 || beresp.status == 503) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_deliver {
#FASTLY deliver

    # If the backend has indicated additional Vary headers to add once the
    # ESI result has been processed, then we'll go ahead and either append them
    # to our existing Vary header or we'll set the Vary header equal to it.
    if (resp.http.Warehouse-ESI-Vary) {
        if (resp.http.Vary) {
            set resp.http.Vary = resp.http.Vary ", " resp.http.Warehouse-ESI-Vary;
        } else {
            set resp.http.Vary = resp.http.Warehouse-ESI-Vary;
        }

        unset resp.http.Warehouse-ESI-Vary;
    }

    return(deliver);
}

sub vcl_error {
#FASTLY error

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    }
}


VCL requires " instead of ' for strings

VCL requires " instead of ' for strings

sub vcl_recv {

    # Some (Older) clients will send a hash fragment as part of the URL even
    # though that is a local only modification. This breaks this badly for the
    # files in S3, and in general it's just not needed.
    set req.url = regsub(req.url, "#.*$", "");

    # Most of the URLs in Warehouse do not support or require any sort of query
    # parameter. If we strip these at the edge then we'll increase our cache
    # efficiency when they won't otherwise change the output of the pages.
    #
    # This will match any URL except those that start with:
    #
    #   * /search/
    #   * /account/login/
    #   * /account/logout/
    #   * /account/register/
    #   * /pypi
    if (req.url !~ "^/(search/|account/(login|logout|register)/|pypi)") {
        set req.url = req.url.path;
    }

    # Sort all of our query parameters, this will ensure that the same query
    # parameters in a different order will end up being represented as the same
    # thing, reducing cache misses due to ordering differences.
    set req.url = boltsort.sort(req.url);

#FASTLY recv

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Canonicalize our domains by redirecting any domain that doesn't match our
    # primary domain to our primary domain. We do this *after* the HTTPS check
    # on purpose.
    if (std.tolower(req.http.host) != std.tolower(req.http.Primary-Domain)) {
        set req.http.Location = "https://" req.http.Primary-Domain req.url;
        error 750 "Redirect to Primary Domain";
    }

    # Requests to /packages/ get dispatched to Amazon instead of to our typical
    # Origin. This requires a a bit of setup to make it work.
    if (req.url ~ "^/packages/") {
        # Setup our environment to better match what S3 expects/needs
        set req.http.Host = req.http.AWS-Bucket-Name ".s3.amazonaws.com";
        set req.http.Date = now;
        set req.url = regsuball(req.url, "\+", urlencode("+"));

        # Compute the Authorization header that S3 requires to be able to
        # access the files stored there.
        set req.http.Authorization = "AWS " req.http.AWS-Access-Key-ID ":" digest.hmac_sha1_base64(req.http.AWS-Secret-Access-Key, "GET" LF LF LF req.http.Date LF "/" req.http.AWS-Bucket-Name req.url.path);

        # We don't want to send our Warehouse-Token to S3, so we'll go ahead
        # and remove it.
        unset req.http.Warehouse-Token;
    }

    # We no longer need any of these variables, which would exist only to
    # shuffle configuration from the Fastly UI into our VCL.
    unset req.http.Primary-Domain;
    unset req.http.AWS-Access-Key-ID;
    unset req.http.AWS-Secret-Access-Key;
    unset req.http.AWS-Bucket-Name;

    # We have a number of items that we'll pass back to the origin, but only
    # if we have a Warehouse-Token that will allow them to be accepted.
    if (req.http.Warehouse-Token) {
        # Set a header to tell the backend if we're using https or http.
        if (req.http.Fastly-SSL) {
            set req.http.Warehouse-Proto = "https";
        } else {
            set req.http.Warehouse-Proto = "http";
        }

        # Pass the client IP address back to the backend.
        if (req.http.Fastly-Client-IP) {
            set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
        }

        # Pass the real host value back to the backend.
        if (req.http.Host) {
            set req.http.Warehouse-Host = req.http.Host;
        }
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {
#FASTLY fetch

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 502 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 502 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (beresp.status == 500 || beresp.status == 503) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_deliver {
#FASTLY deliver

    # Unset headers that we don't need/want to send on to the client because
    # they are not generally useful.
    unset resp.http.Server;
    unset resp.http.Via;

    # Unset a few headers set by Amazon that we don't really have a need/desire
    # to send to clients.
    unset resp.http.X-AMZ-Replication-Status;
    unset resp.http.X-AMZ-Meta-Python-Version;
    unset resp.http.X-AMZ-Meta-Version;
    unset resp.http.X-AMZ-Meta-Package-Type;
    unset resp.http.X-AMZ-Meta-Project;

    # Set our standard security headers, we do this in VCL rather than in
    # Warehouse itself so that we always get these headers, regardless of the
    # origin server being used.
    set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload";
    set resp.http.X-Frame-Options = "deny";
    set resp.http.X-XSS-Protection = "1; mode=block";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-Permitted-Cross-Domain-Policies = "none";

    return(deliver);
}

sub vcl_error {
#FASTLY error

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    } else if (obj.status == 750) {
        set obj.status = 301;
        set obj.http.Location = req.http.Location;
        set obj.http.Content-Type = "text/html; charset=UTF-8";
        synthetic {"<title>301 Moved Permanently</title>
301 Moved Permanently
"};
        return(deliver);
    }

}

sub vcl_recv {

    # Some (Older) clients will send a hash fragment as part of the URL even
    # though that is a local only modification. This breaks this badly for the
    # files in S3, and in general it's just not needed.
    set req.url = regsub(req.url, "#.*$", "");

    # Most of the URLs in Warehouse do not support or require any sort of query
    # parameter. If we strip these at the edge then we'll increase our cache
    # efficiency when they won't otherwise change the output of the pages.
    #
    # This will match any URL except those that start with:
    #
    #   * /search/
    #   * /account/login/
    #   * /account/logout/
    #   * /account/register/
    #   * /pypi
    if (req.url !~ '^/(search/|account/(login|logout|register)/|pypi)') {
        set req.url = req.url.path;
    }

    # Sort all of our query parameters, this will ensure that the same query
    # parameters in a different order will end up being represented as the same
    # thing, reducing cache misses due to ordering differences.
    set req.url = boltsort.sort(req.url);

#FASTLY recv

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Canonicalize our domains by redirecting any domain that doesn't match our
    # primary domain to our primary domain. We do this *after* the HTTPS check
    # on purpose.
    if (std.tolower(req.http.host) != std.tolower(req.http.Primary-Domain)) {
        set req.http.Location = "https://" req.http.Primary-Domain req.url;
        error 750 "Redirect to Primary Domain";
    }

    # Requests to /packages/ get dispatched to Amazon instead of to our typical
    # Origin. This requires a a bit of setup to make it work.
    if (req.url ~ "^/packages/") {
        # Setup our environment to better match what S3 expects/needs
        set req.http.Host = req.http.AWS-Bucket-Name ".s3.amazonaws.com";
        set req.http.Date = now;
        set req.url = regsuball(req.url, "\+", urlencode("+"));

        # Compute the Authorization header that S3 requires to be able to
        # access the files stored there.
        set req.http.Authorization = "AWS " req.http.AWS-Access-Key-ID ":" digest.hmac_sha1_base64(req.http.AWS-Secret-Access-Key, "GET" LF LF LF req.http.Date LF "/" req.http.AWS-Bucket-Name req.url.path);

        # We don't want to send our Warehouse-Token to S3, so we'll go ahead
        # and remove it.
        unset req.http.Warehouse-Token;
    }

    # We no longer need any of these variables, which would exist only to
    # shuffle configuration from the Fastly UI into our VCL.
    unset req.http.Primary-Domain;
    unset req.http.AWS-Access-Key-ID;
    unset req.http.AWS-Secret-Access-Key;
    unset req.http.AWS-Bucket-Name;

    # We have a number of items that we'll pass back to the origin, but only
    # if we have a Warehouse-Token that will allow them to be accepted.
    if (req.http.Warehouse-Token) {
        # Set a header to tell the backend if we're using https or http.
        if (req.http.Fastly-SSL) {
            set req.http.Warehouse-Proto = "https";
        } else {
            set req.http.Warehouse-Proto = "http";
        }

        # Pass the client IP address back to the backend.
        if (req.http.Fastly-Client-IP) {
            set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
        }

        # Pass the real host value back to the backend.
        if (req.http.Host) {
            set req.http.Warehouse-Host = req.http.Host;
        }
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {
#FASTLY fetch

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 502 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 502 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (beresp.status == 500 || beresp.status == 503) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_deliver {
#FASTLY deliver

    # Unset headers that we don't need/want to send on to the client because
    # they are not generally useful.
    unset resp.http.Server;
    unset resp.http.Via;

    # Unset a few headers set by Amazon that we don't really have a need/desire
    # to send to clients.
    unset resp.http.X-AMZ-Replication-Status;
    unset resp.http.X-AMZ-Meta-Python-Version;
    unset resp.http.X-AMZ-Meta-Version;
    unset resp.http.X-AMZ-Meta-Package-Type;
    unset resp.http.X-AMZ-Meta-Project;

    # Set our standard security headers, we do this in VCL rather than in
    # Warehouse itself so that we always get these headers, regardless of the
    # origin server being used.
    set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload";
    set resp.http.X-Frame-Options = "deny";
    set resp.http.X-XSS-Protection = "1; mode=block";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-Permitted-Cross-Domain-Policies = "none";

    return(deliver);
}

sub vcl_error {
#FASTLY error

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    } else if (obj.status == 750) {
        set obj.status = 301;
        set obj.http.Location = req.http.Location;
        set obj.http.Content-Type = "text/html; charset=UTF-8";
        synthetic {"<title>301 Moved Permanently</title>
301 Moved Permanently
"};
        return(deliver);
    }

}


Redirect all domains to the primary domain in Fastly

sub vcl_recv {
#FASTLY recv

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Canonicalize our domains by redirecting any domain that doesn't match our
    # primary domain to our primary domain. We do this *after* the HTTPS check
    # on purpose.
    if (std.tolower(req.http.host) != std.tolower(req.http.Primary-Domain)) {
        error 750 "https://" req.http.Primary-Domain req.url;
    }

    # Set a header to tell the backend if we're using https or http.
    if (req.http.Fastly-SSL) {
        set req.http.Warehouse-Proto = "https";
    } else {
        set req.http.Warehouse-Proto = "http";
    }

    # Pass the client IP address back to the backend.
    if (req.http.Fastly-Client-IP) {
        set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
    }

    # Pass the real host value back to the backend.
    if (req.http.Host) {
        set req.http.Warehouse-Host = req.http.Host;
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {
#FASTLY fetch

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 502 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 502 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (beresp.status == 500 || beresp.status == 503) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_deliver {
#FASTLY deliver

    # Unset headers that we don't need/want to send on to the client because
    # they are not generally useful.
    unset resp.http.Server;
    unset resp.http.Via;

    # Set our standard security headers, we do this in VCL rather than in
    # Warehouse itself so that we always get these headers, regardless of the
    # origin server being used.
    set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload";
    set resp.http.X-Frame-Options = "deny";
    set resp.http.X-XSS-Protection = "1; mode=block";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-Permitted-Cross-Domain-Policies = "none";

    return(deliver);
}

sub vcl_error {
#FASTLY error

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    } else if (obj.status == 750) {
        set obj.status = 301;
        set obj.http.Location = obj.response;
        set obj.http.Content-Type = "text/html; charset=UTF-8";
        synthetic {"<title>301 Moved Permanently</title>
301 Moved Permanently
"};
        return(deliver);
    }

}

sub vcl_recv {
#FASTLY recv

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Set a header to tell the backend if we're using https or http.
    if (req.http.Fastly-SSL) {
        set req.http.Warehouse-Proto = "https";
    } else {
        set req.http.Warehouse-Proto = "http";
    }

    # Pass the client IP address back to the backend.
    if (req.http.Fastly-Client-IP) {
        set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
    }

    # Pass the real host value back to the backend.
    if (req.http.Host) {
        set req.http.Warehouse-Host = req.http.Host;
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {
#FASTLY fetch

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 502 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 502 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (beresp.status == 500 || beresp.status == 503) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_deliver {
#FASTLY deliver

    # Unset headers that we don't need/want to send on to the client because
    # they are not generally useful.
    unset resp.http.Server;
    unset resp.http.Via;

    # Set our standard security headers, we do this in VCL rather than in
    # Warehouse itself so that we always get these headers, regardless of the
    # origin server being used.
    set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload";
    set resp.http.X-Frame-Options = "deny";
    set resp.http.X-XSS-Protection = "1; mode=block";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-Permitted-Cross-Domain-Policies = "none";

    return(deliver);
}

sub vcl_error {
#FASTLY error

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    }

}


Added prerender token to prerender.vcl for crawl tracking

#
# NOTE: Heroku/ELB domains resolve to multiple IP addresses and Varnish
# doesn't like this. The backend host must be a single IP address, a
# hostname that resolves to a single IP address, or a dynamically-
# generated director similar to the one described here:
#
# http://blog.cloudreach.co.uk/2013/01/varnish-and-autoscaling-love-story.html
#

include "prerender_backend.vcl";

sub vcl_recv {
    if (req.url ~ "_escaped_fragment_|prerender=1" ||
         req.http.user-agent ~ "googlebot|yahoo|bingbot|baiduspider|yandex|yeti|yodaobot|gigabot|ia_archiver|facebookexternalhit|twitterbot|developers\.google\.com") {

        if (req.http.user-agent ~ "Prerender") {
            return(pass);
        }

        # FIXME: Add a whitelist of files or filetypes to never prerender

        set req.backend = prerender;

        # When doing SSL offloading in front of Varnish, set X-Scheme header
        # to "https" before passing the request to Varnish
        if (req.http.X-Scheme !~ "^http(s?)") {
            set req.http.X-Scheme = "http";
        }
        set req.url = "/" + req.http.X-Scheme + "://" + req.http.Host + req.url;

        return(lookup);
    }
}

sub vcl_miss {
    if (req.backend == prerender) {
        set bereq.http.Host = "prerender.herokuapp.com";
        set bereq.http.X-Real-IP = client.ip;
        set bereq.http.X-Forwarded-For = client.ip;

        # If you're using hosted prerender.io, set your token here to enable
        # stat collection.
        #
        # IMPORTANT: This information is cached for the lifespan of
        # the Varnish daemon. If you change your token, you must restart Varnish.
        #
        # FIXME: This would ideally be handled via environment variables.
        # I don't think that environment variables are available in Varnish.
        # I'm looking into it.
        set bereq.http.X-Prerender-Token = std.fileread("/etc/varnish/prerender_token.txt");
    }
}

sub vcl_fetch {
    if (req.backend == prerender) {
        # Set the Varnish cache timeout for rendered content
        set beresp.ttl = 60m;
    }
}

#
# NOTE: Heroku/ELB domains resolve to multiple IP addresses and Varnish
# doesn't like this. The backend host must be a single IP address, a
# hostname that resolves to a single IP address, or a dynamically-
# generated director similar to the one described here:
#
# http://blog.cloudreach.co.uk/2013/01/varnish-and-autoscaling-love-story.html
#

include "prerender_backend.vcl";

sub vcl_recv {
    if (req.url ~ "_escaped_fragment_|prerender=1" ||
         req.http.user-agent ~ "googlebot|yahoo|bingbot|baiduspider|yandex|yeti|yodaobot|gigabot|ia_archiver|facebookexternalhit|twitterbot|developers\.google\.com") {

        if (req.http.user-agent ~ "Prerender") {
            return(pass);
        }

        set req.backend = prerender;

        # When doing SSL offloading in front of Varnish, set X-Scheme header
        # to "https" before passing the request to Varnish
        if (req.http.X-Scheme !~ "^http(s?)") {
            set req.http.X-Scheme = "http";
        }
        set req.url = "/" + req.http.X-Scheme + "://" + req.http.Host + req.url;

        return(lookup);
    }
}

sub vcl_miss {
    if (req.backend == prerender) {
        set bereq.http.Host = "prerender.herokuapp.com";
        set bereq.http.X-Real-IP = client.ip;
        set bereq.http.X-Forwarded-For = client.ip;
    }
}

sub vcl_fetch {
    if (req.backend == prerender) {
        # Set the Varnish cache timeout for rendered content
        set beresp.ttl = 60m;
    }
}


Purge Varnish by filename

Purge Varnish by filename

Purging the original or any thumbnail purges everything (original + thumbs)

Bug: T112532
Change-Id: I8d152a23c9080a6baf16c671f352dbb5f09257e3

# vcl_recv is called whenever a request is received
sub vcl_recv {
    # Copy the thumbnail URLs so that they create variants of the same object
    # By later adding X-Url to the Vary header
    if (req.url ~ "^/images/") {
        set req.http.X-Url = req.url;
    }

    # Pass any requests with the "If-None-Match" header directly.
    if (req.http.If-None-Match && !req.url ~ "^/images/thumb/.*\.(jpeg|jpg|png)") {
        return(pass);
    }
}

# Called if the cache does not have a copy of the page.
sub vcl_miss {
    # qlow jpg thumbs
    if (req.url ~ "^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)") {
        set bereq.url = "/unsafe/" + regsub(req.url, "^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)", "\2") + "x/filters:quality(40):sharpen(0.6,0.01,false)/http://127.0.0.1:8080/images/" + regsub(req.url, "^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)", "\1");
    # regular jpg thumbs
    } else if (req.url ~ "^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)") {
        set bereq.url = "/unsafe/" + regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)", "\2") + "x/filters:quality(87):sharpen(0.6,0.01,false)/http://127.0.0.1:8080/images/" + regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)", "\1");
    # png thumbs
    } else if (req.url ~ "^/images/thumb/(.*)/(\d+)px-.*\.png") {
        set bereq.url = "/unsafe/" + regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.png", "\2") + "x/http://127.0.0.1:8080/images/" + regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.png", "\1");
    }
}

sub vcl_deliver {
    # Thumbor doesn't do fine-grained config for the headers it returns
    if (req.url ~ "^/images/thumb/.*\.(jpeg|jpg|png)") {
        unset resp.http.Cache-Control;
        unset resp.http.Expires;
    }
}

sub vcl_fetch {
    if (req.http.X-Url) {
        set beresp.http.X-Url = req.http.X-Url;
        if (!beresp.http.Vary) {
            set beresp.http.Vary = "X-Url";
        } elsif (beresp.http.Vary !~ "(?i)X-Url") {
            set beresp.http.Vary = beresp.http.Vary + ", X-Url";
        }
    }
}

sub vcl_hash {
    # For thumbnails and originals we hash on the filename, to store them all under the same object. This will make purging any of them purge all of them.
    if (req.url ~ "^/images/thumb/") {
        hash_data("Image-" + regsub(req.url, "^/images/thumb/[^/]+/[^/]+/([^/]+)/[^/]+$", "\1"));
    } elsif (req.url ~ "^/images/") {
        hash_data("Image-" + regsub(req.url, "^/images/[^/]+/[^/]+/(.*)", "\1"));
    } else {
        hash_data(req.url);
    }

    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }
    return (hash);
}

# vcl_recv is called whenever a request is received
sub vcl_recv {
    # Pass any requests with the "If-None-Match" header directly.
    if (req.http.If-None-Match && !req.url ~ "^/images/thumb/.*\.(jpeg|jpg|png)") {
        return(pass);
    }
}

# Called if the cache does not have a copy of the page.
sub vcl_miss {
    # qlow jpg thumbs
    if (req.url ~ "^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)") {
        set bereq.url = "/unsafe/" + regsub(req.url, "^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)", "\2") + "x/filters:quality(40):sharpen(0.6,0.01,false)/http://127.0.0.1:8080/images/" + regsub(req.url, "^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)", "\1");
    # regular jpg thumbs
    } else if (req.url ~ "^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)") {
        set bereq.url = "/unsafe/" + regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)", "\2") + "x/filters:quality(87):sharpen(0.6,0.01,false)/http://127.0.0.1:8080/images/" + regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)", "\1");
    # png thumbs
    } else if (req.url ~ "^/images/thumb/(.*)/(\d+)px-.*\.png") {
        set bereq.url = "/unsafe/" + regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.png", "\2") + "x/http://127.0.0.1:8080/images/" + regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.png", "\1");
    }
}

sub vcl_deliver {
    # Thumbor doesn't even fine-grained config for the headers it returns
    if (req.url ~ "^/images/thumb/.*\.(jpeg|jpg|png)") {
        unset resp.http.Cache-Control;
        unset resp.http.Expires;
    }
}


Remove Pragma and Expires headers for immutable objects as they cause unnecessary revalidations on Chrome browsers slowing pages down

    # If we are doing a bypass for Magento Tester return early as not to affect any headers
    if ( req.http.bypass-secret ) {
        set beresp.ttl = 0s;
        return(pass);
    }

    /* handle 5XX (or any other unwanted status code) */
    if (beresp.status >= 500 && beresp.status < 600) {

        /* deliver stale if the object is available */
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    # Remove Set-Cookies from responses for static content to match the cookie removal in recv.
    if (req.http.x-long-cache || req.url ~ "^/(pub/)?(media|static)/") {
        unset beresp.http.set-cookie;

        # Set a short TTL for 404's since those can be temporary during the site build/index
        if (beresp.status == 404) {
            set beresp.ttl = 300s;
            set beresp.http.Cache-Control = "max-age=0";
        } else if (req.http.x-long-cache) {
            # Force caching for signed cached assets.
            set beresp.ttl = 31536000s;
            # Add immutable as it avoids IMS and INM revalidations
            set beresp.http.Cache-Control = "max-age=31536000, immutable";
            # Remove HTTP/1.0 caching headers as they may mess with browser memory caches
            unset beresp.http.Pragma;
            unset beresp.http.Expires;
        }
    }

    # Fix Vary Header in some cases. In 99.9% of cases Varying on User-Agent is counterproductive
    # https://www.varnish-cache.org/trac/wiki/VCLExampleFixupVary
    if (beresp.http.Vary ~ "User-Agent") {
        unset beresp.http.Vary:User-Agent;
        if (beresp.http.Vary == "") {
            unset beresp.http.Vary;
        }
    }

    # All the Magento responses should emit X-Esi headers
    if (beresp.http.x-esi) {
        # enable ESI feature for Magento response by default
        esi;
    } else {
        # enable gzip for all static content except
        if ( http_status_matches(beresp.status, "200,404") && (beresp.http.content-type ~ "^(application/x\-javascript|text/css|text/html|application/javascript|text/javascript|application/json|application/vnd\.ms\-fontobject|application/x\-font\-opentype|application/x\-font\-truetype|application/x\-font\-ttf|application/xml|font/eot|font/opentype|font/otf|image/svg\+xml|image/vnd\.microsoft\.icon|text/plain)\s*($|;)" || req.url.ext ~ "(?i)(css|js|html|eot|ico|otf|ttf|json)" ) ) {
            # always set vary to make sure uncompressed versions dont always win
            if (!beresp.http.Vary ~ "Accept-Encoding") {
                set beresp.http.Vary:Accept-Encoding = "";
            }
            if (req.http.Accept-Encoding == "gzip") {
                set beresp.gzip = true;
            }
        }
    }

    # Add Varying on Store and Content-Currency for GraphQl, X-Magento-Vary otherwise
    if (req.http.graphql) {
        set beresp.http.Vary:Store = "";
        set beresp.http.Vary:Content-Currency = "";
    } else if (beresp.http.Content-Type ~ "text/(html|xml)") {
        set beresp.http.Vary:X-Magento-Vary = "";
        set beresp.http.Vary:Https = "";
    }

    # Just in case the Request Setting for x-pass is missing
    if (req.http.x-pass) {
        return (pass);
    }

    if (beresp.http.Cache-Control ~ "private|no-cache|no-store") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    if (beresp.http.X-Magento-Debug) {
        set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
    }

    # Never cache 302s
    if (beresp.status == 302) {
        return (pass);
    }

    if (beresp.http.x-amz-request-id) {
        # If assets are coming from Amazon they may have no Cache-Control headers which may make them uncacheable
    } else if (!beresp.http.Expires && !beresp.http.Surrogate-Control ~ "max-age" && !beresp.http.Cache-Control ~ "(s-maxage|max-age)") {
        # Varnish sets default TTL if none of the headers above are present. If not set we want to make sure we don't cache it
        set beresp.ttl = 0s;
        set beresp.cacheable = false;
    }

    # validate if we need to cache it and prevent from setting cookie
    # images, css and js are cacheable by default so we have to remove cookie also
    if (beresp.ttl > 0s && (req.request == "GET" || req.request == "HEAD") && !req.http.x-pass ) {
        unset beresp.http.set-cookie;

        # init surrogate keys
        if (beresp.http.X-Magento-Tags) {
            set beresp.http.Surrogate-Key = beresp.http.X-Magento-Tags " text";
        } else {
            set beresp.http.Surrogate-Key = "text";
        }

        # set surrogate keys by content type if they are image/script or CSS
        if (beresp.http.Content-Type ~ "(image|script|css)") {
            set beresp.http.Surrogate-Key = re.group.1;
        }
    }

    # If the cache key in the Magento response doesn't match the one that was sent in the request, don't cache under the request's key
    if (req.http.graphql && req.http.X-Magento-Cache-Id && req.http.X-Magento-Cache-Id != beresp.http.X-Magento-Cache-Id) {
        set beresp.ttl = 0s;
        set beresp.cacheable = false;
    }

    # If we are doing a bypass for Magento Tester return early as not to affect any headers
    if ( req.http.bypass-secret ) {
        set beresp.ttl = 0s;
        return(pass);
    }

    /* handle 5XX (or any other unwanted status code) */
    if (beresp.status >= 500 && beresp.status < 600) {

        /* deliver stale if the object is available */
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    # Remove Set-Cookies from responses for static content to match the cookie removal in recv.
    if (req.http.x-long-cache || req.url ~ "^/(pub/)?(media|static)/") {
        unset beresp.http.set-cookie;

        # Set a short TTL for 404's since those can be temporary during the site build/index
        if (beresp.status == 404) {
            set beresp.ttl = 300s;
            set beresp.http.Cache-Control = "max-age=0";
        } else if (req.http.x-long-cache) {
            # Force caching for signed cached assets.
            set beresp.ttl = 31536000s;
            # Add immutable as it avoids IMS and INM revalidations
            set beresp.http.Cache-Control = "max-age=31536000, immutable";
        }
    }

    # Fix Vary Header in some cases. In 99.9% of cases Varying on User-Agent is counterproductive
    # https://www.varnish-cache.org/trac/wiki/VCLExampleFixupVary
    if (beresp.http.Vary ~ "User-Agent") {
        unset beresp.http.Vary:User-Agent;
        if (beresp.http.Vary == "") {
            unset beresp.http.Vary;
        }
    }

    # All the Magento responses should emit X-Esi headers
    if (beresp.http.x-esi) {
        # enable ESI feature for Magento response by default
        esi;
    } else {
        # enable gzip for all static content except
        if ( http_status_matches(beresp.status, "200,404") && (beresp.http.content-type ~ "^(application/x\-javascript|text/css|text/html|application/javascript|text/javascript|application/json|application/vnd\.ms\-fontobject|application/x\-font\-opentype|application/x\-font\-truetype|application/x\-font\-ttf|application/xml|font/eot|font/opentype|font/otf|image/svg\+xml|image/vnd\.microsoft\.icon|text/plain)\s*($|;)" || req.url.ext ~ "(?i)(css|js|html|eot|ico|otf|ttf|json)" ) ) {
            # always set vary to make sure uncompressed versions dont always win
            if (!beresp.http.Vary ~ "Accept-Encoding") {
                set beresp.http.Vary:Accept-Encoding = "";
            }
            if (req.http.Accept-Encoding == "gzip") {
                set beresp.gzip = true;
            }
        }
    }

    # Add Varying on Store and Content-Currency for GraphQl, X-Magento-Vary otherwise
    if (req.http.graphql) {
        set beresp.http.Vary:Store = "";
        set beresp.http.Vary:Content-Currency = "";
    } else if (beresp.http.Content-Type ~ "text/(html|xml)") {
        set beresp.http.Vary:X-Magento-Vary = "";
        set beresp.http.Vary:Https = "";
    }

    # Just in case the Request Setting for x-pass is missing
    if (req.http.x-pass) {
        return (pass);
    }

    if (beresp.http.Cache-Control ~ "private|no-cache|no-store") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    if (beresp.http.X-Magento-Debug) {
        set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
    }

    # Never cache 302s
    if (beresp.status == 302) {
        return (pass);
    }

    if (beresp.http.x-amz-request-id) {
        # If assets are coming from Amazon they may have no Cache-Control headers which may make them uncacheable
    } else if (!beresp.http.Expires && !beresp.http.Surrogate-Control ~ "max-age" && !beresp.http.Cache-Control ~ "(s-maxage|max-age)") {
        # Varnish sets default TTL if none of the headers above are present. If not set we want to make sure we don't cache it
        set beresp.ttl = 0s;
        set beresp.cacheable = false;
    }

    # validate if we need to cache it and prevent from setting cookie
    # images, css and js are cacheable by default so we have to remove cookie also
    if (beresp.ttl > 0s && (req.request == "GET" || req.request == "HEAD") && !req.http.x-pass ) {
        unset beresp.http.set-cookie;

        # init surrogate keys
        if (beresp.http.X-Magento-Tags) {
            set beresp.http.Surrogate-Key = beresp.http.X-Magento-Tags " text";
        } else {
            set beresp.http.Surrogate-Key = "text";
        }

        # set surrogate keys by content type if they are image/script or CSS
        if (beresp.http.Content-Type ~ "(image|script|css)") {
            set beresp.http.Surrogate-Key = re.group.1;
        }
    }

    # If the cache key in the Magento response doesn't match the one that was sent in the request, don't cache under the request's key
    if (req.http.graphql && req.http.X-Magento-Cache-Id && req.http.X-Magento-Cache-Id != beresp.http.X-Magento-Cache-Id) {
        set beresp.ttl = 0s;
        set beresp.cacheable = false;
    }


remove headers from response which users don't need

sub set_backend {
	# The Fastly macro is inserted before the backend is selected because the
	# macro has the code which defines the values avaiable for req.http.Host
	# but it also contains a default backend which is always set to the EU. I wish we could disable the default backend setting.
	#FASTLY recv

	# Calculate the ideal region to route the request to.
  	declare local var.region STRING;
	if (server.region ~ "(APAC|Asia|North-America|South-America|US-Central|US-East|US-West)") {
		set var.region = "US";
  	} else {
		set var.region = "EU";
  	}

	# Gather the health of the shields and origins.
	declare local var.v3_eu_is_healthy BOOL;
	set req.backend = F_v3_eu;
	set var.v3_eu_is_healthy = req.backend.healthy;

	declare local var.v3_us_is_healthy BOOL;
  	set req.backend = F_v3_us;
  	set var.v3_us_is_healthy = req.backend.healthy;

  	declare local var.shield_eu_is_healthy BOOL;
  	set req.backend = ssl_shield_london_city_uk;
  	set var.shield_eu_is_healthy = req.backend.healthy;

  	declare local var.shield_us_is_healthy BOOL;
  	set req.backend = ssl_shield_iad_va_us;
  	set var.shield_us_is_healthy = req.backend.healthy;

  	# Set some sort of default, that shouldn't get used.
  	set req.backend = F_v3_eu;

	declare local var.EU_shield_server_name STRING;
	set var.EU_shield_server_name = "LCY";

	declare local var.US_shield_server_name STRING;
	set var.US_shield_server_name = "IAD";

	# Route EU requests to the nearest healthy shield or origin.
  	if (var.region == "EU") {
		if (server.datacenter != var.EU_shield_server_name && req.http.Request_Came_From_Shield != var.EU_shield_server_name && var.shield_eu_is_healthy) {
			set req.backend = ssl_shield_london_city_uk;
		} elseif (var.v3_eu_is_healthy) {
			set req.backend = F_v3_eu;
			set req.http.Host = req.http.EU_Host;
		} elseif (var.shield_us_is_healthy) {
			set req.backend = ssl_shield_iad_va_us;
		} elseif (var.v3_us_is_healthy) {
			set req.backend = F_v3_us;
			set req.http.Host = req.http.US_Host;
		} else {
			# Everything is on fire... but lets try the origin anyway just in case
			# it's the probes that are wrong
			# set req.backend = F_origin_last_ditch_eu;
		}
  	}

	# Route US requests to the nearest healthy shield or origin.
  	if (var.region == "US") {
		if (server.datacenter != var.US_shield_server_name && req.http.Request_Came_From_Shield != var.US_shield_server_name && var.shield_us_is_healthy) {
			set req.backend = ssl_shield_iad_va_us;
		} elseif (var.v3_us_is_healthy) {
			set req.backend = F_v3_us;
			set req.http.Host = req.http.US_Host;
		} elseif (var.shield_eu_is_healthy) {
			set req.backend = ssl_shield_london_city_uk;
		} elseif (var.v3_eu_is_healthy) {
			set req.backend = F_v3_eu;
			set req.http.Host = req.http.EU_Host;
		} else {
			# Everything is on fire... but lets try the origin anyway just in case
			# it's the probes that are wrong
			# set req.backend = F_origin_last_ditch_us;
		}
	}

	# Persist the decision so we can debug the result.
  	set req.http.Debug-Backend = req.backend;
}

sub vcl_recv {
	if (req.http.Fastly-Debug) {
		call breadcrumb_recv;
	}

	if (req.method != "GET" && req.method != "HEAD" && req.method != "OPTIONS" && req.method != "FASTLYPURGE" && req.method != "PURGE") {
		error 911;
	}

	if (req.method == "OPTIONS") {
		error 912;
	}

	# Because the old service had a router which allowed any words between /v2/polyfill. and .js
	if (req.url.path ~ "^/v2/polyfill(\.\w+)(\.\w+)?" && req.url.ext == "js") {
		if (re.group.2) {
			set req.url = "/v2/polyfill.min.js?" req.url.qs;
		} else {
			set req.url = "/v2/polyfill.js?" req.url.qs;
		}
	}

	# Override the v3 defaults with the defaults of v2
	if (req.url ~ "^/v2/polyfill(\.min)?\.js") {
		set req.url = regsub(req.url, "^/v2", "/v3");
		set req.url = querystring.set(req.url, "version", "3.25.1");
		declare local var.unknown STRING;
		set var.unknown = subfield(req.url.qs, "unknown", "&");
		set req.url = querystring.set(req.url, "unknown", if(var.unknown != "", var.unknown, "ignore"));
	}

	if (req.url ~ "^/v3/polyfill(\.min)?\.js") {
		call normalise_querystring_parameters_for_polyfill_bundle;
		# Sort the querystring parameters alphabetically to improve chances of hitting a cached copy.
		# If querystring is empty, remove the ? from the url.
		set req.url = querystring.clean(querystring.sort(req.url));
		call set_backend;
	} else {
		# The request is to an endpoint which doesn't use query parameters, let's remove them to increase our cache-hit-ratio
		set req.url = querystring.remove(req.url);
		call set_backend;
	}
}

sub vcl_hash {
	if (req.http.Fastly-Debug) {
		call breadcrumb_hash;
	}

	# We are not adding req.http.host to the hash because we want https://cdn.polyfill.io and https://polyfill.io to be a single object in the cache.
	# set req.hash += req.http.host;
	set req.hash += req.url;
	# We include return(hash) to stop the function falling through to the default VCL built into varnish, which for vcl_hash will add req.url and req.http.Host to the hash.
	return(hash);
}

sub shielding_header {
	if (req.backend == ssl_shield_iad_va_us) {
		set req.http.Request_Came_From_Shield = server.datacenter;
	} elsif (req.backend == ssl_shield_london_city_uk) {
		set req.http.Request_Came_From_Shield = server.datacenter;
	}
}

sub vcl_miss {
	if (req.http.Fastly-Debug) {
		call breadcrumb_miss;
	}
	call shielding_header;
}

sub vcl_pass {
	if (req.http.Fastly-Debug) {
		call breadcrumb_pass;
	}
	call shielding_header;
}

sub vcl_fetch {
	set beresp.http.Request_Came_From_Shield = req.http.Request_Came_From_Shield;
	if (req.http.Fastly-Debug) {
		call breadcrumb_fetch;
	}

	# These header are only required for HTML documents.
	if (beresp.http.Content-Type ~ "text/html") {
		# Enables the cross-site scripting filter built into most modern web browsers.
		set beresp.http.X-XSS-Protection = "1; mode=block";
	}
	# Prevents MIME-sniffing a response away from the declared content type.
	set beresp.http.X-Content-Type-Options = "nosniff";

	# Ensure the site is only served over HTTPS and reduce the chances of someone performing a MITM attack.
	set beresp.http.Strict-Transport-Security = "max-age=31536000; includeSubdomains; preload";

	# The Referrer-Policy header governs which referrer information, sent in the Referer header, should be included with requests made.
	# Send a full URL when performing a same-origin request, but only send the origin of the document for other cases.
	set beresp.http.Referrer-Policy = "origin-when-cross-origin";

	# Enable purging of all objects in the Fastly cache by issuing a purge with the key "polyfill-service".
	if (beresp.http.Surrogate-Key !~ "\bpolyfill-service\b") {
		set beresp.http.Surrogate-Key = if(beresp.http.Surrogate-Key, beresp.http.Surrogate-Key " polyfill-service", "polyfill-service");
	}

	set beresp.http.Timing-Allow-Origin = "*";

	if (req.http.Normalized-User-Agent) {
		set beresp.http.Normalized-User-Agent = req.http.Normalized-User-Agent;
		set beresp.http.Detected-User-Agent = req.http.useragent_parser_family "/"  req.http.useragent_parser_major "." req.http.useragent_parser_minor "." req.http.useragent_parser_patch;
	}
}

sub vcl_deliver {
	if (req.http.Fastly-Debug) {
		call breadcrumb_deliver;
	}

	set req.http.Fastly-Force-Shield = "yes";

	# Allow cross origin GET, HEAD, and OPTIONS requests to be made.
	if (req.http.Origin) {
		set resp.http.Access-Control-Allow-Origin = "*";
		set resp.http.Access-Control-Allow-Methods = "GET,HEAD,OPTIONS";
	}

	if (req.url ~ "^/v3/polyfill(\.min)?\.js" && !resp.http.Request_Came_From_Shield && req.backend != ssl_shield_iad_va_us && req.backend != ssl_shield_london_city_uk) {
		# Need to add "Vary: User-Agent" in after vcl_fetch to avoid the
		# "Vary: User-Agent" entering the Varnish cache.
		# We need "Vary: User-Agent" in the browser cache because a browser
		# may update itself to a version which needs different polyfills
		# So we need to have it ignore the browser cached bundle when the user-agent changes.
		add resp.http.Vary = "User-Agent";
		add resp.http.Vary = "Accept-Encoding";
	}

	add resp.http.Server-Timing = fastly_info.state {", fastly;desc="Edge time";dur="} time.elapsed.msec;

	if (req.http.Fastly-Debug) {
		set resp.http.Debug-Backend = req.http.Debug-Backend;
		set resp.http.Debug-Host = req.http.Host;
		set resp.http.Debug-Fastly-Restarts = req.restarts;
		set resp.http.Debug-Orig-URL = req.http.Orig-URL;
		set resp.http.Debug-VCL-Route = req.http.X-VCL-Route;
		set resp.http.useragent_parser_family = req.http.useragent_parser_family;
		set resp.http.useragent_parser_major = req.http.useragent_parser_major;
		set resp.http.useragent_parser_minor = req.http.useragent_parser_minor;
		set resp.http.useragent_parser_patch = req.http.useragent_parser_patch;
	} else {
		unset resp.http.Server;
		unset resp.http.Via;
		unset resp.http.X-Cache;
		unset resp.http.X-Cache-Hits;
		unset resp.http.X-Served-By;
		unset resp.http.X-Timer;
		unset resp.http.Fastly-Restarts;
		unset resp.http.X-PreFetch-Pass;
		unset resp.http.X-PreFetch-Miss;
		unset resp.http.X-PostFetch;
	}
}

sub vcl_error {
	if (obj.status == 911) {
		set obj.status = 405;
		set obj.response = "METHOD NOT ALLOWED";
		set obj.http.Content-Type = "text/html; charset=UTF-8";
		set obj.http.Cache-Control = "private, no-store";
		synthetic req.method " METHOD NOT ALLOWED";
		return (deliver);
	}
	if (obj.status == 912) {
		set obj.status = 200;
		set obj.response = "OK";
		set obj.http.Allow = "OPTIONS, GET, HEAD";
		return (deliver);
	}
}

sub set_backend {
	# The Fastly macro is inserted before the backend is selected because the
	# macro has the code which defines the values avaiable for req.http.Host
	# but it also contains a default backend which is always set to the EU. I wish we could disable the default backend setting.
	#FASTLY recv

	# Calculate the ideal region to route the request to.
  	declare local var.region STRING;
	if (server.region ~ "(APAC|Asia|North-America|South-America|US-Central|US-East|US-West)") {
		set var.region = "US";
  	} else {
		set var.region = "EU";
  	}

	# Gather the health of the shields and origins.
	declare local var.v3_eu_is_healthy BOOL;
	set req.backend = F_v3_eu;
	set var.v3_eu_is_healthy = req.backend.healthy;

	declare local var.v3_us_is_healthy BOOL;
  	set req.backend = F_v3_us;
  	set var.v3_us_is_healthy = req.backend.healthy;

  	declare local var.shield_eu_is_healthy BOOL;
  	set req.backend = ssl_shield_london_city_uk;
  	set var.shield_eu_is_healthy = req.backend.healthy;

  	declare local var.shield_us_is_healthy BOOL;
  	set req.backend = ssl_shield_iad_va_us;
  	set var.shield_us_is_healthy = req.backend.healthy;

  	# Set some sort of default, that shouldn't get used.
  	set req.backend = F_v3_eu;

	declare local var.EU_shield_server_name STRING;
	set var.EU_shield_server_name = "LCY";

	declare local var.US_shield_server_name STRING;
	set var.US_shield_server_name = "IAD";

	# Route EU requests to the nearest healthy shield or origin.
  	if (var.region == "EU") {
		if (server.datacenter != var.EU_shield_server_name && req.http.Request_Came_From_Shield != var.EU_shield_server_name && var.shield_eu_is_healthy) {
			set req.backend = ssl_shield_london_city_uk;
		} elseif (var.v3_eu_is_healthy) {
			set req.backend = F_v3_eu;
			set req.http.Host = req.http.EU_Host;
		} elseif (var.shield_us_is_healthy) {
			set req.backend = ssl_shield_iad_va_us;
		} elseif (var.v3_us_is_healthy) {
			set req.backend = F_v3_us;
			set req.http.Host = req.http.US_Host;
		} else {
			# Everything is on fire... but lets try the origin anyway just in case
			# it's the probes that are wrong
			# set req.backend = F_origin_last_ditch_eu;
		}
  	}

	# Route US requests to the nearest healthy shield or origin.
  	if (var.region == "US") {
		if (server.datacenter != var.US_shield_server_name && req.http.Request_Came_From_Shield != var.US_shield_server_name && var.shield_us_is_healthy) {
			set req.backend = ssl_shield_iad_va_us;
		} elseif (var.v3_us_is_healthy) {
			set req.backend = F_v3_us;
			set req.http.Host = req.http.US_Host;
		} elseif (var.shield_eu_is_healthy) {
			set req.backend = ssl_shield_london_city_uk;
		} elseif (var.v3_eu_is_healthy) {
			set req.backend = F_v3_eu;
			set req.http.Host = req.http.EU_Host;
		} else {
			# Everything is on fire... but lets try the origin anyway just in case
			# it's the probes that are wrong
			# set req.backend = F_origin_last_ditch_us;
		}
	}

	# Persist the decision so we can debug the result.
  	set req.http.Debug-Backend = req.backend;
}

sub vcl_recv {
	if (req.http.Fastly-Debug) {
		call breadcrumb_recv;
	}

	if (req.method != "GET" && req.method != "HEAD" && req.method != "OPTIONS" && req.method != "FASTLYPURGE" && req.method != "PURGE") {
		error 911;
	}

	if (req.method == "OPTIONS") {
		error 912;
	}

	# Because the old service had a router which allowed any words between /v2/polyfill. and .js
	if (req.url.path ~ "^/v2/polyfill(\.\w+)(\.\w+)?" && req.url.ext == "js") {
		if (re.group.2) {
			set req.url = "/v2/polyfill.min.js?" req.url.qs;
		} else {
			set req.url = "/v2/polyfill.js?" req.url.qs;
		}
	}

	# Override the v3 defaults with the defaults of v2
	if (req.url ~ "^/v2/polyfill(\.min)?\.js") {
		set req.url = regsub(req.url, "^/v2", "/v3");
		set req.url = querystring.set(req.url, "version", "3.25.1");
		declare local var.unknown STRING;
		set var.unknown = subfield(req.url.qs, "unknown", "&");
		set req.url = querystring.set(req.url, "unknown", if(var.unknown != "", var.unknown, "ignore"));
	}

	if (req.url ~ "^/v3/polyfill(\.min)?\.js") {
		call normalise_querystring_parameters_for_polyfill_bundle;
		# Sort the querystring parameters alphabetically to improve chances of hitting a cached copy.
		# If querystring is empty, remove the ? from the url.
		set req.url = querystring.clean(querystring.sort(req.url));
		call set_backend;
	} else {
		# The request is to an endpoint which doesn't use query parameters, let's remove them to increase our cache-hit-ratio
		set req.url = querystring.remove(req.url);
		call set_backend;
	}
}

sub vcl_hash {
	if (req.http.Fastly-Debug) {
		call breadcrumb_hash;
	}

	# We are not adding req.http.host to the hash because we want https://cdn.polyfill.io and https://polyfill.io to be a single object in the cache.
	# set req.hash += req.http.host;
	set req.hash += req.url;
	# We include return(hash) to stop the function falling through to the default VCL built into varnish, which for vcl_hash will add req.url and req.http.Host to the hash.
	return(hash);
}

sub shielding_header {
	if (req.backend == ssl_shield_iad_va_us) {
		set req.http.Request_Came_From_Shield = server.datacenter;
	} elsif (req.backend == ssl_shield_london_city_uk) {
		set req.http.Request_Came_From_Shield = server.datacenter;
	}
}

sub vcl_miss {
	if (req.http.Fastly-Debug) {
		call breadcrumb_miss;
	}
	call shielding_header;
}

sub vcl_pass {
	if (req.http.Fastly-Debug) {
		call breadcrumb_pass;
	}
	call shielding_header;
}

sub vcl_fetch {
	set beresp.http.Request_Came_From_Shield = req.http.Request_Came_From_Shield;
	if (req.http.Fastly-Debug) {
		call breadcrumb_fetch;
	}

	# These header are only required for HTML documents.
	if (beresp.http.Content-Type ~ "text/html") {
		# Enables the cross-site scripting filter built into most modern web browsers.
		set beresp.http.X-XSS-Protection = "1; mode=block";
	}
	# Prevents MIME-sniffing a response away from the declared content type.
	set beresp.http.X-Content-Type-Options = "nosniff";

	# Ensure the site is only served over HTTPS and reduce the chances of someone performing a MITM attack.
	set beresp.http.Strict-Transport-Security = "max-age=31536000; includeSubdomains; preload";

	# The Referrer-Policy header governs which referrer information, sent in the Referer header, should be included with requests made.
	# Send a full URL when performing a same-origin request, but only send the origin of the document for other cases.
	set beresp.http.Referrer-Policy = "origin-when-cross-origin";

	# Enable purging of all objects in the Fastly cache by issuing a purge with the key "polyfill-service".
	if (beresp.http.Surrogate-Key !~ "\bpolyfill-service\b") {
		set beresp.http.Surrogate-Key = if(beresp.http.Surrogate-Key, beresp.http.Surrogate-Key " polyfill-service", "polyfill-service");
	}

	set beresp.http.Timing-Allow-Origin = "*";

	if (req.http.Normalized-User-Agent) {
		set beresp.http.Normalized-User-Agent = req.http.Normalized-User-Agent;
		set beresp.http.Detected-User-Agent = req.http.useragent_parser_family "/"  req.http.useragent_parser_major "." req.http.useragent_parser_minor "." req.http.useragent_parser_patch;
	}
}

sub vcl_deliver {
	if (req.http.Fastly-Debug) {
		call breadcrumb_deliver;
	}

	set req.http.Fastly-Force-Shield = "yes";

	# Allow cross origin GET, HEAD, and OPTIONS requests to be made.
	if (req.http.Origin) {
		set resp.http.Access-Control-Allow-Origin = "*";
		set resp.http.Access-Control-Allow-Methods = "GET,HEAD,OPTIONS";
	}

	if (req.url ~ "^/v3/polyfill(\.min)?\.js" && !resp.http.Request_Came_From_Shield && req.backend != ssl_shield_iad_va_us && req.backend != ssl_shield_london_city_uk) {
		# Need to add "Vary: User-Agent" in after vcl_fetch to avoid the
		# "Vary: User-Agent" entering the Varnish cache.
		# We need "Vary: User-Agent" in the browser cache because a browser
		# may update itself to a version which needs different polyfills
		# So we need to have it ignore the browser cached bundle when the user-agent changes.
		add resp.http.Vary = "User-Agent";
		add resp.http.Vary = "Accept-Encoding";
	}

	add resp.http.Server-Timing = fastly_info.state {", fastly;desc="Edge time";dur="} time.elapsed.msec;

	if (req.http.Fastly-Debug) {
		set resp.http.Debug-Backend = req.http.Debug-Backend;
		set resp.http.Debug-Host = req.http.Host;
		set resp.http.Debug-Fastly-Restarts = req.restarts;
		set resp.http.Debug-Orig-URL = req.http.Orig-URL;
		set resp.http.Debug-VCL-Route = req.http.X-VCL-Route;
		set resp.http.useragent_parser_family = req.http.useragent_parser_family;
		set resp.http.useragent_parser_major = req.http.useragent_parser_major;
		set resp.http.useragent_parser_minor = req.http.useragent_parser_minor;
		set resp.http.useragent_parser_patch = req.http.useragent_parser_patch;
	} else {
		unset resp.http.Server;
		unset resp.http.Via;
		unset resp.http.X-Cache;
		unset resp.http.X-Cache-Hits;
		unset resp.http.X-Served-By;
		unset resp.http.X-Timer;
		unset resp.http.Fastly-Restarts;
	}
}

sub vcl_error {
	if (obj.status == 911) {
		set obj.status = 405;
		set obj.response = "METHOD NOT ALLOWED";
		set obj.http.Content-Type = "text/html; charset=UTF-8";
		set obj.http.Cache-Control = "private, no-store";
		synthetic req.method " METHOD NOT ALLOWED";
		return (deliver);
	}
	if (obj.status == 912) {
		set obj.status = 200;
		set obj.response = "OK";
		set obj.http.Allow = "OPTIONS, GET, HEAD";
		return (deliver);
	}
}


Disable http auth for GraphQL endpoint

Disable http auth for GraphQL endpoint

  # Check Basic auth against a table. /admin URLs are not basic auth protected to avoid the possibility of people
  # locking themselves out. /oauth and /rest have their own auth so we can skip Basic Auth on them as well
  if ( table.lookup(magentomodule_basic_auth, regsub(req.http.Authorization, "^Basic ", ""), "NOTFOUND") == "NOTFOUND" &&
      !req.url ~ "^/(index\.php/)?####ADMIN_PATH####/" &&
      !req.url ~ "^/(index\.php/)?(rest|oauth|graphql)/" &&
      !req.url ~ "^/pub/static/" ) {
      error 771;
  }

  # Check Basic auth against a table. /admin URLs are not basic auth protected to avoid the possibility of people
  # locking themselves out. /oauth and /rest have their own auth so we can skip Basic Auth on them as well
  if ( table.lookup(magentomodule_basic_auth, regsub(req.http.Authorization, "^Basic ", ""), "NOTFOUND") == "NOTFOUND" &&
      !req.url ~ "^/(index\.php/)?####ADMIN_PATH####/" &&
      !req.url ~ "^/(index\.php/)?(rest|oauth)/" &&
      !req.url ~ "^/pub/static/" ) {
      error 771;
  }