Remove caching for static files

Remove caching for static files

# Default backend definition.  Set this to point to your content server.
# all paths relative to varnish option vcl_dir

include "custom.backend.vcl";
include "custom.acl.vcl";

# Handle the HTTP request received by the client
sub vcl_recv {
    # varnish serves stale (but cacheable) objects while retriving object from backend
    if (req.backend.healthy) {
        set req.grace = 30s;
    } else {
        set req.grace = 1h;
    }

    # shortcut for DFind requests
    if (req.url ~ "^/w00tw00t") {
        error 404 "Not Found";
    }

    if (req.restarts == 0) {
        if (req.http.X-Forwarded-For) {
            set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
    }

    # Normalize the header, remove the port (in case you're testing this on various TCP ports)
    set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");

    # Allow purging
    if (req.request == "PURGE") {
        if (!client.ip ~ purge) {
            # Not from an allowed IP? Then die with an error.
            error 405 "This IP is not allowed to send PURGE requests.";
        }

        # If you got this stage (and didn't error out above), do a cache-lookup
        # That will force entry into vcl_hit() or vcl_miss() below and purge the actual cache
        return (lookup);
    }

    # Only deal with "normal" types
    if (req.request != "GET" &&
            req.request != "HEAD" &&
            req.request != "PUT" &&
            req.request != "POST" &&
            req.request != "TRACE" &&
            req.request != "OPTIONS" &&
            req.request != "PATCH" &&
            req.request != "DELETE") {
        /* Non-RFC2616 or CONNECT which is weird. */
        return (pipe);
    }

    if (req.request != "GET" && req.request != "HEAD") {
        # We only deal with GET and HEAD by default
        return (pass);
    }

    # Some generic URL manipulation, useful for all templates that follow
    # First remove the Google Analytics added parameters, useless for our backend
    if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=") {
        set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
        set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
        set req.url = regsub(req.url, "\?&", "?");
        set req.url = regsub(req.url, "\?$", "");
    }

    # Strip hash, server doesn't need it.
    if (req.url ~ "\#") {
        set req.url = regsub(req.url, "\#.*$", "");
    }

    # Strip a trailing ? if it exists
    if (req.url ~ "\?$") {
        set req.url = regsub(req.url, "\?$", "");
    }

    # This is an example to redirect with a 301/302 HTTP status code from within Varnish
    # if (req.http.Host ~ "secure.mysite.tld") {
    #   # We may want to force our users from the secure site to the HTTPs version?
    #   error 720 "https://secure.mysite.tld";
    #   # If you want to keep the URLs intact, this also works:
    #   error 720 "https://" + req.http.Host + req.url;
    # }
    #
    # Or to force a 302 temporary redirect, use error 721
    # if (req.http.Host ~ "temp.mysite.tld") {
    #   # Temporary redirect
    #   error 721 "http://mysite.tld/temp";
    # }
    #

    # Some generic cookie manipulation, useful for all templates that follow

    # Remove any '_' prefixed cookies
    set req.http.Cookie = regsuball(req.http.Cookie, "_[^=]+=[^;]+(; )?", "");

    # Remove a ";" prefix in the cookie if present
    set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");

    # Are there cookies left with only spaces or that are empty?
    if (req.http.cookie ~ "^\s*$") {
        unset req.http.cookie;
    }

    # Normalize Accept-Encoding header
    # straight from the manual: https://www.varnish-cache.org/docs/3.0/tutorial/vary.html
    if (req.http.Accept-Encoding) {
        if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
            # No point in compressing these
            remove req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elsif (req.http.Accept-Encoding ~ "deflate") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            # unkown algorithm
            remove req.http.Accept-Encoding;
        }
    }

    # Include custom vcl_recv logic
    include "conf.d/receive/wordpress.vcl";

    # Remove all cookies for static files
    # A valid discussion could be held on this line: do you really need to cache static files that don't cause load? Only if you have memory left.
    # Sure, there's disk I/O, but chances are your OS will already have these files in their buffers (thus memory).
    # Before you blindly enable this, have a read here: http://mattiasgeniar.be/2012/11/28/stop-caching-static-files/
    # if (req.url ~ "^[^?]*\.(bmp|bz2|css|doc|eot|flv|gif|gz|ico|jpeg|jpg|js|less|mp[34]|pdf|png|rar|rtf|swf|tar|tgz|txt|wav|woff|xml|zip)(\?.*)?$") {
    #     unset req.http.Cookie;
    #     return (lookup);
    # }

    # Send Surrogate-Capability headers to announce ESI support to backend
    set req.http.Surrogate-Capability = "key=ESI/1.0";

    if (req.http.Authorization || req.http.Cookie) {
        # Not cacheable by default
        return (pass);
    }

    return (lookup);
}

sub vcl_pipe {
    # Note that only the first request to the backend will have
    # X-Forwarded-For set.  If you use X-Forwarded-For and want to
    # have it set for all requests, make sure to have:
    # set bereq.http.connection = "close";
    # here.  It is not set by default as it might break some broken web
    # applications, like IIS with NTLM authentication.

    #set bereq.http.Connection = "Close";
    return (pipe);
}

sub vcl_pass {
    return (pass);
}

# The data on which the hashing will take place
sub vcl_hash {
    hash_data(req.url);

    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }

    # hash cookies for object with auth
    if (req.http.Cookie) {
        hash_data(req.http.Cookie);
    }
    if (req.http.Authorization) {
        hash_data(req.http.Authorization);
    }

    return (hash);
}

sub vcl_hit {
    # Allow purges
    if (req.request == "PURGE") {
        purge;
        error 200 "purged";
    }

    return (deliver);
}

sub vcl_miss {
    # Allow purges
    if (req.request == "PURGE") {
        purge;
        error 200 "purged";
    }

    return (fetch);
}

# Handle the HTTP request coming from our backend
sub vcl_fetch {
    set beresp.grace = 1h;

    # Include custom vcl_fetch logic
    include "conf.d/fetch/wordpress.vcl";

    # Parse ESI request and remove Surrogate-Control header
    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
        unset beresp.http.Surrogate-Control;
        set beresp.do_esi = true;
    }

    # If the request to the backend returns a code is 5xx, restart the loop
    # If the number of restarts reaches the value of the parameter max_restarts,
    # the request will be error'ed.  max_restarts defaults to 4.  This prevents
    # an eternal loop in the event that, e.g., the object does not exist at all.
    if (beresp.status >= 500 && beresp.status <= 599){
        return(restart);
    }

    # Enable cache for all static files
    # The same argument as the static caches from above: monitor your cache size, if you get data nuked out of it, consider giving up the static file cache.
    # Before you blindly enable this, have a read here: http://mattiasgeniar.be/2012/11/28/stop-caching-static-files/
    # if (req.url ~ "^[^?]*\.(bmp|bz2|css|doc|eot|flv|gif|gz|ico|jpeg|jpg|js|less|mp[34]|pdf|png|rar|rtf|swf|tar|tgz|txt|wav|woff|xml|zip)(\?.*)?$") {
    #     unset beresp.http.set-cookie;
    # }

    # Sometimes, a 301 or 302 redirect formed via Apache's mod_rewrite can mess with the HTTP port that is being passed along.
    # This often happens with simple rewrite rules in a scenario where Varnish runs on :80 and Apache on :8080 on the same box.
    # A redirect can then often redirect the end-user to a URL on :8080, where it should be :80.
    # This may need finetuning on your setup.
    #
    # To prevent accidental replace, we only filter the 301/302 redirects for now.
    if (beresp.status == 301 || beresp.status == 302) {
        set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", "");
    }

    # Don't cache user & server errors
    if (beresp.status >= 400) {
        return (hit_for_pass);
    }

    # Non private responses without cookies and with a ttl of 0 should be artificially extended to 1 hour
    if (beresp.ttl <= 0s && beresp.http.Cache-Control !~ "private" && (!beresp.http.Set-Cookie)) {
        set beresp.ttl = 1h;
        set beresp.http.X-Cache-Extended = "1";
    }

    # Set 2min cache if unset for static files
    if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Vary == "*") {
        set beresp.ttl = 120s;
        return (hit_for_pass);
    }

    return (deliver);
}

# The routine when we deliver the HTTP request to the user
# Last chance to modify headers that are sent to the client
sub vcl_deliver {
    if (obj.hits > 0) {
        set resp.http.X-Cache = "cached";
    } else {
        set resp.http.x-Cache = "uncached";
    }

    # Remove some headers: PHP version
    unset resp.http.X-Powered-By;

    # Remove some headers: Apache version & OS
    unset resp.http.Server;
    unset resp.http.X-Drupal-Cache;
    unset resp.http.X-Varnish;
    unset resp.http.Via;
    unset resp.http.Link;

    return (deliver);
}

sub vcl_error {
    if (obj.status == 720) {
        # We use this special error status 720 to force redirects with 301 (permanent) redirects
        # To use this, call the following from anywhere in vcl_recv: error 720 "http://host/new.html"
        set obj.status = 301;
        set obj.http.Location = obj.response;
        return (deliver);
    } elseif (obj.status == 721) {
        # And we use error status 721 to force redirects with a 302 (temporary) redirect
        # To use this, call the following from anywhere in vcl_recv: error 720 "http://host/new.html"
        set obj.status = 302;
        set obj.http.Location = obj.response;
        return (deliver);
    }

    return (deliver);
}

sub vcl_init {
    return (ok);
}

sub vcl_fini {
    return (ok);
}

# Default backend definition.  Set this to point to your content server.
# all paths relative to varnish option vcl_dir

include "custom.backend.vcl";
include "custom.acl.vcl";

# Handle the HTTP request received by the client
sub vcl_recv {
    # varnish serves stale (but cacheable) objects while retriving object from backend
    if (req.backend.healthy) {
        set req.grace = 30s;
    } else {
        set req.grace = 1h;
    }

    # shortcut for DFind requests
    if (req.url ~ "^/w00tw00t") {
        error 404 "Not Found";
    }

    if (req.restarts == 0) {
        if (req.http.X-Forwarded-For) {
            set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
    }

    # Normalize the header, remove the port (in case you're testing this on various TCP ports)
    set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");

    # Allow purging
    if (req.request == "PURGE") {
        if (!client.ip ~ purge) {
            # Not from an allowed IP? Then die with an error.
            error 405 "This IP is not allowed to send PURGE requests.";
        }

        # If you got this stage (and didn't error out above), do a cache-lookup
        # That will force entry into vcl_hit() or vcl_miss() below and purge the actual cache
        return (lookup);
    }

    # Only deal with "normal" types
    if (req.request != "GET" &&
            req.request != "HEAD" &&
            req.request != "PUT" &&
            req.request != "POST" &&
            req.request != "TRACE" &&
            req.request != "OPTIONS" &&
            req.request != "PATCH" &&
            req.request != "DELETE") {
        /* Non-RFC2616 or CONNECT which is weird. */
        return (pipe);
    }

    if (req.request != "GET" && req.request != "HEAD") {
        # We only deal with GET and HEAD by default
        return (pass);
    }

    # Some generic URL manipulation, useful for all templates that follow
    # First remove the Google Analytics added parameters, useless for our backend
    if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=") {
        set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
        set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
        set req.url = regsub(req.url, "\?&", "?");
        set req.url = regsub(req.url, "\?$", "");
    }

    # Strip hash, server doesn't need it.
    if (req.url ~ "\#") {
        set req.url = regsub(req.url, "\#.*$", "");
    }

    # Strip a trailing ? if it exists
    if (req.url ~ "\?$") {
        set req.url = regsub(req.url, "\?$", "");
    }

    # This is an example to redirect with a 301/302 HTTP status code from within Varnish
    # if (req.http.Host ~ "secure.mysite.tld") {
    #   # We may want to force our users from the secure site to the HTTPs version?
    #   error 720 "https://secure.mysite.tld";
    #   # If you want to keep the URLs intact, this also works:
    #   error 720 "https://" + req.http.Host + req.url;
    # }
    #
    # Or to force a 302 temporary redirect, use error 721
    # if (req.http.Host ~ "temp.mysite.tld") {
    #   # Temporary redirect
    #   error 721 "http://mysite.tld/temp";
    # }
    #

    # Some generic cookie manipulation, useful for all templates that follow

    # Remove any '_' prefixed cookies
    set req.http.Cookie = regsuball(req.http.Cookie, "_[^=]+=[^;]+(; )?", "");

    # Remove a ";" prefix in the cookie if present
    set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");

    # Are there cookies left with only spaces or that are empty?
    if (req.http.cookie ~ "^\s*$") {
        unset req.http.cookie;
    }

    # Normalize Accept-Encoding header
    # straight from the manual: https://www.varnish-cache.org/docs/3.0/tutorial/vary.html
    if (req.http.Accept-Encoding) {
        if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
            # No point in compressing these
            remove req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elsif (req.http.Accept-Encoding ~ "deflate") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            # unkown algorithm
            remove req.http.Accept-Encoding;
        }
    }

    # Include custom vcl_recv logic
    include "conf.d/receive/wordpress.vcl";

    # Remove all cookies for static files
    # A valid discussion could be held on this line: do you really need to cache static files that don't cause load? Only if you have memory left.
    # Sure, there's disk I/O, but chances are your OS will already have these files in their buffers (thus memory).
    # Before you blindly enable this, have a read here: http://mattiasgeniar.be/2012/11/28/stop-caching-static-files/
    if (req.url ~ "^[^?]*\.(bmp|bz2|css|doc|eot|flv|gif|gz|ico|jpeg|jpg|js|less|mp[34]|pdf|png|rar|rtf|swf|tar|tgz|txt|wav|woff|xml|zip)(\?.*)?$") {
        unset req.http.Cookie;
        return (lookup);
    }

    # Send Surrogate-Capability headers to announce ESI support to backend
    set req.http.Surrogate-Capability = "key=ESI/1.0";

    if (req.http.Authorization || req.http.Cookie) {
        # Not cacheable by default
        return (pass);
    }

    return (lookup);
}

sub vcl_pipe {
    # Note that only the first request to the backend will have
    # X-Forwarded-For set.  If you use X-Forwarded-For and want to
    # have it set for all requests, make sure to have:
    # set bereq.http.connection = "close";
    # here.  It is not set by default as it might break some broken web
    # applications, like IIS with NTLM authentication.

    #set bereq.http.Connection = "Close";
    return (pipe);
}

sub vcl_pass {
    return (pass);
}

# The data on which the hashing will take place
sub vcl_hash {
    hash_data(req.url);

    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }

    # hash cookies for object with auth
    if (req.http.Cookie) {
        hash_data(req.http.Cookie);
    }
    if (req.http.Authorization) {
        hash_data(req.http.Authorization);
    }

    return (hash);
}

sub vcl_hit {
    # Allow purges
    if (req.request == "PURGE") {
        purge;
        error 200 "purged";
    }

    return (deliver);
}

sub vcl_miss {
    # Allow purges
    if (req.request == "PURGE") {
        purge;
        error 200 "purged";
    }

    return (fetch);
}

# Handle the HTTP request coming from our backend
sub vcl_fetch {
    set beresp.grace = 1h;

    # Include custom vcl_fetch logic
    include "conf.d/fetch/wordpress.vcl";

    # Parse ESI request and remove Surrogate-Control header
    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
        unset beresp.http.Surrogate-Control;
        set beresp.do_esi = true;
    }

    # If the request to the backend returns a code is 5xx, restart the loop
    # If the number of restarts reaches the value of the parameter max_restarts,
    # the request will be error'ed.  max_restarts defaults to 4.  This prevents
    # an eternal loop in the event that, e.g., the object does not exist at all.
    if (beresp.status >= 500 && beresp.status <= 599){
        return(restart);
    }

    # Enable cache for all static files
    # The same argument as the static caches from above: monitor your cache size, if you get data nuked out of it, consider giving up the static file cache.
    # Before you blindly enable this, have a read here: http://mattiasgeniar.be/2012/11/28/stop-caching-static-files/
    if (req.url ~ "^[^?]*\.(bmp|bz2|css|doc|eot|flv|gif|gz|ico|jpeg|jpg|js|less|mp[34]|pdf|png|rar|rtf|swf|tar|tgz|txt|wav|woff|xml|zip)(\?.*)?$") {
        unset beresp.http.set-cookie;
    }

    # Sometimes, a 301 or 302 redirect formed via Apache's mod_rewrite can mess with the HTTP port that is being passed along.
    # This often happens with simple rewrite rules in a scenario where Varnish runs on :80 and Apache on :8080 on the same box.
    # A redirect can then often redirect the end-user to a URL on :8080, where it should be :80.
    # This may need finetuning on your setup.
    #
    # To prevent accidental replace, we only filter the 301/302 redirects for now.
    if (beresp.status == 301 || beresp.status == 302) {
        set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", "");
    }

    # Don't cache user & server errors
    if (beresp.status >= 400) {
        return (hit_for_pass);
    }

    # Non private responses without cookies and with a ttl of 0 should be artificially extended to 1 hour
    if (beresp.ttl <= 0s && beresp.http.Cache-Control !~ "private" && (!beresp.http.Set-Cookie)) {
        set beresp.ttl = 1h;
        set beresp.http.X-Cache-Extended = "1";
    }

    # Set 2min cache if unset for static files
    if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Vary == "*") {
        set beresp.ttl = 120s;
        return (hit_for_pass);
    }

    return (deliver);
}

# The routine when we deliver the HTTP request to the user
# Last chance to modify headers that are sent to the client
sub vcl_deliver {
    if (obj.hits > 0) {
        set resp.http.X-Cache = "cached";
    } else {
        set resp.http.x-Cache = "uncached";
    }

    # Remove some headers: PHP version
    unset resp.http.X-Powered-By;

    # Remove some headers: Apache version & OS
    unset resp.http.Server;
    unset resp.http.X-Drupal-Cache;
    unset resp.http.X-Varnish;
    unset resp.http.Via;
    unset resp.http.Link;

    return (deliver);
}

sub vcl_error {
    if (obj.status == 720) {
        # We use this special error status 720 to force redirects with 301 (permanent) redirects
        # To use this, call the following from anywhere in vcl_recv: error 720 "http://host/new.html"
        set obj.status = 301;
        set obj.http.Location = obj.response;
        return (deliver);
    } elseif (obj.status == 721) {
        # And we use error status 721 to force redirects with a 302 (temporary) redirect
        # To use this, call the following from anywhere in vcl_recv: error 720 "http://host/new.html"
        set obj.status = 302;
        set obj.http.Location = obj.response;
        return (deliver);
    }

    return (deliver);
}

sub vcl_init {
    return (ok);
}

sub vcl_fini {
    return (ok);
}


This may be causing unintended effects so removing it

    /* handle 5XX (or any other unwanted status code) */
    if (beresp.status >= 500 && beresp.status < 600) {

        # let SOAP errors pass - better debugging
        if (beresp.http.Content-Type ~ "text/xml") {
            return (deliver);
        }

        /* deliver stale if the object is available */
        if (stale.exists) {
            return(deliver_stale);
        }

        if (req.restarts < 1 && (req.request == "GET" || req.request == "HEAD")) {
            restart;
        }

        /* else go to vcl_error to deliver a synthetic */
        error 503;
    }

    # Remove Set-Cookies from responses for static content
    # to match the cookie removal in recv.
    if (req.url ~ "^/(pub/)?(media|static)/") {
        unset beresp.http.set-cookie;

        # Set a short TTL for 404's since those can be temporary during the site build/index
        if (beresp.status == 404) {
            set beresp.ttl = 300s;
        }

    }

    if (beresp.http.Content-Type ~ "text/(html|xml)") {
        # enable ESI feature for Magento response by default
        esi;
        if (!beresp.http.Vary ~ "X-Magento-Vary,Https") {
            if (beresp.http.Vary) {
                set beresp.http.Vary = beresp.http.Vary ",X-Magento-Vary,Https";
            } else {
                set beresp.http.Vary = "X-Magento-Vary,Https";
            }
        }
        # Since varnish doesn't compress ESIs we need to hint to the HTTP/2 terminators to
        # compress it
        set beresp.http.x-compress-hint = "on";
    } else {
        # enable gzip for all static content
        if (http_status_matches(beresp.status, "200,404") && (beresp.http.content-type ~ "^(application\/x\-javascript|text\/css|application\/javascript|text\/javascript|application\/json|application\/vnd\.ms\-fontobject|application\/x\-font\-opentype|application\/x\-font\-truetype|application\/x\-font\-ttf|application\/xml|font\/eot|font\/opentype|font\/otf|image\/svg\+xml|image\/vnd\.microsoft\.icon|text\/plain)\s*($|;)" || req.url.ext ~ "(?i)(css|js|html|eot|ico|otf|ttf|json)" ) ) {
            # always set vary to make sure uncompressed versions dont always win
            if (!beresp.http.Vary ~ "Accept-Encoding") {
                if (beresp.http.Vary) {
                    set beresp.http.Vary = beresp.http.Vary ", Accept-Encoding";
                } else {
                    set beresp.http.Vary = "Accept-Encoding";
                }
            }
            if (req.http.Accept-Encoding == "gzip") {
                set beresp.gzip = true;
            }
        }
    }

    if (beresp.http.Cache-Control ~ "private|no-cache|no-store") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    if (beresp.http.X-Magento-Debug) {
        set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
    }

    # Never cache 302s
    if (beresp.status == 302) {
        return (pass);
    }

    # validate if we need to cache it and prevent from setting cookie
    # images, css and js are cacheable by default so we have to remove cookie also
    if (beresp.ttl > 0s && (req.request == "GET" || req.request == "HEAD") && !req.http.x-pass ) {
        unset beresp.http.set-cookie;

        # init surrogate keys
        if (beresp.http.X-Magento-Tags) {
            set beresp.http.Surrogate-Key = beresp.http.X-Magento-Tags " text";
        } else {
            set beresp.http.Surrogate-Key = "text";
        }

        # set surrogate keys by content type if they are image/script or CSS
        if (beresp.http.Content-Type ~ "(image|script|css)") {
            set beresp.http.Surrogate-Key = re.group.1;
        }
        return (deliver);
    }

    /* handle 5XX (or any other unwanted status code) */
    if (beresp.status >= 500 && beresp.status < 600) {

        # let SOAP errors pass - better debugging
        if (beresp.http.Content-Type ~ "text/xml") {
            return (deliver);
        }

        /* deliver stale if the object is available */
        if (stale.exists) {
            return(deliver_stale);
        }

        if (req.restarts < 1 && (req.request == "GET" || req.request == "HEAD")) {
            restart;
        }

        /* else go to vcl_error to deliver a synthetic */
        error 503;
    }

    # Remove Set-Cookies from responses for static content
    # to match the cookie removal in recv.
    if (req.url ~ "^/(pub/)?(media|static)/") {
        unset beresp.http.set-cookie;

        # Set a short TTL for 404's since those can be temporary during the site build/index
        if (beresp.status == 404) {
            set beresp.ttl = 300s;
        }

    }

    if (beresp.http.Content-Type ~ "text/(html|xml)") {
        # enable ESI feature for Magento response by default
        esi;
        if (!beresp.http.Vary ~ "X-Magento-Vary,Https") {
            if (beresp.http.Vary) {
                set beresp.http.Vary = beresp.http.Vary ",X-Magento-Vary,Https";
            } else {
                set beresp.http.Vary = "X-Magento-Vary,Https";
            }
        }
        # Since varnish doesn't compress ESIs we need to hint to the HTTP/2 terminators to
        # compress it
        set beresp.http.x-compress-hint = "on";
    } else {
        # enable gzip for all static content
        if (http_status_matches(beresp.status, "200,404") && (beresp.http.content-type ~ "^(application\/x\-javascript|text\/css|application\/javascript|text\/javascript|application\/json|application\/vnd\.ms\-fontobject|application\/x\-font\-opentype|application\/x\-font\-truetype|application\/x\-font\-ttf|application\/xml|font\/eot|font\/opentype|font\/otf|image\/svg\+xml|image\/vnd\.microsoft\.icon|text\/plain)\s*($|;)" || req.url.ext ~ "(?i)(css|js|html|eot|ico|otf|ttf|json)" ) ) {
            # always set vary to make sure uncompressed versions dont always win
            if (!beresp.http.Vary ~ "Accept-Encoding") {
                if (beresp.http.Vary) {
                    set beresp.http.Vary = beresp.http.Vary ", Accept-Encoding";
                } else {
                    set beresp.http.Vary = "Accept-Encoding";
                }
            }
            if (req.http.Accept-Encoding == "gzip") {
                set beresp.gzip = true;
            }
        }
    }

    if (beresp.http.Cache-Control ~ "private|no-cache|no-store") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    if (beresp.http.X-Magento-Debug) {
        set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
    }

    # Never cache 302s
    if (beresp.status == 302) {
        return (pass);
    }

    # Cache non-successful responses for 1 second to avoid onslaught of traffic if backend starts
    # spewing 500s. Also set 5 second grace in case backend is timing out or otherwise messed up
    if ( !http_status_matches(beresp.status, "200,203,300,301,302,404,410")) {
        set req.http.Fastly-Cachetype = "ERROR";
        # Remove Set-Cookies so we don't inadvertenly cache them
        unset beresp.http.Set-Cookie;
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # validate if we need to cache it and prevent from setting cookie
    # images, css and js are cacheable by default so we have to remove cookie also
    if (beresp.ttl > 0s && (req.request == "GET" || req.request == "HEAD") && !req.http.x-pass ) {
        unset beresp.http.set-cookie;

        # init surrogate keys
        if (beresp.http.X-Magento-Tags) {
            set beresp.http.Surrogate-Key = beresp.http.X-Magento-Tags " text";
        } else {
            set beresp.http.Surrogate-Key = "text";
        }

        # set surrogate keys by content type if they are image/script or CSS
        if (beresp.http.Content-Type ~ "(image|script|css)") {
            set beresp.http.Surrogate-Key = re.group.1;
        }
        return (deliver);
    }


Add JikeSpider to list of bot user agents

Observed user-agent was:

User-Agent: Mozilla/5.0 (compatible; JikeSpider; +http://shoulu.jike.com/spider.html)

#
# detectdevice.vcl - regex based device detection for Varnish
# http://github.com/varnish/varnish-devicedetect/
#
# Author: Lasse Karstensen <lasse@varnish-software.com>

sub devicedetect {
    unset req.http.X-UA-Device;
    set req.http.X-UA-Device = "pc";

    if (req.http.User-Agent ~ "(?i)(ads|google|bing|msn|yandex|baidu|ro|career|)bot" ||
	req.http.User-Agent ~ "(?i)(baidu|jike|symantec)spider" ||
	req.http.User-Agent ~ "(?i)scanner" ||
	req.http.User-Agent ~ "(?i)(web)crawler") {
	set req.http.X-UA-Device = "bot"; }
    elsif (req.http.User-Agent ~ "(?i)ip(hone|od)") { set req.http.X-UA-Device = "mobile-iphone"; }
    elsif (req.http.User-Agent ~ "(?i)ipad")        { set req.http.X-UA-Device = "tablet-ipad"; }
    # how do we differ between an android phone and an android tablet?
    # http://stackoverflow.com/questions/5341637/how-do-detect-android-tablets-in-general-useragent
    elsif (req.http.User-Agent ~ "(?i)android.*(mobile|mini)") { set req.http.X-UA-Device = "mobile-android"; }
    # android 3/honeycomb was just about tablet-only, and any phones will probably handle a bigger page layout.
    elsif (req.http.User-Agent ~ "(?i)android 3")              { set req.http.X-UA-Device = "tablet-android"; }
    # may very well give false positives towards android tablets. Suggestions welcome.
    elsif (req.http.User-Agent ~ "(?i)android")         { set req.http.X-UA-Device = "tablet-android"; }

    elsif (req.http.User-Agent ~ "^HTC" ||
        req.http.User-Agent ~ "Fennec" ||
        req.http.User-Agent ~ "IEMobile" ||
        req.http.User-Agent ~ "BlackBerry" ||
        req.http.User-Agent ~ "SymbianOS.*AppleWebKit" ||
        req.http.User-Agent ~ "Opera Mobi") {
        set req.http.X-UA-Device = "mobile-smartphone";
    }
    elsif (req.http.User-Agent ~ "(?i)symbian" ||
        req.http.User-Agent ~ "(?i)^sonyericsson" ||
        req.http.User-Agent ~ "(?i)^nokia" ||
        req.http.User-Agent ~ "(?i)^samsung" ||
        req.http.User-Agent ~ "(?i)^lg" ||
	req.http.User-Agent ~ "(?i)bada" ||
	req.http.User-Agent ~ "(?i)blazer" ||
	req.http.User-Agent ~ "(?i)cellphone" ||
	req.http.User-Agent ~ "(?i)iemobile" ||
	req.http.User-Agent ~ "(?i)midp-2.0" ||
	req.http.User-Agent ~ "(?i)u990" ||
	req.http.User-Agent ~ "(?i)netfront" ||
	req.http.User-Agent ~ "(?i)opera mini" ||
	req.http.User-Agent ~ "(?i)palm" ||
	req.http.User-Agent ~ "(?i)nintendo wii" ||
	req.http.User-Agent ~ "(?i)playstation portable" ||
	req.http.User-Agent ~ "(?i)portalmmm" ||
	req.http.User-Agent ~ "(?i)proxinet" ||
	req.http.User-Agent ~ "(?i)sonyericsson" ||
	req.http.User-Agent ~ "(?i)symbian" ||
	req.http.User-Agent ~ "(?i)windows\ ?ce" ||
	req.http.User-Agent ~ "(?i)winwap" ||
	req.http.User-Agent ~ "(?i)eudoraweb" ||
	req.http.User-Agent ~ "(?i)htc" ||
	req.http.User-Agent ~ "(?i)240x320" ||
	req.http.User-Agent ~ "(?i)avantgo") {
        set req.http.X-UA-Device = "mobile-generic";
    }
    # handle overrides
    if (req.http.Cookie ~ "(i?)X-UA-Device-force") {
        # ;?? means zero or one ;, non-greedy to match the first.
        set req.http.X-UA-Device = regsub(req.http.Cookie, "(?i).*X-UA-Device-force=([^;]+);??.*", "\1");
        # Clean up the cookie header to allow for caching.
        set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *X-UA-Device-force=[^;]+;? *", "\1");
        if (req.http.Cookie ~ "^ *$") { unset req.http.Cookie; }
    }
}

# vim: sw=4:tw=120

#
# detectdevice.vcl - regex based device detection for Varnish
# http://github.com/varnish/varnish-devicedetect/
#
# Author: Lasse Karstensen <lasse@varnish-software.com>

sub devicedetect {
    unset req.http.X-UA-Device;
    set req.http.X-UA-Device = "pc";

    if (req.http.User-Agent ~ "(?i)(ads|google|bing|msn|yandex|baidu|ro|career|)bot" ||
	req.http.User-Agent ~ "(?i)(baidu|symantec)spider" ||
	req.http.User-Agent ~ "(?i)scanner" ||
	req.http.User-Agent ~ "(?i)(web)crawler") {
	set req.http.X-UA-Device = "bot"; }
    elsif (req.http.User-Agent ~ "(?i)ip(hone|od)") { set req.http.X-UA-Device = "mobile-iphone"; }
    elsif (req.http.User-Agent ~ "(?i)ipad")        { set req.http.X-UA-Device = "tablet-ipad"; }
    # how do we differ between an android phone and an android tablet?
    # http://stackoverflow.com/questions/5341637/how-do-detect-android-tablets-in-general-useragent
    elsif (req.http.User-Agent ~ "(?i)android.*(mobile|mini)") { set req.http.X-UA-Device = "mobile-android"; }
    # android 3/honeycomb was just about tablet-only, and any phones will probably handle a bigger page layout.
    elsif (req.http.User-Agent ~ "(?i)android 3")              { set req.http.X-UA-Device = "tablet-android"; }
    # may very well give false positives towards android tablets. Suggestions welcome.
    elsif (req.http.User-Agent ~ "(?i)android")         { set req.http.X-UA-Device = "tablet-android"; }

    elsif (req.http.User-Agent ~ "^HTC" ||
        req.http.User-Agent ~ "Fennec" ||
        req.http.User-Agent ~ "IEMobile" ||
        req.http.User-Agent ~ "BlackBerry" ||
        req.http.User-Agent ~ "SymbianOS.*AppleWebKit" ||
        req.http.User-Agent ~ "Opera Mobi") {
        set req.http.X-UA-Device = "mobile-smartphone";
    }
    elsif (req.http.User-Agent ~ "(?i)symbian" ||
        req.http.User-Agent ~ "(?i)^sonyericsson" ||
        req.http.User-Agent ~ "(?i)^nokia" ||
        req.http.User-Agent ~ "(?i)^samsung" ||
        req.http.User-Agent ~ "(?i)^lg" ||
	req.http.User-Agent ~ "(?i)bada" ||
	req.http.User-Agent ~ "(?i)blazer" ||
	req.http.User-Agent ~ "(?i)cellphone" ||
	req.http.User-Agent ~ "(?i)iemobile" ||
	req.http.User-Agent ~ "(?i)midp-2.0" ||
	req.http.User-Agent ~ "(?i)u990" ||
	req.http.User-Agent ~ "(?i)netfront" ||
	req.http.User-Agent ~ "(?i)opera mini" ||
	req.http.User-Agent ~ "(?i)palm" ||
	req.http.User-Agent ~ "(?i)nintendo wii" ||
	req.http.User-Agent ~ "(?i)playstation portable" ||
	req.http.User-Agent ~ "(?i)portalmmm" ||
	req.http.User-Agent ~ "(?i)proxinet" ||
	req.http.User-Agent ~ "(?i)sonyericsson" ||
	req.http.User-Agent ~ "(?i)symbian" ||
	req.http.User-Agent ~ "(?i)windows\ ?ce" ||
	req.http.User-Agent ~ "(?i)winwap" ||
	req.http.User-Agent ~ "(?i)eudoraweb" ||
	req.http.User-Agent ~ "(?i)htc" ||
	req.http.User-Agent ~ "(?i)240x320" ||
	req.http.User-Agent ~ "(?i)avantgo") {
        set req.http.X-UA-Device = "mobile-generic";
    }
    # handle overrides
    if (req.http.Cookie ~ "(i?)X-UA-Device-force") {
        # ;?? means zero or one ;, non-greedy to match the first.
        set req.http.X-UA-Device = regsub(req.http.Cookie, "(?i).*X-UA-Device-force=([^;]+);??.*", "\1");
        # Clean up the cookie header to allow for caching.
        set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *X-UA-Device-force=[^;]+;? *", "\1");
        if (req.http.Cookie ~ "^ *$") { unset req.http.Cookie; }
    }
}

# vim: sw=4:tw=120


Switch from pypi.io to pypi.org (#1411)

Switch from pypi.io to pypi.org (#1411)

sub vcl_recv {

    # I'm not 100% sure on what this is exactly for, it was taken from the
    # Fastly documentation, however, what I *believe* it does is just ensure
    # that we don't serve a stale copy of the page from the shield node when
    # an edge node is requesting content.
    if (req.http.Fastly-FF) {
        set req.max_stale_while_revalidate = 0s;
    }

    # Some (Older) clients will send a hash fragment as part of the URL even
    # though that is a local only modification. This breaks this badly for the
    # files in S3, and in general it's just not needed.
    set req.url = regsub(req.url, "#.*$", "");

    # Fastly does some normalization of the Accept-Encoding header so that it
    # reduces the number of cached copies (when served with the common,
    # Vary: Accept-Encoding) that are cached for any one URL. This makes a lot
    # of sense, except for the fact that we want to enable brotli compression
    # for our static files. Thus we need to work around the normalized encoding
    # in a way that still minimizes cached copies, but which will allow our
    # static files to be served using brotli.
    if (req.url ~ "^/static/" && req.http.Fastly-Orig-Accept-Encoding) {
        if (req.http.User-Agent ~ "MSIE 6") {
            # For that 0.3% of stubborn users out there
            unset req.http.Accept-Encoding;
        } elsif (req.http.Fastly-Orig-Accept-Encoding ~ "br") {
            set req.http.Accept-Encoding = "br";
        } elsif (req.http.Fastly-Orig-Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } else {
            unset req.http.Accept-Encoding;
        }
    }

    # Most of the URLs in Warehouse do not support or require any sort of query
    # parameter. If we strip these at the edge then we'll increase our cache
    # efficiency when they won't otherwise change the output of the pages.
    #
    # This will match any URL except those that start with:
    #
    #   * /search/
    #   * /account/login/
    #   * /account/logout/
    #   * /account/register/
    #   * /pypi
    if (req.url !~ "^/(search/|account/(login|logout|register)/|pypi)") {
        set req.url = req.url.path;
    }

    # Sort all of our query parameters, this will ensure that the same query
    # parameters in a different order will end up being represented as the same
    # thing, reducing cache misses due to ordering differences.
    set req.url = boltsort.sort(req.url);

#FASTLY recv

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Redirect pypi.io, www.pypi.io, and warehouse.python.org to pypi.io, this
    # is purposely done *after* the HTTPS checks.
    if (std.tolower(req.http.host) ~ "^(www.pypi.org|(www.)?pypi.io|warehouse.python.org)$") {
        set req.http.Location = "https://pypi.org" req.url;
        error 750 "Redirect to Primary Domain";
    }
    # Redirect warehouse-staging.python.org to test.pypi.io.
    if (std.tolower(req.http.host) ~ "^(test.pypi.io|warehouse-staging.python.org)$") {
        set req.http.Location = "https://test.pypi.org" req.url;
        error 750 "Redirect to Primary Domain";
    }

    # Requests to /packages/ get dispatched to Amazon instead of to our typical
    # Origin. This requires a a bit of setup to make it work.
    if (req.http.host ~ "^(test-)?files.pythonhosted.org$"
            && req.url ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/") {
        # Setup our environment to better match what S3 expects/needs
        set req.http.Host = req.http.AWS-Bucket-Name ".s3.amazonaws.com";
        set req.http.Date = now;
        set req.url = regsuball(req.url, "\+", urlencode("+"));

        # Compute the Authorization header that S3 requires to be able to
        # access the files stored there.
        set req.http.Authorization = "AWS " req.http.AWS-Access-Key-ID ":" digest.hmac_sha1_base64(req.http.AWS-Secret-Access-Key, "GET" LF LF LF req.http.Date LF "/" req.http.AWS-Bucket-Name req.url.path);

        # We don't want to send our Warehouse-Token to S3, so we'll go ahead
        # and remove it.
        unset req.http.Warehouse-Token;
    }

    # We no longer need any of these variables, which would exist only to
    # shuffle configuration from the Fastly UI into our VCL.
    unset req.http.AWS-Access-Key-ID;
    unset req.http.AWS-Secret-Access-Key;
    unset req.http.AWS-Bucket-Name;

    # We have a number of items that we'll pass back to the origin, but only
    # if we have a Warehouse-Token that will allow them to be accepted.
    if (req.http.Warehouse-Token) {
        # Set a header to tell the backend if we're using https or http.
        if (req.http.Fastly-SSL) {
            set req.http.Warehouse-Proto = "https";
        } else {
            set req.http.Warehouse-Proto = "http";
        }

        # Pass the client IP address back to the backend.
        if (req.http.Fastly-Client-IP) {
            set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
        }

        # Pass the real host value back to the backend.
        if (req.http.Host) {
            set req.http.Warehouse-Host = req.http.host;
        }
    }

    # On a POST, we want to skip the shielding and hit backends directly.
    if (req.request == "POST") {
        set req.backend = F_Heroku;
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # We don't ever want to cache our health URL. Outside systems should be
    # able to use it to reach past Fastly and get an end to end health check.
    if (req.url == "/_health/") {
        return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {

    # These are newer kinds of redirects which should be able to be cached by
    # default, even though Fastly doesn't currently have them in their default
    # list of cacheable status codes.
    if (http_status_matches(beresp.status, "303,307,308")) {
        set beresp.cacheable = true;
    }

    # For any 5xx status code we want to see if a stale object exists for it,
    # if so we'll go ahead and serve it.
    if (beresp.status >= 500 && beresp.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    # When delivering a 304 response, we don't always have access to all the
    # headers in the resp because a 304 response is supposed to remove most of
    # the headers. So we'll instead stash these headers on the request so that
    # we can log this data from there instead of from the response.
    if (beresp.http.x-amz-meta-project
            || beresp.http.x-amz-meta-version
            || beresp.http.x-amz-meta-package-type) {
        set req.http.Fastly-amz-meta-project = beresp.http.x-amz-meta-project;
        set req.http.Fastly-amz-meta-version = beresp.http.x-amz-meta-version;
        set req.http.Fastly-amz-meta-package-type = beresp.http.x-amz-meta-package-type;
    }

#FASTLY fetch

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 502 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 502 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (http_status_matches(beresp.status, "500,502,503")) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_hit {
#FASTLY hit

    # If the object we have isn't cacheable, then just serve it directly
    # without going through any of the caching mechanisms.
    if (!obj.cacheable) {
        return(pass);
    }

    return(deliver);
}

sub vcl_deliver {
    # If this is an error and we have a stale response available, restart so
    # that we can pick it up and serve it.
    if (resp.status >= 500 && resp.status < 600) {
        if (stale.exists) {
            restart;
        }
    }

#FASTLY deliver

    # Unset headers that we don't need/want to send on to the client because
    # they are not generally useful.
    unset resp.http.Server;
    unset resp.http.Via;

    # Unset a few headers set by Amazon that we don't really have a need/desire
    # to send to clients.
    unset resp.http.X-AMZ-Replication-Status;
    unset resp.http.X-AMZ-Meta-Python-Version;
    unset resp.http.X-AMZ-Meta-Version;
    unset resp.http.X-AMZ-Meta-Package-Type;
    unset resp.http.X-AMZ-Meta-Project;

    # Set our standard security headers, we do this in VCL rather than in
    # Warehouse itself so that we always get these headers, regardless of the
    # origin server being used.
    set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload";
    set resp.http.X-Frame-Options = "deny";
    set resp.http.X-XSS-Protection = "1; mode=block";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-Permitted-Cross-Domain-Policies = "none";

    # Unstash our information about what project/version/package-type a
    # particular file download was for.
    if (req.http.Fastly-amz-meta-project
            || req.http.Fastly-amz-meta-version
            || req.http.Fastly-amz-meta-package-type) {
        set resp.http.x-amz-meta-project = req.http.Fastly-amz-meta-project;
        set resp.http.x-amz-meta-version = req.http.Fastly-amz-meta-version;
        set resp.http.x-amz-meta-package-type = req.http.Fastly-amz-meta-package-type;
    }

    # If we're not executing a shielding request, and the URL is one of our file
    # URLs, and it's a GET request, and the response is either a 200 or a 304
    # then we want to log an event stating that a download has taken place.
    if (!req.http.Fastly-FF
            && std.tolower(req.http.host) == "files.pythonhosted.org"
            && req.url.path ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/"
            && req.request == "GET"
            && http_status_matches(resp.status, "200,304")) {
        log {"syslog "} req.service_id {" linehaul :: "} "2@" now "|" geoip.country_code "|" req.url.path "|" tls.client.protocol "|" tls.client.cipher "|" resp.http.x-amz-meta-project "|" resp.http.x-amz-meta-version "|" resp.http.x-amz-meta-package-type "|" req.http.user-agent;
        log {"syslog "} req.service_id {" downloads :: "} "2@" now "|" geoip.country_code "|" req.url.path "|" tls.client.protocol "|" tls.client.cipher "|" resp.http.x-amz-meta-project "|" resp.http.x-amz-meta-version "|" resp.http.x-amz-meta-package-type "|" req.http.user-agent;
    }

    return(deliver);
}

sub vcl_error {
#FASTLY error

    # If we have a 5xx error and there is a stale object available, then we
    # will deliver that stale object.
    if (obj.status >= 500 && obj.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    } else if (obj.status == 750) {
        set obj.status = 301;
        set obj.http.Location = req.http.Location;
        set obj.http.Content-Type = "text/html; charset=UTF-8";
        synthetic {"<title>301 Moved Permanently</title>
301 Moved Permanently
"};
        return(deliver);
    }

}

sub vcl_recv {

    # I'm not 100% sure on what this is exactly for, it was taken from the
    # Fastly documentation, however, what I *believe* it does is just ensure
    # that we don't serve a stale copy of the page from the shield node when
    # an edge node is requesting content.
    if (req.http.Fastly-FF) {
        set req.max_stale_while_revalidate = 0s;
    }

    # Some (Older) clients will send a hash fragment as part of the URL even
    # though that is a local only modification. This breaks this badly for the
    # files in S3, and in general it's just not needed.
    set req.url = regsub(req.url, "#.*$", "");

    # Fastly does some normalization of the Accept-Encoding header so that it
    # reduces the number of cached copies (when served with the common,
    # Vary: Accept-Encoding) that are cached for any one URL. This makes a lot
    # of sense, except for the fact that we want to enable brotli compression
    # for our static files. Thus we need to work around the normalized encoding
    # in a way that still minimizes cached copies, but which will allow our
    # static files to be served using brotli.
    if (req.url ~ "^/static/" && req.http.Fastly-Orig-Accept-Encoding) {
        if (req.http.User-Agent ~ "MSIE 6") {
            # For that 0.3% of stubborn users out there
            unset req.http.Accept-Encoding;
        } elsif (req.http.Fastly-Orig-Accept-Encoding ~ "br") {
            set req.http.Accept-Encoding = "br";
        } elsif (req.http.Fastly-Orig-Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } else {
            unset req.http.Accept-Encoding;
        }
    }

    # Most of the URLs in Warehouse do not support or require any sort of query
    # parameter. If we strip these at the edge then we'll increase our cache
    # efficiency when they won't otherwise change the output of the pages.
    #
    # This will match any URL except those that start with:
    #
    #   * /search/
    #   * /account/login/
    #   * /account/logout/
    #   * /account/register/
    #   * /pypi
    if (req.url !~ "^/(search/|account/(login|logout|register)/|pypi)") {
        set req.url = req.url.path;
    }

    # Sort all of our query parameters, this will ensure that the same query
    # parameters in a different order will end up being represented as the same
    # thing, reducing cache misses due to ordering differences.
    set req.url = boltsort.sort(req.url);

#FASTLY recv

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Redirect www.pypi.io and warehouse.python.org to pypi.io, this is
    # purposely done *after* the HTTPS checks.
    if (std.tolower(req.http.host) ~ "^(www.pypi.io|warehouse.python.org)$") {
        set req.http.Location = "https://pypi.io" req.url;
        error 750 "Redirect to Primary Domain";
    }
    # Redirect warehouse-staging.python.org to test.pypi.io.
    if (std.tolower(req.http.host) ~ "^(test.pypi.io|warehouse-staging.python.org)$") {
        set req.http.Location = "https://test.pypi.org" req.url;
        error 750 "Redirect to Primary Domain";
    }

    # Requests to /packages/ get dispatched to Amazon instead of to our typical
    # Origin. This requires a a bit of setup to make it work.
    if (req.http.host ~ "^(test-)?files.pythonhosted.org$"
            && req.url ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/") {
        # Setup our environment to better match what S3 expects/needs
        set req.http.Host = req.http.AWS-Bucket-Name ".s3.amazonaws.com";
        set req.http.Date = now;
        set req.url = regsuball(req.url, "\+", urlencode("+"));

        # Compute the Authorization header that S3 requires to be able to
        # access the files stored there.
        set req.http.Authorization = "AWS " req.http.AWS-Access-Key-ID ":" digest.hmac_sha1_base64(req.http.AWS-Secret-Access-Key, "GET" LF LF LF req.http.Date LF "/" req.http.AWS-Bucket-Name req.url.path);

        # We don't want to send our Warehouse-Token to S3, so we'll go ahead
        # and remove it.
        unset req.http.Warehouse-Token;
    }

    # We no longer need any of these variables, which would exist only to
    # shuffle configuration from the Fastly UI into our VCL.
    unset req.http.AWS-Access-Key-ID;
    unset req.http.AWS-Secret-Access-Key;
    unset req.http.AWS-Bucket-Name;

    # We have a number of items that we'll pass back to the origin, but only
    # if we have a Warehouse-Token that will allow them to be accepted.
    if (req.http.Warehouse-Token) {
        # Set a header to tell the backend if we're using https or http.
        if (req.http.Fastly-SSL) {
            set req.http.Warehouse-Proto = "https";
        } else {
            set req.http.Warehouse-Proto = "http";
        }

        # Pass the client IP address back to the backend.
        if (req.http.Fastly-Client-IP) {
            set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
        }

        # Pass the real host value back to the backend.
        if (req.http.Host) {
            set req.http.Warehouse-Host = req.http.host;
        }
    }

    # On a POST, we want to skip the shielding and hit backends directly.
    if (req.request == "POST") {
        set req.backend = F_Heroku;
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # We don't ever want to cache our health URL. Outside systems should be
    # able to use it to reach past Fastly and get an end to end health check.
    if (req.url == "/_health/") {
        return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {

    # These are newer kinds of redirects which should be able to be cached by
    # default, even though Fastly doesn't currently have them in their default
    # list of cacheable status codes.
    if (http_status_matches(beresp.status, "303,307,308")) {
        set beresp.cacheable = true;
    }

    # For any 5xx status code we want to see if a stale object exists for it,
    # if so we'll go ahead and serve it.
    if (beresp.status >= 500 && beresp.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    # When delivering a 304 response, we don't always have access to all the
    # headers in the resp because a 304 response is supposed to remove most of
    # the headers. So we'll instead stash these headers on the request so that
    # we can log this data from there instead of from the response.
    if (beresp.http.x-amz-meta-project
            || beresp.http.x-amz-meta-version
            || beresp.http.x-amz-meta-package-type) {
        set req.http.Fastly-amz-meta-project = beresp.http.x-amz-meta-project;
        set req.http.Fastly-amz-meta-version = beresp.http.x-amz-meta-version;
        set req.http.Fastly-amz-meta-package-type = beresp.http.x-amz-meta-package-type;
    }

#FASTLY fetch

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 502 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 502 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (http_status_matches(beresp.status, "500,502,503")) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_hit {
#FASTLY hit

    # If the object we have isn't cacheable, then just serve it directly
    # without going through any of the caching mechanisms.
    if (!obj.cacheable) {
        return(pass);
    }

    return(deliver);
}

sub vcl_deliver {
    # If this is an error and we have a stale response available, restart so
    # that we can pick it up and serve it.
    if (resp.status >= 500 && resp.status < 600) {
        if (stale.exists) {
            restart;
        }
    }

#FASTLY deliver

    # Unset headers that we don't need/want to send on to the client because
    # they are not generally useful.
    unset resp.http.Server;
    unset resp.http.Via;

    # Unset a few headers set by Amazon that we don't really have a need/desire
    # to send to clients.
    unset resp.http.X-AMZ-Replication-Status;
    unset resp.http.X-AMZ-Meta-Python-Version;
    unset resp.http.X-AMZ-Meta-Version;
    unset resp.http.X-AMZ-Meta-Package-Type;
    unset resp.http.X-AMZ-Meta-Project;

    # Set our standard security headers, we do this in VCL rather than in
    # Warehouse itself so that we always get these headers, regardless of the
    # origin server being used.
    set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload";
    set resp.http.X-Frame-Options = "deny";
    set resp.http.X-XSS-Protection = "1; mode=block";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-Permitted-Cross-Domain-Policies = "none";

    # Unstash our information about what project/version/package-type a
    # particular file download was for.
    if (req.http.Fastly-amz-meta-project
            || req.http.Fastly-amz-meta-version
            || req.http.Fastly-amz-meta-package-type) {
        set resp.http.x-amz-meta-project = req.http.Fastly-amz-meta-project;
        set resp.http.x-amz-meta-version = req.http.Fastly-amz-meta-version;
        set resp.http.x-amz-meta-package-type = req.http.Fastly-amz-meta-package-type;
    }

    # If we're not executing a shielding request, and the URL is one of our file
    # URLs, and it's a GET request, and the response is either a 200 or a 304
    # then we want to log an event stating that a download has taken place.
    if (!req.http.Fastly-FF
            && std.tolower(req.http.host) == "files.pythonhosted.org"
            && req.url.path ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/"
            && req.request == "GET"
            && http_status_matches(resp.status, "200,304")) {
        log {"syslog "} req.service_id {" linehaul :: "} "2@" now "|" geoip.country_code "|" req.url.path "|" tls.client.protocol "|" tls.client.cipher "|" resp.http.x-amz-meta-project "|" resp.http.x-amz-meta-version "|" resp.http.x-amz-meta-package-type "|" req.http.user-agent;
        log {"syslog "} req.service_id {" downloads :: "} "2@" now "|" geoip.country_code "|" req.url.path "|" tls.client.protocol "|" tls.client.cipher "|" resp.http.x-amz-meta-project "|" resp.http.x-amz-meta-version "|" resp.http.x-amz-meta-package-type "|" req.http.user-agent;
    }

    return(deliver);
}

sub vcl_error {
#FASTLY error

    # If we have a 5xx error and there is a stale object available, then we
    # will deliver that stale object.
    if (obj.status >= 500 && obj.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    } else if (obj.status == 750) {
        set obj.status = 301;
        set obj.http.Location = req.http.Location;
        set obj.http.Content-Type = "text/html; charset=UTF-8";
        synthetic {"<title>301 Moved Permanently</title>
301 Moved Permanently
"};
        return(deliver);
    }

}


Various fixes

Various fixes

Spelling. While here also fix indentation and make sure we are using
the right name for the variables and not the backward compat ones.

/*-
 * Copyright (c) 2006 Verdens Gang AS
 * Copyright (c) 2006-2014 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *

 *
 * The built-in (previously called default) VCL code.
 *
 * NB! You do NOT need to copy & paste all of these functions into your
 * own vcl code, if you do not provide a definition of one of these
 * functions, the compiler will automatically fall back to the default
 * code from this file.
 *
 * This code will be prefixed with a backend declaration built from the
 * -b argument.
 */

vcl 4.0;

sub vcl_recv {
    if (req.restarts == 0) {
	if (req.http.x-forwarded-for) {
	    set req.http.X-Forwarded-For =
		req.http.X-Forwarded-For + ", " + client.ip;
	} else {
	    set req.http.X-Forwarded-For = client.ip;
	}
    }
    if (req.method != "GET" &&
      req.method != "HEAD" &&
      req.method != "PUT" &&
      req.method != "POST" &&
      req.method != "TRACE" &&
      req.method != "OPTIONS" &&
      req.method != "DELETE") {
        /* Non-RFC2616 or CONNECT which is weird. */
        return (pipe);
    }
    if (req.method != "GET" && req.method != "HEAD") {
        /* We only deal with GET and HEAD by default */
        return (pass);
    }
    if (req.http.Authorization || req.http.Cookie) {
        /* Not cacheable by default */
        return (pass);
    }
    return (hash);
}

sub vcl_pipe {
    # Note that only the first request to the backend will have
    # X-Forwarded-For set.  If you use X-Forwarded-For and want to
    # have it set for all requests, make sure to have:
    # set bereq.http.connection = "close";
    # here.  It is not set by default as it might break some broken web
    # applications, like IIS with NTLM authentication.
    return (pipe);
}

sub vcl_pass {
    return (fetch);
}

sub vcl_hash {
    hash_data(req.url);
    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }
    return (lookup);
}

sub vcl_purge {
    return (error(200, "Purged"));
}

sub vcl_hit {
    if (obj.ttl >= 0s) {
	// A pure unadultered hit, deliver it
	return (deliver);
    }
    if (obj.ttl + obj.grace > 0s) {
	// Object is in grace, deliver it
	// Automatically triggers a background fetch
	return (deliver);
    }
    // fetch & deliver once we get the result
    return (fetch);
}

sub vcl_miss {
    return (fetch);
}

sub vcl_backend_fetch {
    return (fetch);
}

sub vcl_backend_response {
    if (beresp.ttl <= 0s ||
        beresp.http.Set-Cookie ||
        beresp.http.Surrogate-control ~ "no-store" ||
        (!beresp.http.Surrogate-Control &&
          beresp.http.Cache-Control ~ "no-cache|no-store|private") ||
        beresp.http.Vary == "*") {
		/*
		 * Mark as "Hit-For-Pass" for the next 2 minutes
		 */
		set beresp.ttl = 120s;
		set beresp.uncacheable = true;
    }
    return (deliver);
}

sub vcl_deliver {
    /*
     * These two write to the stored object causing extra page faults
     * Enable them only if you need them.
     *
     * set obj.hits = obj.hits + 1;
     * set obj.last_use = now;
     */
    return (deliver);
}

/*
 * We can come here "invisibly" with the following errors: 413, 417 & 503
 */
sub vcl_error {
    set obj.http.Content-Type = "text/html; charset=utf-8";
    set obj.http.Retry-After = "5";
    synthetic {"
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

    <title>"} + obj.status + " " + obj.reason + {"</title>

Error "} + obj.status + " " + obj.reason + {"

"} + obj.reason + {"

Guru Meditation:

XID: "} + req.xid + {"

Varnish cache server

"};
    return (deliver);
}

sub vcl_init {
    return (ok);
}

sub vcl_fini {
    return (ok);
}

/*-
 * Copyright (c) 2006 Verdens Gang AS
 * Copyright (c) 2006-2014 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *

 *
 * The built-in (previously called default) VCL code.
 *
 * NB! You do NOT need to copy & paste all of these functions into your
 * own vcl code, if you do not provide a definition of one of these
 * functions, the compiler will automatically fall back to the default
 * code from this file.
 *
 * This code will be prefixed with a backend declaration built from the
 * -b argument.
 */

vcl 4.0;

sub vcl_recv {
    if (req.restarts == 0) {
	if (req.http.x-forwarded-for) {
	    set req.http.X-Forwarded-For =
		req.http.X-Forwarded-For + ", " + client.ip;
	} else {
	    set req.http.X-Forwarded-For = client.ip;
	}
    }
    if (req.method != "GET" &&
      req.method != "HEAD" &&
      req.method != "PUT" &&
      req.method != "POST" &&
      req.method != "TRACE" &&
      req.method != "OPTIONS" &&
      req.method != "DELETE") {
        /* Non-RFC2616 or CONNECT which is weird. */
        return (pipe);
    }
    if (req.method != "GET" && req.method != "HEAD") {
        /* We only deal with GET and HEAD by default */
        return (pass);
    }
    if (req.http.Authorization || req.http.Cookie) {
        /* Not cacheable by default */
        return (pass);
    }
    return (hash);
}

sub vcl_pipe {
    # Note that only the first request to the backend will have
    # X-Forwarded-For set.  If you use X-Forwarded-For and want to
    # have it set for all requests, make sure to have:
    # set bereq.http.connection = "close";
    # here.  It is not set by default as it might break some broken web
    # applications, like IIS with NTLM authentication.
    return (pipe);
}

sub vcl_pass {
    return (fetch);
}

sub vcl_hash {
    hash_data(req.url);
    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }
    return (lookup);
}

sub vcl_purge {
    return (error(200, "Purged"));
}

sub vcl_hit {
    if (obj.ttl >= 0s) {
	// A pure unadultered hit, deliver it
	return (deliver);
    }
    if (obj.ttl + obj.grace > 0s) {
	// Object is in grace, delive it
	// Automatically triggers a background fetch
	return (deliver);
    }
    // fetch & deliver once we get the result
    return (fetch);
}

sub vcl_miss {
    return (fetch);
}

sub vcl_backend_fetch {
    return (fetch);
}

sub vcl_backend_response {
    if (beresp.ttl <= 0s ||
        beresp.http.Set-Cookie ||
        beresp.http.Surrogate-control ~ "no-store" ||
        (!beresp.http.Surrogate-Control &&
          beresp.http.Cache-Control ~ "no-cache|no-store|private") ||
        beresp.http.Vary == "*") {
		/*
		 * Mark as "Hit-For-Pass" for the next 2 minutes
		 */
		set beresp.ttl = 120 s;
		set beresp.uncacheable = true;
    }
    return (deliver);
}

sub vcl_deliver {
    /*
     * These two write to the stored object causing extra page faults
     * Enable them only if you need them.
     *
     * set obj.hits = obj.hits + 1;
     * set obj.last_use = now;
     */
    return (deliver);
}

/*
 * We can come here "invisibly" with the following errors:  413, 417 & 503
 */
sub vcl_error {
    set obj.http.Content-Type = "text/html; charset=utf-8";
    set obj.http.Retry-After = "5";
    synthetic {"
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

    <title>"} + obj.status + " " + obj.response + {"</title>

Error "} + obj.status + " " + obj.response + {"

"} + obj.response + {"

Guru Meditation:

XID: "} + req.xid + {"

Varnish cache server

"};
    return (deliver);
}

sub vcl_init {
	return (ok);
}

sub vcl_fini {
	return (ok);
}


Added a test command

Added a test command

# Voice commands for atom

Test Command = "This Is a Test Command";

include folders.vch;
include letters.vch;

# Commands for developing D3: D3 visualizations, D3 Sandwiches
Save and Reload =  {Ctrl+s}  Wait(100) SendSystemKeys("{Alt+Tab}")  {Ctrl+r};
Save and Really Reload =  {Ctrl+s}  Wait(100) SendSystemKeys("{Alt+Tab}") {Ctrl+Shift+r};
Save and Run =  {Ctrl+s}  Wait(100) SendSystemKeys("{Alt+Tab}")  {Up}{Enter};

# File commands
Find (File | Document) = {Ctrl+t};
Save As = {Ctrl+Shift+s};
Next Window = {Ctrl+Tab};
Last Window = {Ctrl+Shift+Tab};
Latest File = {Ctrl+Shift+t};
(Tree View | List Files) = {Alt+o};	# NOTE: to make this work, I had to create a new keyboard shortcut in Atom (my personal keymap.cson)

# --- Text navigation commands ------------------------------------
<delimiters> := (Quote = '"' | 'Double Quote' = '"' | 'Single Quote' = "'" | Comma = ',' | 'Equal Sign' = '=' | Equals = '='
	| Period = '.' | Colon = ':' | 'Semi-Colon' = ';' | Hyphen = '-' |  Underscore = '_' | Slash = '/'
	| Paren = "(" | "Close Paren" = ")" | Parentheses = "(" | "Open Parenthesis" = "(" | "Close Parentheses" = ")"
	| Bracket = '[' | "Close Bracket" = ']' | Brace = '{' | 'Close Brace' = '}'	);

Line 1..200 = {Ctrl+g} $1 {Enter};
(Left = 'b' | Right = 'f') Word = {Alt+$1};
Top = {Ctrl+Home};
Bottom = {Ctrl+End};
(Find = 'Right' | 'Insert After' = 'Right' | 'Insert Before' = 'Left') (Next = 'Enter' | Last = 'Shift+F3') <delimiters>
		= {Ctrl+f} $3 {$2} {Esc}   Wait(100) {$1};
Insert (Before = 'Left' | After = 'Right') <delimiters> =  {Ctrl+f} $2 {Enter} {Esc}   Wait(100) {$1};
Insert (Before = 'Left' | After = 'Right') 1..20 <delimiters> =  {Ctrl+f} $3 {Enter} {Esc}   Wait(100) {F3_$2} {Shift+F3} Wait(100) {$1};
Insert (Before = 'Left' | After = 'Right') Enda <delimiters> = {End} {Ctrl+f} $2 {Enter} {Esc} {Shift+F3}  Wait(100) {$1};

Fine Alpha <letters> = {Ctrl+f} $1 {Enter} {Esc}   Wait(100) {Left};
Insert (Before = 'Left' | After = 'Right') Alpha <letters> =  {Ctrl+f} $2 {Enter} {Esc}   Wait(100) {$1};
Fine Number 0..9 = {Ctrl+f} $1 {Enter} {Esc} Wait(100) {Right};
(Find | Insert) Again = {F3} {Right};
(Find | Insert) Again 1..20 Times = {F3_$1} {Right};

(Set | Add | Clear | Hide | Delete) Bookmark = {Ctrl+Alt+F2};
(Next = 'F2' | Last = 'Shift+F2') Bookmark = {$1};

# --- Editing commands ----------------------------------------
Select Line = {Ctrl+l};
Select (Next = 'Down' | Forward = 'Down' | Last = 'Up' | Back = 'Up' | Previous = 'Up') 1..20 Lines = {Home} {Shift+$1_$2};
Move Line (Up | Down) 1..20  = {Ctrl+$1_$2};
Add 1..20 (Lines = 'Down' | Line = 'Down' | Pages = 'PgDn' | Page = 'PgDn') = {Shift+$2_$1};
Start Adding 1..20 Lines = {Home}{Shift+Down_$1};

Select between (Delimiters | Brackets | Tags) ={Ctrl+m} {Ctrl+Alt+m};
Delete between Tags = {Ctrl+m} {Ctrl+Alt+m}  {Del};
Delete Tags = {Ctrl+Alt+F4};
Replace With (Header | Heading) 1..7 = {Ctrl+Alt+F4}  '<h' $2 '>' {End} '</h' $2 '>' {Home};
Replace With (para = 'p') = {Ctrl+Alt+F4} '<' $1 '>' {End} '</' $1 '>';

Delete Word = {Ctrl+Del};
Delete 1..20 Words = {Ctrl+Del_$1};
Delete (Back | Last) Word = {Ctrl+Backspace};
Delete (Back | Last) 1..20 Words = {Ctrl+Backspace_$2};
Delete Enda Word = {End} {Ctrl+Backspace};
Kill Word = {Ctrl+Backspace}{Ctrl+Del};

(Delete Line | Dede) = {Ctrl+Shift+k};
Delete 1..20 Lines = {Ctrl+Shift+k_$1};
(Cut = 'x' | Paste = 'v' | Copy = 'c') That = {Ctrl+$1};

Comment (That | Line) = {Ctrl+/};
Comment 1..20 Lines = {Home} {Shift+Down_$1} {Ctrl+/};

Replace = {Ctrl+f};
Replace All = {Ctrl+Enter};

(Join = 'j' | Indent = ']' | Dedent = '[')  That = {Ctrl+$1};

# --- Commands to create code/tags -------------------------------------------
# NOTE: basic HTML commands are stored in my global Vocala commands

Start (	# HTML commands
	'HTML page' = 'html' | pre = "pre" | 'Comment' = 'comment-html'
	| div = 'div'

	# D3 Recipe commands
	| 'Recipe Page' = 'recipe-page' | 'Recipe Row' = 'recipe-row'
	| 'Recipe Toy' = 'recipe-toy' | Toy = 'recipe-toy'
	| 'Bold ID'

	# D3 and JavaScript commands
	| Console = 'console' | Log = 'console'
	| Var = 'variable' | Variable = 'variable'
	# Coffeescript commands
	| Snippet = 'my-snippet'
	| 'Fix Line' = 'fix-current-line' | 'Fix Current Line' = 'fix-current-line'
) = $1 {Tab};

Stop (	#HTML commands
	'div comment' = 'comment-div-end' |  Comment = 'comment-html-end'
) = $1 {Tab};

# HTML commands
Dot HTML = '.html';
Start (Header | Heading) 1..7 = h $2 {Tab};
Add (Header | Heading) 1..7 =  '<h' $2 '>' {End} '</h' $2 '>';
Add (para = 'p') =  '<' $1 '>' {End} '</' $1 '>';

# D3 Recipe commands
Pre-Format Code =  {Ctrl+Alt+F5};
Reload Atom = {Ctrl+s} Wait(100)  {Shift+Ctrl+Alt+F5};
Command Palette = {Ctrl+Shift+p};
Keyboard Shortcuts = {Ctrl+.};
Run Specs = {Ctrl+Alt+p};
Show Console = {Ctrl+Alt+i};

# D3 commands
Paste Function = 'function ' {Ctrl+v} ' {' {Enter} {Enter} '};' {Up};

# Coffeescript commands

# Python commands

# --- D3 commands -------------------------------------------

# --- Python commands -------------------------------------------

# --- Coffeescript commands -------------------------------------------

# Voice commands for atom

include folders.vch;
include letters.vch;

# Commands for developing D3: D3 visualizations, D3 Sandwiches
Save and Reload =  {Ctrl+s}  Wait(100) SendSystemKeys("{Alt+Tab}")  {Ctrl+r};
Save and Really Reload =  {Ctrl+s}  Wait(100) SendSystemKeys("{Alt+Tab}") {Ctrl+Shift+r};
Save and Run =  {Ctrl+s}  Wait(100) SendSystemKeys("{Alt+Tab}")  {Up}{Enter};

# File commands
Find (File | Document) = {Ctrl+t};
Save As = {Ctrl+Shift+s};
Next Window = {Ctrl+Tab};
Last Window = {Ctrl+Shift+Tab};
Latest File = {Ctrl+Shift+t};
(Tree View | List Files) = {Alt+o};	# NOTE: to make this work, I had to create a new keyboard shortcut in Atom (my personal keymap.cson)

# --- Text navigation commands ------------------------------------
<delimiters> := (Quote = '"' | 'Double Quote' = '"' | 'Single Quote' = "'" | Comma = ',' | 'Equal Sign' = '=' | Equals = '='
	| Period = '.' | Colon = ':' | 'Semi-Colon' = ';' | Hyphen = '-' |  Underscore = '_' | Slash = '/'
	| Paren = "(" | "Close Paren" = ")" | Parentheses = "(" | "Open Parenthesis" = "(" | "Close Parentheses" = ")"
	| Bracket = '[' | "Close Bracket" = ']' | Brace = '{' | 'Close Brace' = '}'	);

Line 1..200 = {Ctrl+g} $1 {Enter};
(Left = 'b' | Right = 'f') Word = {Alt+$1};
Top = {Ctrl+Home};
Bottom = {Ctrl+End};
(Find = 'Right' | 'Insert After' = 'Right' | 'Insert Before' = 'Left') (Next = 'Enter' | Last = 'Shift+F3') <delimiters>
		= {Ctrl+f} $3 {$2} {Esc}   Wait(100) {$1};
Insert (Before = 'Left' | After = 'Right') <delimiters> =  {Ctrl+f} $2 {Enter} {Esc}   Wait(100) {$1};
Insert (Before = 'Left' | After = 'Right') 1..20 <delimiters> =  {Ctrl+f} $3 {Enter} {Esc}   Wait(100) {F3_$2} {Shift+F3} Wait(100) {$1};
Insert (Before = 'Left' | After = 'Right') Enda <delimiters> = {End} {Ctrl+f} $2 {Enter} {Esc} {Shift+F3}  Wait(100) {$1};

Fine Alpha <letters> = {Ctrl+f} $1 {Enter} {Esc}   Wait(100) {Left};
Insert (Before = 'Left' | After = 'Right') Alpha <letters> =  {Ctrl+f} $2 {Enter} {Esc}   Wait(100) {$1};
Fine Number 0..9 = {Ctrl+f} $1 {Enter} {Esc} Wait(100) {Right};
(Find | Insert) Again = {F3} {Right};
(Find | Insert) Again 1..20 Times = {F3_$1} {Right};

(Set | Add | Clear | Hide | Delete) Bookmark = {Ctrl+Alt+F2};
(Next = 'F2' | Last = 'Shift+F2') Bookmark = {$1};

# --- Editing commands ----------------------------------------
Select Line = {Ctrl+l};
Select (Next = 'Down' | Forward = 'Down' | Last = 'Up' | Back = 'Up' | Previous = 'Up') 1..20 Lines = {Home} {Shift+$1_$2};
Move Line (Up | Down) 1..20  = {Ctrl+$1_$2};
Add 1..20 (Lines = 'Down' | Line = 'Down' | Pages = 'PgDn' | Page = 'PgDn') = {Shift+$2_$1};
Start Adding 1..20 Lines = {Home}{Shift+Down_$1};

Select between (Delimiters | Brackets | Tags) ={Ctrl+m} {Ctrl+Alt+m};
Delete between Tags = {Ctrl+m} {Ctrl+Alt+m}  {Del};
Delete Tags = {Ctrl+Alt+F4};
Replace With (Header | Heading) 1..7 = {Ctrl+Alt+F4}  '<h' $2 '>' {End} '</h' $2 '>' {Home};
Replace With (para = 'p') = {Ctrl+Alt+F4} '<' $1 '>' {End} '</' $1 '>';

Delete Word = {Ctrl+Del};
Delete 1..20 Words = {Ctrl+Del_$1};
Delete (Back | Last) Word = {Ctrl+Backspace};
Delete (Back | Last) 1..20 Words = {Ctrl+Backspace_$2};
Delete Enda Word = {End} {Ctrl+Backspace};
Kill Word = {Ctrl+Backspace}{Ctrl+Del};

(Delete Line | Dede) = {Ctrl+Shift+k};
Delete 1..20 Lines = {Ctrl+Shift+k_$1};
(Cut = 'x' | Paste = 'v' | Copy = 'c') That = {Ctrl+$1};

Comment (That | Line) = {Ctrl+/};
Comment 1..20 Lines = {Home} {Shift+Down_$1} {Ctrl+/};

Replace = {Ctrl+f};
Replace All = {Ctrl+Enter};

(Join = 'j' | Indent = ']' | Dedent = '[')  That = {Ctrl+$1};

# --- Commands to create code/tags -------------------------------------------
# NOTE: basic HTML commands are stored in my global Vocala commands

Start (	# HTML commands
	'HTML page' = 'html' | pre = "pre" | 'Comment' = 'comment-html'
	| div = 'div'

	# D3 Recipe commands
	| 'Recipe Page' = 'recipe-page' | 'Recipe Row' = 'recipe-row'
	| 'Recipe Toy' = 'recipe-toy' | Toy = 'recipe-toy'
	| 'Bold ID'

	# D3 and JavaScript commands
	| Console = 'console' | Log = 'console'
	| Var = 'variable' | Variable = 'variable'
	# Coffeescript commands
	| Snippet = 'my-snippet'
	| 'Fix Line' = 'fix-current-line' | 'Fix Current Line' = 'fix-current-line'
) = $1 {Tab};

Stop (	#HTML commands
	'div comment' = 'comment-div-end' |  Comment = 'comment-html-end'
) = $1 {Tab};

# HTML commands
Dot HTML = '.html';
Start (Header | Heading) 1..7 = h $2 {Tab};
Add (Header | Heading) 1..7 =  '<h' $2 '>' {End} '</h' $2 '>';
Add (para = 'p') =  '<' $1 '>' {End} '</' $1 '>';

# D3 Recipe commands
Pre-Format Code =  {Ctrl+Alt+F5};
Reload Atom = {Ctrl+s} Wait(100)  {Shift+Ctrl+Alt+F5};
Command Palette = {Ctrl+Shift+p};
Keyboard Shortcuts = {Ctrl+.};
Run Specs = {Ctrl+Alt+p};
Show Console = {Ctrl+Alt+i};

# D3 commands
Paste Function = 'function ' {Ctrl+v} ' {' {Enter} {Enter} '};' {Up};

# Coffeescript commands

# Python commands

# --- D3 commands -------------------------------------------

# --- Python commands -------------------------------------------

# --- Coffeescript commands -------------------------------------------


Stop doing UA normalisation on v1 endpoints

sub vcl_recv {
#FASTLY recv

	if (req.request != "HEAD" && req.request != "GET" && req.request != "FASTLYPURGE") {
		return(pass);
	}

	if (req.url ~ "^/v2/polyfill\." && req.url !~ "[\?\&]ua=") {
		set req.http.X-Orig-URL = req.url;
		set req.url = "/v2/normalizeUa?ua=" urlencode(req.http.User-Agent);
	}

	return(lookup);
}

sub vcl_deliver {
#FASTLY deliver

	# If the response is to a normalise request and there's a parked "original request", use the normalised UA response to modify the original request and restart it
	if (req.url ~ "^/v\d/normalizeUa" && resp.status == 200 && req.http.X-Orig-URL) {
		set req.http.Fastly-force-Shield = "1";
		if (req.http.X-Orig-URL ~ "\?") {
			set req.url = req.http.X-Orig-URL "&ua=" resp.http.Normalized-User-Agent;
		} else {
			set req.url = req.http.X-Orig-URL "?ua=" resp.http.Normalized-User-Agent;
		}
		restart;

	# If the original request didn't specify a UA override, and we added one when the request was sent to the backend, the backend won't have included a Vary:user-agent header (correctly) but we need to add one
	} else if (req.url ~ "^/v\d/polyfill\..*[\?\&]ua=" && req.http.X-Orig-URL && req.http.X-Orig-URL !~ "[\?\&]ua=") {
		add resp.http.Vary = "User-Agent";
	}
	return(deliver);
}

sub vcl_recv {
#FASTLY recv

	if (req.request != "HEAD" && req.request != "GET" && req.request != "FASTLYPURGE") {
		return(pass);
	}

	if (req.url ~ "^/v\d/polyfill\." && req.url !~ "[\?\&]ua=") {
		set req.http.X-Orig-URL = req.url;
		set req.url = "/v2/normalizeUa?ua=" urlencode(req.http.User-Agent);
	}

	return(lookup);
}

sub vcl_deliver {
#FASTLY deliver

	# If the response is to a normalise request and there's a parked "original request", use the normalised UA response to modify the original request and restart it
	if (req.url ~ "^/v\d/normalizeUa" && resp.status == 200 && req.http.X-Orig-URL) {
		set req.http.Fastly-force-Shield = "1";
		if (req.http.X-Orig-URL ~ "\?") {
			set req.url = req.http.X-Orig-URL "&ua=" resp.http.Normalized-User-Agent;
		} else {
			set req.url = req.http.X-Orig-URL "?ua=" resp.http.Normalized-User-Agent;
		}
		restart;

	# If the original request didn't specify a UA override, and we added one when the request was sent to the backend, the backend won't have included a Vary:user-agent header (correctly) but we need to add one
	} else if (req.url ~ "^/v\d/polyfill\..*[\?\&]ua=" && req.http.X-Orig-URL && req.http.X-Orig-URL !~ "[\?\&]ua=") {
		add resp.http.Vary = "User-Agent";
	}
	return(deliver);
}