Fix 400 bad request of varnish with nginx and apache 2.4

Update vcl from https://github.com/cosimo/varnish-accept-language
./gen_vcl.pl pt de eo es fr hy it pt ru < accept-language.c > new.vcl

(ActionItem3186)

C{

/* ------------------------------------------------------------------ */
/* THIS FILE IS AUTOMATICALLY GENERATED BY ./gen_vcl.pl. DO NOT EDIT. */

/*
 * Accept-language header normalization
 *
 * - Parses client Accept-Language HTTP header
 * - Tries to find the best match with the supported languages
 * - Writes the best match as req.http.X-Varnish-Accept-Language
 *
 * First version: Cosimo, 21/Jan/2010
 * Last update:   Cosimo, 03/Nov/2011
 *
 * http://github.com/cosimo/varnish-accept-language
 *
 */

#include <ctype.h>  /* isupper */
#include <stdio.h>
#include <stdlib.h> /* qsort */
#include <string.h>

#define DEFAULT_LANGUAGE "en"
#define SUPPORTED_LANGUAGES ":de:eo:es:fr:hy:it:pt:ru:"

#define vcl_string char
#define LANG_LIST_SIZE 16
#define HDR_MAXLEN 256
#define LANG_MAXLEN 8
#define RETURN_LANG(x) { \
    strncpy(lang, x, LANG_MAXLEN); \
    return; \
}
#define RETURN_DEFAULT_LANG RETURN_LANG(DEFAULT_LANGUAGE)
#define PUSH_LANG(x,y) { \
    /* fprintf(stderr, "Pushing lang [%d] %s %.4f\n", curr_lang, x, y); */ \
    /* We have to copy, otherwise root_lang will be the same every time */ \
    strncpy(pl[curr_lang].lang, x, LANG_MAXLEN); \
    pl[curr_lang].q = y;       \
    curr_lang++;               \
}

struct lang_list {
    vcl_string lang[LANG_MAXLEN];
    float q;
};

/* In-place lowercase of a string */
static void strtolower(char *s) {
    register char *c;
    for (c=s; *c; c++) {
        if (isupper(*c)) {
            *c = tolower(*c);
        }
    }
    return;
}

/* Checks if a given language is in the static list of the ones we support */
int is_supported(vcl_string *lang) {
    vcl_string *supported_languages = SUPPORTED_LANGUAGES;
    vcl_string match_str[LANG_MAXLEN + 3] = "";  /* :, :, \0 = 3 */
    int is_supported = 0;

    /* We want to match 'zh-cn' and 'zh-CN' too */
    strtolower(lang);

    /* Search ":<lang>:" in supported languages string */
    strncpy(match_str, ":", 1);
    strncat(match_str, lang, LANG_MAXLEN);
    strncat(match_str, ":\0", 2);

    if (strstr(supported_languages, match_str))
        is_supported = 1;

    return is_supported;
}

/* Used by qsort() below */
int sort_by_q(const void *x, const void *y) {
    struct lang_list *a = (struct lang_list *)x;
    struct lang_list *b = (struct lang_list *)y;
    if (a->q > b->q) return -1;
    if (a->q < b->q) return 1;
    return 0;
}

/* Reads Accept-Language, parses it, and finds the first match
   among the supported languages. In case of no match,
   returns the default language.
*/
void select_language(const vcl_string *incoming_header, char *lang) {

    struct lang_list pl[LANG_LIST_SIZE];
    vcl_string *lang_tok = NULL;
    vcl_string root_lang[3];
    vcl_string *header;
    vcl_string header_copy[HDR_MAXLEN];
    vcl_string *pos = NULL;
    vcl_string *q_spec = NULL;
    unsigned int curr_lang = 0, i = 0;
    float q;

    /* Empty or default string, return default language immediately */
    if (
        !incoming_header
        || (0 == strcmp(incoming_header, "en-US"))
        || (0 == strcmp(incoming_header, "en-GB"))
        || (0 == strcmp(incoming_header, DEFAULT_LANGUAGE))
        || (0 == strcmp(incoming_header, ""))
    )
        RETURN_DEFAULT_LANG;

    /* Tokenize Accept-Language */
    header = strncpy(header_copy, incoming_header, sizeof(header_copy));

    while ((lang_tok = strtok_r(header, " ,", &pos))) {

        q = 1.0;

        if ((q_spec = strstr(lang_tok, ";q="))) {
            /* Truncate language name before ';' */
            *q_spec = '\0';
            /* Get q value */
            sscanf(q_spec + 3, "%f", &q);
        }

        /* Wildcard language '*' should be last in list */
        if ((*lang_tok) == '*') q = 0.0;

        /* Push in the prioritized list */
        PUSH_LANG(lang_tok, q);

        /* For cases like 'en-GB', we also want the root language in the final list */
        if ('-' == lang_tok[2]) {
            root_lang[0] = lang_tok[0];
            root_lang[1] = lang_tok[1];
            root_lang[2] = '\0';
            PUSH_LANG(root_lang, q - 0.001);
        }

        /* For strtok_r() to proceed from where it left off */
        header = NULL;

        /* Break out if stored max no. of languages */
        if (curr_lang >= LANG_LIST_SIZE)
            break;
    }

    /* Sort by priority */
    qsort(pl, curr_lang, sizeof(struct lang_list), &sort_by_q);

    /* Match with supported languages */
    for (i = 0; i < curr_lang; i++) {
        if (is_supported(pl[i].lang))
            RETURN_LANG(pl[i].lang);
    }

    RETURN_DEFAULT_LANG;
}

/* Reads req.http.Accept-Language and writes X-Varnish-Accept-Language */
void vcl_rewrite_accept_language(const struct sess *sp) {
    vcl_string *in_hdr;
    vcl_string lang[LANG_MAXLEN];

    /* Get Accept-Language header from client */
    in_hdr = VRT_GetHdr(sp, HDR_REQ, "\020Accept-Language:");

    /* Normalize and filter out by list of supported languages */
    memset(lang, 0, sizeof(lang));
    select_language(in_hdr, lang);

    /* By default, use a different header name: don't mess with backend logic */
    VRT_SetHdr(sp, HDR_REQ, "\032X-Varnish-Accept-Language:", lang, vrt_magic_string_end);

    return;
}

/* vim: syn=c ts=4 et sts=4 sw=4 tw=0
*/

/* THIS FILE IS AUTOMATICALLY GENERATED BY ./gen_vcl.pl. DO NOT EDIT. */
/* ------------------------------------------------------------------ */
}C

sub vcl_recv {
  C{
    vcl_rewrite_accept_language(sp);
  }C
}

sub vcl_fetch {
  if (beresp.http.Vary) {
    set beresp.http.Vary = beresp.http.Vary + ", X-Varnish-Accept-Language";
  } else {
    set beresp.http.Vary = "X-Varnish-Accept-Language";
  }
}

C{

/* ------------------------------------------------------------------ */
/* THIS FILE IS AUTOMATICALLY GENERATED BY ./gen_vcl.pl. DO NOT EDIT. */

/*
 * Accept-language header normalization
 *
 * Cosimo, 21/01/2010
 *
 */

#include <ctype.h>  /* isupper */
#include <stdio.h>
#include <stdlib.h> /* qsort */
#include <string.h>

#define DEFAULT_LANGUAGE "en"
#define SUPPORTED_LANGUAGES ":de:fr:es:ru:pt:hy:en:"

#define vcl_string char
#define LANG_LIST_SIZE 16
#define LANG_MAXLEN 16
#define RETURN_LANG(x) { \
    strncpy(lang, x, LANG_MAXLEN); \
    return; \
}
#define RETURN_DEFAULT_LANG RETURN_LANG(DEFAULT_LANGUAGE)
#define PUSH_LANG(x,y) { \
    /* fprintf(stderr, "Pushing lang [%d] %s %.4f\n", curr_lang, x, y); */ \
    /* We have to copy, otherwise root_lang will be the same every time */ \
    strncpy(pl[curr_lang].lang, x, LANG_MAXLEN); \
    pl[curr_lang].q = y;       \
    curr_lang++;               \
}

struct lang_list {
    vcl_string lang[LANG_MAXLEN];
    float q;
};

/* In-place lowercase of a string */
static void strtolower(char *s) {
    register char *c;
    for (c=s; *c; c++) {
        if (isupper(*c)) {
            *c = tolower(*c);
        }
    }
    return;
}

/* Checks if a given language is in the static list of the ones we support */
int is_supported(vcl_string *lang) {
    vcl_string *supported_languages = SUPPORTED_LANGUAGES;
    vcl_string match_str[LANG_MAXLEN + 3] = "";  /* :, :, \0 = 3 */
    int is_supported = 0;

    /* We want to match 'zh-cn' and 'zh-CN' too */
    strtolower(lang);

    /* Search ":<lang>:" in supported languages string */
    strncpy(match_str, ":", 1);
    strncat(match_str, lang, LANG_MAXLEN);
    strncat(match_str, ":\0", 2);

    if (strstr(supported_languages, match_str)) {
        is_supported = 1;
    }

    return is_supported;
}

/* Used by qsort() below */
int sort_by_q(const void *x, const void *y) {
    struct lang_list *a = (struct lang_list *)x;
    struct lang_list *b = (struct lang_list *)y;
    if (a->q > b->q) return -1;
    if (a->q < b->q) return 1;
    return 0;
}

/* Reads Accept-Language, parses it, and finds the first match
   among the supported languages. In case of no match,
   returns the default language.
*/
void select_language(const vcl_string *incoming_header, char *lang) {

    struct lang_list pl[LANG_LIST_SIZE];
    vcl_string *lang_tok = NULL;
    vcl_string root_lang[3];
    vcl_string *header;
    vcl_string *pos = NULL;
    vcl_string *q_spec = NULL;
    unsigned int curr_lang = 0, i = 0;
    float q;

    /* Empty or default string, return default language immediately */
    if (
        !incoming_header
        || (0 == strcmp(incoming_header, "en-US"))
        || (0 == strcmp(incoming_header, "en-GB"))
        || (0 == strcmp(incoming_header, DEFAULT_LANGUAGE))
        || (0 == strcmp(incoming_header, ""))
    )
        RETURN_DEFAULT_LANG;

    /* Tokenize Accept-Language */
    header = (vcl_string *) incoming_header;

    while ((lang_tok = strtok_r(header, " ,", &pos))) {

        q = 1.0;

        if ((q_spec = strstr(lang_tok, ";q="))) {
            /* Truncate language name before ';' */
            *q_spec = '\0';
            /* Get q value */
            sscanf(q_spec + 3, "%f", &q);
        }

        /* Wildcard language '*' should be last in list */
        if ((*lang_tok) == '*') q = 0.0;

        /* Push in the prioritized list */
        PUSH_LANG(lang_tok, q);

        /* For cases like 'en-GB', we also want the root language in the final list */
        if ('-' == lang_tok[2]) {
            root_lang[0] = lang_tok[0];
            root_lang[1] = lang_tok[1];
            root_lang[2] = '\0';
            PUSH_LANG(root_lang, q - 0.001);
        }

        /* For strtok_r() to proceed from where it left off */
        header = NULL;

        /* Break out if stored max no. of languages */
        if (curr_lang >= LANG_MAXLEN) break;
    }

    /* Sort by priority */
    qsort(pl, curr_lang, sizeof(struct lang_list), &sort_by_q);

    /* Match with supported languages */
    for (i = 0; i < curr_lang; i++) {
        if (is_supported(pl[i].lang))
            RETURN_LANG(pl[i].lang);
    }

    RETURN_DEFAULT_LANG;
}

/* Reads req.http.Accept-Language and writes X-Varnish-Accept-Language */
void vcl_rewrite_accept_language(const struct sess *sp) {
    vcl_string *in_hdr;
    vcl_string lang[LANG_MAXLEN];

    memset(lang, 0, LANG_MAXLEN);

    /* Get Accept-Language header from client */
    in_hdr = VRT_GetHdr(sp, HDR_REQ, "\020Accept-Language:");

    /* Normalize and filter out by list of supported languages */
    select_language(in_hdr, lang);

    /* By default, use a different header name: don't mess with backend logic */
    VRT_SetHdr(sp, HDR_REQ, "\032X-Varnish-Accept-Language:", lang, vrt_magic_string_end);

    return;
}

/* vim: syn=c ts=4 et sts=4 sw=4 tw=0
*/

/* THIS FILE IS AUTOMATICALLY GENERATED BY ./gen_vcl.pl. DO NOT EDIT. */
/* ------------------------------------------------------------------ */
}C

sub vcl_recv {
  C{
    vcl_rewrite_accept_language(sp);
  }C
}

sub vcl_fetch {
  if (beresp.http.Vary) {
    set beresp.http.Vary = beresp.http.Vary + ", X-Varnish-Accept-Language";
  } else {
    set beresp.http.Vary = "X-Varnish-Accept-Language";
  }
}


Fixed syslog include in example VCL

Fixed syslog include in example VCL

import std;
import abtest;

C{
    #include <syslog.h>
}C

 backend default {
    .host = "127.0.0.1";
    .port = "3000";
 }

/* Only permit localhost to manipulate abtest configuration */
acl abconfig {
    "localhost";
}

sub vcl_init {
    if (abtest.load_config("/tmp/abtest.cfg") != 0) {
        C{ syslog(LOG_ALERT, "Unable to load AB config from /tmp/abtest.cfg"); }C
    }
    abtest.set_rule(".*", "a:1;b:1;86400;"); // Set default rule with a duration of 24H

    return (ok);
}

sub vcl_fini {
}

sub vcl_recv {
    if (req.http.X-AB-Cfg) {
        if (!client.ip ~ abconfig) {
            error 405 "Not allowed.";
        } else {
            // curl localhost:8080 -X PUT -H "X-AB-Cfg:base" -H "X-AB-Cfg-Val:a:25;b:75;"
            if (req.request == "PUT") {
                std.log("AB Config PUT request: " + req.http.X-AB-Cfg + "|" + req.http.X-AB-Cfg-Val);
                abtest.set_rule(req.http.X-AB-Cfg, req.http.X-AB-Cfg-Val);
                if (abtest.save_config("/tmp/abtest.cfg") != 0) {
                    std.log("ABTest - Error, could not save the configuration");
                }
            }

            // curl localhost:8080 -X DELETE -H "X-AB-Cfg:base"
            if (req.request == "DELETE") {
                std.log("AB Config DELETE request: " + req.http.X-AB-Cfg);
                abtest.rem_rule(req.http.X-AB-Cfg);
            }
        }
    }

    if(!req.http.Cookie ~ "abtesting") {
        std.log("No AB cookie, setting it.");
        set req.http.Cookie = "abtesting=" + abtest.get_rand(req.url) + "; path=/; expires=" + abtest.get_expire(req.url);
        set req.http.X-Set-AB-Cookie = req.http.Cookie;
    } else {
        std.log("AB cookie found.");
    }

    return (lookup);
}

sub vcl_pipe {
}

sub vcl_pass {
}

sub vcl_hash {
}

sub vcl_hit {
}

sub vcl_miss {
}

sub vcl_fetch {
}

sub vcl_deliver {
    if (req.http.X-Set-AB-Cookie) {
        std.log("Setting cookie to " + req.http.X-Set-AB-Cookie);
        set resp.http.Set-Cookie = req.http.X-Set-AB-Cookie;
    }

    if (req.http.X-AB-Cfg && client.ip ~ abconfig) {
        if (req.request == "GET") {
            // curl localhost:8080 -X GET -H "X-AB-Cfg:;"
            set resp.http.X-AB-Cfg = abtest.get_rules();
        }
    }
}

sub vcl_error {
}

import std;
import abtest;

 backend default {
    .host = "127.0.0.1";
    .port = "3000";
 }

/* Only permit localhost to manipulate abtest configuration */
acl abconfig {
    "localhost";
}

sub vcl_init {
    if (abtest.load_config("/tmp/abtest.cfg") != 0) {
        C{ syslog(LOG_ALERT, "Unable to load AB config from /tmp/abtest.cfg"); }C
    }
    abtest.set_rule(".*", "a:1;b:1;86400;"); // Set default rule with a duration of 24H

    return (ok);
}

sub vcl_fini {
}

sub vcl_recv {
    if (req.http.X-AB-Cfg) {
        if (!client.ip ~ abconfig) {
            error 405 "Not allowed.";
        } else {
            // curl localhost:8080 -X PUT -H "X-AB-Cfg:base" -H "X-AB-Cfg-Val:a:25;b:75;"
            if (req.request == "PUT") {
                std.log("AB Config PUT request: " + req.http.X-AB-Cfg + "|" + req.http.X-AB-Cfg-Val);
                abtest.set_rule(req.http.X-AB-Cfg, req.http.X-AB-Cfg-Val);
                if (abtest.save_config("/tmp/abtest.cfg") != 0) {
                    std.log("ABTest - Error, could not save the configuration");
                }
            }

            // curl localhost:8080 -X DELETE -H "X-AB-Cfg:base"
            if (req.request == "DELETE") {
                std.log("AB Config DELETE request: " + req.http.X-AB-Cfg);
                abtest.rem_rule(req.http.X-AB-Cfg);
            }
        }
    }

    if(!req.http.Cookie ~ "abtesting") {
        std.log("No AB cookie, setting it.");
        set req.http.Cookie = "abtesting=" + abtest.get_rand(req.url) + "; path=/; expires=" + abtest.get_expire(req.url);
        set req.http.X-Set-AB-Cookie = req.http.Cookie;
    } else {
        std.log("AB cookie found.");
    }

    return (lookup);
}

sub vcl_pipe {
}

sub vcl_pass {
}

sub vcl_hash {
}

sub vcl_hit {
}

sub vcl_miss {
}

sub vcl_fetch {
}

sub vcl_deliver {
    if (req.http.X-Set-AB-Cookie) {
        std.log("Setting cookie to " + req.http.X-Set-AB-Cookie);
        set resp.http.Set-Cookie = req.http.X-Set-AB-Cookie;
    }

    if (req.http.X-AB-Cfg && client.ip ~ abconfig) {
        if (req.request == "GET") {
            // curl localhost:8080 -X GET -H "X-AB-Cfg:;"
            set resp.http.X-AB-Cfg = abtest.get_rules();
        }
    }
}

sub vcl_error {
}


Fix incorrect VCL in example.vcl.

Fix incorrect VCL in example.vcl.

Old habits die hard.

Thanks to Federico Schwindt for pointing this out.

#
# This is an example VCL file for Varnish.
# It does not do anything by default, delegating control to the builtin vcl.
# The builtin VCL is called when there is no explicit explicit return
# statement.

# See the VCL tutorial at https://www.varnish-cache.org/docs/trunk/tutorial/
# See http://varnish-cache.org/trac/wiki/VCLExamples for more examples.

# Marker to tell the VCL compiler that this VCL has been adapted to the new 4.0
# format.
vcl 4.0;

# Default backend definition. Set this to point to your content server.
backend default {
    .host = "127.0.0.1";
    .port = "8080";
}

sub vcl_recv {
    # Happens before we check if we have this in cache already.
    # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_recv
}

sub vcl_backend_response {
    # Happens after we have read the response headers from the backend.
    # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_fetch
#
}

sub vcl_deliver {
    # Happens when we have all the pieces we need, and are about to send the
    # response to the client.
    # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_fetch
}

#
# This is an example VCL file for Varnish.
# It does not do anything by default, delegating control to the builtin vcl.
# The builtin VCL is called when there is no explicit explicit return
# statement.

# See the VCL tutorial at https://www.varnish-cache.org/docs/trunk/tutorial/
# See http://varnish-cache.org/trac/wiki/VCLExamples for more examples.

# Marker to tell the VCL compiler that this VCL has been adapted to the new 4.0
# format.
vcl 4.0;

# Default backend definition. Set this to point to your content server.
backend default {
    .host = "127.0.0.1";
    .port = "8080";
}

sub vcl_recv {
    # Happens before we check if we have this in cache already.
    # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_recv
}

sub vcl_fetch {
    # Happens after we have read the response headers from the backend.
    # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_fetch
#
}

sub vcl_deliver {
    # Happens when we have all the pieces we need, and are about to send the
    # response to the client.
    # See http://www.varnish-cache.org/docs/3.0/tutorial/vcl.html#vcl_fetch
}


Lowercase the req.http.host variable

Lowercase the req.http.host variable

sub vcl_recv {

    # I'm not 100% sure on what this is exactly for, it was taken from the
    # Fastly documentation, however, what I *believe* it does is just ensure
    # that we don't serve a stale copy of the page from the shield node when
    # an edge node is requesting content.
    if (req.http.Fastly-FF) {
        set req.max_stale_while_revalidate = 0s;
    }

    # Some (Older) clients will send a hash fragment as part of the URL even
    # though that is a local only modification. This breaks this badly for the
    # files in S3, and in general it's just not needed.
    set req.url = regsub(req.url, "#.*$", "");

    # Most of the URLs in Warehouse do not support or require any sort of query
    # parameter. If we strip these at the edge then we'll increase our cache
    # efficiency when they won't otherwise change the output of the pages.
    #
    # This will match any URL except those that start with:
    #
    #   * /search/
    #   * /account/login/
    #   * /account/logout/
    #   * /account/register/
    #   * /pypi
    if (req.url !~ "^/(search/|account/(login|logout|register)/|pypi)") {
        set req.url = req.url.path;
    }

    # Sort all of our query parameters, this will ensure that the same query
    # parameters in a different order will end up being represented as the same
    # thing, reducing cache misses due to ordering differences.
    set req.url = boltsort.sort(req.url);

#FASTLY recv

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Canonicalize our domains by redirecting any domain that doesn't match our
    # primary domain to our primary domain. We do this *after* the HTTPS check
    # on purpose.
    if (std.tolower(req.http.host) != std.tolower(req.http.Primary-Domain)) {
        set req.http.Location = "https://" req.http.Primary-Domain req.url;
        error 750 "Redirect to Primary Domain";
    }

    # Requests to /packages/ get dispatched to Amazon instead of to our typical
    # Origin. This requires a a bit of setup to make it work.
    if (req.url ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/") {
        # Setup our environment to better match what S3 expects/needs
        set req.http.Host = req.http.AWS-Bucket-Name ".s3.amazonaws.com";
        set req.http.Date = now;
        set req.url = regsuball(req.url, "\+", urlencode("+"));

        # Compute the Authorization header that S3 requires to be able to
        # access the files stored there.
        set req.http.Authorization = "AWS " req.http.AWS-Access-Key-ID ":" digest.hmac_sha1_base64(req.http.AWS-Secret-Access-Key, "GET" LF LF LF req.http.Date LF "/" req.http.AWS-Bucket-Name req.url.path);

        # We don't want to send our Warehouse-Token to S3, so we'll go ahead
        # and remove it.
        unset req.http.Warehouse-Token;
    }

    # We no longer need any of these variables, which would exist only to
    # shuffle configuration from the Fastly UI into our VCL.
    unset req.http.Primary-Domain;
    unset req.http.AWS-Access-Key-ID;
    unset req.http.AWS-Secret-Access-Key;
    unset req.http.AWS-Bucket-Name;

    # We have a number of items that we'll pass back to the origin, but only
    # if we have a Warehouse-Token that will allow them to be accepted.
    if (req.http.Warehouse-Token) {
        # Set a header to tell the backend if we're using https or http.
        if (req.http.Fastly-SSL) {
            set req.http.Warehouse-Proto = "https";
        } else {
            set req.http.Warehouse-Proto = "http";
        }

        # Pass the client IP address back to the backend.
        if (req.http.Fastly-Client-IP) {
            set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
        }

        # Pass the real host value back to the backend.
        if (req.http.Host) {
            set req.http.Warehouse-Host = req.http.host;
        }
    }

    # On a POST, we want to skip the shielding and hit backends directly.
    if (req.request == "POST") {
        set req.backend = F_Heroku;
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # We don't ever want to cache our health URL. Outside systems should be
    # able to use it to reach past Fastly and get an end to end health check.
    if (req.url == "/_health/") {
        return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {

    # These are newer kinds of redirects which should be able to be cached by
    # default, even though Fastly doesn't currently have them in their default
    # list of cacheable status codes.
    if (http_status_matches(beresp.status, "303,307,308")) {
        set beresp.cacheable = true;
    }

    # For any 5xx status code we want to see if a stale object exists for it,
    # if so we'll go ahead and serve it.
    if (beresp.status >= 500 && beresp.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

#FASTLY fetch

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 502 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 502 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (http_status_matches(beresp.status, "500,502,503")) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_hit {
#FASTLY hit

    # If the object we have isn't cacheable, then just serve it directly
    # without going through any of the caching mechanisms.
    if (!obj.cacheable) {
        return(pass);
    }

    return(deliver);
}

sub vcl_deliver {
    # If this is an error and we have a stale response available, restart so
    # that we can pick it up and serve it.
    if (resp.status >= 500 && resp.status < 600) {
        if (stale.exists) {
            restart;
        }
    }

#FASTLY deliver

    # Unset headers that we don't need/want to send on to the client because
    # they are not generally useful.
    unset resp.http.Server;
    unset resp.http.Via;

    # Unset a few headers set by Amazon that we don't really have a need/desire
    # to send to clients.
    unset resp.http.X-AMZ-Replication-Status;
    unset resp.http.X-AMZ-Meta-Python-Version;
    unset resp.http.X-AMZ-Meta-Version;
    unset resp.http.X-AMZ-Meta-Package-Type;
    unset resp.http.X-AMZ-Meta-Project;

    # Set our standard security headers, we do this in VCL rather than in
    # Warehouse itself so that we always get these headers, regardless of the
    # origin server being used.
    set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload";
    set resp.http.X-Frame-Options = "deny";
    set resp.http.X-XSS-Protection = "1; mode=block";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-Permitted-Cross-Domain-Policies = "none";

    return(deliver);
}

sub vcl_error {
#FASTLY error

    # If we have a 5xx error and there is a stale object available, then we
    # will deliver that stale object.
    if (obj.status >= 500 && obj.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    } else if (obj.status == 750) {
        set obj.status = 301;
        set obj.http.Location = req.http.Location;
        set obj.http.Content-Type = "text/html; charset=UTF-8";
        synthetic {"<title>301 Moved Permanently</title>
301 Moved Permanently
"};
        return(deliver);
    }

}

sub vcl_recv {

    # I'm not 100% sure on what this is exactly for, it was taken from the
    # Fastly documentation, however, what I *believe* it does is just ensure
    # that we don't serve a stale copy of the page from the shield node when
    # an edge node is requesting content.
    if (req.http.Fastly-FF) {
        set req.max_stale_while_revalidate = 0s;
    }

    # Some (Older) clients will send a hash fragment as part of the URL even
    # though that is a local only modification. This breaks this badly for the
    # files in S3, and in general it's just not needed.
    set req.url = regsub(req.url, "#.*$", "");

    # Most of the URLs in Warehouse do not support or require any sort of query
    # parameter. If we strip these at the edge then we'll increase our cache
    # efficiency when they won't otherwise change the output of the pages.
    #
    # This will match any URL except those that start with:
    #
    #   * /search/
    #   * /account/login/
    #   * /account/logout/
    #   * /account/register/
    #   * /pypi
    if (req.url !~ "^/(search/|account/(login|logout|register)/|pypi)") {
        set req.url = req.url.path;
    }

    # Sort all of our query parameters, this will ensure that the same query
    # parameters in a different order will end up being represented as the same
    # thing, reducing cache misses due to ordering differences.
    set req.url = boltsort.sort(req.url);

#FASTLY recv

    # We want to Force SSL for the WebUI by redirecting to the HTTPS version of
    # the page, however for API calls we want to return an error code directing
    # people to instead use HTTPS.
    if (!req.http.Fastly-SSL) {

        # The /simple/ and /packages/ API.
        if (req.url ~ "^/(simple|packages)") {
            error 803 "SSL is required";
        }

        # The Legacy JSON API.
        if (req.url ~ "^/pypi/.+/json$") {
            error 803 "SSL is required";
        }

        # The Legacy ?:action= API.
        if (req.url ~ "^/pypi.*(\?|&)=:action") {
            error 803 "SSL is required";
        }

        # If we're on the /pypi page and we've received something other than a
        # GET or HEAD request, then we have no way to determine if a particular
        # request is an API call or not because it'll be in the request body
        # and that isn't available to us here. So in those cases, we won't
        # do a redirect.
        if (req.url ~ "^/pypi") {
            if (req.request == "GET" || req.request == "HEAD") {
                error 801 "Force SSL";
            }
        }
        else {
            # This isn't a /pypi URL so we'll just unconditionally redirect to
            # HTTPS.
            error 801 "Force SSL";
        }
    }

    # Canonicalize our domains by redirecting any domain that doesn't match our
    # primary domain to our primary domain. We do this *after* the HTTPS check
    # on purpose.
    if (std.tolower(req.http.host) != std.tolower(req.http.Primary-Domain)) {
        set req.http.Location = "https://" req.http.Primary-Domain req.url;
        error 750 "Redirect to Primary Domain";
    }

    # Requests to /packages/ get dispatched to Amazon instead of to our typical
    # Origin. This requires a a bit of setup to make it work.
    if (req.url ~ "^/packages/[a-f0-9]{2}/[a-f0-9]{2}/[a-f0-9]{60}/") {
        # Setup our environment to better match what S3 expects/needs
        set req.http.Host = req.http.AWS-Bucket-Name ".s3.amazonaws.com";
        set req.http.Date = now;
        set req.url = regsuball(req.url, "\+", urlencode("+"));

        # Compute the Authorization header that S3 requires to be able to
        # access the files stored there.
        set req.http.Authorization = "AWS " req.http.AWS-Access-Key-ID ":" digest.hmac_sha1_base64(req.http.AWS-Secret-Access-Key, "GET" LF LF LF req.http.Date LF "/" req.http.AWS-Bucket-Name req.url.path);

        # We don't want to send our Warehouse-Token to S3, so we'll go ahead
        # and remove it.
        unset req.http.Warehouse-Token;
    }

    # We no longer need any of these variables, which would exist only to
    # shuffle configuration from the Fastly UI into our VCL.
    unset req.http.Primary-Domain;
    unset req.http.AWS-Access-Key-ID;
    unset req.http.AWS-Secret-Access-Key;
    unset req.http.AWS-Bucket-Name;

    # We have a number of items that we'll pass back to the origin, but only
    # if we have a Warehouse-Token that will allow them to be accepted.
    if (req.http.Warehouse-Token) {
        # Set a header to tell the backend if we're using https or http.
        if (req.http.Fastly-SSL) {
            set req.http.Warehouse-Proto = "https";
        } else {
            set req.http.Warehouse-Proto = "http";
        }

        # Pass the client IP address back to the backend.
        if (req.http.Fastly-Client-IP) {
            set req.http.Warehouse-IP = req.http.Fastly-Client-IP;
        }

        # Pass the real host value back to the backend.
        if (req.http.Host) {
            set req.http.Warehouse-Host = req.http.Host;
        }
    }

    # On a POST, we want to skip the shielding and hit backends directly.
    if (req.request == "POST") {
        set req.backend = F_Heroku;
    }

    # Do not bother to attempt to run the caching mechanisms for methods that
    # are not generally safe to cache.
    if (req.request != "HEAD" &&
        req.request != "GET" &&
        req.request != "FASTLYPURGE") {
      return(pass);
    }

    # We don't ever want to cache our health URL. Outside systems should be
    # able to use it to reach past Fastly and get an end to end health check.
    if (req.url == "/_health/") {
        return(pass);
    }

    # Finally, return the default lookup action.
    return(lookup);
}

sub vcl_fetch {

    # These are newer kinds of redirects which should be able to be cached by
    # default, even though Fastly doesn't currently have them in their default
    # list of cacheable status codes.
    if (http_status_matches(beresp.status, "303,307,308")) {
        set beresp.cacheable = true;
    }

    # For any 5xx status code we want to see if a stale object exists for it,
    # if so we'll go ahead and serve it.
    if (beresp.status >= 500 && beresp.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

#FASTLY fetch

    # Trigger a "SSL is required" error if the backend has indicated to do so.
    if (beresp.http.X-Fastly-Error == "803") {
        error 803 "SSL is required";
    }

    # If we've gotten a 502 or a 503 from the backend, we'll go ahead and retry
    # the request.
    if ((beresp.status == 502 || beresp.status == 503) &&
            req.restarts < 1 &&
            (req.request == "GET" || req.request == "HEAD")) {
        restart;
    }

    # If we've restarted, then we'll record the number of restarts.
    if(req.restarts > 0 ) {
        set beresp.http.Fastly-Restarts = req.restarts;
    }

    # If there is a Set-Cookie header, we'll ensure that we do not cache the
    # response.
    if (beresp.http.Set-Cookie) {
        set req.http.Fastly-Cachetype = "SETCOOKIE";
        return (pass);
    }

    # If the response has the private Cache-Control directive then we won't
    # cache it.
    if (beresp.http.Cache-Control ~ "private") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    # If we've gotten an error after the restarts we'll deliver the response
    # with a very short cache time.
    if (http_status_matches(beresp.status, "500,502,503")) {
        set req.http.Fastly-Cachetype = "ERROR";
        set beresp.ttl = 1s;
        set beresp.grace = 5s;
        return (deliver);
    }

    # Apply a default TTL if there isn't a max-age or s-maxage.
    if (beresp.http.Expires ||
            beresp.http.Surrogate-Control ~ "max-age" ||
            beresp.http.Cache-Control ~"(s-maxage|max-age)") {
        # Keep the ttl here
    }
    else {
        # Apply the default ttl
        set beresp.ttl = 60s;
    }

    # Actually deliver the fetched response.
    return(deliver);
}

sub vcl_hit {
#FASTLY hit

    # If the object we have isn't cacheable, then just serve it directly
    # without going through any of the caching mechanisms.
    if (!obj.cacheable) {
        return(pass);
    }

    return(deliver);
}

sub vcl_deliver {
    # If this is an error and we have a stale response available, restart so
    # that we can pick it up and serve it.
    if (resp.status >= 500 && resp.status < 600) {
        if (stale.exists) {
            restart;
        }
    }

#FASTLY deliver

    # Unset headers that we don't need/want to send on to the client because
    # they are not generally useful.
    unset resp.http.Server;
    unset resp.http.Via;

    # Unset a few headers set by Amazon that we don't really have a need/desire
    # to send to clients.
    unset resp.http.X-AMZ-Replication-Status;
    unset resp.http.X-AMZ-Meta-Python-Version;
    unset resp.http.X-AMZ-Meta-Version;
    unset resp.http.X-AMZ-Meta-Package-Type;
    unset resp.http.X-AMZ-Meta-Project;

    # Set our standard security headers, we do this in VCL rather than in
    # Warehouse itself so that we always get these headers, regardless of the
    # origin server being used.
    set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload";
    set resp.http.X-Frame-Options = "deny";
    set resp.http.X-XSS-Protection = "1; mode=block";
    set resp.http.X-Content-Type-Options = "nosniff";
    set resp.http.X-Permitted-Cross-Domain-Policies = "none";

    return(deliver);
}

sub vcl_error {
#FASTLY error

    # If we have a 5xx error and there is a stale object available, then we
    # will deliver that stale object.
    if (obj.status >= 500 && obj.status < 600) {
        if (stale.exists) {
            return(deliver_stale);
        }
    }

    if (obj.status == 803) {
        set obj.status = 403;
        set obj.response = "SSL is required";
        set obj.http.Content-Type = "text/plain; charset=UTF-8";
        synthetic {"SSL is required."};
        return (deliver);
    } else if (obj.status == 750) {
        set obj.status = 301;
        set obj.http.Location = req.http.Location;
        set obj.http.Content-Type = "text/html; charset=UTF-8";
        synthetic {"<title>301 Moved Permanently</title>
301 Moved Permanently
"};
        return(deliver);
    }

}


Decode consumer key; unset secret after signature generation; minor refactors

table consumer_secrets {
	"foo": "super_secret"
}

sub vcl_recv {
#FASTLY recv
#DEPLOY recv
	set req.url = boltsort.sort(req.url);
	set req.http.X-Consumer-Key = urldecode(regsub(req.url, "(?i).*oauth_consumer_key=([^&]*).*", "\1"));
	set req.http.X-Consumer-Secret = table.lookup(consumer_secrets, req.http.X-Consumer-Key);
	set req.http.X-Provided-Signature = urldecode(regsub(req.url, "(?i).*oauth_signature=([^&]*).*", "\1"));
	set req.http.X-Parameters = regsub(regsub(req.url, ".*\?", ""), "&oauth_signature=[^&]*", "");
	set req.http.X-Base-String-Uri =
		if(req.http.Fastly-SSL, "https", "http")
		"://"
		std.tolower(req.http.host)
		req.url.path;

	# Construct according to https://tools.ietf.org/html/rfc5849#page-18
	set req.http.X-Signature-Base-String =
		req.request
		"&"
		urlencode(req.http.X-Base-String-Uri)
		"&"
		urlencode(req.http.X-Parameters);

	set req.http.X-Calculated-Signature = digest.hmac_sha1_base64(
		req.http.X-Consumer-Secret "&", req.http.X-Signature-Base-String);

	unset req.http.X-Consumer-Secret;
	unset req.http.X-Base-String-Uri;

	if(req.http.X-Provided-Signature != req.http.X-Calculated-Signature) {
		error 401 "Invalid OAuth signature";
	}

	return(lookup);
}

sub vcl_fetch {
#FASTLY fetch
	return(deliver);
}

sub vcl_hit {
#FASTLY hit
	if (!obj.cacheable) {
		return(pass);
	}
	return(deliver);
}

sub vcl_miss {
#FASTLY miss
	return(fetch);
}

sub vcl_deliver{
#FASTLY deliver
	set resp.http.X-Consumer-Key = req.http.X-Consumer-Key;
	set resp.http.X-Provided-Signature = req.http.X-Provided-Signature;
	set resp.http.X-Calculated-Signature = req.http.X-Calculated-Signature;
	set resp.http.X-Parameters = req.http.X-Parameters;
	set resp.http.X-Signature-Base-String = req.http.X-Signature-Base-String;
	return(deliver);
}

sub vcl_error {
#FASTLY error
#DEPLOY error
}

sub vcl_pass {
#FASTLY pass
}

table consumer_secrets {
	"foo": "super_secret"
}

sub vcl_recv {
#FASTLY recv
#DEPLOY recv
	set req.url = boltsort.sort(req.url);
	set req.http.X-Consumer-Key = regsub(req.url, "(?i).*oauth_consumer_key=([^&]*).*", "\1");
	set req.http.X-Consumer-Secret = table.lookup(consumer_secrets, req.http.X-Consumer-Key);
	set req.http.X-Provided-Signature = regsub(req.url, "(?i).*oauth_signature=([^&]*).*", "\1");
	set req.http.X-Parameters = regsub(req.url, ".*\?", "");
	set req.http.X-Parameters = regsub(req.http.X-Parameters, "&oauth_signature=[^&]*", "");

	set req.http.X-Base-String-Uri =
		if(req.http.Fastly-SSL,"https","http")
		"://"
		std.tolower(req.http.host)
		req.url.path;

	# Construct according to https://tools.ietf.org/html/rfc5849#page-18
	set req.http.X-Signature-Base-String =
		req.request
		"&"
		urlencode(req.http.X-Base-String-Uri)
		"&"
		urlencode(req.http.X-Parameters);

	set req.http.X-Calculated-Signature = urlencode(digest.hmac_sha1_base64(
		req.http.X-Consumer-Secret "&", req.http.X-Signature-Base-String));

	if(req.http.X-Provided-Signature != req.http.X-Calculated-Signature) {
		error 401 "Invalid OAuth signature";
	}

	return(lookup);
}

sub vcl_fetch {
#FASTLY fetch
	return(deliver);
}

sub vcl_hit {
#FASTLY hit
	if (!obj.cacheable) {
		return(pass);
	}
	return(deliver);
}

sub vcl_miss {
#FASTLY miss
	return(fetch);
}

sub vcl_deliver{
#FASTLY deliver
	set resp.http.X-Consumer-Key = req.http.X-Consumer-Key;
	set resp.http.X-Consumer-Secret = req.http.X-Consumer-Secret;
	set resp.http.X-Provided-Signature = req.http.X-Provided-Signature;
	set resp.http.X-Calculated-Signature = req.http.X-Calculated-Signature;
	set resp.http.X-Parameters = req.http.X-Parameters;
	set resp.http.X-Base-String-Uri = req.http.X-Base-String-Uri;
	set resp.http.X-Signature-Base-String = req.http.X-Signature-Base-String;
	return(deliver);
}

sub vcl_error {
#FASTLY error
#DEPLOY error
}

sub vcl_pass {
#FASTLY pass
}


Enable load-balancing logic for qa service (#1191)

import boltsort;

sub vcl_recv {
#FASTLY recv

	# Enable API key authentication for URL purge requests
	if ( req.request == "FASTLYPURGE" ) {
		set req.http.Fastly-Purge-Requires-Auth = "1";
	}

	if (req.request != "HEAD" && req.request != "GET" && req.request != "FASTLYPURGE") {
		return(pass);
	}

	if (req.http.Host ~ "polyfills.io") {
		# Do the canonicalise check before the SSL check to avoid a double redirect
		error 751 "Canonicalise";
	}

	if (!req.http.Fastly-SSL) {
		# 801 is a special error code that Fastly uses to Force SSL on the request
		error 801 "Redirect to prod HTTPS";
	}

	if (req.url ~ "^/v2/(polyfill\.|recordRumData)" && req.url !~ "[\?\&]ua=") {
		set req.http.X-Orig-URL = req.url;
		set req.url = "/v2/normalizeUa?ua=" urlencode(req.http.User-Agent);
	}

	if (req.url ~ "^/v2/recordRumData" && req.http.Normalized-User-Agent) {
		set req.http.Log = regsub(req.url, "^.*?\?(.*)$", "\1") "&ip=" client.ip "&refer_domain=" regsub(req.http.Referer, "^(https?\:\/\/)?(www\.)?(.+?)(\:\d+)?([\/\?].*)?$", "\3") "&country=" geoip.country_code "&data_center=" if(req.http.Cookie:FastlyDC, req.http.Cookie:FastlyDC, server.datacenter);
		error 204 "No Content";
	}

	set req.url = boltsort.sort(req.url);

	if (req.restarts == 0) {
		set req.http.X-Original-Host = req.http.Host;

		if (req.http.Host == "qa.polyfill.io") {
			if (req.http.X-Geoip-Continent ~ "(NA|SA|OC|AS)") {
				set req.backend = origami_polyfill_service_us;
				set req.http.Host = "ft-polyfill-service-us-qa.herokuapp.com";

				if (!req.backend.healthy) {
					set req.backend = origami_polyfill_service_eu;
					set req.http.Host = "ft-polyfill-service-qa.herokuapp.com";
				}

			} else {
				set req.backend = origami_polyfill_service_eu;
				set req.http.Host = "ft-polyfill-service-qa.herokuapp.com";

				if (!req.backend.healthy) {
					set req.backend = origami_polyfill_service_us;
					set req.http.Host = "ft-polyfill-service-us-qa.herokuapp.com";
				}
			}
		} else {
			if (req.http.X-Geoip-Continent ~ "(NA|SA|OC|AS)") {
				set req.backend = origami_polyfill_service_us;
				set req.http.Host = "ft-polyfill-service-us.herokuapp.com";

				if (!req.backend.healthy) {
					set req.backend = origami_polyfill_service_eu;
					set req.http.Host = "ft-polyfill-service.herokuapp.com";
				}

			} else {
				set req.backend = origami_polyfill_service_eu;
				set req.http.Host = "ft-polyfill-service.herokuapp.com";

				if (!req.backend.healthy) {
					set req.backend = origami_polyfill_service_us;
					set req.http.Host = "ft-polyfill-service-us.herokuapp.com";
				}
			}
		}
	}

	# https://community.fastly.com/t/brotli-compression-support/578/6
	if (req.http.Fastly-Orig-Accept-Encoding) {
		if (req.http.User-Agent ~ "MSIE 6") {
			# For that 0.3% of stubborn users out there
			unset req.http.Accept-Encoding;
		} elsif (req.http.Fastly-Orig-Accept-Encoding ~ "br") {
			set req.http.Accept-Encoding = "br";
		} elsif (req.http.Fastly-Orig-Accept-Encoding ~ "gzip") {
			set req.http.Accept-Encoding = "gzip";
		} else {
			unset req.http.Accept-Encoding;
		}
	}

	return(lookup);
}

sub vcl_deliver {
#FASTLY deliver

	# If the response is to a normalise request and there's a parked "original request", use the normalised UA response to modify the original request and restart it
	if (req.url ~ "^/v\d/normalizeUa" && resp.status == 200 && req.http.X-Orig-URL ~ "^/v2/(polyfill\.|recordRumData)") {
		set req.http.Fastly-force-Shield = "1";
		set req.http.Normalized-User-Agent = resp.http.Normalized-User-Agent;
		if (req.http.X-Orig-URL ~ "\?") {
			set req.url = req.http.X-Orig-URL "&ua=" resp.http.Normalized-User-Agent;
		} else {
			set req.url = req.http.X-Orig-URL "?ua=" resp.http.Normalized-User-Agent;
		}
		restart;

	# If the original request didn't specify a UA override, and we added one when the request was sent to the backend, the backend won't have included a Vary:user-agent header (correctly) but we need to add one
	} else if (req.url ~ "^/v\d/polyfill\..*[\?\&]ua=" && req.http.X-Orig-URL && req.http.X-Orig-URL !~ "[\?\&]ua=") {
		add resp.http.Vary = "User-Agent";
	}

	if (req.url ~ "[\&\?]rum=1") {
		add resp.http.Set-Cookie = "FastlyDC=" server.datacenter "; Path=/; HttpOnly; max-age=60";
	}

	if (req.http.Fastly-Debug) {
		set resp.http.Debug-Host = req.http.Host;
		set resp.http.Debug-X-Original-Host = req.http.X-Original-Host;
	}

	return(deliver);
}

sub vcl_error {
#FASTLY error

	# Redirect to canonical prod/qa origins
	if (obj.status == 751) {
		set obj.status = 301;
		set obj.response = "Moved Permanently";
		set obj.http.Location = "https://polyfill.io" req.url;
		synthetic {""};
		return (deliver);
	}
}

import boltsort;

sub vcl_recv {
#FASTLY recv

	# Enable API key authentication for URL purge requests
	if ( req.request == "FASTLYPURGE" ) {
		set req.http.Fastly-Purge-Requires-Auth = "1";
	}

	if (req.request != "HEAD" && req.request != "GET" && req.request != "FASTLYPURGE") {
		return(pass);
	}

	if (req.http.Host ~ "polyfills.io") {
		# Do the canonicalise check before the SSL check to avoid a double redirect
		error 751 "Canonicalise";
	}

	if (!req.http.Fastly-SSL) {
		# 801 is a special error code that Fastly uses to Force SSL on the request
		error 801 "Redirect to prod HTTPS";
	}

	if (req.url ~ "^/v2/(polyfill\.|recordRumData)" && req.url !~ "[\?\&]ua=") {
		set req.http.X-Orig-URL = req.url;
		set req.url = "/v2/normalizeUa?ua=" urlencode(req.http.User-Agent);
	}

	if (req.url ~ "^/v2/recordRumData" && req.http.Normalized-User-Agent) {
		set req.http.Log = regsub(req.url, "^.*?\?(.*)$", "\1") "&ip=" client.ip "&refer_domain=" regsub(req.http.Referer, "^(https?\:\/\/)?(www\.)?(.+?)(\:\d+)?([\/\?].*)?$", "\3") "&country=" geoip.country_code "&data_center=" if(req.http.Cookie:FastlyDC, req.http.Cookie:FastlyDC, server.datacenter);
		error 204 "No Content";
	}

	set req.url = boltsort.sort(req.url);

	declare local var.isQA BOOL;

	set var.isQA = (req.http.host == "qa.polyfill.io");

	if (req.http.X-Geoip-Continent ~ "(NA|SA|OC|AS)") {
		set req.backend = origami_polyfill_service_us;
		set req.http.Host = "ft-polyfill-service-us.herokuapp.com";

		if (!req.backend.healthy) {
			set req.backend = origami_polyfill_service_eu;
			set req.http.Host = "ft-polyfill-service.herokuapp.com";
		}

	} else {
		set req.backend = origami_polyfill_service_eu;
		set req.http.Host = "ft-polyfill-service.herokuapp.com";

		if (!req.backend.healthy) {
			set req.backend = origami_polyfill_service_us;
			set req.http.Host = "ft-polyfill-service-us.herokuapp.com";
		}
	}

	if (var.isQA) {
		if (req.backend == origami_polyfill_service_us) {
			set req.http.Host = "ft-polyfill-service-us-qa.herokuapp.com";
		} else {
			set req.http.Host = "ft-polyfill-service-qa.herokuapp.com";
		}
	}

	# https://community.fastly.com/t/brotli-compression-support/578/6
	if (req.http.Fastly-Orig-Accept-Encoding) {
		if (req.http.User-Agent ~ "MSIE 6") {
			# For that 0.3% of stubborn users out there
			unset req.http.Accept-Encoding;
		} elsif (req.http.Fastly-Orig-Accept-Encoding ~ "br") {
			set req.http.Accept-Encoding = "br";
		} elsif (req.http.Fastly-Orig-Accept-Encoding ~ "gzip") {
			set req.http.Accept-Encoding = "gzip";
		} else {
			unset req.http.Accept-Encoding;
		}
	}

	return(lookup);
}

sub vcl_deliver {
#FASTLY deliver

	# If the response is to a normalise request and there's a parked "original request", use the normalised UA response to modify the original request and restart it
	if (req.url ~ "^/v\d/normalizeUa" && resp.status == 200 && req.http.X-Orig-URL ~ "^/v2/(polyfill\.|recordRumData)") {
		set req.http.Fastly-force-Shield = "1";
		set req.http.Normalized-User-Agent = resp.http.Normalized-User-Agent;
		if (req.http.X-Orig-URL ~ "\?") {
			set req.url = req.http.X-Orig-URL "&ua=" resp.http.Normalized-User-Agent;
		} else {
			set req.url = req.http.X-Orig-URL "?ua=" resp.http.Normalized-User-Agent;
		}
		restart;

	# If the original request didn't specify a UA override, and we added one when the request was sent to the backend, the backend won't have included a Vary:user-agent header (correctly) but we need to add one
	} else if (req.url ~ "^/v\d/polyfill\..*[\?\&]ua=" && req.http.X-Orig-URL && req.http.X-Orig-URL !~ "[\?\&]ua=") {
		add resp.http.Vary = "User-Agent";
	}

	if (req.url ~ "[\&\?]rum=1") {
		add resp.http.Set-Cookie = "FastlyDC=" server.datacenter "; Path=/; HttpOnly; max-age=60";
	}

	if (req.http.Fastly-Debug) {
		set resp.http.Debug-Host = req.http.Host;
	}

	return(deliver);
}

sub vcl_error {
#FASTLY error

	# Redirect to canonical prod/qa origins
	if (obj.status == 751) {
		set obj.status = 301;
		set obj.response = "Moved Permanently";
		set obj.http.Location = "https://polyfill.io" req.url;
		synthetic {""};
		return (deliver);
	}
}


Set TTL to 0 if there are no TTL headers from origin

    /* handle 5XX (or any other unwanted status code) */
    if (beresp.status >= 500 && beresp.status < 600) {

        # let SOAP errors pass - better debugging
        if (beresp.http.Content-Type ~ "text/xml") {
            return (deliver);
        }

        /* deliver stale if the object is available */
        if (stale.exists) {
            return(deliver_stale);
        }

        if (req.restarts < 1 && (req.request == "GET" || req.request == "HEAD")) {
            restart;
        }

        /* else go to vcl_error to deliver a synthetic */
        error beresp.status beresp.response;
    }

    # Remove Set-Cookies from responses for static content
    # to match the cookie removal in recv.
    if (req.url ~ "^/(pub/)?(media|static)/") {
        unset beresp.http.set-cookie;

        # Set a short TTL for 404's since those can be temporary during the site build/index
        if (beresp.status == 404) {
            set beresp.ttl = 300s;
        }

    }

    if (beresp.http.Content-Type ~ "text/(html|xml)") {
        # enable ESI feature for Magento response by default
        esi;
        if (!beresp.http.Vary ~ "X-Magento-Vary,Https") {
            if (beresp.http.Vary) {
                set beresp.http.Vary = beresp.http.Vary ",X-Magento-Vary,Https";
            } else {
                set beresp.http.Vary = "X-Magento-Vary,Https";
            }
        }
        # Since varnish doesn't compress ESIs we need to hint to the HTTP/2 terminators to
        # compress it
        set beresp.http.x-compress-hint = "on";
    } else {
        # enable gzip for all static content
        if (http_status_matches(beresp.status, "200,404") && (beresp.http.content-type ~ "^(application\/x\-javascript|text\/css|application\/javascript|text\/javascript|application\/json|application\/vnd\.ms\-fontobject|application\/x\-font\-opentype|application\/x\-font\-truetype|application\/x\-font\-ttf|application\/xml|font\/eot|font\/opentype|font\/otf|image\/svg\+xml|image\/vnd\.microsoft\.icon|text\/plain)\s*($|;)" || req.url.ext ~ "(?i)(css|js|html|eot|ico|otf|ttf|json)" ) ) {
            # always set vary to make sure uncompressed versions dont always win
            if (!beresp.http.Vary ~ "Accept-Encoding") {
                if (beresp.http.Vary) {
                    set beresp.http.Vary = beresp.http.Vary ", Accept-Encoding";
                } else {
                    set beresp.http.Vary = "Accept-Encoding";
                }
            }
            if (req.http.Accept-Encoding == "gzip") {
                set beresp.gzip = true;
            }
        }
    }

    if (beresp.http.Cache-Control ~ "private|no-cache|no-store") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    if (beresp.http.X-Magento-Debug) {
        set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
    }

    # Never cache 302s
    if (beresp.status == 302) {
        return (pass);
    }

    # Just in case the Request Setting for x-pass is missing
    if (req.http.x-pass) {
        return (pass);
    }

    # Varnish sets default TTL if none of these are present
    if (!beresp.http.Expires && !beresp.http.Surrogate-Control ~ "max-age" && !beresp.http.Cache-Control ~ "(s-maxage|max-age)") {
        set beresp.ttl = 0s;
    }

    # validate if we need to cache it and prevent from setting cookie
    # images, css and js are cacheable by default so we have to remove cookie also
    if (beresp.ttl > 0s && (req.request == "GET" || req.request == "HEAD") && !req.http.x-pass ) {
        unset beresp.http.set-cookie;

        # init surrogate keys
        if (beresp.http.X-Magento-Tags) {
            set beresp.http.Surrogate-Key = beresp.http.X-Magento-Tags " text";
        } else {
            set beresp.http.Surrogate-Key = "text";
        }

        # set surrogate keys by content type if they are image/script or CSS
        if (beresp.http.Content-Type ~ "(image|script|css)") {
            set beresp.http.Surrogate-Key = re.group.1;
        }
        return (deliver);
    }

    /* handle 5XX (or any other unwanted status code) */
    if (beresp.status >= 500 && beresp.status < 600) {

        # let SOAP errors pass - better debugging
        if (beresp.http.Content-Type ~ "text/xml") {
            return (deliver);
        }

        /* deliver stale if the object is available */
        if (stale.exists) {
            return(deliver_stale);
        }

        if (req.restarts < 1 && (req.request == "GET" || req.request == "HEAD")) {
            restart;
        }

        /* else go to vcl_error to deliver a synthetic */
        error beresp.status beresp.response;
    }

    # Remove Set-Cookies from responses for static content
    # to match the cookie removal in recv.
    if (req.url ~ "^/(pub/)?(media|static)/") {
        unset beresp.http.set-cookie;

        # Set a short TTL for 404's since those can be temporary during the site build/index
        if (beresp.status == 404) {
            set beresp.ttl = 300s;
        }

    }

    if (beresp.http.Content-Type ~ "text/(html|xml)") {
        # enable ESI feature for Magento response by default
        esi;
        if (!beresp.http.Vary ~ "X-Magento-Vary,Https") {
            if (beresp.http.Vary) {
                set beresp.http.Vary = beresp.http.Vary ",X-Magento-Vary,Https";
            } else {
                set beresp.http.Vary = "X-Magento-Vary,Https";
            }
        }
        # Since varnish doesn't compress ESIs we need to hint to the HTTP/2 terminators to
        # compress it
        set beresp.http.x-compress-hint = "on";
    } else {
        # enable gzip for all static content
        if (http_status_matches(beresp.status, "200,404") && (beresp.http.content-type ~ "^(application\/x\-javascript|text\/css|application\/javascript|text\/javascript|application\/json|application\/vnd\.ms\-fontobject|application\/x\-font\-opentype|application\/x\-font\-truetype|application\/x\-font\-ttf|application\/xml|font\/eot|font\/opentype|font\/otf|image\/svg\+xml|image\/vnd\.microsoft\.icon|text\/plain)\s*($|;)" || req.url.ext ~ "(?i)(css|js|html|eot|ico|otf|ttf|json)" ) ) {
            # always set vary to make sure uncompressed versions dont always win
            if (!beresp.http.Vary ~ "Accept-Encoding") {
                if (beresp.http.Vary) {
                    set beresp.http.Vary = beresp.http.Vary ", Accept-Encoding";
                } else {
                    set beresp.http.Vary = "Accept-Encoding";
                }
            }
            if (req.http.Accept-Encoding == "gzip") {
                set beresp.gzip = true;
            }
        }
    }

    if (beresp.http.Cache-Control ~ "private|no-cache|no-store") {
        set req.http.Fastly-Cachetype = "PRIVATE";
        return (pass);
    }

    if (beresp.http.X-Magento-Debug) {
        set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
    }

    # Never cache 302s
    if (beresp.status == 302) {
        return (pass);
    }

    # Just in case the Request Setting for x-pass is missing
    if (req.http.x-pass) {
        return (pass);
    }

    # validate if we need to cache it and prevent from setting cookie
    # images, css and js are cacheable by default so we have to remove cookie also
    if (beresp.ttl > 0s && (req.request == "GET" || req.request == "HEAD") && !req.http.x-pass ) {
        unset beresp.http.set-cookie;

        # init surrogate keys
        if (beresp.http.X-Magento-Tags) {
            set beresp.http.Surrogate-Key = beresp.http.X-Magento-Tags " text";
        } else {
            set beresp.http.Surrogate-Key = "text";
        }

        # set surrogate keys by content type if they are image/script or CSS
        if (beresp.http.Content-Type ~ "(image|script|css)") {
            set beresp.http.Surrogate-Key = re.group.1;
        }
        return (deliver);
    }