def download_static_assets(doc, destination, base_url,
        request_fn=make_request, url_blacklist=[], js_middleware=None,
        css_middleware=None, derive_filename=_derive_filename):
    """
    Download all static assets referenced from an HTML page.
    The goal is to easily create HTML5 apps! Downloads JS, CSS, images, and
    audio clips.

    Args:
        doc: The HTML page source as a string or BeautifulSoup instance.
        destination: The folder to download the static assets to!
        base_url: The base URL where assets will be downloaded from.
        request_fn: The function to be called to make requests, passed to
            ricecooker.utils.html.download_file(). Pass in a custom one for custom
            caching logic.
        url_blacklist: A list of keywords of files to not include in downloading.
            Will do substring matching, so e.g. 'acorn.js' will match
            '/some/path/to/acorn.js'.
        js_middleware: If specificed, JS content will be passed into this callback
            which is expected to return JS content with any modifications.
        css_middleware: If specificed, CSS content will be passed into this callback
            which is expected to return CSS content with any modifications.

    Return the modified page HTML with links rewritten to the locations of the
    downloaded static files, as a BeautifulSoup object. (Call str() on it to
    extract the raw HTML.)
    """
    if not isinstance(doc, BeautifulSoup):
        doc = BeautifulSoup(doc, "html.parser")

    # Helper function to download all assets for a given CSS selector.
    def download_assets(selector, attr, url_middleware=None,
            content_middleware=None, node_filter=None):
        nodes = doc.select(selector)

        for i, node in enumerate(nodes):

            if node_filter:
                if not node_filter(node):
                    src = node[attr]
                    node[attr] = ''
                    print('        Skipping node with src ', src)
                    continue

            if node[attr].startswith('data:'):
                continue

            url = urljoin(base_url, node[attr])

            if _is_blacklisted(url, url_blacklist):
                print('        Skipping downloading blacklisted url', url)
                node[attr] = ""
                continue

            if url_middleware:
                url = url_middleware(url)

            filename = derive_filename(url)
            node[attr] = filename

            print("        Downloading", url, "to filename", filename)
            download_file(url, destination, request_fn=request_fn,
                    filename=filename, middleware_callbacks=content_middleware)

    def js_content_middleware(content, url, **kwargs):
        if js_middleware:
            content = js_middleware(content, url, **kwargs)

        # Polyfill localStorage and document.cookie as iframes can't access
        # them
        return (content
            .replace("localStorage", "_localStorage")
            .replace('document.cookie.split', '"".split')
            .replace('document.cookie', 'window._document_cookie'))

    def css_node_filter(node):
        return "stylesheet" in node["rel"]

    def css_content_middleware(content, url, **kwargs):
        if css_middleware:
            content = css_middleware(content, url, **kwargs)

        file_dir = os.path.dirname(urlparse(url).path)

        # Download linked fonts and images
        def repl(match):
            src = match.group(1)
            if src.startswith('//localhost'):
                return 'url()'
            # Don't download data: files
            if src.startswith('data:'):
                return match.group(0)
            src_url = urljoin(base_url, os.path.join(file_dir, src))

            if _is_blacklisted(src_url, url_blacklist):
                print('        Skipping downloading blacklisted url', src_url)
                return 'url()'

            derived_filename = derive_filename(src_url)
            download_file(src_url, destination, request_fn=request_fn,
                    filename=derived_filename)
            return 'url("%s")' % derived_filename

        return _CSS_URL_RE.sub(repl, content)

    # Download all linked static assets.
    download_assets("img[src]", "src")  # Images
    download_assets("link[href]", "href",
            content_middleware=css_content_middleware,
            node_filter=css_node_filter)  # CSS
    download_assets("script[src]", "src",
            content_middleware=js_content_middleware) # JS
    download_assets("source[src]", "src") # Potentially audio
    download_assets("source[srcset]", "srcset") # Potentially audio

    # ... and also run the middleware on CSS/JS embedded in the page source to
    # get linked files.
    for node in doc.select('style'):
        node.string = css_content_middleware(node.get_text(), url='')

    for node in doc.select('script'):
        if not node.attrs.get('src'):
            node.string = js_content_middleware(node.get_text(), url='')

    return doc
Download all static assets referenced from an HTML page.
    The goal is to easily create HTML5 apps! Downloads JS, CSS, images, and
    audio clips.

    Args:
        doc: The HTML page source as a string or BeautifulSoup instance.
        destination: The folder to download the static assets to!
        base_url: The base URL where assets will be downloaded from.
        request_fn: The function to be called to make requests, passed to
            ricecooker.utils.html.download_file(). Pass in a custom one for custom
            caching logic.
        url_blacklist: A list of keywords of files to not include in downloading.
            Will do substring matching, so e.g. 'acorn.js' will match
            '/some/path/to/acorn.js'.
        js_middleware: If specificed, JS content will be passed into this callback
            which is expected to return JS content with any modifications.
        css_middleware: If specificed, CSS content will be passed into this callback
            which is expected to return CSS content with any modifications.

    Return the modified page HTML with links rewritten to the locations of the
    downloaded static files, as a BeautifulSoup object. (Call str() on it to
    extract the raw HTML.)