prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Make Docs builds fallback in case external docs sources are missing
  Every now and then our docs builds start to fail because of external dependency (latest example here #14985). And while we are doing caching now of that information, it does not help when the initial retrieval fails. This information does not change often but with the number of dependencies we have it will continue to fail regularly simply because many of those depenencies are not very reliable - they are just a web page hosted somewhere. They are nowhere near the stabilty of even PyPI or Apt sources and we have no mirroring in case of problem.
  
  Maybe we could 
  
  a) see if we can use some kind of mirroring scheme (do those sites have mirrrors ? )
  b) if not, simply write a simple script that will dump the cached content for those to S3, refresh it in the CI scheduled (nightly) master builds ad have a fallback mechanism to download that from there in case of any problems in CI?
  
   
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `docs/exts/docs_build/fetch_inventories.py`
  Content:
  ```
  1 # Licensed to the Apache Software Foundation (ASF) under one
  2 # or more contributor license agreements.  See the NOTICE file
  3 # distributed with this work for additional information
  4 # regarding copyright ownership.  The ASF licenses this file
  5 # to you under the Apache License, Version 2.0 (the
  6 # "License"); you may not use this file except in compliance
  7 # with the License.  You may obtain a copy of the License at
  8 #
  9 #   http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing,
  12 # software distributed under the License is distributed on an
  13 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  14 # KIND, either express or implied.  See the License for the
  15 # specific language governing permissions and limitations
  16 # under the License.
  17 
  18 import concurrent
  19 import concurrent.futures
  20 import datetime
  21 import os
  22 import shutil
  23 from itertools import repeat
  24 from typing import Iterator, List, Tuple
  25 
  26 import requests
  27 from requests.adapters import DEFAULT_POOLSIZE
  28 
  29 from airflow.utils.helpers import partition
  30 from docs.exts.docs_build.docs_builder import (  # pylint: disable=no-name-in-module
  31     get_available_providers_packages,
  32 )
  33 from docs.exts.docs_build.third_party_inventories import (  # pylint: disable=no-name-in-module
  34     THIRD_PARTY_INDEXES,
  35 )
  36 
  37 CURRENT_DIR = os.path.dirname(__file__)
  38 ROOT_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir, os.pardir))
  39 DOCS_DIR = os.path.join(ROOT_DIR, 'docs')
  40 CACHE_DIR = os.path.join(DOCS_DIR, '_inventory_cache')
  41 EXPIRATION_DATE_PATH = os.path.join(DOCS_DIR, '_inventory_cache', "expiration-date")
  42 
  43 S3_DOC_URL = "http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com"
  44 S3_DOC_URL_VERSIONED = S3_DOC_URL + "/docs/{package_name}/latest/objects.inv"
  45 S3_DOC_URL_NON_VERSIONED = S3_DOC_URL + "/docs/{package_name}/objects.inv"
  46 
  47 
  48 def _fetch_file(session: requests.Session, package_name: str, url: str, path: str) -> Tuple[str, bool]:
  49     """
  50     Download a file and returns status information as a tuple with package
  51     name and success status(bool value).
  52     """
  53     response = session.get(url, allow_redirects=True, stream=True)
  54     if not response.ok:
  55         print(f"Failed to fetch inventory: {url}")
  56         return package_name, False
  57 
  58     os.makedirs(os.path.dirname(path), exist_ok=True)
  59     with open(path, 'wb') as f:
  60         response.raw.decode_content = True
  61         shutil.copyfileobj(response.raw, f)
  62     print(f"Fetched inventory: {url}")
  63     return package_name, True
  64 
  65 
  66 def _is_outdated(path: str):
  67     if not os.path.exists(path):
  68         return True
  69     delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(path))
  70     return delta > datetime.timedelta(hours=12)
  71 
  72 
  73 def fetch_inventories():
  74     """Fetch all inventories for Airflow documentation packages and store in cache."""
  75     os.makedirs(os.path.dirname(CACHE_DIR), exist_ok=True)
  76     to_download: List[Tuple[str, str, str]] = []
  77 
  78     for pkg_name in get_available_providers_packages():
  79         to_download.append(
  80             (
  81                 pkg_name,
  82                 S3_DOC_URL_VERSIONED.format(package_name=pkg_name),
  83                 f'{CACHE_DIR}/{pkg_name}/objects.inv',
  84             )
  85         )
  86     to_download.append(
  87         (
  88             "apache-airflow",
  89             S3_DOC_URL_VERSIONED.format(package_name='apache-airflow'),
  90             f'{CACHE_DIR}/apache-airflow/objects.inv',
  91         )
  92     )
  93     for pkg_name in ['apache-airflow-providers', 'docker-stack']:
  94         to_download.append(
  95             (
  96                 pkg_name,
  97                 S3_DOC_URL_NON_VERSIONED.format(package_name=pkg_name),
  98                 f'{CACHE_DIR}/{pkg_name}/objects.inv',
  99             )
  100         )
  101     to_download.extend(
  102         (
  103             pkg_name,
  104             f"{doc_url}/objects.inv",
  105             f'{CACHE_DIR}/{pkg_name}/objects.inv',
  106         )
  107         for pkg_name, doc_url in THIRD_PARTY_INDEXES.items()
  108     )
  109 
  110     to_download = [(pkg_name, url, path) for pkg_name, url, path in to_download if _is_outdated(path)]
  111     if not to_download:
  112         print("Nothing to do")
  113         return []
  114 
  115     print(f"To download {len(to_download)} inventorie(s)")
  116 
  117     with requests.Session() as session, concurrent.futures.ThreadPoolExecutor(DEFAULT_POOLSIZE) as pool:
  118         download_results: Iterator[Tuple[str, bool]] = pool.map(
  119             _fetch_file,
  120             repeat(session, len(to_download)),
  121             (pkg_name for pkg_name, _, _ in to_download),
  122             (url for _, url, _ in to_download),
  123             (path for _, _, path in to_download),
  124         )
  125     failed, success = partition(lambda d: d[1], download_results)
  126     failed, success = list(failed), list(success)
  127     print(f"Result: {len(success)} success, {len(failed)} failed")
  128     if failed:
  129         print("Failed packages:")
  130         for pkg_no, (pkg_name, _) in enumerate(failed, start=1):
  131             print(f"{pkg_no}. {pkg_name}")
  132 
  133     return [pkg_name for pkg_name, status in failed]
  134
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/docs/exts/docs_build/fetch_inventories.py b/docs/exts/docs_build/fetch_inventories.py
  --- a/docs/exts/docs_build/fetch_inventories.py
  +++ b/docs/exts/docs_build/fetch_inventories.py
  @@ -20,10 +20,13 @@
   import datetime
   import os
   import shutil
  +import sys
  +import traceback
   from itertools import repeat
   from typing import Iterator, List, Tuple
   
   import requests
  +import urllib3.exceptions
   from requests.adapters import DEFAULT_POOLSIZE
   
   from airflow.utils.helpers import partition
  @@ -50,9 +53,15 @@
       Download a file and returns status information as a tuple with package
       name and success status(bool value).
       """
  -    response = session.get(url, allow_redirects=True, stream=True)
  +    try:
  +        response = session.get(url, allow_redirects=True, stream=True)
  +    except (requests.RequestException, urllib3.exceptions.HTTPError):
  +        print(f"Failed to fetch inventory: {url}")
  +        traceback.print_exc(file=sys.stderr)
  +        return package_name, False
       if not response.ok:
           print(f"Failed to fetch inventory: {url}")
  +        print(f"Failed with status: {response.status_code}", file=sys.stderr)
           return package_name, False
   
       os.makedirs(os.path.dirname(path), exist_ok=True)
  
verification_info: {"golden_diff": "diff --git a/docs/exts/docs_build/fetch_inventories.py b/docs/exts/docs_build/fetch_inventories.py\n--- a/docs/exts/docs_build/fetch_inventories.py\n+++ b/docs/exts/docs_build/fetch_inventories.py\n@@ -20,10 +20,13 @@\n import datetime\n import os\n import shutil\n+import sys\n+import traceback\n from itertools import repeat\n from typing import Iterator, List, Tuple\n \n import requests\n+import urllib3.exceptions\n from requests.adapters import DEFAULT_POOLSIZE\n \n from airflow.utils.helpers import partition\n@@ -50,9 +53,15 @@\n     Download a file and returns status information as a tuple with package\n     name and success status(bool value).\n     \"\"\"\n-    response = session.get(url, allow_redirects=True, stream=True)\n+    try:\n+        response = session.get(url, allow_redirects=True, stream=True)\n+    except (requests.RequestException, urllib3.exceptions.HTTPError):\n+        print(f\"Failed to fetch inventory: {url}\")\n+        traceback.print_exc(file=sys.stderr)\n+        return package_name, False\n     if not response.ok:\n         print(f\"Failed to fetch inventory: {url}\")\n+        print(f\"Failed with status: {response.status_code}\", file=sys.stderr)\n         return package_name, False\n \n     os.makedirs(os.path.dirname(path), exist_ok=True)\n", "issue": "Make Docs builds fallback in case external docs sources are missing\nEvery now and then our docs builds start to fail because of external dependency (latest example here #14985). And while we are doing caching now of that information, it does not help when the initial retrieval fails. This information does not change often but with the number of dependencies we have it will continue to fail regularly simply because many of those depenencies are not very reliable - they are just a web page hosted somewhere. They are nowhere near the stabilty of even PyPI or Apt sources and we have no mirroring in case of problem.\r\n\r\nMaybe we could \r\n\r\na) see if we can use some kind of mirroring scheme (do those sites have mirrrors ? )\r\nb) if not, simply write a simple script that will dump the cached content for those to S3, refresh it in the CI scheduled (nightly) master builds ad have a fallback mechanism to download that from there in case of any problems in CI?\r\n\r\n \n", "before_files": [{"content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport concurrent\nimport concurrent.futures\nimport datetime\nimport os\nimport shutil\nfrom itertools import repeat\nfrom typing import Iterator, List, Tuple\n\nimport requests\nfrom requests.adapters import DEFAULT_POOLSIZE\n\nfrom airflow.utils.helpers import partition\nfrom docs.exts.docs_build.docs_builder import (  # pylint: disable=no-name-in-module\n    get_available_providers_packages,\n)\nfrom docs.exts.docs_build.third_party_inventories import (  # pylint: disable=no-name-in-module\n    THIRD_PARTY_INDEXES,\n)\n\nCURRENT_DIR = os.path.dirname(__file__)\nROOT_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir, os.pardir))\nDOCS_DIR = os.path.join(ROOT_DIR, 'docs')\nCACHE_DIR = os.path.join(DOCS_DIR, '_inventory_cache')\nEXPIRATION_DATE_PATH = os.path.join(DOCS_DIR, '_inventory_cache', \"expiration-date\")\n\nS3_DOC_URL = \"http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com\"\nS3_DOC_URL_VERSIONED = S3_DOC_URL + \"/docs/{package_name}/latest/objects.inv\"\nS3_DOC_URL_NON_VERSIONED = S3_DOC_URL + \"/docs/{package_name}/objects.inv\"\n\n\ndef _fetch_file(session: requests.Session, package_name: str, url: str, path: str) -> Tuple[str, bool]:\n    \"\"\"\n    Download a file and returns status information as a tuple with package\n    name and success status(bool value).\n    \"\"\"\n    response = session.get(url, allow_redirects=True, stream=True)\n    if not response.ok:\n        print(f\"Failed to fetch inventory: {url}\")\n        return package_name, False\n\n    os.makedirs(os.path.dirname(path), exist_ok=True)\n    with open(path, 'wb') as f:\n        response.raw.decode_content = True\n        shutil.copyfileobj(response.raw, f)\n    print(f\"Fetched inventory: {url}\")\n    return package_name, True\n\n\ndef _is_outdated(path: str):\n    if not os.path.exists(path):\n        return True\n    delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(path))\n    return delta > datetime.timedelta(hours=12)\n\n\ndef fetch_inventories():\n    \"\"\"Fetch all inventories for Airflow documentation packages and store in cache.\"\"\"\n    os.makedirs(os.path.dirname(CACHE_DIR), exist_ok=True)\n    to_download: List[Tuple[str, str, str]] = []\n\n    for pkg_name in get_available_providers_packages():\n        to_download.append(\n            (\n                pkg_name,\n                S3_DOC_URL_VERSIONED.format(package_name=pkg_name),\n                f'{CACHE_DIR}/{pkg_name}/objects.inv',\n            )\n        )\n    to_download.append(\n        (\n            \"apache-airflow\",\n            S3_DOC_URL_VERSIONED.format(package_name='apache-airflow'),\n            f'{CACHE_DIR}/apache-airflow/objects.inv',\n        )\n    )\n    for pkg_name in ['apache-airflow-providers', 'docker-stack']:\n        to_download.append(\n            (\n                pkg_name,\n                S3_DOC_URL_NON_VERSIONED.format(package_name=pkg_name),\n                f'{CACHE_DIR}/{pkg_name}/objects.inv',\n            )\n        )\n    to_download.extend(\n        (\n            pkg_name,\n            f\"{doc_url}/objects.inv\",\n            f'{CACHE_DIR}/{pkg_name}/objects.inv',\n        )\n        for pkg_name, doc_url in THIRD_PARTY_INDEXES.items()\n    )\n\n    to_download = [(pkg_name, url, path) for pkg_name, url, path in to_download if _is_outdated(path)]\n    if not to_download:\n        print(\"Nothing to do\")\n        return []\n\n    print(f\"To download {len(to_download)} inventorie(s)\")\n\n    with requests.Session() as session, concurrent.futures.ThreadPoolExecutor(DEFAULT_POOLSIZE) as pool:\n        download_results: Iterator[Tuple[str, bool]] = pool.map(\n            _fetch_file,\n            repeat(session, len(to_download)),\n            (pkg_name for pkg_name, _, _ in to_download),\n            (url for _, url, _ in to_download),\n            (path for _, _, path in to_download),\n        )\n    failed, success = partition(lambda d: d[1], download_results)\n    failed, success = list(failed), list(success)\n    print(f\"Result: {len(success)} success, {len(failed)} failed\")\n    if failed:\n        print(\"Failed packages:\")\n        for pkg_no, (pkg_name, _) in enumerate(failed, start=1):\n            print(f\"{pkg_no}. {pkg_name}\")\n\n    return [pkg_name for pkg_name, status in failed]\n", "path": "docs/exts/docs_build/fetch_inventories.py"}], "after_files": [{"content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport concurrent\nimport concurrent.futures\nimport datetime\nimport os\nimport shutil\nimport sys\nimport traceback\nfrom itertools import repeat\nfrom typing import Iterator, List, Tuple\n\nimport requests\nimport urllib3.exceptions\nfrom requests.adapters import DEFAULT_POOLSIZE\n\nfrom airflow.utils.helpers import partition\nfrom docs.exts.docs_build.docs_builder import (  # pylint: disable=no-name-in-module\n    get_available_providers_packages,\n)\nfrom docs.exts.docs_build.third_party_inventories import (  # pylint: disable=no-name-in-module\n    THIRD_PARTY_INDEXES,\n)\n\nCURRENT_DIR = os.path.dirname(__file__)\nROOT_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir, os.pardir))\nDOCS_DIR = os.path.join(ROOT_DIR, 'docs')\nCACHE_DIR = os.path.join(DOCS_DIR, '_inventory_cache')\nEXPIRATION_DATE_PATH = os.path.join(DOCS_DIR, '_inventory_cache', \"expiration-date\")\n\nS3_DOC_URL = \"http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com\"\nS3_DOC_URL_VERSIONED = S3_DOC_URL + \"/docs/{package_name}/latest/objects.inv\"\nS3_DOC_URL_NON_VERSIONED = S3_DOC_URL + \"/docs/{package_name}/objects.inv\"\n\n\ndef _fetch_file(session: requests.Session, package_name: str, url: str, path: str) -> Tuple[str, bool]:\n    \"\"\"\n    Download a file and returns status information as a tuple with package\n    name and success status(bool value).\n    \"\"\"\n    try:\n        response = session.get(url, allow_redirects=True, stream=True)\n    except (requests.RequestException, urllib3.exceptions.HTTPError):\n        print(f\"Failed to fetch inventory: {url}\")\n        traceback.print_exc(file=sys.stderr)\n        return package_name, False\n    if not response.ok:\n        print(f\"Failed to fetch inventory: {url}\")\n        print(f\"Failed with status: {response.status_code}\", file=sys.stderr)\n        return package_name, False\n\n    os.makedirs(os.path.dirname(path), exist_ok=True)\n    with open(path, 'wb') as f:\n        response.raw.decode_content = True\n        shutil.copyfileobj(response.raw, f)\n    print(f\"Fetched inventory: {url}\")\n    return package_name, True\n\n\ndef _is_outdated(path: str):\n    if not os.path.exists(path):\n        return True\n    delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(path))\n    return delta > datetime.timedelta(hours=12)\n\n\ndef fetch_inventories():\n    \"\"\"Fetch all inventories for Airflow documentation packages and store in cache.\"\"\"\n    os.makedirs(os.path.dirname(CACHE_DIR), exist_ok=True)\n    to_download: List[Tuple[str, str, str]] = []\n\n    for pkg_name in get_available_providers_packages():\n        to_download.append(\n            (\n                pkg_name,\n                S3_DOC_URL_VERSIONED.format(package_name=pkg_name),\n                f'{CACHE_DIR}/{pkg_name}/objects.inv',\n            )\n        )\n    to_download.append(\n        (\n            \"apache-airflow\",\n            S3_DOC_URL_VERSIONED.format(package_name='apache-airflow'),\n            f'{CACHE_DIR}/apache-airflow/objects.inv',\n        )\n    )\n    for pkg_name in ['apache-airflow-providers', 'docker-stack']:\n        to_download.append(\n            (\n                pkg_name,\n                S3_DOC_URL_NON_VERSIONED.format(package_name=pkg_name),\n                f'{CACHE_DIR}/{pkg_name}/objects.inv',\n            )\n        )\n    to_download.extend(\n        (\n            pkg_name,\n            f\"{doc_url}/objects.inv\",\n            f'{CACHE_DIR}/{pkg_name}/objects.inv',\n        )\n        for pkg_name, doc_url in THIRD_PARTY_INDEXES.items()\n    )\n\n    to_download = [(pkg_name, url, path) for pkg_name, url, path in to_download if _is_outdated(path)]\n    if not to_download:\n        print(\"Nothing to do\")\n        return []\n\n    print(f\"To download {len(to_download)} inventorie(s)\")\n\n    with requests.Session() as session, concurrent.futures.ThreadPoolExecutor(DEFAULT_POOLSIZE) as pool:\n        download_results: Iterator[Tuple[str, bool]] = pool.map(\n            _fetch_file,\n            repeat(session, len(to_download)),\n            (pkg_name for pkg_name, _, _ in to_download),\n            (url for _, url, _ in to_download),\n            (path for _, _, path in to_download),\n        )\n    failed, success = partition(lambda d: d[1], download_results)\n    failed, success = list(failed), list(success)\n    print(f\"Result: {len(success)} success, {len(failed)} failed\")\n    if failed:\n        print(\"Failed packages:\")\n        for pkg_no, (pkg_name, _) in enumerate(failed, start=1):\n            print(f\"{pkg_no}. {pkg_name}\")\n\n    return [pkg_name for pkg_name, status in failed]\n", "path": "docs/exts/docs_build/fetch_inventories.py"}]}