We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Make Docs builds fallback in case external docs sources are missing
Every now and then our docs builds start to fail because of external dependency (latest example here #14985). And while we are doing caching now of that information, it does not help when the initial retrieval fails. This information does not change often but with the number of dependencies we have it will continue to fail regularly simply because many of those depenencies are not very reliable - they are just a web page hosted somewhere. They are nowhere near the stabilty of even PyPI or Apt sources and we have no mirroring in case of problem.

Maybe we could 

a) see if we can use some kind of mirroring scheme (do those sites have mirrrors ? )
b) if not, simply write a simple script that will dump the cached content for those to S3, refresh it in the CI scheduled (nightly) master builds ad have a fallback mechanism to download that from there in case of any problems in CI?

 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `docs/exts/docs_build/fetch_inventories.py`
Content:
```
1 # Licensed to the Apache Software Foundation (ASF) under one
2 # or more contributor license agreements.  See the NOTICE file
3 # distributed with this work for additional information
4 # regarding copyright ownership.  The ASF licenses this file
5 # to you under the Apache License, Version 2.0 (the
6 # "License"); you may not use this file except in compliance
7 # with the License.  You may obtain a copy of the License at
8 #
9 #   http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing,
12 # software distributed under the License is distributed on an
13 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 # KIND, either express or implied.  See the License for the
15 # specific language governing permissions and limitations
16 # under the License.
17 
18 import concurrent
19 import concurrent.futures
20 import datetime
21 import os
22 import shutil
23 from itertools import repeat
24 from typing import Iterator, List, Tuple
25 
26 import requests
27 from requests.adapters import DEFAULT_POOLSIZE
28 
29 from airflow.utils.helpers import partition
30 from docs.exts.docs_build.docs_builder import (  # pylint: disable=no-name-in-module
31     get_available_providers_packages,
32 )
33 from docs.exts.docs_build.third_party_inventories import (  # pylint: disable=no-name-in-module
34     THIRD_PARTY_INDEXES,
35 )
36 
37 CURRENT_DIR = os.path.dirname(__file__)
38 ROOT_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir, os.pardir))
39 DOCS_DIR = os.path.join(ROOT_DIR, 'docs')
40 CACHE_DIR = os.path.join(DOCS_DIR, '_inventory_cache')
41 EXPIRATION_DATE_PATH = os.path.join(DOCS_DIR, '_inventory_cache', "expiration-date")
42 
43 S3_DOC_URL = "http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com"
44 S3_DOC_URL_VERSIONED = S3_DOC_URL + "/docs/{package_name}/latest/objects.inv"
45 S3_DOC_URL_NON_VERSIONED = S3_DOC_URL + "/docs/{package_name}/objects.inv"
46 
47 
48 def _fetch_file(session: requests.Session, package_name: str, url: str, path: str) -> Tuple[str, bool]:
49     """
50     Download a file and returns status information as a tuple with package
51     name and success status(bool value).
52     """
53     response = session.get(url, allow_redirects=True, stream=True)
54     if not response.ok:
55         print(f"Failed to fetch inventory: {url}")
56         return package_name, False
57 
58     os.makedirs(os.path.dirname(path), exist_ok=True)
59     with open(path, 'wb') as f:
60         response.raw.decode_content = True
61         shutil.copyfileobj(response.raw, f)
62     print(f"Fetched inventory: {url}")
63     return package_name, True
64 
65 
66 def _is_outdated(path: str):
67     if not os.path.exists(path):
68         return True
69     delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(path))
70     return delta > datetime.timedelta(hours=12)
71 
72 
73 def fetch_inventories():
74     """Fetch all inventories for Airflow documentation packages and store in cache."""
75     os.makedirs(os.path.dirname(CACHE_DIR), exist_ok=True)
76     to_download: List[Tuple[str, str, str]] = []
77 
78     for pkg_name in get_available_providers_packages():
79         to_download.append(
80             (
81                 pkg_name,
82                 S3_DOC_URL_VERSIONED.format(package_name=pkg_name),
83                 f'{CACHE_DIR}/{pkg_name}/objects.inv',
84             )
85         )
86     to_download.append(
87         (
88             "apache-airflow",
89             S3_DOC_URL_VERSIONED.format(package_name='apache-airflow'),
90             f'{CACHE_DIR}/apache-airflow/objects.inv',
91         )
92     )
93     for pkg_name in ['apache-airflow-providers', 'docker-stack']:
94         to_download.append(
95             (
96                 pkg_name,
97                 S3_DOC_URL_NON_VERSIONED.format(package_name=pkg_name),
98                 f'{CACHE_DIR}/{pkg_name}/objects.inv',
99             )
100         )
101     to_download.extend(
102         (
103             pkg_name,
104             f"{doc_url}/objects.inv",
105             f'{CACHE_DIR}/{pkg_name}/objects.inv',
106         )
107         for pkg_name, doc_url in THIRD_PARTY_INDEXES.items()
108     )
109 
110     to_download = [(pkg_name, url, path) for pkg_name, url, path in to_download if _is_outdated(path)]
111     if not to_download:
112         print("Nothing to do")
113         return []
114 
115     print(f"To download {len(to_download)} inventorie(s)")
116 
117     with requests.Session() as session, concurrent.futures.ThreadPoolExecutor(DEFAULT_POOLSIZE) as pool:
118         download_results: Iterator[Tuple[str, bool]] = pool.map(
119             _fetch_file,
120             repeat(session, len(to_download)),
121             (pkg_name for pkg_name, _, _ in to_download),
122             (url for _, url, _ in to_download),
123             (path for _, _, path in to_download),
124         )
125     failed, success = partition(lambda d: d[1], download_results)
126     failed, success = list(failed), list(success)
127     print(f"Result: {len(success)} success, {len(failed)} failed")
128     if failed:
129         print("Failed packages:")
130         for pkg_no, (pkg_name, _) in enumerate(failed, start=1):
131             print(f"{pkg_no}. {pkg_name}")
132 
133     return [pkg_name for pkg_name, status in failed]
134
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/docs/exts/docs_build/fetch_inventories.py b/docs/exts/docs_build/fetch_inventories.py
--- a/docs/exts/docs_build/fetch_inventories.py
+++ b/docs/exts/docs_build/fetch_inventories.py
@@ -20,10 +20,13 @@
 import datetime
 import os
 import shutil
+import sys
+import traceback
 from itertools import repeat
 from typing import Iterator, List, Tuple
 
 import requests
+import urllib3.exceptions
 from requests.adapters import DEFAULT_POOLSIZE
 
 from airflow.utils.helpers import partition
@@ -50,9 +53,15 @@
     Download a file and returns status information as a tuple with package
     name and success status(bool value).
     """
-    response = session.get(url, allow_redirects=True, stream=True)
+    try:
+        response = session.get(url, allow_redirects=True, stream=True)
+    except (requests.RequestException, urllib3.exceptions.HTTPError):
+        print(f"Failed to fetch inventory: {url}")
+        traceback.print_exc(file=sys.stderr)
+        return package_name, False
     if not response.ok:
         print(f"Failed to fetch inventory: {url}")
+        print(f"Failed with status: {response.status_code}", file=sys.stderr)
         return package_name, False
 
     os.makedirs(os.path.dirname(path), exist_ok=True)

{"golden_diff": "diff --git a/docs/exts/docs_build/fetch_inventories.py b/docs/exts/docs_build/fetch_inventories.py\n--- a/docs/exts/docs_build/fetch_inventories.py\n+++ b/docs/exts/docs_build/fetch_inventories.py\n@@ -20,10 +20,13 @@\n import datetime\n import os\n import shutil\n+import sys\n+import traceback\n from itertools import repeat\n from typing import Iterator, List, Tuple\n \n import requests\n+import urllib3.exceptions\n from requests.adapters import DEFAULT_POOLSIZE\n \n from airflow.utils.helpers import partition\n@@ -50,9 +53,15 @@\n     Download a file and returns status information as a tuple with package\n     name and success status(bool value).\n     \"\"\"\n-    response = session.get(url, allow_redirects=True, stream=True)\n+    try:\n+        response = session.get(url, allow_redirects=True, stream=True)\n+    except (requests.RequestException, urllib3.exceptions.HTTPError):\n+        print(f\"Failed to fetch inventory: {url}\")\n+        traceback.print_exc(file=sys.stderr)\n+        return package_name, False\n     if not response.ok:\n         print(f\"Failed to fetch inventory: {url}\")\n+        print(f\"Failed with status: {response.status_code}\", file=sys.stderr)\n         return package_name, False\n \n     os.makedirs(os.path.dirname(path), exist_ok=True)\n", "issue": "Make Docs builds fallback in case external docs sources are missing\nEvery now and then our docs builds start to fail because of external dependency (latest example here #14985). And while we are doing caching now of that information, it does not help when the initial retrieval fails. This information does not change often but with the number of dependencies we have it will continue to fail regularly simply because many of those depenencies are not very reliable - they are just a web page hosted somewhere. They are nowhere near the stabilty of even PyPI or Apt sources and we have no mirroring in case of problem.\r\n\r\nMaybe we could \r\n\r\na) see if we can use some kind of mirroring scheme (do those sites have mirrrors ? )\r\nb) if not, simply write a simple script that will dump the cached content for those to S3, refresh it in the CI scheduled (nightly) master builds ad have a fallback mechanism to download that from there in case of any problems in CI?\r\n\r\n \n", "before_files": [{"content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport concurrent\nimport concurrent.futures\nimport datetime\nimport os\nimport shutil\nfrom itertools import repeat\nfrom typing import Iterator, List, Tuple\n\nimport requests\nfrom requests.adapters import DEFAULT_POOLSIZE\n\nfrom airflow.utils.helpers import partition\nfrom docs.exts.docs_build.docs_builder import (  # pylint: disable=no-name-in-module\n    get_available_providers_packages,\n)\nfrom docs.exts.docs_build.third_party_inventories import (  # pylint: disable=no-name-in-module\n    THIRD_PARTY_INDEXES,\n)\n\nCURRENT_DIR = os.path.dirname(__file__)\nROOT_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir, os.pardir))\nDOCS_DIR = os.path.join(ROOT_DIR, 'docs')\nCACHE_DIR = os.path.join(DOCS_DIR, '_inventory_cache')\nEXPIRATION_DATE_PATH = os.path.join(DOCS_DIR, '_inventory_cache', \"expiration-date\")\n\nS3_DOC_URL = \"http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com\"\nS3_DOC_URL_VERSIONED = S3_DOC_URL + \"/docs/{package_name}/latest/objects.inv\"\nS3_DOC_URL_NON_VERSIONED = S3_DOC_URL + \"/docs/{package_name}/objects.inv\"\n\n\ndef _fetch_file(session: requests.Session, package_name: str, url: str, path: str) -> Tuple[str, bool]:\n    \"\"\"\n    Download a file and returns status information as a tuple with package\n    name and success status(bool value).\n    \"\"\"\n    response = session.get(url, allow_redirects=True, stream=True)\n    if not response.ok:\n        print(f\"Failed to fetch inventory: {url}\")\n        return package_name, False\n\n    os.makedirs(os.path.dirname(path), exist_ok=True)\n    with open(path, 'wb') as f:\n        response.raw.decode_content = True\n        shutil.copyfileobj(response.raw, f)\n    print(f\"Fetched inventory: {url}\")\n    return package_name, True\n\n\ndef _is_outdated(path: str):\n    if not os.path.exists(path):\n        return True\n    delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(path))\n    return delta > datetime.timedelta(hours=12)\n\n\ndef fetch_inventories():\n    \"\"\"Fetch all inventories for Airflow documentation packages and store in cache.\"\"\"\n    os.makedirs(os.path.dirname(CACHE_DIR), exist_ok=True)\n    to_download: List[Tuple[str, str, str]] = []\n\n    for pkg_name in get_available_providers_packages():\n        to_download.append(\n            (\n                pkg_name,\n                S3_DOC_URL_VERSIONED.format(package_name=pkg_name),\n                f'{CACHE_DIR}/{pkg_name}/objects.inv',\n            )\n        )\n    to_download.append(\n        (\n            \"apache-airflow\",\n            S3_DOC_URL_VERSIONED.format(package_name='apache-airflow'),\n            f'{CACHE_DIR}/apache-airflow/objects.inv',\n        )\n    )\n    for pkg_name in ['apache-airflow-providers', 'docker-stack']:\n        to_download.append(\n            (\n                pkg_name,\n                S3_DOC_URL_NON_VERSIONED.format(package_name=pkg_name),\n                f'{CACHE_DIR}/{pkg_name}/objects.inv',\n            )\n        )\n    to_download.extend(\n        (\n            pkg_name,\n            f\"{doc_url}/objects.inv\",\n            f'{CACHE_DIR}/{pkg_name}/objects.inv',\n        )\n        for pkg_name, doc_url in THIRD_PARTY_INDEXES.items()\n    )\n\n    to_download = [(pkg_name, url, path) for pkg_name, url, path in to_download if _is_outdated(path)]\n    if not to_download:\n        print(\"Nothing to do\")\n        return []\n\n    print(f\"To download {len(to_download)} inventorie(s)\")\n\n    with requests.Session() as session, concurrent.futures.ThreadPoolExecutor(DEFAULT_POOLSIZE) as pool:\n        download_results: Iterator[Tuple[str, bool]] = pool.map(\n            _fetch_file,\n            repeat(session, len(to_download)),\n            (pkg_name for pkg_name, _, _ in to_download),\n            (url for _, url, _ in to_download),\n            (path for _, _, path in to_download),\n        )\n    failed, success = partition(lambda d: d[1], download_results)\n    failed, success = list(failed), list(success)\n    print(f\"Result: {len(success)} success, {len(failed)} failed\")\n    if failed:\n        print(\"Failed packages:\")\n        for pkg_no, (pkg_name, _) in enumerate(failed, start=1):\n            print(f\"{pkg_no}. {pkg_name}\")\n\n    return [pkg_name for pkg_name, status in failed]\n", "path": "docs/exts/docs_build/fetch_inventories.py"}], "after_files": [{"content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n#   http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\nimport concurrent\nimport concurrent.futures\nimport datetime\nimport os\nimport shutil\nimport sys\nimport traceback\nfrom itertools import repeat\nfrom typing import Iterator, List, Tuple\n\nimport requests\nimport urllib3.exceptions\nfrom requests.adapters import DEFAULT_POOLSIZE\n\nfrom airflow.utils.helpers import partition\nfrom docs.exts.docs_build.docs_builder import (  # pylint: disable=no-name-in-module\n    get_available_providers_packages,\n)\nfrom docs.exts.docs_build.third_party_inventories import (  # pylint: disable=no-name-in-module\n    THIRD_PARTY_INDEXES,\n)\n\nCURRENT_DIR = os.path.dirname(__file__)\nROOT_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir, os.pardir))\nDOCS_DIR = os.path.join(ROOT_DIR, 'docs')\nCACHE_DIR = os.path.join(DOCS_DIR, '_inventory_cache')\nEXPIRATION_DATE_PATH = os.path.join(DOCS_DIR, '_inventory_cache', \"expiration-date\")\n\nS3_DOC_URL = \"http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com\"\nS3_DOC_URL_VERSIONED = S3_DOC_URL + \"/docs/{package_name}/latest/objects.inv\"\nS3_DOC_URL_NON_VERSIONED = S3_DOC_URL + \"/docs/{package_name}/objects.inv\"\n\n\ndef _fetch_file(session: requests.Session, package_name: str, url: str, path: str) -> Tuple[str, bool]:\n    \"\"\"\n    Download a file and returns status information as a tuple with package\n    name and success status(bool value).\n    \"\"\"\n    try:\n        response = session.get(url, allow_redirects=True, stream=True)\n    except (requests.RequestException, urllib3.exceptions.HTTPError):\n        print(f\"Failed to fetch inventory: {url}\")\n        traceback.print_exc(file=sys.stderr)\n        return package_name, False\n    if not response.ok:\n        print(f\"Failed to fetch inventory: {url}\")\n        print(f\"Failed with status: {response.status_code}\", file=sys.stderr)\n        return package_name, False\n\n    os.makedirs(os.path.dirname(path), exist_ok=True)\n    with open(path, 'wb') as f:\n        response.raw.decode_content = True\n        shutil.copyfileobj(response.raw, f)\n    print(f\"Fetched inventory: {url}\")\n    return package_name, True\n\n\ndef _is_outdated(path: str):\n    if not os.path.exists(path):\n        return True\n    delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(path))\n    return delta > datetime.timedelta(hours=12)\n\n\ndef fetch_inventories():\n    \"\"\"Fetch all inventories for Airflow documentation packages and store in cache.\"\"\"\n    os.makedirs(os.path.dirname(CACHE_DIR), exist_ok=True)\n    to_download: List[Tuple[str, str, str]] = []\n\n    for pkg_name in get_available_providers_packages():\n        to_download.append(\n            (\n                pkg_name,\n                S3_DOC_URL_VERSIONED.format(package_name=pkg_name),\n                f'{CACHE_DIR}/{pkg_name}/objects.inv',\n            )\n        )\n    to_download.append(\n        (\n            \"apache-airflow\",\n            S3_DOC_URL_VERSIONED.format(package_name='apache-airflow'),\n            f'{CACHE_DIR}/apache-airflow/objects.inv',\n        )\n    )\n    for pkg_name in ['apache-airflow-providers', 'docker-stack']:\n        to_download.append(\n            (\n                pkg_name,\n                S3_DOC_URL_NON_VERSIONED.format(package_name=pkg_name),\n                f'{CACHE_DIR}/{pkg_name}/objects.inv',\n            )\n        )\n    to_download.extend(\n        (\n            pkg_name,\n            f\"{doc_url}/objects.inv\",\n            f'{CACHE_DIR}/{pkg_name}/objects.inv',\n        )\n        for pkg_name, doc_url in THIRD_PARTY_INDEXES.items()\n    )\n\n    to_download = [(pkg_name, url, path) for pkg_name, url, path in to_download if _is_outdated(path)]\n    if not to_download:\n        print(\"Nothing to do\")\n        return []\n\n    print(f\"To download {len(to_download)} inventorie(s)\")\n\n    with requests.Session() as session, concurrent.futures.ThreadPoolExecutor(DEFAULT_POOLSIZE) as pool:\n        download_results: Iterator[Tuple[str, bool]] = pool.map(\n            _fetch_file,\n            repeat(session, len(to_download)),\n            (pkg_name for pkg_name, _, _ in to_download),\n            (url for _, url, _ in to_download),\n            (path for _, _, path in to_download),\n        )\n    failed, success = partition(lambda d: d[1], download_results)\n    failed, success = list(failed), list(success)\n    print(f\"Result: {len(success)} success, {len(failed)} failed\")\n    if failed:\n        print(\"Failed packages:\")\n        for pkg_no, (pkg_name, _) in enumerate(failed, start=1):\n            print(f\"{pkg_no}. {pkg_name}\")\n\n    return [pkg_name for pkg_name, status in failed]\n", "path": "docs/exts/docs_build/fetch_inventories.py"}]}