We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Anonymous credentials for single model serving with s3
/kind feature

**Describe the solution you'd like**
[A clear and concise description of what you want to happen.]
Update [download from s3](https://github.com/kubeflow/kfserving/blob/4fd401e348174f954fee7596dc3d54f197f4aa8c/python/kfserving/kfserving/storage.py#L86) to configure boto3 to use anon credentials if anonymous credentials env is set to true.

**Anything else you would like to add:**
[Miscellaneous information that will assist in solving the issue.]
Would be nice if anon e2e test were also added.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `python/kfserving/kfserving/storage.py`
Content:
```
1 # Copyright 2020 kubeflow.org.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #    http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 import glob
16 import gzip
17 import logging
18 import mimetypes
19 import os
20 import re
21 import json
22 import shutil
23 import tarfile
24 import tempfile
25 import zipfile
26 from urllib.parse import urlparse
27 
28 import boto3
29 import requests
30 from azure.storage.blob import BlockBlobService
31 from google.auth import exceptions
32 from google.cloud import storage
33 
34 from kfserving.kfmodel_repository import MODEL_MOUNT_DIRS
35 
36 _GCS_PREFIX = "gs://"
37 _S3_PREFIX = "s3://"
38 _BLOB_RE = "https://(.+?).blob.core.windows.net/(.+)"
39 _LOCAL_PREFIX = "file://"
40 _URI_RE = "https?://(.+)/(.+)"
41 _HTTP_PREFIX = "http(s)://"
42 _HEADERS_SUFFIX = "-headers"
43 
44 
45 class Storage(object):  # pylint: disable=too-few-public-methods
46     @staticmethod
47     def download(uri: str, out_dir: str = None) -> str:
48         logging.info("Copying contents of %s to local", uri)
49 
50         is_local = False
51         if uri.startswith(_LOCAL_PREFIX) or os.path.exists(uri):
52             is_local = True
53 
54         if out_dir is None:
55             if is_local:
56                 # noop if out_dir is not set and the path is local
57                 return Storage._download_local(uri)
58             out_dir = tempfile.mkdtemp()
59         elif not os.path.exists(out_dir):
60             os.mkdir(out_dir)
61 
62         if uri.startswith(_GCS_PREFIX):
63             Storage._download_gcs(uri, out_dir)
64         elif uri.startswith(_S3_PREFIX):
65             Storage._download_s3(uri, out_dir)
66         elif re.search(_BLOB_RE, uri):
67             Storage._download_blob(uri, out_dir)
68         elif is_local:
69             return Storage._download_local(uri, out_dir)
70         elif re.search(_URI_RE, uri):
71             return Storage._download_from_uri(uri, out_dir)
72         elif uri.startswith(MODEL_MOUNT_DIRS):
73             # Don't need to download models if this InferenceService is running in the multi-model
74             # serving mode. The model agent will download models.
75             return out_dir
76         else:
77             raise Exception("Cannot recognize storage type for " + uri +
78                             "\n'%s', '%s', '%s', and '%s' are the current available storage type." %
79                             (_GCS_PREFIX, _S3_PREFIX, _LOCAL_PREFIX, _HTTP_PREFIX))
80 
81         logging.info("Successfully copied %s to %s", uri, out_dir)
82         return out_dir
83 
84     @staticmethod
85     def _download_s3(uri, temp_dir: str):
86         s3 = boto3.resource('s3', endpoint_url=os.getenv("AWS_ENDPOINT_URL", "http://s3.amazonaws.com"))
87         parsed = urlparse(uri, scheme='s3')
88         bucket_name = parsed.netloc
89         bucket_path = parsed.path.lstrip('/')
90 
91         bucket = s3.Bucket(bucket_name)
92         for obj in bucket.objects.filter(Prefix=bucket_path):
93             # Skip where boto3 lists the directory as an object
94             if obj.key.endswith("/"):
95                 continue
96             # In the case where bucket_path points to a single object, set the target key to bucket_path
97             # Otherwise, remove the bucket_path prefix, strip any extra slashes, then prepend the target_dir
98             target_key = (
99                 obj.key
100                 if bucket_path == obj.key
101                 else obj.key.replace(bucket_path, "", 1).lstrip("/")
102             )
103             target = f"{temp_dir}/{target_key}"
104             if not os.path.exists(os.path.dirname(target)):
105                 os.makedirs(os.path.dirname(target), exist_ok=True)
106             bucket.download_file(obj.key, target)
107 
108     @staticmethod
109     def _download_gcs(uri, temp_dir: str):
110         try:
111             storage_client = storage.Client()
112         except exceptions.DefaultCredentialsError:
113             storage_client = storage.Client.create_anonymous_client()
114         bucket_args = uri.replace(_GCS_PREFIX, "", 1).split("/", 1)
115         bucket_name = bucket_args[0]
116         bucket_path = bucket_args[1] if len(bucket_args) > 1 else ""
117         bucket = storage_client.bucket(bucket_name)
118         prefix = bucket_path
119         if not prefix.endswith("/"):
120             prefix = prefix + "/"
121         blobs = bucket.list_blobs(prefix=prefix)
122         count = 0
123         for blob in blobs:
124             # Replace any prefix from the object key with temp_dir
125             subdir_object_key = blob.name.replace(bucket_path, "", 1).strip("/")
126 
127             # Create necessary subdirectory to store the object locally
128             if "/" in subdir_object_key:
129                 local_object_dir = os.path.join(temp_dir, subdir_object_key.rsplit("/", 1)[0])
130                 if not os.path.isdir(local_object_dir):
131                     os.makedirs(local_object_dir, exist_ok=True)
132             if subdir_object_key.strip() != "":
133                 dest_path = os.path.join(temp_dir, subdir_object_key)
134                 logging.info("Downloading: %s", dest_path)
135                 blob.download_to_filename(dest_path)
136             count = count + 1
137         if count == 0:
138             raise RuntimeError("Failed to fetch model. \
139 The path or model %s does not exist." % uri)
140 
141     @staticmethod
142     def _download_blob(uri, out_dir: str):  # pylint: disable=too-many-locals
143         match = re.search(_BLOB_RE, uri)
144         account_name = match.group(1)
145         storage_url = match.group(2)
146         container_name, prefix = storage_url.split("/", 1)
147 
148         logging.info("Connecting to BLOB account: [%s], container: [%s], prefix: [%s]",
149                      account_name,
150                      container_name,
151                      prefix)
152         try:
153             block_blob_service = BlockBlobService(account_name=account_name)
154             blobs = block_blob_service.list_blobs(container_name, prefix=prefix)
155         except Exception:  # pylint: disable=broad-except
156             token = Storage._get_azure_storage_token()
157             if token is None:
158                 logging.warning("Azure credentials not found, retrying anonymous access")
159             block_blob_service = BlockBlobService(account_name=account_name, token_credential=token)
160             blobs = block_blob_service.list_blobs(container_name, prefix=prefix)
161         count = 0
162         for blob in blobs:
163             dest_path = os.path.join(out_dir, blob.name)
164             if "/" in blob.name:
165                 head, tail = os.path.split(blob.name)
166                 if prefix is not None:
167                     head = head[len(prefix):]
168                 if head.startswith('/'):
169                     head = head[1:]
170                 dir_path = os.path.join(out_dir, head)
171                 dest_path = os.path.join(dir_path, tail)
172                 if not os.path.isdir(dir_path):
173                     os.makedirs(dir_path)
174 
175             logging.info("Downloading: %s to %s", blob.name, dest_path)
176             block_blob_service.get_blob_to_path(container_name, blob.name, dest_path)
177             count = count + 1
178         if count == 0:
179             raise RuntimeError("Failed to fetch model. \
180 The path or model %s does not exist." % (uri))
181 
182     @staticmethod
183     def _get_azure_storage_token():
184         tenant_id = os.getenv("AZ_TENANT_ID", "")
185         client_id = os.getenv("AZ_CLIENT_ID", "")
186         client_secret = os.getenv("AZ_CLIENT_SECRET", "")
187         subscription_id = os.getenv("AZ_SUBSCRIPTION_ID", "")
188 
189         if tenant_id == "" or client_id == "" or client_secret == "" or subscription_id == "":
190             return None
191 
192         # note the SP must have "Storage Blob Data Owner" perms for this to work
193         import adal
194         from azure.storage.common import TokenCredential
195 
196         authority_url = "https://login.microsoftonline.com/" + tenant_id
197 
198         context = adal.AuthenticationContext(authority_url)
199 
200         token = context.acquire_token_with_client_credentials(
201             "https://storage.azure.com/",
202             client_id,
203             client_secret)
204 
205         token_credential = TokenCredential(token["accessToken"])
206 
207         logging.info("Retrieved SP token credential for client_id: %s", client_id)
208 
209         return token_credential
210 
211     @staticmethod
212     def _download_local(uri, out_dir=None):
213         local_path = uri.replace(_LOCAL_PREFIX, "", 1)
214         if not os.path.exists(local_path):
215             raise RuntimeError("Local path %s does not exist." % (uri))
216 
217         if out_dir is None:
218             return local_path
219         elif not os.path.isdir(out_dir):
220             os.makedirs(out_dir)
221 
222         if os.path.isdir(local_path):
223             local_path = os.path.join(local_path, "*")
224 
225         for src in glob.glob(local_path):
226             _, tail = os.path.split(src)
227             dest_path = os.path.join(out_dir, tail)
228             logging.info("Linking: %s to %s", src, dest_path)
229             os.symlink(src, dest_path)
230         return out_dir
231 
232     @staticmethod
233     def _download_from_uri(uri, out_dir=None):
234         url = urlparse(uri)
235         filename = os.path.basename(url.path)
236         mimetype, encoding = mimetypes.guess_type(url.path)
237         local_path = os.path.join(out_dir, filename)
238 
239         if filename == '':
240             raise ValueError('No filename contained in URI: %s' % (uri))
241 
242         # Get header information from host url
243         headers = {}
244         host_uri = url.hostname
245 
246         headers_json = os.getenv(host_uri + _HEADERS_SUFFIX, "{}")
247         headers = json.loads(headers_json)
248 
249         with requests.get(uri, stream=True, headers=headers) as response:
250             if response.status_code != 200:
251                 raise RuntimeError("URI: %s returned a %s response code." % (uri, response.status_code))
252             if mimetype == 'application/zip' and not response.headers.get('Content-Type', '')\
253                     .startswith('application/zip'):
254                 raise RuntimeError("URI: %s did not respond with \'Content-Type\': \'application/zip\'" % uri)
255             if mimetype == 'application/x-tar' and not response.headers.get('Content-Type', '')\
256                     .startswith('application/x-tar'):
257                 raise RuntimeError("URI: %s did not respond with \'Content-Type\': \'application/x-tar\'" % uri)
258             if (mimetype != 'application/zip' and mimetype != 'application/x-tar') and \
259                     not response.headers.get('Content-Type', '').startswith('application/octet-stream'):
260                 raise RuntimeError("URI: %s did not respond with \'Content-Type\': \'application/octet-stream\'"
261                                    % uri)
262 
263             if encoding == 'gzip':
264                 stream = gzip.GzipFile(fileobj=response.raw)
265                 local_path = os.path.join(out_dir, f'{filename}.tar')
266             else:
267                 stream = response.raw
268             with open(local_path, 'wb') as out:
269                 shutil.copyfileobj(stream, out)
270 
271         if mimetype in ["application/x-tar", "application/zip"]:
272             if mimetype == "application/x-tar":
273                 archive = tarfile.open(local_path, 'r', encoding='utf-8')
274             else:
275                 archive = zipfile.ZipFile(local_path, 'r')
276             archive.extractall(out_dir)
277             archive.close()
278             os.remove(local_path)
279 
280         return out_dir
281
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/python/kfserving/kfserving/storage.py b/python/kfserving/kfserving/storage.py
--- a/python/kfserving/kfserving/storage.py
+++ b/python/kfserving/kfserving/storage.py
@@ -25,6 +25,8 @@
 import zipfile
 from urllib.parse import urlparse
 
+from botocore.client import Config
+from botocore import UNSIGNED
 import boto3
 import requests
 from azure.storage.blob import BlockBlobService
@@ -81,9 +83,26 @@
         logging.info("Successfully copied %s to %s", uri, out_dir)
         return out_dir
 
+    @staticmethod
+    def get_S3_config():
+        # anon environment variable defined in s3_secret.go
+        anon = ("True" == os.getenv("awsAnonymousCredential", "false").capitalize())
+        if anon:
+            return Config(signature_version=UNSIGNED)
+        else:
+            return None
+
     @staticmethod
     def _download_s3(uri, temp_dir: str):
-        s3 = boto3.resource('s3', endpoint_url=os.getenv("AWS_ENDPOINT_URL", "http://s3.amazonaws.com"))
+        # Boto3 looks at various configuration locations until it finds configuration values.
+        # lookup order:
+        # 1. Config object passed in as the config parameter when creating S3 resource
+        #    if awsAnonymousCredential env var true, passed in via config
+        # 2. Environment variables
+        # 3. ~/.aws/config file
+        s3 = boto3.resource('s3',
+                            endpoint_url=os.getenv("AWS_ENDPOINT_URL", "http://s3.amazonaws.com"),
+                            config=Storage.get_S3_config())
         parsed = urlparse(uri, scheme='s3')
         bucket_name = parsed.netloc
         bucket_path = parsed.path.lstrip('/')

{"golden_diff": "diff --git a/python/kfserving/kfserving/storage.py b/python/kfserving/kfserving/storage.py\n--- a/python/kfserving/kfserving/storage.py\n+++ b/python/kfserving/kfserving/storage.py\n@@ -25,6 +25,8 @@\n import zipfile\n from urllib.parse import urlparse\n \n+from botocore.client import Config\n+from botocore import UNSIGNED\n import boto3\n import requests\n from azure.storage.blob import BlockBlobService\n@@ -81,9 +83,26 @@\n         logging.info(\"Successfully copied %s to %s\", uri, out_dir)\n         return out_dir\n \n+    @staticmethod\n+    def get_S3_config():\n+        # anon environment variable defined in s3_secret.go\n+        anon = (\"True\" == os.getenv(\"awsAnonymousCredential\", \"false\").capitalize())\n+        if anon:\n+            return Config(signature_version=UNSIGNED)\n+        else:\n+            return None\n+\n     @staticmethod\n     def _download_s3(uri, temp_dir: str):\n-        s3 = boto3.resource('s3', endpoint_url=os.getenv(\"AWS_ENDPOINT_URL\", \"http://s3.amazonaws.com\"))\n+        # Boto3 looks at various configuration locations until it finds configuration values.\n+        # lookup order:\n+        # 1. Config object passed in as the config parameter when creating S3 resource\n+        #    if awsAnonymousCredential env var true, passed in via config\n+        # 2. Environment variables\n+        # 3. ~/.aws/config file\n+        s3 = boto3.resource('s3',\n+                            endpoint_url=os.getenv(\"AWS_ENDPOINT_URL\", \"http://s3.amazonaws.com\"),\n+                            config=Storage.get_S3_config())\n         parsed = urlparse(uri, scheme='s3')\n         bucket_name = parsed.netloc\n         bucket_path = parsed.path.lstrip('/')\n", "issue": "Anonymous credentials for single model serving with s3\n/kind feature\r\n\r\n**Describe the solution you'd like**\r\n[A clear and concise description of what you want to happen.]\r\nUpdate [download from s3](https://github.com/kubeflow/kfserving/blob/4fd401e348174f954fee7596dc3d54f197f4aa8c/python/kfserving/kfserving/storage.py#L86) to configure boto3 to use anon credentials if anonymous credentials env is set to true.\r\n\r\n**Anything else you would like to add:**\r\n[Miscellaneous information that will assist in solving the issue.]\r\nWould be nice if anon e2e test were also added.\n", "before_files": [{"content": "# Copyright 2020 kubeflow.org.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport glob\nimport gzip\nimport logging\nimport mimetypes\nimport os\nimport re\nimport json\nimport shutil\nimport tarfile\nimport tempfile\nimport zipfile\nfrom urllib.parse import urlparse\n\nimport boto3\nimport requests\nfrom azure.storage.blob import BlockBlobService\nfrom google.auth import exceptions\nfrom google.cloud import storage\n\nfrom kfserving.kfmodel_repository import MODEL_MOUNT_DIRS\n\n_GCS_PREFIX = \"gs://\"\n_S3_PREFIX = \"s3://\"\n_BLOB_RE = \"https://(.+?).blob.core.windows.net/(.+)\"\n_LOCAL_PREFIX = \"file://\"\n_URI_RE = \"https?://(.+)/(.+)\"\n_HTTP_PREFIX = \"http(s)://\"\n_HEADERS_SUFFIX = \"-headers\"\n\n\nclass Storage(object):  # pylint: disable=too-few-public-methods\n    @staticmethod\n    def download(uri: str, out_dir: str = None) -> str:\n        logging.info(\"Copying contents of %s to local\", uri)\n\n        is_local = False\n        if uri.startswith(_LOCAL_PREFIX) or os.path.exists(uri):\n            is_local = True\n\n        if out_dir is None:\n            if is_local:\n                # noop if out_dir is not set and the path is local\n                return Storage._download_local(uri)\n            out_dir = tempfile.mkdtemp()\n        elif not os.path.exists(out_dir):\n            os.mkdir(out_dir)\n\n        if uri.startswith(_GCS_PREFIX):\n            Storage._download_gcs(uri, out_dir)\n        elif uri.startswith(_S3_PREFIX):\n            Storage._download_s3(uri, out_dir)\n        elif re.search(_BLOB_RE, uri):\n            Storage._download_blob(uri, out_dir)\n        elif is_local:\n            return Storage._download_local(uri, out_dir)\n        elif re.search(_URI_RE, uri):\n            return Storage._download_from_uri(uri, out_dir)\n        elif uri.startswith(MODEL_MOUNT_DIRS):\n            # Don't need to download models if this InferenceService is running in the multi-model\n            # serving mode. The model agent will download models.\n            return out_dir\n        else:\n            raise Exception(\"Cannot recognize storage type for \" + uri +\n                            \"\\n'%s', '%s', '%s', and '%s' are the current available storage type.\" %\n                            (_GCS_PREFIX, _S3_PREFIX, _LOCAL_PREFIX, _HTTP_PREFIX))\n\n        logging.info(\"Successfully copied %s to %s\", uri, out_dir)\n        return out_dir\n\n    @staticmethod\n    def _download_s3(uri, temp_dir: str):\n        s3 = boto3.resource('s3', endpoint_url=os.getenv(\"AWS_ENDPOINT_URL\", \"http://s3.amazonaws.com\"))\n        parsed = urlparse(uri, scheme='s3')\n        bucket_name = parsed.netloc\n        bucket_path = parsed.path.lstrip('/')\n\n        bucket = s3.Bucket(bucket_name)\n        for obj in bucket.objects.filter(Prefix=bucket_path):\n            # Skip where boto3 lists the directory as an object\n            if obj.key.endswith(\"/\"):\n                continue\n            # In the case where bucket_path points to a single object, set the target key to bucket_path\n            # Otherwise, remove the bucket_path prefix, strip any extra slashes, then prepend the target_dir\n            target_key = (\n                obj.key\n                if bucket_path == obj.key\n                else obj.key.replace(bucket_path, \"\", 1).lstrip(\"/\")\n            )\n            target = f\"{temp_dir}/{target_key}\"\n            if not os.path.exists(os.path.dirname(target)):\n                os.makedirs(os.path.dirname(target), exist_ok=True)\n            bucket.download_file(obj.key, target)\n\n    @staticmethod\n    def _download_gcs(uri, temp_dir: str):\n        try:\n            storage_client = storage.Client()\n        except exceptions.DefaultCredentialsError:\n            storage_client = storage.Client.create_anonymous_client()\n        bucket_args = uri.replace(_GCS_PREFIX, \"\", 1).split(\"/\", 1)\n        bucket_name = bucket_args[0]\n        bucket_path = bucket_args[1] if len(bucket_args) > 1 else \"\"\n        bucket = storage_client.bucket(bucket_name)\n        prefix = bucket_path\n        if not prefix.endswith(\"/\"):\n            prefix = prefix + \"/\"\n        blobs = bucket.list_blobs(prefix=prefix)\n        count = 0\n        for blob in blobs:\n            # Replace any prefix from the object key with temp_dir\n            subdir_object_key = blob.name.replace(bucket_path, \"\", 1).strip(\"/\")\n\n            # Create necessary subdirectory to store the object locally\n            if \"/\" in subdir_object_key:\n                local_object_dir = os.path.join(temp_dir, subdir_object_key.rsplit(\"/\", 1)[0])\n                if not os.path.isdir(local_object_dir):\n                    os.makedirs(local_object_dir, exist_ok=True)\n            if subdir_object_key.strip() != \"\":\n                dest_path = os.path.join(temp_dir, subdir_object_key)\n                logging.info(\"Downloading: %s\", dest_path)\n                blob.download_to_filename(dest_path)\n            count = count + 1\n        if count == 0:\n            raise RuntimeError(\"Failed to fetch model. \\\nThe path or model %s does not exist.\" % uri)\n\n    @staticmethod\n    def _download_blob(uri, out_dir: str):  # pylint: disable=too-many-locals\n        match = re.search(_BLOB_RE, uri)\n        account_name = match.group(1)\n        storage_url = match.group(2)\n        container_name, prefix = storage_url.split(\"/\", 1)\n\n        logging.info(\"Connecting to BLOB account: [%s], container: [%s], prefix: [%s]\",\n                     account_name,\n                     container_name,\n                     prefix)\n        try:\n            block_blob_service = BlockBlobService(account_name=account_name)\n            blobs = block_blob_service.list_blobs(container_name, prefix=prefix)\n        except Exception:  # pylint: disable=broad-except\n            token = Storage._get_azure_storage_token()\n            if token is None:\n                logging.warning(\"Azure credentials not found, retrying anonymous access\")\n            block_blob_service = BlockBlobService(account_name=account_name, token_credential=token)\n            blobs = block_blob_service.list_blobs(container_name, prefix=prefix)\n        count = 0\n        for blob in blobs:\n            dest_path = os.path.join(out_dir, blob.name)\n            if \"/\" in blob.name:\n                head, tail = os.path.split(blob.name)\n                if prefix is not None:\n                    head = head[len(prefix):]\n                if head.startswith('/'):\n                    head = head[1:]\n                dir_path = os.path.join(out_dir, head)\n                dest_path = os.path.join(dir_path, tail)\n                if not os.path.isdir(dir_path):\n                    os.makedirs(dir_path)\n\n            logging.info(\"Downloading: %s to %s\", blob.name, dest_path)\n            block_blob_service.get_blob_to_path(container_name, blob.name, dest_path)\n            count = count + 1\n        if count == 0:\n            raise RuntimeError(\"Failed to fetch model. \\\nThe path or model %s does not exist.\" % (uri))\n\n    @staticmethod\n    def _get_azure_storage_token():\n        tenant_id = os.getenv(\"AZ_TENANT_ID\", \"\")\n        client_id = os.getenv(\"AZ_CLIENT_ID\", \"\")\n        client_secret = os.getenv(\"AZ_CLIENT_SECRET\", \"\")\n        subscription_id = os.getenv(\"AZ_SUBSCRIPTION_ID\", \"\")\n\n        if tenant_id == \"\" or client_id == \"\" or client_secret == \"\" or subscription_id == \"\":\n            return None\n\n        # note the SP must have \"Storage Blob Data Owner\" perms for this to work\n        import adal\n        from azure.storage.common import TokenCredential\n\n        authority_url = \"https://login.microsoftonline.com/\" + tenant_id\n\n        context = adal.AuthenticationContext(authority_url)\n\n        token = context.acquire_token_with_client_credentials(\n            \"https://storage.azure.com/\",\n            client_id,\n            client_secret)\n\n        token_credential = TokenCredential(token[\"accessToken\"])\n\n        logging.info(\"Retrieved SP token credential for client_id: %s\", client_id)\n\n        return token_credential\n\n    @staticmethod\n    def _download_local(uri, out_dir=None):\n        local_path = uri.replace(_LOCAL_PREFIX, \"\", 1)\n        if not os.path.exists(local_path):\n            raise RuntimeError(\"Local path %s does not exist.\" % (uri))\n\n        if out_dir is None:\n            return local_path\n        elif not os.path.isdir(out_dir):\n            os.makedirs(out_dir)\n\n        if os.path.isdir(local_path):\n            local_path = os.path.join(local_path, \"*\")\n\n        for src in glob.glob(local_path):\n            _, tail = os.path.split(src)\n            dest_path = os.path.join(out_dir, tail)\n            logging.info(\"Linking: %s to %s\", src, dest_path)\n            os.symlink(src, dest_path)\n        return out_dir\n\n    @staticmethod\n    def _download_from_uri(uri, out_dir=None):\n        url = urlparse(uri)\n        filename = os.path.basename(url.path)\n        mimetype, encoding = mimetypes.guess_type(url.path)\n        local_path = os.path.join(out_dir, filename)\n\n        if filename == '':\n            raise ValueError('No filename contained in URI: %s' % (uri))\n\n        # Get header information from host url\n        headers = {}\n        host_uri = url.hostname\n\n        headers_json = os.getenv(host_uri + _HEADERS_SUFFIX, \"{}\")\n        headers = json.loads(headers_json)\n\n        with requests.get(uri, stream=True, headers=headers) as response:\n            if response.status_code != 200:\n                raise RuntimeError(\"URI: %s returned a %s response code.\" % (uri, response.status_code))\n            if mimetype == 'application/zip' and not response.headers.get('Content-Type', '')\\\n                    .startswith('application/zip'):\n                raise RuntimeError(\"URI: %s did not respond with \\'Content-Type\\': \\'application/zip\\'\" % uri)\n            if mimetype == 'application/x-tar' and not response.headers.get('Content-Type', '')\\\n                    .startswith('application/x-tar'):\n                raise RuntimeError(\"URI: %s did not respond with \\'Content-Type\\': \\'application/x-tar\\'\" % uri)\n            if (mimetype != 'application/zip' and mimetype != 'application/x-tar') and \\\n                    not response.headers.get('Content-Type', '').startswith('application/octet-stream'):\n                raise RuntimeError(\"URI: %s did not respond with \\'Content-Type\\': \\'application/octet-stream\\'\"\n                                   % uri)\n\n            if encoding == 'gzip':\n                stream = gzip.GzipFile(fileobj=response.raw)\n                local_path = os.path.join(out_dir, f'{filename}.tar')\n            else:\n                stream = response.raw\n            with open(local_path, 'wb') as out:\n                shutil.copyfileobj(stream, out)\n\n        if mimetype in [\"application/x-tar\", \"application/zip\"]:\n            if mimetype == \"application/x-tar\":\n                archive = tarfile.open(local_path, 'r', encoding='utf-8')\n            else:\n                archive = zipfile.ZipFile(local_path, 'r')\n            archive.extractall(out_dir)\n            archive.close()\n            os.remove(local_path)\n\n        return out_dir\n", "path": "python/kfserving/kfserving/storage.py"}], "after_files": [{"content": "# Copyright 2020 kubeflow.org.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport glob\nimport gzip\nimport logging\nimport mimetypes\nimport os\nimport re\nimport json\nimport shutil\nimport tarfile\nimport tempfile\nimport zipfile\nfrom urllib.parse import urlparse\n\nfrom botocore.client import Config\nfrom botocore import UNSIGNED\nimport boto3\nimport requests\nfrom azure.storage.blob import BlockBlobService\nfrom google.auth import exceptions\nfrom google.cloud import storage\n\nfrom kfserving.kfmodel_repository import MODEL_MOUNT_DIRS\n\n_GCS_PREFIX = \"gs://\"\n_S3_PREFIX = \"s3://\"\n_BLOB_RE = \"https://(.+?).blob.core.windows.net/(.+)\"\n_LOCAL_PREFIX = \"file://\"\n_URI_RE = \"https?://(.+)/(.+)\"\n_HTTP_PREFIX = \"http(s)://\"\n_HEADERS_SUFFIX = \"-headers\"\n\n\nclass Storage(object):  # pylint: disable=too-few-public-methods\n    @staticmethod\n    def download(uri: str, out_dir: str = None) -> str:\n        logging.info(\"Copying contents of %s to local\", uri)\n\n        is_local = False\n        if uri.startswith(_LOCAL_PREFIX) or os.path.exists(uri):\n            is_local = True\n\n        if out_dir is None:\n            if is_local:\n                # noop if out_dir is not set and the path is local\n                return Storage._download_local(uri)\n            out_dir = tempfile.mkdtemp()\n        elif not os.path.exists(out_dir):\n            os.mkdir(out_dir)\n\n        if uri.startswith(_GCS_PREFIX):\n            Storage._download_gcs(uri, out_dir)\n        elif uri.startswith(_S3_PREFIX):\n            Storage._download_s3(uri, out_dir)\n        elif re.search(_BLOB_RE, uri):\n            Storage._download_blob(uri, out_dir)\n        elif is_local:\n            return Storage._download_local(uri, out_dir)\n        elif re.search(_URI_RE, uri):\n            return Storage._download_from_uri(uri, out_dir)\n        elif uri.startswith(MODEL_MOUNT_DIRS):\n            # Don't need to download models if this InferenceService is running in the multi-model\n            # serving mode. The model agent will download models.\n            return out_dir\n        else:\n            raise Exception(\"Cannot recognize storage type for \" + uri +\n                            \"\\n'%s', '%s', '%s', and '%s' are the current available storage type.\" %\n                            (_GCS_PREFIX, _S3_PREFIX, _LOCAL_PREFIX, _HTTP_PREFIX))\n\n        logging.info(\"Successfully copied %s to %s\", uri, out_dir)\n        return out_dir\n\n    @staticmethod\n    def get_S3_config():\n        # anon environment variable defined in s3_secret.go\n        anon = (\"True\" == os.getenv(\"awsAnonymousCredential\", \"false\").capitalize())\n        if anon:\n            return Config(signature_version=UNSIGNED)\n        else:\n            return None\n\n    @staticmethod\n    def _download_s3(uri, temp_dir: str):\n        # Boto3 looks at various configuration locations until it finds configuration values.\n        # lookup order:\n        # 1. Config object passed in as the config parameter when creating S3 resource\n        #    if awsAnonymousCredential env var true, passed in via config\n        # 2. Environment variables\n        # 3. ~/.aws/config file\n        s3 = boto3.resource('s3',\n                            endpoint_url=os.getenv(\"AWS_ENDPOINT_URL\", \"http://s3.amazonaws.com\"),\n                            config=Storage.get_S3_config())\n        parsed = urlparse(uri, scheme='s3')\n        bucket_name = parsed.netloc\n        bucket_path = parsed.path.lstrip('/')\n\n        bucket = s3.Bucket(bucket_name)\n        for obj in bucket.objects.filter(Prefix=bucket_path):\n            # Skip where boto3 lists the directory as an object\n            if obj.key.endswith(\"/\"):\n                continue\n            # In the case where bucket_path points to a single object, set the target key to bucket_path\n            # Otherwise, remove the bucket_path prefix, strip any extra slashes, then prepend the target_dir\n            target_key = (\n                obj.key\n                if bucket_path == obj.key\n                else obj.key.replace(bucket_path, \"\", 1).lstrip(\"/\")\n            )\n            target = f\"{temp_dir}/{target_key}\"\n            if not os.path.exists(os.path.dirname(target)):\n                os.makedirs(os.path.dirname(target), exist_ok=True)\n            bucket.download_file(obj.key, target)\n\n    @staticmethod\n    def _download_gcs(uri, temp_dir: str):\n        try:\n            storage_client = storage.Client()\n        except exceptions.DefaultCredentialsError:\n            storage_client = storage.Client.create_anonymous_client()\n        bucket_args = uri.replace(_GCS_PREFIX, \"\", 1).split(\"/\", 1)\n        bucket_name = bucket_args[0]\n        bucket_path = bucket_args[1] if len(bucket_args) > 1 else \"\"\n        bucket = storage_client.bucket(bucket_name)\n        prefix = bucket_path\n        if not prefix.endswith(\"/\"):\n            prefix = prefix + \"/\"\n        blobs = bucket.list_blobs(prefix=prefix)\n        count = 0\n        for blob in blobs:\n            # Replace any prefix from the object key with temp_dir\n            subdir_object_key = blob.name.replace(bucket_path, \"\", 1).strip(\"/\")\n\n            # Create necessary subdirectory to store the object locally\n            if \"/\" in subdir_object_key:\n                local_object_dir = os.path.join(temp_dir, subdir_object_key.rsplit(\"/\", 1)[0])\n                if not os.path.isdir(local_object_dir):\n                    os.makedirs(local_object_dir, exist_ok=True)\n            if subdir_object_key.strip() != \"\":\n                dest_path = os.path.join(temp_dir, subdir_object_key)\n                logging.info(\"Downloading: %s\", dest_path)\n                blob.download_to_filename(dest_path)\n            count = count + 1\n        if count == 0:\n            raise RuntimeError(\"Failed to fetch model. \\\nThe path or model %s does not exist.\" % uri)\n\n    @staticmethod\n    def _download_blob(uri, out_dir: str):  # pylint: disable=too-many-locals\n        match = re.search(_BLOB_RE, uri)\n        account_name = match.group(1)\n        storage_url = match.group(2)\n        container_name, prefix = storage_url.split(\"/\", 1)\n\n        logging.info(\"Connecting to BLOB account: [%s], container: [%s], prefix: [%s]\",\n                     account_name,\n                     container_name,\n                     prefix)\n        try:\n            block_blob_service = BlockBlobService(account_name=account_name)\n            blobs = block_blob_service.list_blobs(container_name, prefix=prefix)\n        except Exception:  # pylint: disable=broad-except\n            token = Storage._get_azure_storage_token()\n            if token is None:\n                logging.warning(\"Azure credentials not found, retrying anonymous access\")\n            block_blob_service = BlockBlobService(account_name=account_name, token_credential=token)\n            blobs = block_blob_service.list_blobs(container_name, prefix=prefix)\n        count = 0\n        for blob in blobs:\n            dest_path = os.path.join(out_dir, blob.name)\n            if \"/\" in blob.name:\n                head, tail = os.path.split(blob.name)\n                if prefix is not None:\n                    head = head[len(prefix):]\n                if head.startswith('/'):\n                    head = head[1:]\n                dir_path = os.path.join(out_dir, head)\n                dest_path = os.path.join(dir_path, tail)\n                if not os.path.isdir(dir_path):\n                    os.makedirs(dir_path)\n\n            logging.info(\"Downloading: %s to %s\", blob.name, dest_path)\n            block_blob_service.get_blob_to_path(container_name, blob.name, dest_path)\n            count = count + 1\n        if count == 0:\n            raise RuntimeError(\"Failed to fetch model. \\\nThe path or model %s does not exist.\" % (uri))\n\n    @staticmethod\n    def _get_azure_storage_token():\n        tenant_id = os.getenv(\"AZ_TENANT_ID\", \"\")\n        client_id = os.getenv(\"AZ_CLIENT_ID\", \"\")\n        client_secret = os.getenv(\"AZ_CLIENT_SECRET\", \"\")\n        subscription_id = os.getenv(\"AZ_SUBSCRIPTION_ID\", \"\")\n\n        if tenant_id == \"\" or client_id == \"\" or client_secret == \"\" or subscription_id == \"\":\n            return None\n\n        # note the SP must have \"Storage Blob Data Owner\" perms for this to work\n        import adal\n        from azure.storage.common import TokenCredential\n\n        authority_url = \"https://login.microsoftonline.com/\" + tenant_id\n\n        context = adal.AuthenticationContext(authority_url)\n\n        token = context.acquire_token_with_client_credentials(\n            \"https://storage.azure.com/\",\n            client_id,\n            client_secret)\n\n        token_credential = TokenCredential(token[\"accessToken\"])\n\n        logging.info(\"Retrieved SP token credential for client_id: %s\", client_id)\n\n        return token_credential\n\n    @staticmethod\n    def _download_local(uri, out_dir=None):\n        local_path = uri.replace(_LOCAL_PREFIX, \"\", 1)\n        if not os.path.exists(local_path):\n            raise RuntimeError(\"Local path %s does not exist.\" % (uri))\n\n        if out_dir is None:\n            return local_path\n        elif not os.path.isdir(out_dir):\n            os.makedirs(out_dir)\n\n        if os.path.isdir(local_path):\n            local_path = os.path.join(local_path, \"*\")\n\n        for src in glob.glob(local_path):\n            _, tail = os.path.split(src)\n            dest_path = os.path.join(out_dir, tail)\n            logging.info(\"Linking: %s to %s\", src, dest_path)\n            os.symlink(src, dest_path)\n        return out_dir\n\n    @staticmethod\n    def _download_from_uri(uri, out_dir=None):\n        url = urlparse(uri)\n        filename = os.path.basename(url.path)\n        mimetype, encoding = mimetypes.guess_type(url.path)\n        local_path = os.path.join(out_dir, filename)\n\n        if filename == '':\n            raise ValueError('No filename contained in URI: %s' % (uri))\n\n        # Get header information from host url\n        headers = {}\n        host_uri = url.hostname\n\n        headers_json = os.getenv(host_uri + _HEADERS_SUFFIX, \"{}\")\n        headers = json.loads(headers_json)\n\n        with requests.get(uri, stream=True, headers=headers) as response:\n            if response.status_code != 200:\n                raise RuntimeError(\"URI: %s returned a %s response code.\" % (uri, response.status_code))\n            if mimetype == 'application/zip' and not response.headers.get('Content-Type', '')\\\n                    .startswith('application/zip'):\n                raise RuntimeError(\"URI: %s did not respond with \\'Content-Type\\': \\'application/zip\\'\" % uri)\n            if mimetype == 'application/x-tar' and not response.headers.get('Content-Type', '')\\\n                    .startswith('application/x-tar'):\n                raise RuntimeError(\"URI: %s did not respond with \\'Content-Type\\': \\'application/x-tar\\'\" % uri)\n            if (mimetype != 'application/zip' and mimetype != 'application/x-tar') and \\\n                    not response.headers.get('Content-Type', '').startswith('application/octet-stream'):\n                raise RuntimeError(\"URI: %s did not respond with \\'Content-Type\\': \\'application/octet-stream\\'\"\n                                   % uri)\n\n            if encoding == 'gzip':\n                stream = gzip.GzipFile(fileobj=response.raw)\n                local_path = os.path.join(out_dir, f'{filename}.tar')\n            else:\n                stream = response.raw\n            with open(local_path, 'wb') as out:\n                shutil.copyfileobj(stream, out)\n\n        if mimetype in [\"application/x-tar\", \"application/zip\"]:\n            if mimetype == \"application/x-tar\":\n                archive = tarfile.open(local_path, 'r', encoding='utf-8')\n            else:\n                archive = zipfile.ZipFile(local_path, 'r')\n            archive.extractall(out_dir)\n            archive.close()\n            os.remove(local_path)\n\n        return out_dir\n", "path": "python/kfserving/kfserving/storage.py"}]}