We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
KFServing should have a consistent way of supporting model download across inference server implementations
/kind feature

**Describe the solution you'd like**
KFServing should expose a consistent way to download models across inference servers and clouds. The current implementation depends on the features of individual inference servers expose. E.g. see #137 

**Anything else you would like to add:**
Proposed solution design is documented here: https://docs.google.com/document/d/1xqBOkoQ6Vzc5gv4O5MgVVNE3qILbKuMkC-DN5zp5w28/edit?usp=sharing


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `python/kfserving/kfserving/storage.py`
Content:
```
1 # Copyright 2019 kubeflow.org.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #    http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 import logging
16 import tempfile
17 import os
18 import re
19 from minio import Minio
20 from google.cloud import storage
21 from google.auth import exceptions
22 
23 _GCS_PREFIX = "gs://"
24 _S3_PREFIX = "s3://"
25 _LOCAL_PREFIX = "file://"
26 
27 
28 class Storage(object): # pylint: disable=too-few-public-methods
29     @staticmethod
30     def download(uri: str) -> str:
31         logging.info("Copying contents of %s to local", uri)
32         if uri.startswith(_LOCAL_PREFIX) or os.path.exists(uri):
33             return Storage._download_local(uri)
34 
35         temp_dir = tempfile.mkdtemp()
36         if uri.startswith(_GCS_PREFIX):
37             Storage._download_gcs(uri, temp_dir)
38         elif uri.startswith(_S3_PREFIX):
39             Storage._download_s3(uri, temp_dir)
40         else:
41             raise Exception("Cannot recognize storage type for " + uri +
42                             "\n'%s', '%s', and '%s' are the current available storage type." %
43                             (_GCS_PREFIX, _S3_PREFIX, _LOCAL_PREFIX))
44 
45         logging.info("Successfully copied %s to %s", uri, temp_dir)
46         return temp_dir
47 
48     @staticmethod
49     def _download_s3(uri, temp_dir: str):
50         client = Storage._create_minio_client()
51         bucket_args = uri.replace(_S3_PREFIX, "", 1).split("/", 1)
52         bucket_name = bucket_args[0]
53         bucket_path = bucket_args[1] if len(bucket_args) > 1 else ""
54         objects = client.list_objects(bucket_name, prefix=bucket_path, recursive=True)
55         for obj in objects:
56             # Replace any prefix from the object key with temp_dir
57             subdir_object_key = obj.object_name.replace(bucket_path, "", 1).strip("/")
58             client.fget_object(bucket_name, obj.object_name,
59                                os.path.join(temp_dir, subdir_object_key))
60 
61     @staticmethod
62     def _download_gcs(uri, temp_dir: str):
63         try:
64             storage_client = storage.Client()
65         except exceptions.DefaultCredentialsError:
66             storage_client = storage.Client.create_anonymous_client()
67         bucket_args = uri.replace(_GCS_PREFIX, "", 1).split("/", 1)
68         bucket_name = bucket_args[0]
69         bucket_path = bucket_args[1] if len(bucket_args) > 1 else ""
70         bucket = storage_client.bucket(bucket_name)
71         blobs = bucket.list_blobs(prefix=bucket_path)
72         for blob in blobs:
73             # Replace any prefix from the object key with temp_dir
74             subdir_object_key = blob.name.replace(bucket_path, "", 1).strip("/")
75             # Create necessary subdirectory to store the object locally
76             if "/" in subdir_object_key:
77                 local_object_dir = os.path.join(temp_dir, subdir_object_key.rsplit("/", 1)[0])
78                 if not os.path.isdir(local_object_dir):
79                     os.makedirs(local_object_dir, exist_ok=True)
80             blob.download_to_filename(os.path.join(temp_dir, subdir_object_key))
81 
82     @staticmethod
83     def _download_local(uri):
84         local_path = uri.replace(_LOCAL_PREFIX, "", 1)
85         if not os.path.exists(local_path):
86             raise Exception("Local path %s does not exist." % (uri))
87         return local_path
88 
89     @staticmethod
90     def _create_minio_client():
91         # Remove possible http scheme for Minio
92         url = re.compile(r"https?://")
93         minioClient = Minio(url.sub("", os.getenv("S3_ENDPOINT", "")),
94                             access_key=os.getenv("AWS_ACCESS_KEY_ID", ""),
95                             secret_key=os.getenv("AWS_SECRET_ACCESS_KEY", ""),
96                             secure=True)
97         return minioClient
98
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/python/kfserving/kfserving/storage.py b/python/kfserving/kfserving/storage.py
--- a/python/kfserving/kfserving/storage.py
+++ b/python/kfserving/kfserving/storage.py
@@ -27,23 +27,25 @@
 
 class Storage(object): # pylint: disable=too-few-public-methods
     @staticmethod
-    def download(uri: str) -> str:
+    def download(uri: str, out_dir: str = None) -> str:
         logging.info("Copying contents of %s to local", uri)
         if uri.startswith(_LOCAL_PREFIX) or os.path.exists(uri):
             return Storage._download_local(uri)
 
-        temp_dir = tempfile.mkdtemp()
+        if out_dir is None:
+            out_dir = tempfile.mkdtemp()
+
         if uri.startswith(_GCS_PREFIX):
-            Storage._download_gcs(uri, temp_dir)
+            Storage._download_gcs(uri, out_dir)
         elif uri.startswith(_S3_PREFIX):
-            Storage._download_s3(uri, temp_dir)
+            Storage._download_s3(uri, out_dir)
         else:
             raise Exception("Cannot recognize storage type for " + uri +
                             "\n'%s', '%s', and '%s' are the current available storage type." %
                             (_GCS_PREFIX, _S3_PREFIX, _LOCAL_PREFIX))
 
-        logging.info("Successfully copied %s to %s", uri, temp_dir)
-        return temp_dir
+        logging.info("Successfully copied %s to %s", uri, out_dir)
+        return out_dir
 
     @staticmethod
     def _download_s3(uri, temp_dir: str):
@@ -68,16 +70,23 @@
         bucket_name = bucket_args[0]
         bucket_path = bucket_args[1] if len(bucket_args) > 1 else ""
         bucket = storage_client.bucket(bucket_name)
-        blobs = bucket.list_blobs(prefix=bucket_path)
+        prefix = bucket_path
+        if not prefix.endswith("/"):
+            prefix = prefix + "/"
+        blobs = bucket.list_blobs(prefix=prefix)
         for blob in blobs:
             # Replace any prefix from the object key with temp_dir
             subdir_object_key = blob.name.replace(bucket_path, "", 1).strip("/")
+
             # Create necessary subdirectory to store the object locally
             if "/" in subdir_object_key:
                 local_object_dir = os.path.join(temp_dir, subdir_object_key.rsplit("/", 1)[0])
                 if not os.path.isdir(local_object_dir):
                     os.makedirs(local_object_dir, exist_ok=True)
-            blob.download_to_filename(os.path.join(temp_dir, subdir_object_key))
+            if subdir_object_key.strip() != "":
+                dest_path = os.path.join(temp_dir, subdir_object_key)
+                logging.info("Downloading: %s", dest_path)
+                blob.download_to_filename(dest_path)
 
     @staticmethod
     def _download_local(uri):

{"golden_diff": "diff --git a/python/kfserving/kfserving/storage.py b/python/kfserving/kfserving/storage.py\n--- a/python/kfserving/kfserving/storage.py\n+++ b/python/kfserving/kfserving/storage.py\n@@ -27,23 +27,25 @@\n \n class Storage(object): # pylint: disable=too-few-public-methods\n     @staticmethod\n-    def download(uri: str) -> str:\n+    def download(uri: str, out_dir: str = None) -> str:\n         logging.info(\"Copying contents of %s to local\", uri)\n         if uri.startswith(_LOCAL_PREFIX) or os.path.exists(uri):\n             return Storage._download_local(uri)\n \n-        temp_dir = tempfile.mkdtemp()\n+        if out_dir is None:\n+            out_dir = tempfile.mkdtemp()\n+\n         if uri.startswith(_GCS_PREFIX):\n-            Storage._download_gcs(uri, temp_dir)\n+            Storage._download_gcs(uri, out_dir)\n         elif uri.startswith(_S3_PREFIX):\n-            Storage._download_s3(uri, temp_dir)\n+            Storage._download_s3(uri, out_dir)\n         else:\n             raise Exception(\"Cannot recognize storage type for \" + uri +\n                             \"\\n'%s', '%s', and '%s' are the current available storage type.\" %\n                             (_GCS_PREFIX, _S3_PREFIX, _LOCAL_PREFIX))\n \n-        logging.info(\"Successfully copied %s to %s\", uri, temp_dir)\n-        return temp_dir\n+        logging.info(\"Successfully copied %s to %s\", uri, out_dir)\n+        return out_dir\n \n     @staticmethod\n     def _download_s3(uri, temp_dir: str):\n@@ -68,16 +70,23 @@\n         bucket_name = bucket_args[0]\n         bucket_path = bucket_args[1] if len(bucket_args) > 1 else \"\"\n         bucket = storage_client.bucket(bucket_name)\n-        blobs = bucket.list_blobs(prefix=bucket_path)\n+        prefix = bucket_path\n+        if not prefix.endswith(\"/\"):\n+            prefix = prefix + \"/\"\n+        blobs = bucket.list_blobs(prefix=prefix)\n         for blob in blobs:\n             # Replace any prefix from the object key with temp_dir\n             subdir_object_key = blob.name.replace(bucket_path, \"\", 1).strip(\"/\")\n+\n             # Create necessary subdirectory to store the object locally\n             if \"/\" in subdir_object_key:\n                 local_object_dir = os.path.join(temp_dir, subdir_object_key.rsplit(\"/\", 1)[0])\n                 if not os.path.isdir(local_object_dir):\n                     os.makedirs(local_object_dir, exist_ok=True)\n-            blob.download_to_filename(os.path.join(temp_dir, subdir_object_key))\n+            if subdir_object_key.strip() != \"\":\n+                dest_path = os.path.join(temp_dir, subdir_object_key)\n+                logging.info(\"Downloading: %s\", dest_path)\n+                blob.download_to_filename(dest_path)\n \n     @staticmethod\n     def _download_local(uri):\n", "issue": "KFServing should have a consistent way of supporting model download across inference server implementations\n/kind feature\r\n\r\n**Describe the solution you'd like**\r\nKFServing should expose a consistent way to download models across inference servers and clouds. The current implementation depends on the features of individual inference servers expose. E.g. see #137 \r\n\r\n**Anything else you would like to add:**\r\nProposed solution design is documented here: https://docs.google.com/document/d/1xqBOkoQ6Vzc5gv4O5MgVVNE3qILbKuMkC-DN5zp5w28/edit?usp=sharing\r\n\n", "before_files": [{"content": "# Copyright 2019 kubeflow.org.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport tempfile\nimport os\nimport re\nfrom minio import Minio\nfrom google.cloud import storage\nfrom google.auth import exceptions\n\n_GCS_PREFIX = \"gs://\"\n_S3_PREFIX = \"s3://\"\n_LOCAL_PREFIX = \"file://\"\n\n\nclass Storage(object): # pylint: disable=too-few-public-methods\n    @staticmethod\n    def download(uri: str) -> str:\n        logging.info(\"Copying contents of %s to local\", uri)\n        if uri.startswith(_LOCAL_PREFIX) or os.path.exists(uri):\n            return Storage._download_local(uri)\n\n        temp_dir = tempfile.mkdtemp()\n        if uri.startswith(_GCS_PREFIX):\n            Storage._download_gcs(uri, temp_dir)\n        elif uri.startswith(_S3_PREFIX):\n            Storage._download_s3(uri, temp_dir)\n        else:\n            raise Exception(\"Cannot recognize storage type for \" + uri +\n                            \"\\n'%s', '%s', and '%s' are the current available storage type.\" %\n                            (_GCS_PREFIX, _S3_PREFIX, _LOCAL_PREFIX))\n\n        logging.info(\"Successfully copied %s to %s\", uri, temp_dir)\n        return temp_dir\n\n    @staticmethod\n    def _download_s3(uri, temp_dir: str):\n        client = Storage._create_minio_client()\n        bucket_args = uri.replace(_S3_PREFIX, \"\", 1).split(\"/\", 1)\n        bucket_name = bucket_args[0]\n        bucket_path = bucket_args[1] if len(bucket_args) > 1 else \"\"\n        objects = client.list_objects(bucket_name, prefix=bucket_path, recursive=True)\n        for obj in objects:\n            # Replace any prefix from the object key with temp_dir\n            subdir_object_key = obj.object_name.replace(bucket_path, \"\", 1).strip(\"/\")\n            client.fget_object(bucket_name, obj.object_name,\n                               os.path.join(temp_dir, subdir_object_key))\n\n    @staticmethod\n    def _download_gcs(uri, temp_dir: str):\n        try:\n            storage_client = storage.Client()\n        except exceptions.DefaultCredentialsError:\n            storage_client = storage.Client.create_anonymous_client()\n        bucket_args = uri.replace(_GCS_PREFIX, \"\", 1).split(\"/\", 1)\n        bucket_name = bucket_args[0]\n        bucket_path = bucket_args[1] if len(bucket_args) > 1 else \"\"\n        bucket = storage_client.bucket(bucket_name)\n        blobs = bucket.list_blobs(prefix=bucket_path)\n        for blob in blobs:\n            # Replace any prefix from the object key with temp_dir\n            subdir_object_key = blob.name.replace(bucket_path, \"\", 1).strip(\"/\")\n            # Create necessary subdirectory to store the object locally\n            if \"/\" in subdir_object_key:\n                local_object_dir = os.path.join(temp_dir, subdir_object_key.rsplit(\"/\", 1)[0])\n                if not os.path.isdir(local_object_dir):\n                    os.makedirs(local_object_dir, exist_ok=True)\n            blob.download_to_filename(os.path.join(temp_dir, subdir_object_key))\n\n    @staticmethod\n    def _download_local(uri):\n        local_path = uri.replace(_LOCAL_PREFIX, \"\", 1)\n        if not os.path.exists(local_path):\n            raise Exception(\"Local path %s does not exist.\" % (uri))\n        return local_path\n\n    @staticmethod\n    def _create_minio_client():\n        # Remove possible http scheme for Minio\n        url = re.compile(r\"https?://\")\n        minioClient = Minio(url.sub(\"\", os.getenv(\"S3_ENDPOINT\", \"\")),\n                            access_key=os.getenv(\"AWS_ACCESS_KEY_ID\", \"\"),\n                            secret_key=os.getenv(\"AWS_SECRET_ACCESS_KEY\", \"\"),\n                            secure=True)\n        return minioClient\n", "path": "python/kfserving/kfserving/storage.py"}], "after_files": [{"content": "# Copyright 2019 kubeflow.org.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport tempfile\nimport os\nimport re\nfrom minio import Minio\nfrom google.cloud import storage\nfrom google.auth import exceptions\n\n_GCS_PREFIX = \"gs://\"\n_S3_PREFIX = \"s3://\"\n_LOCAL_PREFIX = \"file://\"\n\n\nclass Storage(object): # pylint: disable=too-few-public-methods\n    @staticmethod\n    def download(uri: str, out_dir: str = None) -> str:\n        logging.info(\"Copying contents of %s to local\", uri)\n        if uri.startswith(_LOCAL_PREFIX) or os.path.exists(uri):\n            return Storage._download_local(uri)\n\n        if out_dir is None:\n            out_dir = tempfile.mkdtemp()\n\n        if uri.startswith(_GCS_PREFIX):\n            Storage._download_gcs(uri, out_dir)\n        elif uri.startswith(_S3_PREFIX):\n            Storage._download_s3(uri, out_dir)\n        else:\n            raise Exception(\"Cannot recognize storage type for \" + uri +\n                            \"\\n'%s', '%s', and '%s' are the current available storage type.\" %\n                            (_GCS_PREFIX, _S3_PREFIX, _LOCAL_PREFIX))\n\n        logging.info(\"Successfully copied %s to %s\", uri, out_dir)\n        return out_dir\n\n    @staticmethod\n    def _download_s3(uri, temp_dir: str):\n        client = Storage._create_minio_client()\n        bucket_args = uri.replace(_S3_PREFIX, \"\", 1).split(\"/\", 1)\n        bucket_name = bucket_args[0]\n        bucket_path = bucket_args[1] if len(bucket_args) > 1 else \"\"\n        objects = client.list_objects(bucket_name, prefix=bucket_path, recursive=True)\n        for obj in objects:\n            # Replace any prefix from the object key with temp_dir\n            subdir_object_key = obj.object_name.replace(bucket_path, \"\", 1).strip(\"/\")\n            client.fget_object(bucket_name, obj.object_name,\n                               os.path.join(temp_dir, subdir_object_key))\n\n    @staticmethod\n    def _download_gcs(uri, temp_dir: str):\n        try:\n            storage_client = storage.Client()\n        except exceptions.DefaultCredentialsError:\n            storage_client = storage.Client.create_anonymous_client()\n        bucket_args = uri.replace(_GCS_PREFIX, \"\", 1).split(\"/\", 1)\n        bucket_name = bucket_args[0]\n        bucket_path = bucket_args[1] if len(bucket_args) > 1 else \"\"\n        bucket = storage_client.bucket(bucket_name)\n        prefix = bucket_path\n        if not prefix.endswith(\"/\"):\n            prefix = prefix + \"/\"\n        blobs = bucket.list_blobs(prefix=prefix)\n        for blob in blobs:\n            # Replace any prefix from the object key with temp_dir\n            subdir_object_key = blob.name.replace(bucket_path, \"\", 1).strip(\"/\")\n\n            # Create necessary subdirectory to store the object locally\n            if \"/\" in subdir_object_key:\n                local_object_dir = os.path.join(temp_dir, subdir_object_key.rsplit(\"/\", 1)[0])\n                if not os.path.isdir(local_object_dir):\n                    os.makedirs(local_object_dir, exist_ok=True)\n            if subdir_object_key.strip() != \"\":\n                dest_path = os.path.join(temp_dir, subdir_object_key)\n                logging.info(\"Downloading: %s\", dest_path)\n                blob.download_to_filename(dest_path)\n\n    @staticmethod\n    def _download_local(uri):\n        local_path = uri.replace(_LOCAL_PREFIX, \"\", 1)\n        if not os.path.exists(local_path):\n            raise Exception(\"Local path %s does not exist.\" % (uri))\n        return local_path\n\n    @staticmethod\n    def _create_minio_client():\n        # Remove possible http scheme for Minio\n        url = re.compile(r\"https?://\")\n        minioClient = Minio(url.sub(\"\", os.getenv(\"S3_ENDPOINT\", \"\")),\n                            access_key=os.getenv(\"AWS_ACCESS_KEY_ID\", \"\"),\n                            secret_key=os.getenv(\"AWS_SECRET_ACCESS_KEY\", \"\"),\n                            secure=True)\n        return minioClient\n", "path": "python/kfserving/kfserving/storage.py"}]}