prompt: You will be provided with a partial code base and an issue statement explaining a problem to resolve.
  <issue>
  Python Model download for GCS and S3
  Downloading from GCS and S3 needs to be completed.
  
  https://github.com/kubeflow/kfserving/blob/2f8d33d1a9773c5694a22ba749192163251fe287/python/kfserving/kfserving/storage.py#L27-L33
  
  </issue>
  <code>
  [start of python/kfserving/kfserving/storage.py]
  1 # Copyright 2019 kubeflow.org.
  2 #
  3 # Licensed under the Apache License, Version 2.0 (the "License");
  4 # you may not use this file except in compliance with the License.
  5 # You may obtain a copy of the License at
  6 #
  7 #    http://www.apache.org/licenses/LICENSE-2.0
  8 #
  9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 
  15 import logging
  16 import tempfile
  17 import os
  18 
  19 _GCS_PREFIX = "gs://"
  20 _S3_PREFIX = "s3://"
  21 _LOCAL_PREFIX = "file://"
  22 
  23 
  24 class Storage(object):
  25     @staticmethod
  26     def download(uri: str) -> str:
  27         logging.info("Copying contents of %s to local" % uri)
  28         if uri.startswith(_LOCAL_PREFIX) or os.path.exists(uri):
  29             return Storage._download_local(uri)
  30 
  31         temp_dir = tempfile.mkdtemp()
  32         if uri.startswith(_GCS_PREFIX):
  33             Storage._download_gcs(uri, temp_dir)
  34         elif uri.startswith(_S3_PREFIX):
  35             Storage._download_s3(uri, temp_dir)
  36         else:
  37             raise Exception("Cannot recognize storage type for " + uri +
  38                             "\n'%s', '%s', and '%s' are the current available storage type." %
  39                             (_GCS_PREFIX, _S3_PREFIX, _LOCAL_PREFIX))
  40 
  41         logging.info("Successfully copied %s to %s" % (uri, temp_dir))
  42         return temp_dir
  43 
  44     @staticmethod
  45     def _download_s3(uri, temp_dir: str):
  46         raise NotImplementedError
  47 
  48     @staticmethod
  49     def _download_gcs(uri, temp_dir: str):
  50         raise NotImplementedError
  51 
  52     @staticmethod
  53     def _download_local(uri):
  54         local_path = uri.replace(_LOCAL_PREFIX, "", 1)
  55         if not os.path.exists(local_path):
  56             raise Exception("Local path %s does not exist." % (uri))
  57         return local_path
  58
  [end of python/kfserving/kfserving/storage.py]
  [start of python/kfserving/setup.py]
  1 # Copyright 2019 kubeflow.org.
  2 #
  3 # Licensed under the Apache License, Version 2.0 (the "License");
  4 # you may not use this file except in compliance with the License.
  5 # You may obtain a copy of the License at
  6 #
  7 #    http://www.apache.org/licenses/LICENSE-2.0
  8 #
  9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 
  15 from setuptools import setup, find_packages
  16 
  17 tests_require = [
  18     'pytest',
  19     'pytest-tornasync',
  20     'mypy'
  21 ]
  22 
  23 setup(
  24     name='kfserver',
  25     version='0.1.0',
  26     author_email='ellisbigelow@google.com',
  27     license='../../LICENSE.txt',
  28     url='https://github.com/kubeflow/kfserving/python/kfserving/kfserving',
  29     description='Model Server for arbitrary python ML frameworks.',
  30     long_description=open('README.md').read(),
  31     python_requires='>3.4',
  32     packages=find_packages("kfserving"),
  33     install_requires=[
  34         "tornado >= 1.4.1",
  35         "argparse >= 1.4.0",
  36         "numpy"
  37     ],
  38     tests_require=tests_require,
  39     extras_require={'test': tests_require}
  40 )
  41
  [end of python/kfserving/setup.py]
  </code>
  
  I need you to solve this issue by generating a single patch file that I can apply directly to this repository using git apply. Please respond with a single patch file in the following format.
  <patch>
  diff --git a/file.py b/file.py
  --- a/file.py
  +++ b/file.py
  @@ -1,27 +1,35 @@
   def euclidean(a, b):
  -    while b:
  -        a, b = b, a % b
  -    return a
  +    if b == 0:
  +        return a
  +    return euclidean(b, a % b)
   
   
   def bresenham(x0, y0, x1, y1):
       points = []
       dx = abs(x1 - x0)
       dy = abs(y1 - y0)
  -    sx = 1 if x0 < x1 else -1
  -    sy = 1 if y0 < y1 else -1
  -    err = dx - dy
  +    x, y = x0, y0
  +    sx = -1 if x0 > x1 else 1
  +    sy = -1 if y0 > y1 else 1
   
  -    while True:
  -        points.append((x0, y0))
  -        if x0 == x1 and y0 == y1:
  -            break
  -        e2 = 2 * err
  -        if e2 > -dy:
  -            err -= dy
  -            x0 += sx
  -        if e2 < dx:
  -            err += dx
  -            y0 += sy
  +    if dx > dy:
  +        err = dx / 2.0
  +        while x != x1:
  +            points.append((x, y))
  +            err -= dy
  +            if err < 0:
  +                y += sy
  +                err += dx
  +            x += sx
  +    else:
  +        err = dy / 2.0
  +        while y != y1:
  +            points.append((x, y))
  +            err -= dx
  +            if err < 0:
  +                x += sx
  +                err += dy
  +            y += sy
  +
  +    points.append((x, y))
       return points
  
  </patch>
golden_diff: diff --git a/python/kfserving/kfserving/storage.py b/python/kfserving/kfserving/storage.py
  --- a/python/kfserving/kfserving/storage.py
  +++ b/python/kfserving/kfserving/storage.py
  @@ -15,6 +15,10 @@
   import logging
   import tempfile
   import os
  +import re
  +from minio import Minio
  +from google.cloud import storage
  +from google.auth import exceptions
   
   _GCS_PREFIX = "gs://"
   _S3_PREFIX = "s3://"
  @@ -43,11 +47,36 @@
   
       @staticmethod
       def _download_s3(uri, temp_dir: str):
  -        raise NotImplementedError
  +        client = Storage._create_minio_client()
  +        bucket_args = uri.replace(_S3_PREFIX, "", 1).split("/", 1)
  +        bucket_name = bucket_args[0]
  +        bucket_path = bucket_args[1] if len(bucket_args) > 1 else ""
  +        objects = client.list_objects(bucket_name, prefix=bucket_path, recursive=True)
  +        for obj in objects:
  +            # Replace any prefix from the object key with temp_dir
  +            subdir_object_key = obj.object_name.replace(bucket_path, "", 1).strip("/")
  +            client.fget_object(bucket_name, obj.object_name, os.path.join(temp_dir, subdir_object_key))
   
       @staticmethod
       def _download_gcs(uri, temp_dir: str):
  -        raise NotImplementedError
  +        try:
  +            storage_client = storage.Client()
  +        except exceptions.DefaultCredentialsError as e:
  +            storage_client = storage.Client.create_anonymous_client()
  +        bucket_args = uri.replace(_GCS_PREFIX, "", 1).split("/", 1)
  +        bucket_name = bucket_args[0]
  +        bucket_path = bucket_args[1] if len(bucket_args) > 1 else ""
  +        bucket = storage_client.bucket(bucket_name)
  +        blobs = bucket.list_blobs(prefix=bucket_path)
  +        for blob in blobs:
  +            # Replace any prefix from the object key with temp_dir
  +            subdir_object_key = blob.name.replace(bucket_path, "", 1).strip("/")
  +            # Create necessary subdirectory to store the object locally
  +            if "/" in subdir_object_key:
  +                local_object_dir = os.path.join(temp_dir, subdir_object_key.rsplit("/", 1)[0])
  +                if not os.path.isdir(local_object_dir):
  +                    os.makedirs(local_object_dir, exist_ok=True)
  +            blob.download_to_filename(os.path.join(temp_dir, subdir_object_key))
   
       @staticmethod
       def _download_local(uri):
  @@ -55,3 +84,13 @@
           if not os.path.exists(local_path):
               raise Exception("Local path %s does not exist." % (uri))
           return local_path
  +
  +    @staticmethod
  +    def _create_minio_client():
  +        # Remove possible http scheme for Minio
  +        url = re.compile(r"https?://")
  +        minioClient = Minio(url.sub("", os.getenv("S3_ENDPOINT", "")),
  +                            access_key=os.getenv("AWS_ACCESS_KEY_ID", ""),
  +                            secret_key=os.getenv("AWS_SECRET_ACCESS_KEY", ""),
  +                            secure=True)
  +        return minioClient
  diff --git a/python/kfserving/setup.py b/python/kfserving/setup.py
  --- a/python/kfserving/setup.py
  +++ b/python/kfserving/setup.py
  @@ -33,6 +33,8 @@
       install_requires=[
           "tornado >= 1.4.1",
           "argparse >= 1.4.0",
  +        "minio >= 4.0.9",
  +        "google-cloud-storage >= 1.16.0",
           "numpy"
       ],
       tests_require=tests_require,
  
verification_info: {"golden_diff": "diff --git a/python/kfserving/kfserving/storage.py b/python/kfserving/kfserving/storage.py\n--- a/python/kfserving/kfserving/storage.py\n+++ b/python/kfserving/kfserving/storage.py\n@@ -15,6 +15,10 @@\n import logging\n import tempfile\n import os\n+import re\n+from minio import Minio\n+from google.cloud import storage\n+from google.auth import exceptions\n \n _GCS_PREFIX = \"gs://\"\n _S3_PREFIX = \"s3://\"\n@@ -43,11 +47,36 @@\n \n     @staticmethod\n     def _download_s3(uri, temp_dir: str):\n-        raise NotImplementedError\n+        client = Storage._create_minio_client()\n+        bucket_args = uri.replace(_S3_PREFIX, \"\", 1).split(\"/\", 1)\n+        bucket_name = bucket_args[0]\n+        bucket_path = bucket_args[1] if len(bucket_args) > 1 else \"\"\n+        objects = client.list_objects(bucket_name, prefix=bucket_path, recursive=True)\n+        for obj in objects:\n+            # Replace any prefix from the object key with temp_dir\n+            subdir_object_key = obj.object_name.replace(bucket_path, \"\", 1).strip(\"/\")\n+            client.fget_object(bucket_name, obj.object_name, os.path.join(temp_dir, subdir_object_key))\n \n     @staticmethod\n     def _download_gcs(uri, temp_dir: str):\n-        raise NotImplementedError\n+        try:\n+            storage_client = storage.Client()\n+        except exceptions.DefaultCredentialsError as e:\n+            storage_client = storage.Client.create_anonymous_client()\n+        bucket_args = uri.replace(_GCS_PREFIX, \"\", 1).split(\"/\", 1)\n+        bucket_name = bucket_args[0]\n+        bucket_path = bucket_args[1] if len(bucket_args) > 1 else \"\"\n+        bucket = storage_client.bucket(bucket_name)\n+        blobs = bucket.list_blobs(prefix=bucket_path)\n+        for blob in blobs:\n+            # Replace any prefix from the object key with temp_dir\n+            subdir_object_key = blob.name.replace(bucket_path, \"\", 1).strip(\"/\")\n+            # Create necessary subdirectory to store the object locally\n+            if \"/\" in subdir_object_key:\n+                local_object_dir = os.path.join(temp_dir, subdir_object_key.rsplit(\"/\", 1)[0])\n+                if not os.path.isdir(local_object_dir):\n+                    os.makedirs(local_object_dir, exist_ok=True)\n+            blob.download_to_filename(os.path.join(temp_dir, subdir_object_key))\n \n     @staticmethod\n     def _download_local(uri):\n@@ -55,3 +84,13 @@\n         if not os.path.exists(local_path):\n             raise Exception(\"Local path %s does not exist.\" % (uri))\n         return local_path\n+\n+    @staticmethod\n+    def _create_minio_client():\n+        # Remove possible http scheme for Minio\n+        url = re.compile(r\"https?://\")\n+        minioClient = Minio(url.sub(\"\", os.getenv(\"S3_ENDPOINT\", \"\")),\n+                            access_key=os.getenv(\"AWS_ACCESS_KEY_ID\", \"\"),\n+                            secret_key=os.getenv(\"AWS_SECRET_ACCESS_KEY\", \"\"),\n+                            secure=True)\n+        return minioClient\ndiff --git a/python/kfserving/setup.py b/python/kfserving/setup.py\n--- a/python/kfserving/setup.py\n+++ b/python/kfserving/setup.py\n@@ -33,6 +33,8 @@\n     install_requires=[\n         \"tornado >= 1.4.1\",\n         \"argparse >= 1.4.0\",\n+        \"minio >= 4.0.9\",\n+        \"google-cloud-storage >= 1.16.0\",\n         \"numpy\"\n     ],\n     tests_require=tests_require,\n", "issue": "Python Model download for GCS and S3\nDownloading from GCS and S3 needs to be completed.\r\n\r\nhttps://github.com/kubeflow/kfserving/blob/2f8d33d1a9773c5694a22ba749192163251fe287/python/kfserving/kfserving/storage.py#L27-L33\n", "before_files": [{"content": "# Copyright 2019 kubeflow.org.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport tempfile\nimport os\n\n_GCS_PREFIX = \"gs://\"\n_S3_PREFIX = \"s3://\"\n_LOCAL_PREFIX = \"file://\"\n\n\nclass Storage(object):\n    @staticmethod\n    def download(uri: str) -> str:\n        logging.info(\"Copying contents of %s to local\" % uri)\n        if uri.startswith(_LOCAL_PREFIX) or os.path.exists(uri):\n            return Storage._download_local(uri)\n\n        temp_dir = tempfile.mkdtemp()\n        if uri.startswith(_GCS_PREFIX):\n            Storage._download_gcs(uri, temp_dir)\n        elif uri.startswith(_S3_PREFIX):\n            Storage._download_s3(uri, temp_dir)\n        else:\n            raise Exception(\"Cannot recognize storage type for \" + uri +\n                            \"\\n'%s', '%s', and '%s' are the current available storage type.\" %\n                            (_GCS_PREFIX, _S3_PREFIX, _LOCAL_PREFIX))\n\n        logging.info(\"Successfully copied %s to %s\" % (uri, temp_dir))\n        return temp_dir\n\n    @staticmethod\n    def _download_s3(uri, temp_dir: str):\n        raise NotImplementedError\n\n    @staticmethod\n    def _download_gcs(uri, temp_dir: str):\n        raise NotImplementedError\n\n    @staticmethod\n    def _download_local(uri):\n        local_path = uri.replace(_LOCAL_PREFIX, \"\", 1)\n        if not os.path.exists(local_path):\n            raise Exception(\"Local path %s does not exist.\" % (uri))\n        return local_path\n", "path": "python/kfserving/kfserving/storage.py"}, {"content": "# Copyright 2019 kubeflow.org.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom setuptools import setup, find_packages\n\ntests_require = [\n    'pytest',\n    'pytest-tornasync',\n    'mypy'\n]\n\nsetup(\n    name='kfserver',\n    version='0.1.0',\n    author_email='ellisbigelow@google.com',\n    license='../../LICENSE.txt',\n    url='https://github.com/kubeflow/kfserving/python/kfserving/kfserving',\n    description='Model Server for arbitrary python ML frameworks.',\n    long_description=open('README.md').read(),\n    python_requires='>3.4',\n    packages=find_packages(\"kfserving\"),\n    install_requires=[\n        \"tornado >= 1.4.1\",\n        \"argparse >= 1.4.0\",\n        \"numpy\"\n    ],\n    tests_require=tests_require,\n    extras_require={'test': tests_require}\n)\n", "path": "python/kfserving/setup.py"}]}