files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  import copy
  import os
  import time
  import base64
  import hmac
  
  from hashlib import sha256
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import b
  from libcloud.utils.xml import fixxpath
  
  from libcloud.utils.py3 import ET
  from libcloud.common.types import InvalidCredsError
  from libcloud.common.types import LibcloudError, MalformedResponseError
  from libcloud.common.base import ConnectionUserAndKey, RawResponse
  from libcloud.common.base import CertificateConnection
  from libcloud.common.base import XmlResponse
  from libcloud.common.base import BaseDriver
  
  # The time format for headers in Azure requests
  AZURE_TIME_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
  
  
  class AzureRedirectException(Exception):
  
      def __init__(self, response):
          self.location = response.headers['location']
  
  
  class AzureResponse(XmlResponse):
  
      valid_response_codes = [
          httplib.NOT_FOUND,
          httplib.CONFLICT,
          httplib.BAD_REQUEST,
          # added TEMPORARY_REDIRECT as this can sometimes be
          # sent by azure instead of a success or fail response
          httplib.TEMPORARY_REDIRECT,
          # Used by Azure Blobs range downloads
          httplib.PARTIAL_CONTENT
      ]
  
      def success(self):
          i = int(self.status)
          return 200 <= i <= 299 or i in self.valid_response_codes
  
      def parse_error(self, msg=None):
          error_msg = 'Unknown error'
  
          try:
              # Azure does give some meaningful errors, but is inconsistent
              # Some APIs respond with an XML error. Others just dump HTML
              body = self.parse_body()
  
              # pylint: disable=no-member
              if type(body) == ET.Element:
                  code = body.findtext(fixxpath(xpath='Code'))
                  message = body.findtext(fixxpath(xpath='Message'))
                  message = message.split('\n')[0]
                  error_msg = '%s: %s' % (code, message)
  
          except MalformedResponseError:
              pass
  
          if msg:
              error_msg = '%s - %s' % (msg, error_msg)
  
          if self.status in [httplib.UNAUTHORIZED, httplib.FORBIDDEN]:
              raise InvalidCredsError(error_msg)
  
          raise LibcloudError(
              '%s Status code: %d.' % (error_msg, self.status),
              driver=self
          )
  
      def parse_body(self):
          is_redirect = int(self.status) == httplib.TEMPORARY_REDIRECT
  
          if is_redirect and self.connection.driver.follow_redirects:
              raise AzureRedirectException(self)
          else:
              return super(AzureResponse, self).parse_body()
  
  
  class AzureRawResponse(RawResponse):
      pass
  
  
  class AzureConnection(ConnectionUserAndKey):
      """
      Represents a single connection to Azure
      """
  
      responseCls = AzureResponse
      rawResponseCls = AzureRawResponse
  
      API_VERSION = '2012-02-12'
  
      def add_default_params(self, params):
          return params
  
      def pre_connect_hook(self, params, headers):
          headers = copy.deepcopy(headers)
  
          # We have to add a date header in GMT
          headers['x-ms-date'] = time.strftime(AZURE_TIME_FORMAT, time.gmtime())
          headers['x-ms-version'] = self.API_VERSION
  
          # Add the authorization header
          headers['Authorization'] = self._get_azure_auth_signature(
              method=self.method,
              headers=headers,
              params=params,
              account=self.user_id,
              secret_key=self.key,
              path=self.action
          )
  
          # Azure cribs about this in 'raw' connections
          headers.pop('Host', None)
  
          return params, headers
  
      def _get_azure_auth_signature(self,
                                    method,
                                    headers,
                                    params,
                                    account,
                                    secret_key,
                                    path='/'):
          """
          Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID,
                              UTF-8-Encoding-Of( StringToSign ) ) ) );
  
          StringToSign = HTTP-VERB + "\n" +
              Content-Encoding + "\n" +
              Content-Language + "\n" +
              Content-Length + "\n" +
              Content-MD5 + "\n" +
              Content-Type + "\n" +
              Date + "\n" +
              If-Modified-Since + "\n" +
              If-Match + "\n" +
              If-None-Match + "\n" +
              If-Unmodified-Since + "\n" +
              Range + "\n" +
              CanonicalizedHeaders +
              CanonicalizedResource;
          """
          xms_header_values = []
          param_list = []
  
          # Split the x-ms headers and normal headers and make everything
          # lower case
          headers_copy = {}
          for header, value in headers.items():
              header = header.lower()
              value = str(value).strip()
              if header.startswith('x-ms-'):
                  xms_header_values.append((header, value))
              else:
                  headers_copy[header] = value
  
          # Get the values for the headers in the specific order
          special_header_values = self._format_special_header_values(
              headers_copy, method)
  
          # Prepare the first section of the string to be signed
          values_to_sign = [method] + special_header_values
          # string_to_sign = '\n'.join([method] + special_header_values)
  
          # The x-ms-* headers have to be in lower case and sorted
          xms_header_values.sort()
  
          for header, value in xms_header_values:
              values_to_sign.append('%s:%s' % (header, value))
  
          # Add the canonicalized path
          values_to_sign.append('/%s%s' % (account, path))
  
          # URL query parameters (sorted and lower case)
          for key, value in params.items():
              param_list.append((key.lower(), str(value).strip()))
  
          param_list.sort()
  
          for key, value in param_list:
              values_to_sign.append('%s:%s' % (key, value))
  
          string_to_sign = b('\n'.join(values_to_sign))
          secret_key = b(secret_key)
          b64_hmac = base64.b64encode(
              hmac.new(secret_key, string_to_sign, digestmod=sha256).digest()
          )
  
          return 'SharedKey %s:%s' % (self.user_id, b64_hmac.decode('utf-8'))
  
      def _format_special_header_values(self, headers, method):
          is_change = method not in ('GET', 'HEAD')
          is_old_api = self.API_VERSION <= '2014-02-14'
  
          special_header_keys = [
              'content-encoding',
              'content-language',
              'content-length',
              'content-md5',
              'content-type',
              'date',
              'if-modified-since',
              'if-match',
              'if-none-match',
              'if-unmodified-since',
              'range'
          ]
  
          special_header_values = []
  
          for header in special_header_keys:
              header = header.lower()  # Just for safety
              if header in headers:
                  special_header_values.append(headers[header])
              elif header == 'content-length' and is_change and is_old_api:
                  # For old API versions, the Content-Length header must be '0'
                  # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key#content-length-header-in-version-2014-02-14-and-earlier
                  special_header_values.append('0')
              else:
                  special_header_values.append('')
  
          return special_header_values
  
  
  class AzureBaseDriver(BaseDriver):
      name = "Microsoft Azure Service Management API"
  
  
  class AzureServiceManagementConnection(CertificateConnection):
      # This needs the following approach -
      # 1. Make request using LibcloudHTTPSConnection which is a overloaded
      # class which takes in a client certificate
      # 2. Depending on the type of operation use a PollingConnection
      # when the response id is returned
      # 3. The Response can be used in an AzureServiceManagementResponse
  
      """
      Authentication class for "Service Account" authentication.
      """
  
      driver = AzureBaseDriver
      responseCls = AzureResponse
      rawResponseCls = AzureRawResponse
      name = 'Azure Service Management API Connection'
      host = 'management.core.windows.net'
      keyfile = ""
  
      def __init__(self, subscription_id, key_file, *args, **kwargs):
          """
          Check to see if PyCrypto is available, and convert key file path into a
          key string if the key is in a file.
  
          :param  subscription_id: Azure subscription ID.
          :type   subscription_id: ``str``
  
          :param  key_file: The PEM file used to authenticate with the service.
          :type   key_file: ``str``
          """
  
          super(AzureServiceManagementConnection, self).__init__(
              key_file,
              *args,
              **kwargs
          )
  
          self.subscription_id = subscription_id
  
          keypath = os.path.expanduser(key_file)
          self.keyfile = keypath
          is_file_path = os.path.exists(keypath) and os.path.isfile(keypath)
          if not is_file_path:
              raise InvalidCredsError(
                  'You need an certificate PEM file to authenticate with '
                  'Microsoft Azure. This can be found in the portal.'
              )
          self.key_file = key_file
  
      def add_default_headers(self, headers):
          """
          @inherits: :class:`Connection.add_default_headers`
          TODO: move to constant..
          """
          headers['x-ms-version'] = "2014-05-01"
          headers['x-ms-date'] = time.strftime(AZURE_TIME_FORMAT, time.gmtime())
          #  headers['host'] = self.host
          return headers
  
files: libcloud/common/azure.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  import copy
  import os
  import time
  import base64
  import hmac
  
  from hashlib import sha256
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import b
  from libcloud.utils.xml import fixxpath
  
  from libcloud.utils.py3 import ET
  from libcloud.common.types import InvalidCredsError
  from libcloud.common.types import LibcloudError, MalformedResponseError
  from libcloud.common.base import ConnectionUserAndKey, RawResponse
  from libcloud.common.base import CertificateConnection
  from libcloud.common.base import XmlResponse
  from libcloud.common.base import BaseDriver
  
  # The time format for headers in Azure requests
  AZURE_TIME_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
  
  
  class AzureRedirectException(Exception):
  
      def __init__(self, response):
          self.location = response.headers['location']
  
  
  class AzureResponse(XmlResponse):
  
      valid_response_codes = [
          httplib.NOT_FOUND,
          httplib.CONFLICT,
          httplib.BAD_REQUEST,
          httplib.TEMPORARY_REDIRECT
          # added TEMPORARY_REDIRECT as this can sometimes be
          # sent by azure instead of a success or fail response
      ]
  
      def success(self):
          i = int(self.status)
          return 200 <= i <= 299 or i in self.valid_response_codes
  
      def parse_error(self, msg=None):
          error_msg = 'Unknown error'
  
          try:
              # Azure does give some meaningful errors, but is inconsistent
              # Some APIs respond with an XML error. Others just dump HTML
              body = self.parse_body()
  
              # pylint: disable=no-member
              if type(body) == ET.Element:
                  code = body.findtext(fixxpath(xpath='Code'))
                  message = body.findtext(fixxpath(xpath='Message'))
                  message = message.split('\n')[0]
                  error_msg = '%s: %s' % (code, message)
  
          except MalformedResponseError:
              pass
  
          if msg:
              error_msg = '%s - %s' % (msg, error_msg)
  
          if self.status in [httplib.UNAUTHORIZED, httplib.FORBIDDEN]:
              raise InvalidCredsError(error_msg)
  
          raise LibcloudError(
              '%s Status code: %d.' % (error_msg, self.status),
              driver=self
          )
  
      def parse_body(self):
          is_redirect = int(self.status) == httplib.TEMPORARY_REDIRECT
  
          if is_redirect and self.connection.driver.follow_redirects:
              raise AzureRedirectException(self)
          else:
              return super(AzureResponse, self).parse_body()
  
  
  class AzureRawResponse(RawResponse):
      pass
  
  
  class AzureConnection(ConnectionUserAndKey):
      """
      Represents a single connection to Azure
      """
  
      responseCls = AzureResponse
      rawResponseCls = AzureRawResponse
  
      API_VERSION = '2012-02-12'
  
      def add_default_params(self, params):
          return params
  
      def pre_connect_hook(self, params, headers):
          headers = copy.deepcopy(headers)
  
          # We have to add a date header in GMT
          headers['x-ms-date'] = time.strftime(AZURE_TIME_FORMAT, time.gmtime())
          headers['x-ms-version'] = self.API_VERSION
  
          # Add the authorization header
          headers['Authorization'] = self._get_azure_auth_signature(
              method=self.method,
              headers=headers,
              params=params,
              account=self.user_id,
              secret_key=self.key,
              path=self.action
          )
  
          # Azure cribs about this in 'raw' connections
          headers.pop('Host', None)
  
          return params, headers
  
      def _get_azure_auth_signature(self,
                                    method,
                                    headers,
                                    params,
                                    account,
                                    secret_key,
                                    path='/'):
          """
          Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID,
                              UTF-8-Encoding-Of( StringToSign ) ) ) );
  
          StringToSign = HTTP-VERB + "\n" +
              Content-Encoding + "\n" +
              Content-Language + "\n" +
              Content-Length + "\n" +
              Content-MD5 + "\n" +
              Content-Type + "\n" +
              Date + "\n" +
              If-Modified-Since + "\n" +
              If-Match + "\n" +
              If-None-Match + "\n" +
              If-Unmodified-Since + "\n" +
              Range + "\n" +
              CanonicalizedHeaders +
              CanonicalizedResource;
          """
          xms_header_values = []
          param_list = []
  
          # Split the x-ms headers and normal headers and make everything
          # lower case
          headers_copy = {}
          for header, value in headers.items():
              header = header.lower()
              value = str(value).strip()
              if header.startswith('x-ms-'):
                  xms_header_values.append((header, value))
              else:
                  headers_copy[header] = value
  
          # Get the values for the headers in the specific order
          special_header_values = self._format_special_header_values(
              headers_copy, method)
  
          # Prepare the first section of the string to be signed
          values_to_sign = [method] + special_header_values
          # string_to_sign = '\n'.join([method] + special_header_values)
  
          # The x-ms-* headers have to be in lower case and sorted
          xms_header_values.sort()
  
          for header, value in xms_header_values:
              values_to_sign.append('%s:%s' % (header, value))
  
          # Add the canonicalized path
          values_to_sign.append('/%s%s' % (account, path))
  
          # URL query parameters (sorted and lower case)
          for key, value in params.items():
              param_list.append((key.lower(), str(value).strip()))
  
          param_list.sort()
  
          for key, value in param_list:
              values_to_sign.append('%s:%s' % (key, value))
  
          string_to_sign = b('\n'.join(values_to_sign))
          secret_key = b(secret_key)
          b64_hmac = base64.b64encode(
              hmac.new(secret_key, string_to_sign, digestmod=sha256).digest()
          )
  
          return 'SharedKey %s:%s' % (self.user_id, b64_hmac.decode('utf-8'))
  
      def _format_special_header_values(self, headers, method):
          is_change = method not in ('GET', 'HEAD')
          is_old_api = self.API_VERSION <= '2014-02-14'
  
          special_header_keys = [
              'content-encoding',
              'content-language',
              'content-length',
              'content-md5',
              'content-type',
              'date',
              'if-modified-since',
              'if-match',
              'if-none-match',
              'if-unmodified-since',
              'range'
          ]
  
          special_header_values = []
  
          for header in special_header_keys:
              header = header.lower()  # Just for safety
              if header in headers:
                  special_header_values.append(headers[header])
              elif header == 'content-length' and is_change and is_old_api:
                  # For old API versions, the Content-Length header must be '0'
                  # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key#content-length-header-in-version-2014-02-14-and-earlier
                  special_header_values.append('0')
              else:
                  special_header_values.append('')
  
          return special_header_values
  
  
  class AzureBaseDriver(BaseDriver):
      name = "Microsoft Azure Service Management API"
  
  
  class AzureServiceManagementConnection(CertificateConnection):
      # This needs the following approach -
      # 1. Make request using LibcloudHTTPSConnection which is a overloaded
      # class which takes in a client certificate
      # 2. Depending on the type of operation use a PollingConnection
      # when the response id is returned
      # 3. The Response can be used in an AzureServiceManagementResponse
  
      """
      Authentication class for "Service Account" authentication.
      """
  
      driver = AzureBaseDriver
      responseCls = AzureResponse
      rawResponseCls = AzureRawResponse
      name = 'Azure Service Management API Connection'
      host = 'management.core.windows.net'
      keyfile = ""
  
      def __init__(self, subscription_id, key_file, *args, **kwargs):
          """
          Check to see if PyCrypto is available, and convert key file path into a
          key string if the key is in a file.
  
          :param  subscription_id: Azure subscription ID.
          :type   subscription_id: ``str``
  
          :param  key_file: The PEM file used to authenticate with the service.
          :type   key_file: ``str``
          """
  
          super(AzureServiceManagementConnection, self).__init__(
              key_file,
              *args,
              **kwargs
          )
  
          self.subscription_id = subscription_id
  
          keypath = os.path.expanduser(key_file)
          self.keyfile = keypath
          is_file_path = os.path.exists(keypath) and os.path.isfile(keypath)
          if not is_file_path:
              raise InvalidCredsError(
                  'You need an certificate PEM file to authenticate with '
                  'Microsoft Azure. This can be found in the portal.'
              )
          self.key_file = key_file
  
      def add_default_headers(self, headers):
          """
          @inherits: :class:`Connection.add_default_headers`
          TODO: move to constant..
          """
          headers['x-ms-version'] = "2014-05-01"
          headers['x-ms-date'] = time.strftime(AZURE_TIME_FORMAT, time.gmtime())
          #  headers['host'] = self.host
          return headers
  
files: libcloud/common/azure.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  """
  Provides base classes for working with storage
  """
  
  # Backward compatibility for Python 2.5
  from __future__ import with_statement
  
  from typing import Dict
  from typing import Iterator
  from typing import List
  from typing import Optional
  from typing import Type
  
  import os.path                          # pylint: disable-msg=W0404
  import hashlib
  import warnings
  import errno
  from os.path import join as pjoin
  
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import b
  
  import libcloud.utils.files
  from libcloud.common.types import LibcloudError
  from libcloud.common.base import Connection
  from libcloud.common.base import ConnectionUserAndKey, BaseDriver
  from libcloud.storage.types import ObjectDoesNotExistError
  
  __all__ = [
      'Object',
      'Container',
      'StorageDriver',
  
      'CHUNK_SIZE',
      'DEFAULT_CONTENT_TYPE'
  ]
  
  CHUNK_SIZE = 8096
  
  # Default Content-Type which is sent when uploading an object if one is not
  # supplied and can't be detected when using non-strict mode.
  DEFAULT_CONTENT_TYPE = 'application/octet-stream'
  
  
  class Object(object):
      """
      Represents an object (BLOB).
      """
  
      def __init__(self,
                   name,  # type: str
                   size,  # type: int
                   hash,  # type: str
                   extra,  # type: dict
                   meta_data,  # type: dict
                   container,  # type: Container
                   driver,  # type: StorageDriver
                   ):
          """
          :param name: Object name (must be unique per container).
          :type  name: ``str``
  
          :param size: Object size in bytes.
          :type  size: ``int``
  
          :param hash: Object hash.
          :type  hash: ``str``
  
          :param container: Object container.
          :type  container: :class:`libcloud.storage.base.Container`
  
          :param extra: Extra attributes.
          :type  extra: ``dict``
  
          :param meta_data: Optional object meta data.
          :type  meta_data: ``dict``
  
          :param driver: StorageDriver instance.
          :type  driver: :class:`libcloud.storage.base.StorageDriver`
          """
  
          self.name = name
          self.size = size
          self.hash = hash
          self.container = container
          self.extra = extra or {}
          self.meta_data = meta_data or {}
          self.driver = driver
  
      def get_cdn_url(self):
          # type: () -> str
          return self.driver.get_object_cdn_url(obj=self)
  
      def enable_cdn(self):
          # type: () -> bool
          return self.driver.enable_object_cdn(obj=self)
  
      def download(self, destination_path, overwrite_existing=False,
                   delete_on_failure=True):
          # type: (str, bool, bool) -> bool
          return self.driver.download_object(
              obj=self,
              destination_path=destination_path,
              overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def as_stream(self, chunk_size=None):
          # type: (Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_as_stream(obj=self,
                                                       chunk_size=chunk_size)
  
      def download_range(self, destination_path, start_bytes, end_bytes=None,
                         overwrite_existing=False,
                         delete_on_failure=True):
          # type: (str, int, Optional[int], bool, bool) -> bool
          return self.driver.download_object_range(
              obj=self,
              destination_path=destination_path,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def range_as_stream(self, start_bytes, end_bytes=None, chunk_size=None):
          # type: (int, Optional[int], Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_range_as_stream(
              obj=self,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              chunk_size=chunk_size)
  
      def delete(self):
          # type: () -> bool
          return self.driver.delete_object(self)
  
      def __repr__(self):
          return ('<Object: name=%s, size=%s, hash=%s, provider=%s ...>' %
                  (self.name, self.size, self.hash, self.driver.name))
  
  
  class Container(object):
      """
      Represents a container (bucket) which can hold multiple objects.
      """
  
      def __init__(self,
                   name,  # type: str
                   extra,  # type: dict
                   driver,  # type: StorageDriver
                   ):
          """
          :param name: Container name (must be unique).
          :type name: ``str``
  
          :param extra: Extra attributes.
          :type extra: ``dict``
  
          :param driver: StorageDriver instance.
          :type driver: :class:`libcloud.storage.base.StorageDriver`
          """
  
          self.name = name
          self.extra = extra or {}
          self.driver = driver
  
      def iterate_objects(self, prefix=None, ex_prefix=None):
          # type: (Optional[str], Optional[str]) -> Iterator[Object]
          return self.driver.iterate_container_objects(container=self,
                                                       prefix=prefix,
                                                       ex_prefix=ex_prefix)
  
      def list_objects(self, prefix=None, ex_prefix=None):
          # type: (Optional[str], Optional[str]) -> List[Object]
          return self.driver.list_container_objects(container=self,
                                                    prefix=prefix,
                                                    ex_prefix=ex_prefix)
  
      def get_cdn_url(self):
          # type: () -> str
          return self.driver.get_container_cdn_url(container=self)
  
      def enable_cdn(self):
          # type: () -> bool
          return self.driver.enable_container_cdn(container=self)
  
      def get_object(self, object_name):
          # type: (str) -> Object
          return self.driver.get_object(container_name=self.name,
                                        object_name=object_name)
  
      def upload_object(self, file_path, object_name, extra=None,
                        verify_hash=True, headers=None):
          # type: (str, str, Optional[dict], bool, Optional[Dict[str, str]]) -> Object  # noqa: E501
          return self.driver.upload_object(
              file_path, self, object_name, extra=extra,
              verify_hash=verify_hash, headers=headers)
  
      def upload_object_via_stream(self, iterator, object_name, extra=None,
                                   headers=None):
          # type: (Iterator[bytes], str, Optional[dict], Optional[Dict[str, str]]) -> Object  # noqa: E501
          return self.driver.upload_object_via_stream(
              iterator, self, object_name, extra=extra, headers=headers)
  
      def download_object(self, obj, destination_path, overwrite_existing=False,
                          delete_on_failure=True):
          # type: (Object, str, bool, bool) -> bool
          return self.driver.download_object(
              obj, destination_path, overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def download_object_as_stream(self, obj, chunk_size=None):
          # type: (Object, Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_as_stream(obj, chunk_size)
  
      def download_object_range(self, obj, destination_path, start_bytes,
                                end_bytes=None, overwrite_existing=False,
                                delete_on_failure=True):
          # type: (Object, str, int, Optional[int], bool, bool) -> bool
          return self.driver.download_object_range(
              obj=obj,
              destination_path=destination_path,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None,
                                          chunk_size=None):
          # type: (Object, int, Optional[int], Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_range_as_stream(
              obj=obj,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              chunk_size=chunk_size)
  
      def delete_object(self, obj):
          # type: (Object) -> bool
          return self.driver.delete_object(obj)
  
      def delete(self):
          # type: () -> bool
          return self.driver.delete_container(self)
  
      def __repr__(self):
          return ('<Container: name=%s, provider=%s>'
                  % (self.name, self.driver.name))
  
  
  class StorageDriver(BaseDriver):
      """
      A base StorageDriver to derive from.
      """
  
      connectionCls = ConnectionUserAndKey  # type: Type[Connection]
      name = None  # type: str
      hash_type = 'md5'  # type: str
      supports_chunked_encoding = False  # type: bool
  
      # When strict mode is used, exception will be thrown if no content type is
      # provided and none can be detected when uploading an object
      strict_mode = False  # type: bool
  
      def iterate_containers(self):
          # type: () -> Iterator[Container]
          """
          Return a iterator of containers for the given account
  
          :return: A iterator of Container instances.
          :rtype: ``iterator`` of :class:`libcloud.storage.base.Container`
          """
          raise NotImplementedError(
              'iterate_containers not implemented for this driver')
  
      def list_containers(self):
          # type: () -> List[Container]
          """
          Return a list of containers.
  
          :return: A list of Container instances.
          :rtype: ``list`` of :class:`Container`
          """
          return list(self.iterate_containers())
  
      def iterate_container_objects(self, container, prefix=None,
                                    ex_prefix=None):
          # type: (Container, Optional[str], Optional[str]) -> Iterator[Object]
          """
          Return a iterator of objects for the given container.
  
          :param container: Container instance
          :type container: :class:`libcloud.storage.base.Container`
  
          :param prefix: Filter objects starting with a prefix.
          :type  prefix: ``str``
  
          :param ex_prefix: (Deprecated.) Filter objects starting with a prefix.
          :type  ex_prefix: ``str``
  
          :return: A iterator of Object instances.
          :rtype: ``iterator`` of :class:`libcloud.storage.base.Object`
          """
          raise NotImplementedError(
              'iterate_container_objects not implemented for this driver')
  
      def list_container_objects(self, container, prefix=None, ex_prefix=None):
          # type: (Container, Optional[str], Optional[str]) -> List[Object]
          """
          Return a list of objects for the given container.
  
          :param container: Container instance.
          :type container: :class:`libcloud.storage.base.Container`
  
          :param prefix: Filter objects starting with a prefix.
          :type  prefix: ``str``
  
          :param ex_prefix: (Deprecated.) Filter objects starting with a prefix.
          :type  ex_prefix: ``str``
  
          :return: A list of Object instances.
          :rtype: ``list`` of :class:`libcloud.storage.base.Object`
          """
          return list(self.iterate_container_objects(container,
                                                     prefix=prefix,
                                                     ex_prefix=ex_prefix))
  
      def _normalize_prefix_argument(self, prefix, ex_prefix):
          if ex_prefix:
              warnings.warn('The ``ex_prefix`` argument is deprecated - '
                            'please update code to use ``prefix``',
                            DeprecationWarning)
              return ex_prefix
  
          return prefix
  
      def _filter_listed_container_objects(self, objects, prefix):
          if prefix is not None:
              warnings.warn('Driver %s does not implement native object '
                            'filtering; falling back to filtering the full '
                            'object stream.' % self.__class__.__name__)
  
          for obj in objects:
              if prefix is None or obj.name.startswith(prefix):
                  yield obj
  
      def get_container(self, container_name):
          # type: (str) -> Container
          """
          Return a container instance.
  
          :param container_name: Container name.
          :type container_name: ``str``
  
          :return: :class:`Container` instance.
          :rtype: :class:`libcloud.storage.base.Container`
          """
          raise NotImplementedError(
              'get_object not implemented for this driver')
  
      def get_container_cdn_url(self, container):
          # type: (Container) -> str
          """
          Return a container CDN URL.
  
          :param container: Container instance
          :type  container: :class:`libcloud.storage.base.Container`
  
          :return: A CDN URL for this container.
          :rtype: ``str``
          """
          raise NotImplementedError(
              'get_container_cdn_url not implemented for this driver')
  
      def get_object(self, container_name, object_name):
          # type: (str, str) -> Object
          """
          Return an object instance.
  
          :param container_name: Container name.
          :type  container_name: ``str``
  
          :param object_name: Object name.
          :type  object_name: ``str``
  
          :return: :class:`Object` instance.
          :rtype: :class:`libcloud.storage.base.Object`
          """
          raise NotImplementedError(
              'get_object not implemented for this driver')
  
      def get_object_cdn_url(self, obj):
          # type: (Object) -> str
          """
          Return an object CDN URL.
  
          :param obj: Object instance
          :type  obj: :class:`libcloud.storage.base.Object`
  
          :return: A CDN URL for this object.
          :rtype: ``str``
          """
          raise NotImplementedError(
              'get_object_cdn_url not implemented for this driver')
  
      def enable_container_cdn(self, container):
          # type: (Container) -> bool
          """
          Enable container CDN.
  
          :param container: Container instance
          :type  container: :class:`libcloud.storage.base.Container`
  
          :rtype: ``bool``
          """
          raise NotImplementedError(
              'enable_container_cdn not implemented for this driver')
  
      def enable_object_cdn(self, obj):
          # type: (Object) -> bool
          """
          Enable object CDN.
  
          :param obj: Object instance
          :type  obj: :class:`libcloud.storage.base.Object`
  
          :rtype: ``bool``
          """
          raise NotImplementedError(
              'enable_object_cdn not implemented for this driver')
  
      def download_object(self, obj, destination_path, overwrite_existing=False,
                          delete_on_failure=True):
          # type: (Object, str, bool, bool) -> bool
          """
          Download an object to the specified destination path.
  
          :param obj: Object instance.
          :type obj: :class:`libcloud.storage.base.Object`
  
          :param destination_path: Full path to a file or a directory where the
                                   incoming file will be saved.
          :type destination_path: ``str``
  
          :param overwrite_existing: True to overwrite an existing file,
                                     defaults to False.
          :type overwrite_existing: ``bool``
  
          :param delete_on_failure: True to delete a partially downloaded file if
                                     the download was not successful (hash
                                     mismatch / file size).
          :type delete_on_failure: ``bool``
  
          :return: True if an object has been successfully downloaded, False
                   otherwise.
          :rtype: ``bool``
          """
          raise NotImplementedError(
              'download_object not implemented for this driver')
  
      def download_object_as_stream(self, obj, chunk_size=None):
          # type: (Object, Optional[int]) -> Iterator[bytes]
          """
          Return a iterator which yields object data.
  
          :param obj: Object instance
          :type obj: :class:`libcloud.storage.base.Object`
  
          :param chunk_size: Optional chunk size (in bytes).
          :type chunk_size: ``int``
  
          :rtype: ``iterator`` of ``bytes``
          """
          raise NotImplementedError(
              'download_object_as_stream not implemented for this driver')
  
      def download_object_range(self, obj, destination_path, start_bytes,
                                end_bytes=None, overwrite_existing=False,
                                delete_on_failure=True):
          # type: (Object, str, int, Optional[int], bool, bool) -> bool
          """
          Download part of an object.
  
          :param obj: Object instance.
          :type obj: :class:`libcloud.storage.base.Object`
  
          :param destination_path: Full path to a file or a directory where the
                                   incoming file will be saved.
          :type destination_path: ``str``
  
          :param start_bytes: Start byte offset (inclusive) for the range
                              download. Offset is 0 index based so the first
                              byte in file file is "0".
          :type start_bytes: ``int``
  
          :param end_bytes: End byte offset (non-inclusive) for the range
                            download. If not provided, it will default to the
                            end of the file.
          :type end_bytes: ``int``
  
          :param overwrite_existing: True to overwrite an existing file,
                                     defaults to False.
          :type overwrite_existing: ``bool``
  
          :param delete_on_failure: True to delete a partially downloaded file if
                                     the download was not successful (hash
                                     mismatch / file size).
          :type delete_on_failure: ``bool``
  
          :return: True if an object has been successfully downloaded, False
                   otherwise.
          :rtype: ``bool``
  
          """
          raise NotImplementedError(
              'download_object_range not implemented for this driver')
  
      def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None,
                                          chunk_size=None):
          # type: (Object, int, Optional[int], Optional[int]) -> Iterator[bytes]
          """
          Return a iterator which yields range / part of the object data.
  
          :param obj: Object instance
          :type obj: :class:`libcloud.storage.base.Object`
  
          :param start_bytes: Start byte offset (inclusive) for the range
                              download. Offset is 0 index based so the first
                              byte in file file is "0".
          :type start_bytes: ``int``
  
          :param end_bytes: End byte offset (non-inclusive) for the range
                            download. If not provided, it will default to the
                            end of the file.
          :type end_bytes: ``int``
  
          :param chunk_size: Optional chunk size (in bytes).
          :type chunk_size: ``int``
  
          :rtype: ``iterator`` of ``bytes``
          """
          raise NotImplementedError(
              'download_object_range_as_stream not implemented for this driver')
  
      def upload_object(self, file_path, container, object_name, extra=None,
                        verify_hash=True, headers=None):
          # type: (str, Container, str, Optional[dict], bool, Optional[Dict[str, str]]) -> Object  # noqa: E501
          """
          Upload an object currently located on a disk.
  
          :param file_path: Path to the object on disk.
          :type file_path: ``str``
  
          :param container: Destination container.
          :type container: :class:`libcloud.storage.base.Container`
  
          :param object_name: Object name.
          :type object_name: ``str``
  
          :param verify_hash: Verify hash
          :type verify_hash: ``bool``
  
          :param extra: Extra attributes (driver specific). (optional)
          :type extra: ``dict``
  
          :param headers: (optional) Additional request headers,
              such as CORS headers. For example:
              headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'}
          :type headers: ``dict``
  
          :rtype: :class:`libcloud.storage.base.Object`
          """
          raise NotImplementedError(
              'upload_object not implemented for this driver')
  
      def upload_object_via_stream(self, iterator, container,
                                   object_name,
                                   extra=None,
                                   headers=None):
          # type: (Iterator[bytes], Container, str, Optional[dict], Optional[Dict[str, str]]) -> Object  # noqa: E501
          """
          Upload an object using an iterator.
  
          If a provider supports it, chunked transfer encoding is used and you
          don't need to know in advance the amount of data to be uploaded.
  
          Otherwise if a provider doesn't support it, iterator will be exhausted
          so a total size for data to be uploaded can be determined.
  
          Note: Exhausting the iterator means that the whole data must be
          buffered in memory which might result in memory exhausting when
          uploading a very large object.
  
          If a file is located on a disk you are advised to use upload_object
          function which uses fs.stat function to determine the file size and it
          doesn't need to buffer whole object in the memory.
  
          :param iterator: An object which implements the iterator interface.
          :type iterator: :class:`object`
  
          :param container: Destination container.
          :type container: :class:`libcloud.storage.base.Container`
  
          :param object_name: Object name.
          :type object_name: ``str``
  
          :param extra: (optional) Extra attributes (driver specific). Note:
              This dictionary must contain a 'content_type' key which represents
              a content type of the stored object.
          :type extra: ``dict``
  
          :param headers: (optional) Additional request headers,
              such as CORS headers. For example:
              headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'}
          :type headers: ``dict``
  
          :rtype: ``libcloud.storage.base.Object``
          """
          raise NotImplementedError(
              'upload_object_via_stream not implemented for this driver')
  
      def delete_object(self, obj):
          # type: (Object) -> bool
          """
          Delete an object.
  
          :param obj: Object instance.
          :type obj: :class:`libcloud.storage.base.Object`
  
          :return: ``bool`` True on success.
          :rtype: ``bool``
          """
          raise NotImplementedError(
              'delete_object not implemented for this driver')
  
      def create_container(self, container_name):
          # type: (str) -> Container
          """
          Create a new container.
  
          :param container_name: Container name.
          :type container_name: ``str``
  
          :return: Container instance on success.
          :rtype: :class:`libcloud.storage.base.Container`
          """
          raise NotImplementedError(
              'create_container not implemented for this driver')
  
      def delete_container(self, container):
          # type: (Container) -> bool
          """
          Delete a container.
  
          :param container: Container instance
          :type container: :class:`libcloud.storage.base.Container`
  
          :return: ``True`` on success, ``False`` otherwise.
          :rtype: ``bool``
          """
          raise NotImplementedError(
              'delete_container not implemented for this driver')
  
      def _get_object(self, obj, callback, callback_kwargs, response,
                      success_status_code=None):
          """
          Call passed callback and start transfer of the object'
  
          :param obj: Object instance.
          :type obj: :class:`Object`
  
          :param callback: Function which is called with the passed
              callback_kwargs
          :type callback: :class:`function`
  
          :param callback_kwargs: Keyword arguments which are passed to the
               callback.
          :type callback_kwargs: ``dict``
  
          :param response: Response instance.
          :type response: :class:`Response`
  
          :param success_status_code: Status code which represents a successful
                                      transfer (defaults to httplib.OK)
          :type success_status_code: ``int``
  
          :return: ``True`` on success, ``False`` otherwise.
          :rtype: ``bool``
          """
          success_status_code = success_status_code or httplib.OK
  
          if response.status == success_status_code:
              return callback(**callback_kwargs)
          elif response.status == httplib.NOT_FOUND:
              raise ObjectDoesNotExistError(object_name=obj.name,
                                            value='', driver=self)
  
          raise LibcloudError(value='Unexpected status code: %s' %
                                    (response.status),
                              driver=self)
  
      def _save_object(self, response, obj, destination_path,
                       overwrite_existing=False, delete_on_failure=True,
                       chunk_size=None, partial_download=False):
          """
          Save object to the provided path.
  
          :param response: RawResponse instance.
          :type response: :class:`RawResponse`
  
          :param obj: Object instance.
          :type obj: :class:`Object`
  
          :param destination_path: Destination directory.
          :type destination_path: ``str``
  
          :param delete_on_failure: True to delete partially downloaded object if
                                    the download fails.
          :type delete_on_failure: ``bool``
  
          :param overwrite_existing: True to overwrite a local path if it already
                                     exists.
          :type overwrite_existing: ``bool``
  
          :param chunk_size: Optional chunk size
              (defaults to ``libcloud.storage.base.CHUNK_SIZE``, 8kb)
          :type chunk_size: ``int``
  
          :param partial_download: True if this is a range (partial) save,
                                   False otherwise.
          :type partial_download: ``bool``
  
          :return: ``True`` on success, ``False`` otherwise.
          :rtype: ``bool``
          """
  
          chunk_size = chunk_size or CHUNK_SIZE
  
          base_name = os.path.basename(destination_path)
  
          if not base_name and not os.path.exists(destination_path):
              raise LibcloudError(
                  value='Path %s does not exist' % (destination_path),
                  driver=self)
  
          if not base_name:
              file_path = pjoin(destination_path, obj.name)
          else:
              file_path = destination_path
  
          if os.path.exists(file_path) and not overwrite_existing:
              raise LibcloudError(
                  value='File %s already exists, but ' % (file_path) +
                  'overwrite_existing=False',
                  driver=self)
  
          bytes_transferred = 0
  
          with open(file_path, 'wb') as file_handle:
              for chunk in response._response.iter_content(chunk_size):
                  file_handle.write(b(chunk))
                  bytes_transferred += len(chunk)
  
          if not partial_download and int(obj.size) != int(bytes_transferred):
              # Transfer failed, support retry?
              # NOTE: We only perform this check if this is a regular and not a
              # partial / range download
              if delete_on_failure:
                  try:
                      os.unlink(file_path)
                  except Exception:
                      pass
  
              return False
  
          return True
  
      def _upload_object(self, object_name, content_type, request_path,
                         request_method='PUT',
                         headers=None, file_path=None, stream=None,
                         chunked=False, multipart=False):
          """
          Helper function for setting common request headers and calling the
          passed in callback which uploads an object.
          """
          headers = headers or {}
  
          if file_path and not os.path.exists(file_path):
              raise OSError('File %s does not exist' % (file_path))
  
          if stream is not None and not hasattr(stream, 'next') and not \
                  hasattr(stream, '__next__'):
              raise AttributeError('iterator object must implement next() ' +
                                   'method.')
  
          headers['Content-Type'] = self._determine_content_type(
              content_type, object_name, file_path=file_path)
  
          if stream:
              response = self.connection.request(
                  request_path,
                  method=request_method, data=stream,
                  headers=headers, raw=True)
              stream_hash, stream_length = self._hash_buffered_stream(
                  stream,
                  self._get_hash_function())
          else:
              with open(file_path, 'rb') as file_stream:
                  response = self.connection.request(
                      request_path,
                      method=request_method, data=file_stream,
                      headers=headers, raw=True)
              with open(file_path, 'rb') as file_stream:
                  stream_hash, stream_length = self._hash_buffered_stream(
                      file_stream,
                      self._get_hash_function())
  
          if not response.success():
              response.parse_error()
  
          return {'response': response,
                  'bytes_transferred': stream_length,
                  'data_hash': stream_hash}
  
      def _determine_content_type(self, content_type, object_name,
                                  file_path=None):
          if content_type:
              return content_type
  
          name = file_path or object_name
          content_type, _ = libcloud.utils.files.guess_file_mime_type(name)
  
          if self.strict_mode and not content_type:
              raise AttributeError('File content-type could not be guessed for '
                                   '"%s" and no content_type value is provided'
                                   % name)
  
          return content_type or DEFAULT_CONTENT_TYPE
  
      def _hash_buffered_stream(self, stream, hasher, blocksize=65536):
          total_len = 0
  
          if hasattr(stream, '__next__') or hasattr(stream, 'next'):
              # Ensure we start from the begining of a stream in case stream is
              # not at the beginning
              if hasattr(stream, 'seek'):
                  try:
                      stream.seek(0)
                  except OSError as e:
                      if e.errno != errno.ESPIPE:
                          # This represents "OSError: [Errno 29] Illegal seek"
                          # error. This could either mean that the underlying
                          # handle doesn't support seek operation (e.g. pipe) or
                          # that the invalid seek position is provided. Sadly
                          # there is no good robust way to distinghuish that so
                          # we simply ignore all the "Illeal seek" errors so
                          # this function works correctly with pipes.
                          # See https://github.com/apache/libcloud/pull/1427 for
                          # details
                          raise e
  
              for chunk in libcloud.utils.files.read_in_chunks(iterator=stream):
                  hasher.update(b(chunk))
                  total_len += len(chunk)
  
              return (hasher.hexdigest(), total_len)
  
          if not hasattr(stream, '__exit__'):
              for s in stream:
                  hasher.update(s)
                  total_len = total_len + len(s)
              return (hasher.hexdigest(), total_len)
  
          with stream:
              buf = stream.read(blocksize)
              while len(buf) > 0:
                  total_len = total_len + len(buf)
                  hasher.update(buf)
                  buf = stream.read(blocksize)
  
          return (hasher.hexdigest(), total_len)
  
      def _get_hash_function(self):
          """
          Return instantiated hash function for the hash type supported by
          the provider.
          """
          try:
              func = getattr(hashlib, self.hash_type)()
          except AttributeError:
              raise RuntimeError('Invalid or unsupported hash type: %s' %
                                 (self.hash_type))
  
          return func
  
      def _validate_start_and_end_bytes(self, start_bytes, end_bytes=None):
          # type: (int, Optional[int]) -> bool
          """
          Method which validates that start_bytes and end_bytes arguments contain
          valid values.
          """
          if start_bytes < 0:
              raise ValueError('start_bytes must be greater than 0')
  
          if end_bytes and start_bytes > end_bytes:
              raise ValueError('start_bytes must be smaller than end_bytes')
  
          return True
  
      def _get_standard_range_str(self, start_bytes, end_bytes=None):
          # type: (int, Optional[int]) -> str
          """
          Return range string which is used as a Range header value for range
          requests for drivers which follow standard Range header notation
  
          This returns range string in the following format:
          bytes=<start_bytes>-<end bytes>.
  
          For example:
  
          bytes=1-10
          bytes=0-2
          bytes=5-
          bytes=100-5000
          """
          range_str = 'bytes=%s-' % (start_bytes)
  
          if end_bytes is not None:
              range_str += str(end_bytes)
  
          return range_str
  
files: libcloud/storage/base.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  """
  Provides base classes for working with storage
  """
  
  # Backward compatibility for Python 2.5
  from __future__ import with_statement
  
  from typing import Dict
  from typing import Iterator
  from typing import List
  from typing import Optional
  from typing import Type
  
  import os.path                          # pylint: disable-msg=W0404
  import hashlib
  import warnings
  import errno
  from os.path import join as pjoin
  
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import b
  
  import libcloud.utils.files
  from libcloud.common.types import LibcloudError
  from libcloud.common.base import Connection
  from libcloud.common.base import ConnectionUserAndKey, BaseDriver
  from libcloud.storage.types import ObjectDoesNotExistError
  
  __all__ = [
      'Object',
      'Container',
      'StorageDriver',
  
      'CHUNK_SIZE',
      'DEFAULT_CONTENT_TYPE'
  ]
  
  CHUNK_SIZE = 8096
  
  # Default Content-Type which is sent when uploading an object if one is not
  # supplied and can't be detected when using non-strict mode.
  DEFAULT_CONTENT_TYPE = 'application/octet-stream'
  
  
  class Object(object):
      """
      Represents an object (BLOB).
      """
  
      def __init__(self,
                   name,  # type: str
                   size,  # type: int
                   hash,  # type: str
                   extra,  # type: dict
                   meta_data,  # type: dict
                   container,  # type: Container
                   driver,  # type: StorageDriver
                   ):
          """
          :param name: Object name (must be unique per container).
          :type  name: ``str``
  
          :param size: Object size in bytes.
          :type  size: ``int``
  
          :param hash: Object hash.
          :type  hash: ``str``
  
          :param container: Object container.
          :type  container: :class:`libcloud.storage.base.Container`
  
          :param extra: Extra attributes.
          :type  extra: ``dict``
  
          :param meta_data: Optional object meta data.
          :type  meta_data: ``dict``
  
          :param driver: StorageDriver instance.
          :type  driver: :class:`libcloud.storage.base.StorageDriver`
          """
  
          self.name = name
          self.size = size
          self.hash = hash
          self.container = container
          self.extra = extra or {}
          self.meta_data = meta_data or {}
          self.driver = driver
  
      def get_cdn_url(self):
          # type: () -> str
          return self.driver.get_object_cdn_url(obj=self)
  
      def enable_cdn(self):
          # type: () -> bool
          return self.driver.enable_object_cdn(obj=self)
  
      def download(self, destination_path, overwrite_existing=False,
                   delete_on_failure=True):
          # type: (str, bool, bool) -> bool
          return self.driver.download_object(
              obj=self,
              destination_path=destination_path,
              overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def as_stream(self, chunk_size=None):
          # type: (Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_as_stream(obj=self,
                                                       chunk_size=chunk_size)
  
      def download_range(self, destination_path, start_bytes, end_bytes=None,
                         overwrite_existing=False,
                         delete_on_failure=True):
          # type: (str, int, Optional[int], bool, bool) -> bool
          return self.driver.download_object_range(
              obj=self,
              destination_path=destination_path,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def range_as_stream(self, start_bytes, end_bytes=None, chunk_size=None):
          # type: (int, Optional[int], Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_range_as_stream(
              obj=self,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              chunk_size=chunk_size)
  
      def delete(self):
          # type: () -> bool
          return self.driver.delete_object(self)
  
      def __repr__(self):
          return ('<Object: name=%s, size=%s, hash=%s, provider=%s ...>' %
                  (self.name, self.size, self.hash, self.driver.name))
  
  
  class Container(object):
      """
      Represents a container (bucket) which can hold multiple objects.
      """
  
      def __init__(self,
                   name,  # type: str
                   extra,  # type: dict
                   driver,  # type: StorageDriver
                   ):
          """
          :param name: Container name (must be unique).
          :type name: ``str``
  
          :param extra: Extra attributes.
          :type extra: ``dict``
  
          :param driver: StorageDriver instance.
          :type driver: :class:`libcloud.storage.base.StorageDriver`
          """
  
          self.name = name
          self.extra = extra or {}
          self.driver = driver
  
      def iterate_objects(self, prefix=None, ex_prefix=None):
          # type: (Optional[str], Optional[str]) -> Iterator[Object]
          return self.driver.iterate_container_objects(container=self,
                                                       prefix=prefix,
                                                       ex_prefix=ex_prefix)
  
      def list_objects(self, prefix=None, ex_prefix=None):
          # type: (Optional[str], Optional[str]) -> List[Object]
          return self.driver.list_container_objects(container=self,
                                                    prefix=prefix,
                                                    ex_prefix=ex_prefix)
  
      def get_cdn_url(self):
          # type: () -> str
          return self.driver.get_container_cdn_url(container=self)
  
      def enable_cdn(self):
          # type: () -> bool
          return self.driver.enable_container_cdn(container=self)
  
      def get_object(self, object_name):
          # type: (str) -> Object
          return self.driver.get_object(container_name=self.name,
                                        object_name=object_name)
  
      def upload_object(self, file_path, object_name, extra=None,
                        verify_hash=True, headers=None):
          # type: (str, str, Optional[dict], bool, Optional[Dict[str, str]]) -> Object  # noqa: E501
          return self.driver.upload_object(
              file_path, self, object_name, extra=extra,
              verify_hash=verify_hash, headers=headers)
  
      def upload_object_via_stream(self, iterator, object_name, extra=None,
                                   headers=None):
          # type: (Iterator[bytes], str, Optional[dict], Optional[Dict[str, str]]) -> Object  # noqa: E501
          return self.driver.upload_object_via_stream(
              iterator, self, object_name, extra=extra, headers=headers)
  
      def download_object(self, obj, destination_path, overwrite_existing=False,
                          delete_on_failure=True):
          # type: (Object, str, bool, bool) -> bool
          return self.driver.download_object(
              obj, destination_path, overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def download_object_as_stream(self, obj, chunk_size=None):
          # type: (Object, Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_as_stream(obj, chunk_size)
  
      def download_object_range(self, obj, destination_path, start_bytes,
                                end_bytes=None, overwrite_existing=False,
                                delete_on_failure=True):
          # type: (Object, str, int, Optional[int], bool, bool) -> bool
          return self.driver.download_object_range(
              obj=obj,
              destination_path=destination_path,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None,
                                          chunk_size=None):
          # type: (Object, int, Optional[int], Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_range_as_stream(
              obj=obj,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              chunk_size=chunk_size)
  
      def delete_object(self, obj):
          # type: (Object) -> bool
          return self.driver.delete_object(obj)
  
      def delete(self):
          # type: () -> bool
          return self.driver.delete_container(self)
  
      def __repr__(self):
          return ('<Container: name=%s, provider=%s>'
                  % (self.name, self.driver.name))
  
  
  class StorageDriver(BaseDriver):
      """
      A base StorageDriver to derive from.
      """
  
      connectionCls = ConnectionUserAndKey  # type: Type[Connection]
      name = None  # type: str
      hash_type = 'md5'  # type: str
      supports_chunked_encoding = False  # type: bool
  
      # When strict mode is used, exception will be thrown if no content type is
      # provided and none can be detected when uploading an object
      strict_mode = False  # type: bool
  
      def iterate_containers(self):
          # type: () -> Iterator[Container]
          """
          Return a iterator of containers for the given account
  
          :return: A iterator of Container instances.
          :rtype: ``iterator`` of :class:`libcloud.storage.base.Container`
          """
          raise NotImplementedError(
              'iterate_containers not implemented for this driver')
  
      def list_containers(self):
          # type: () -> List[Container]
          """
          Return a list of containers.
  
          :return: A list of Container instances.
          :rtype: ``list`` of :class:`Container`
          """
          return list(self.iterate_containers())
  
      def iterate_container_objects(self, container, prefix=None,
                                    ex_prefix=None):
          # type: (Container, Optional[str], Optional[str]) -> Iterator[Object]
          """
          Return a iterator of objects for the given container.
  
          :param container: Container instance
          :type container: :class:`libcloud.storage.base.Container`
  
          :param prefix: Filter objects starting with a prefix.
          :type  prefix: ``str``
  
          :param ex_prefix: (Deprecated.) Filter objects starting with a prefix.
          :type  ex_prefix: ``str``
  
          :return: A iterator of Object instances.
          :rtype: ``iterator`` of :class:`libcloud.storage.base.Object`
          """
          raise NotImplementedError(
              'iterate_container_objects not implemented for this driver')
  
      def list_container_objects(self, container, prefix=None, ex_prefix=None):
          # type: (Container, Optional[str], Optional[str]) -> List[Object]
          """
          Return a list of objects for the given container.
  
          :param container: Container instance.
          :type container: :class:`libcloud.storage.base.Container`
  
          :param prefix: Filter objects starting with a prefix.
          :type  prefix: ``str``
  
          :param ex_prefix: (Deprecated.) Filter objects starting with a prefix.
          :type  ex_prefix: ``str``
  
          :return: A list of Object instances.
          :rtype: ``list`` of :class:`libcloud.storage.base.Object`
          """
          return list(self.iterate_container_objects(container,
                                                     prefix=prefix,
                                                     ex_prefix=ex_prefix))
  
      def _normalize_prefix_argument(self, prefix, ex_prefix):
          if ex_prefix:
              warnings.warn('The ``ex_prefix`` argument is deprecated - '
                            'please update code to use ``prefix``',
                            DeprecationWarning)
              return ex_prefix
  
          return prefix
  
      def _filter_listed_container_objects(self, objects, prefix):
          if prefix is not None:
              warnings.warn('Driver %s does not implement native object '
                            'filtering; falling back to filtering the full '
                            'object stream.' % self.__class__.__name__)
  
          for obj in objects:
              if prefix is None or obj.name.startswith(prefix):
                  yield obj
  
      def get_container(self, container_name):
          # type: (str) -> Container
          """
          Return a container instance.
  
          :param container_name: Container name.
          :type container_name: ``str``
  
          :return: :class:`Container` instance.
          :rtype: :class:`libcloud.storage.base.Container`
          """
          raise NotImplementedError(
              'get_object not implemented for this driver')
  
      def get_container_cdn_url(self, container):
          # type: (Container) -> str
          """
          Return a container CDN URL.
  
          :param container: Container instance
          :type  container: :class:`libcloud.storage.base.Container`
  
          :return: A CDN URL for this container.
          :rtype: ``str``
          """
          raise NotImplementedError(
              'get_container_cdn_url not implemented for this driver')
  
      def get_object(self, container_name, object_name):
          # type: (str, str) -> Object
          """
          Return an object instance.
  
          :param container_name: Container name.
          :type  container_name: ``str``
  
          :param object_name: Object name.
          :type  object_name: ``str``
  
          :return: :class:`Object` instance.
          :rtype: :class:`libcloud.storage.base.Object`
          """
          raise NotImplementedError(
              'get_object not implemented for this driver')
  
      def get_object_cdn_url(self, obj):
          # type: (Object) -> str
          """
          Return an object CDN URL.
  
          :param obj: Object instance
          :type  obj: :class:`libcloud.storage.base.Object`
  
          :return: A CDN URL for this object.
          :rtype: ``str``
          """
          raise NotImplementedError(
              'get_object_cdn_url not implemented for this driver')
  
      def enable_container_cdn(self, container):
          # type: (Container) -> bool
          """
          Enable container CDN.
  
          :param container: Container instance
          :type  container: :class:`libcloud.storage.base.Container`
  
          :rtype: ``bool``
          """
          raise NotImplementedError(
              'enable_container_cdn not implemented for this driver')
  
      def enable_object_cdn(self, obj):
          # type: (Object) -> bool
          """
          Enable object CDN.
  
          :param obj: Object instance
          :type  obj: :class:`libcloud.storage.base.Object`
  
          :rtype: ``bool``
          """
          raise NotImplementedError(
              'enable_object_cdn not implemented for this driver')
  
      def download_object(self, obj, destination_path, overwrite_existing=False,
                          delete_on_failure=True):
          # type: (Object, str, bool, bool) -> bool
          """
          Download an object to the specified destination path.
  
          :param obj: Object instance.
          :type obj: :class:`libcloud.storage.base.Object`
  
          :param destination_path: Full path to a file or a directory where the
                                   incoming file will be saved.
          :type destination_path: ``str``
  
          :param overwrite_existing: True to overwrite an existing file,
                                     defaults to False.
          :type overwrite_existing: ``bool``
  
          :param delete_on_failure: True to delete a partially downloaded file if
                                     the download was not successful (hash
                                     mismatch / file size).
          :type delete_on_failure: ``bool``
  
          :return: True if an object has been successfully downloaded, False
                   otherwise.
          :rtype: ``bool``
          """
          raise NotImplementedError(
              'download_object not implemented for this driver')
  
      def download_object_as_stream(self, obj, chunk_size=None):
          # type: (Object, Optional[int]) -> Iterator[bytes]
          """
          Return a iterator which yields object data.
  
          :param obj: Object instance
          :type obj: :class:`libcloud.storage.base.Object`
  
          :param chunk_size: Optional chunk size (in bytes).
          :type chunk_size: ``int``
  
          :rtype: ``iterator`` of ``bytes``
          """
          raise NotImplementedError(
              'download_object_as_stream not implemented for this driver')
  
      def download_object_range(self, obj, destination_path, start_bytes,
                                end_bytes=None, overwrite_existing=False,
                                delete_on_failure=True):
          # type: (Object, str, int, Optional[int], bool, bool) -> bool
          """
          Download part of an object.
  
          :param obj: Object instance.
          :type obj: :class:`libcloud.storage.base.Object`
  
          :param destination_path: Full path to a file or a directory where the
                                   incoming file will be saved.
          :type destination_path: ``str``
  
          :param start_bytes: Start byte offset (inclusive) for the range
                              download. Offset is 0 index based so the first
                              byte in file file is "0".
          :type start_bytes: ``int``
  
          :param end_bytes: End byte offset (non-inclusive) for the range
                            download. If not provided, it will default to the
                            end of the file.
          :type end_bytes: ``int``
  
          :param overwrite_existing: True to overwrite an existing file,
                                     defaults to False.
          :type overwrite_existing: ``bool``
  
          :param delete_on_failure: True to delete a partially downloaded file if
                                     the download was not successful (hash
                                     mismatch / file size).
          :type delete_on_failure: ``bool``
  
          :return: True if an object has been successfully downloaded, False
                   otherwise.
          :rtype: ``bool``
  
          """
          raise NotImplementedError(
              'download_object_range not implemented for this driver')
  
      def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None,
                                          chunk_size=None):
          # type: (Object, int, Optional[int], Optional[int]) -> Iterator[bytes]
          """
          Return a iterator which yields range / part of the object data.
  
          :param obj: Object instance
          :type obj: :class:`libcloud.storage.base.Object`
  
          :param start_bytes: Start byte offset (inclusive) for the range
                              download. Offset is 0 index based so the first
                              byte in file file is "0".
          :type start_bytes: ``int``
  
          :param end_bytes: End byte offset (non-inclusive) for the range
                            download. If not provided, it will default to the
                            end of the file.
          :type end_bytes: ``int``
  
          :param chunk_size: Optional chunk size (in bytes).
          :type chunk_size: ``int``
  
          :rtype: ``iterator`` of ``bytes``
          """
          raise NotImplementedError(
              'download_object_range_as_stream not implemented for this driver')
  
      def upload_object(self, file_path, container, object_name, extra=None,
                        verify_hash=True, headers=None):
          # type: (str, Container, str, Optional[dict], bool, Optional[Dict[str, str]]) -> Object  # noqa: E501
          """
          Upload an object currently located on a disk.
  
          :param file_path: Path to the object on disk.
          :type file_path: ``str``
  
          :param container: Destination container.
          :type container: :class:`libcloud.storage.base.Container`
  
          :param object_name: Object name.
          :type object_name: ``str``
  
          :param verify_hash: Verify hash
          :type verify_hash: ``bool``
  
          :param extra: Extra attributes (driver specific). (optional)
          :type extra: ``dict``
  
          :param headers: (optional) Additional request headers,
              such as CORS headers. For example:
              headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'}
          :type headers: ``dict``
  
          :rtype: :class:`libcloud.storage.base.Object`
          """
          raise NotImplementedError(
              'upload_object not implemented for this driver')
  
      def upload_object_via_stream(self, iterator, container,
                                   object_name,
                                   extra=None,
                                   headers=None):
          # type: (Iterator[bytes], Container, str, Optional[dict], Optional[Dict[str, str]]) -> Object  # noqa: E501
          """
          Upload an object using an iterator.
  
          If a provider supports it, chunked transfer encoding is used and you
          don't need to know in advance the amount of data to be uploaded.
  
          Otherwise if a provider doesn't support it, iterator will be exhausted
          so a total size for data to be uploaded can be determined.
  
          Note: Exhausting the iterator means that the whole data must be
          buffered in memory which might result in memory exhausting when
          uploading a very large object.
  
          If a file is located on a disk you are advised to use upload_object
          function which uses fs.stat function to determine the file size and it
          doesn't need to buffer whole object in the memory.
  
          :param iterator: An object which implements the iterator interface.
          :type iterator: :class:`object`
  
          :param container: Destination container.
          :type container: :class:`libcloud.storage.base.Container`
  
          :param object_name: Object name.
          :type object_name: ``str``
  
          :param extra: (optional) Extra attributes (driver specific). Note:
              This dictionary must contain a 'content_type' key which represents
              a content type of the stored object.
          :type extra: ``dict``
  
          :param headers: (optional) Additional request headers,
              such as CORS headers. For example:
              headers = {'Access-Control-Allow-Origin': 'http://mozilla.com'}
          :type headers: ``dict``
  
          :rtype: ``libcloud.storage.base.Object``
          """
          raise NotImplementedError(
              'upload_object_via_stream not implemented for this driver')
  
      def delete_object(self, obj):
          # type: (Object) -> bool
          """
          Delete an object.
  
          :param obj: Object instance.
          :type obj: :class:`libcloud.storage.base.Object`
  
          :return: ``bool`` True on success.
          :rtype: ``bool``
          """
          raise NotImplementedError(
              'delete_object not implemented for this driver')
  
      def create_container(self, container_name):
          # type: (str) -> Container
          """
          Create a new container.
  
          :param container_name: Container name.
          :type container_name: ``str``
  
          :return: Container instance on success.
          :rtype: :class:`libcloud.storage.base.Container`
          """
          raise NotImplementedError(
              'create_container not implemented for this driver')
  
      def delete_container(self, container):
          # type: (Container) -> bool
          """
          Delete a container.
  
          :param container: Container instance
          :type container: :class:`libcloud.storage.base.Container`
  
          :return: ``True`` on success, ``False`` otherwise.
          :rtype: ``bool``
          """
          raise NotImplementedError(
              'delete_container not implemented for this driver')
  
      def _get_object(self, obj, callback, callback_kwargs, response,
                      success_status_code=None):
          """
          Call passed callback and start transfer of the object'
  
          :param obj: Object instance.
          :type obj: :class:`Object`
  
          :param callback: Function which is called with the passed
              callback_kwargs
          :type callback: :class:`function`
  
          :param callback_kwargs: Keyword arguments which are passed to the
               callback.
          :type callback_kwargs: ``dict``
  
          :param response: Response instance.
          :type response: :class:`Response`
  
          :param success_status_code: Status code which represents a successful
                                      transfer (defaults to httplib.OK)
          :type success_status_code: ``int``
  
          :return: ``True`` on success, ``False`` otherwise.
          :rtype: ``bool``
          """
          success_status_code = success_status_code or httplib.OK
  
          if response.status == success_status_code:
              return callback(**callback_kwargs)
          elif response.status == httplib.NOT_FOUND:
              raise ObjectDoesNotExistError(object_name=obj.name,
                                            value='', driver=self)
  
          raise LibcloudError(value='Unexpected status code: %s' %
                                    (response.status),
                              driver=self)
  
      def _save_object(self, response, obj, destination_path,
                       overwrite_existing=False, delete_on_failure=True,
                       chunk_size=None, partial_download=False):
          """
          Save object to the provided path.
  
          :param response: RawResponse instance.
          :type response: :class:`RawResponse`
  
          :param obj: Object instance.
          :type obj: :class:`Object`
  
          :param destination_path: Destination directory.
          :type destination_path: ``str``
  
          :param delete_on_failure: True to delete partially downloaded object if
                                    the download fails.
          :type delete_on_failure: ``bool``
  
          :param overwrite_existing: True to overwrite a local path if it already
                                     exists.
          :type overwrite_existing: ``bool``
  
          :param chunk_size: Optional chunk size
              (defaults to ``libcloud.storage.base.CHUNK_SIZE``, 8kb)
          :type chunk_size: ``int``
  
          :param partial_download: True if this is a range (partial) save,
                                   False otherwise.
          :type partial_download: ``bool``
  
          :return: ``True`` on success, ``False`` otherwise.
          :rtype: ``bool``
          """
  
          chunk_size = chunk_size or CHUNK_SIZE
  
          base_name = os.path.basename(destination_path)
  
          if not base_name and not os.path.exists(destination_path):
              raise LibcloudError(
                  value='Path %s does not exist' % (destination_path),
                  driver=self)
  
          if not base_name:
              file_path = pjoin(destination_path, obj.name)
          else:
              file_path = destination_path
  
          if os.path.exists(file_path) and not overwrite_existing:
              raise LibcloudError(
                  value='File %s already exists, but ' % (file_path) +
                  'overwrite_existing=False',
                  driver=self)
  
          bytes_transferred = 0
  
          with open(file_path, 'wb') as file_handle:
              for chunk in response._response.iter_content(chunk_size):
                  file_handle.write(b(chunk))
                  bytes_transferred += len(chunk)
  
          if not partial_download and int(obj.size) != int(bytes_transferred):
              # Transfer failed, support retry?
              # NOTE: We only perform this check if this is a regular and not a
              # partial / range download
              if delete_on_failure:
                  try:
                      os.unlink(file_path)
                  except Exception:
                      pass
  
              return False
  
          return True
  
      def _upload_object(self, object_name, content_type, request_path,
                         request_method='PUT',
                         headers=None, file_path=None, stream=None,
                         chunked=False, multipart=False):
          """
          Helper function for setting common request headers and calling the
          passed in callback which uploads an object.
          """
          headers = headers or {}
  
          if file_path and not os.path.exists(file_path):
              raise OSError('File %s does not exist' % (file_path))
  
          if stream is not None and not hasattr(stream, 'next') and not \
                  hasattr(stream, '__next__'):
              raise AttributeError('iterator object must implement next() ' +
                                   'method.')
  
          headers['Content-Type'] = self._determine_content_type(
              content_type, object_name, file_path=file_path)
  
          if stream:
              response = self.connection.request(
                  request_path,
                  method=request_method, data=stream,
                  headers=headers, raw=True)
              stream_hash, stream_length = self._hash_buffered_stream(
                  stream,
                  self._get_hash_function())
          else:
              with open(file_path, 'rb') as file_stream:
                  response = self.connection.request(
                      request_path,
                      method=request_method, data=file_stream,
                      headers=headers, raw=True)
              with open(file_path, 'rb') as file_stream:
                  stream_hash, stream_length = self._hash_buffered_stream(
                      file_stream,
                      self._get_hash_function())
  
          if not response.success():
              response.parse_error()
  
          return {'response': response,
                  'bytes_transferred': stream_length,
                  'data_hash': stream_hash}
  
      def _determine_content_type(self, content_type, object_name,
                                  file_path=None):
          if content_type:
              return content_type
  
          name = file_path or object_name
          content_type, _ = libcloud.utils.files.guess_file_mime_type(name)
  
          if self.strict_mode and not content_type:
              raise AttributeError('File content-type could not be guessed for '
                                   '"%s" and no content_type value is provided'
                                   % name)
  
          return content_type or DEFAULT_CONTENT_TYPE
  
      def _hash_buffered_stream(self, stream, hasher, blocksize=65536):
          total_len = 0
  
          if hasattr(stream, '__next__') or hasattr(stream, 'next'):
              # Ensure we start from the begining of a stream in case stream is
              # not at the beginning
              if hasattr(stream, 'seek'):
                  try:
                      stream.seek(0)
                  except OSError as e:
                      if e.errno != errno.ESPIPE:
                          # This represents "OSError: [Errno 29] Illegal seek"
                          # error. This could either mean that the underlying
                          # handle doesn't support seek operation (e.g. pipe) or
                          # that the invalid seek position is provided. Sadly
                          # there is no good robust way to distinghuish that so
                          # we simply ignore all the "Illeal seek" errors so
                          # this function works correctly with pipes.
                          # See https://github.com/apache/libcloud/pull/1427 for
                          # details
                          raise e
  
              for chunk in libcloud.utils.files.read_in_chunks(iterator=stream):
                  hasher.update(b(chunk))
                  total_len += len(chunk)
  
              return (hasher.hexdigest(), total_len)
  
          if not hasattr(stream, '__exit__'):
              for s in stream:
                  hasher.update(s)
                  total_len = total_len + len(s)
              return (hasher.hexdigest(), total_len)
  
          with stream:
              buf = stream.read(blocksize)
              while len(buf) > 0:
                  total_len = total_len + len(buf)
                  hasher.update(buf)
                  buf = stream.read(blocksize)
  
          return (hasher.hexdigest(), total_len)
  
      def _get_hash_function(self):
          """
          Return instantiated hash function for the hash type supported by
          the provider.
          """
          try:
              func = getattr(hashlib, self.hash_type)()
          except AttributeError:
              raise RuntimeError('Invalid or unsupported hash type: %s' %
                                 (self.hash_type))
  
          return func
  
      def _validate_start_and_end_bytes(self, start_bytes, end_bytes=None):
          # type: (int, Optional[int]) -> bool
          """
          Method which validates that start_bytes and end_bytes arguments contain
          valid values.
          """
          if start_bytes < 0:
              raise ValueError('start_bytes must be greater than 0')
  
          if end_bytes and start_bytes > end_bytes:
              raise ValueError('start_bytes must be smaller than end_bytes')
  
          return True
  
files: libcloud/storage/base.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  from __future__ import with_statement
  
  import base64
  import hashlib
  import hmac
  import os
  import binascii
  from datetime import datetime, timedelta
  
  from libcloud.utils.py3 import ET
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import urlencode
  from libcloud.utils.py3 import urlquote
  from libcloud.utils.py3 import tostring
  from libcloud.utils.py3 import b
  
  from libcloud.utils.xml import fixxpath
  from libcloud.utils.files import read_in_chunks
  from libcloud.common.types import LibcloudError
  from libcloud.common.azure import AzureConnection
  
  from libcloud.storage.base import Object, Container, StorageDriver
  from libcloud.storage.types import ContainerIsNotEmptyError
  from libcloud.storage.types import ContainerAlreadyExistsError
  from libcloud.storage.types import InvalidContainerNameError
  from libcloud.storage.types import ContainerDoesNotExistError
  from libcloud.storage.types import ObjectDoesNotExistError
  from libcloud.storage.types import ObjectHashMismatchError
  
  # Desired number of items in each response inside a paginated request
  RESPONSES_PER_REQUEST = 100
  
  # According to the Azure Docs:
  # > The block must be less than or equal to 100 MB in size for version
  # > 2016-05-31 and later (4 MB for older versions).
  # However for performance reasons, using a lower upload chunk size
  # usually leads to fewer dropped requests and retries.
  AZURE_UPLOAD_CHUNK_SIZE = int(
      os.getenv('LIBCLOUD_AZURE_UPLOAD_CHUNK_SIZE_MB', '4')
  ) * 1024 * 1024
  
  AZURE_DOWNLOAD_CHUNK_SIZE = int(
      os.getenv('LIBCLOUD_AZURE_DOWNLOAD_CHUNK_SIZE_MB', '4')
  ) * 1024 * 1024
  
  # The time period (in seconds) for which a lease must be obtained.
  # If set as -1, we get an infinite lease, but that is a bad idea. If
  # after getting an infinite lease, there was an issue in releasing the
  # lease, the object will remain 'locked' forever, unless the lease is
  # released using the lease_id (which is not exposed to the user)
  AZURE_LEASE_PERIOD = int(
      os.getenv('LIBCLOUD_AZURE_LEASE_PERIOD_SECONDS', '60')
  )
  
  AZURE_STORAGE_HOST_SUFFIX = 'blob.core.windows.net'
  
  AZURE_STORAGE_CDN_URL_DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
  
  AZURE_STORAGE_CDN_URL_START_MINUTES = float(
      os.getenv('LIBCLOUD_AZURE_STORAGE_CDN_URL_START_MINUTES', '5')
  )
  
  AZURE_STORAGE_CDN_URL_EXPIRY_HOURS = float(
      os.getenv('LIBCLOUD_AZURE_STORAGE_CDN_URL_EXPIRY_HOURS', '24')
  )
  
  
  class AzureBlobLease(object):
      """
      A class to help in leasing an azure blob and renewing the lease
      """
      def __init__(self, driver, object_path, use_lease):
          """
          :param driver: The Azure storage driver that is being used
          :type driver: :class:`AzureStorageDriver`
  
          :param object_path: The path of the object we need to lease
          :type object_path: ``str``
  
          :param use_lease: Indicates if we must take a lease or not
          :type use_lease: ``bool``
          """
          self.object_path = object_path
          self.driver = driver
          self.use_lease = use_lease
          self.lease_id = None
          self.params = {'comp': 'lease'}
  
      def renew(self):
          """
          Renew the lease if it is older than a predefined time period
          """
          if self.lease_id is None:
              return
  
          headers = {'x-ms-lease-action': 'renew',
                     'x-ms-lease-id': self.lease_id,
                     'x-ms-lease-duration': '60'}
  
          response = self.driver.connection.request(self.object_path,
                                                    headers=headers,
                                                    params=self.params,
                                                    method='PUT')
  
          if response.status != httplib.OK:
              raise LibcloudError('Unable to obtain lease', driver=self)
  
      def update_headers(self, headers):
          """
          Update the lease id in the headers
          """
          if self.lease_id:
              headers['x-ms-lease-id'] = self.lease_id
  
      def __enter__(self):
          if not self.use_lease:
              return self
  
          headers = {'x-ms-lease-action': 'acquire',
                     'x-ms-lease-duration': '60'}
  
          response = self.driver.connection.request(self.object_path,
                                                    headers=headers,
                                                    params=self.params,
                                                    method='PUT')
  
          if response.status == httplib.NOT_FOUND:
              return self
          elif response.status != httplib.CREATED:
              raise LibcloudError('Unable to obtain lease', driver=self)
  
          self.lease_id = response.headers['x-ms-lease-id']
          return self
  
      def __exit__(self, type, value, traceback):
          if self.lease_id is None:
              return
  
          headers = {'x-ms-lease-action': 'release',
                     'x-ms-lease-id': self.lease_id}
          response = self.driver.connection.request(self.object_path,
                                                    headers=headers,
                                                    params=self.params,
                                                    method='PUT')
  
          if response.status != httplib.OK:
              raise LibcloudError('Unable to release lease', driver=self)
  
  
  class AzureBlobsConnection(AzureConnection):
      """
      Represents a single connection to Azure Blobs.
  
      The main Azure Blob Storage service uses a prefix in the hostname to
      distinguish between accounts, e.g. ``theaccount.blob.core.windows.net``.
      However, some custom deployments of the service, such as the Azurite
      emulator, instead use a URL prefix such as ``/theaccount``. To support
      these deployments, the parameter ``account_prefix`` must be set on the
      connection. This is done by instantiating the driver with arguments such
      as ``host='somewhere.tld'`` and ``key='theaccount'``. To specify a custom
      host without an account prefix, e.g. for use-cases where the custom host
      implements an auditing proxy or similar, the driver can be instantiated
      with ``host='theaccount.somewhere.tld'`` and ``key=''``.
  
      :param account_prefix: Optional prefix identifying the sotrage account.
                             Used when connecting to a custom deployment of the
                             storage service like Azurite or IoT Edge Storage.
      :type account_prefix: ``str``
      """
      def __init__(self, *args, **kwargs):
          self.account_prefix = kwargs.pop('account_prefix', None)
          super(AzureBlobsConnection, self).__init__(*args, **kwargs)
  
      def morph_action_hook(self, action):
          action = super(AzureBlobsConnection, self).morph_action_hook(action)
  
          if self.account_prefix is not None:
              action = '/%s%s' % (self.account_prefix, action)
  
          return action
  
      API_VERSION = '2018-11-09'
  
  
  class AzureBlobsStorageDriver(StorageDriver):
      name = 'Microsoft Azure (blobs)'
      website = 'http://windows.azure.com/'
      connectionCls = AzureBlobsConnection
      hash_type = 'md5'
      supports_chunked_encoding = False
  
      def __init__(self, key, secret=None, secure=True, host=None, port=None,
                   **kwargs):
          self._host_argument_set = bool(host)
  
          # B64decode() this key and keep it, so that we don't have to do
          # so for every request. Minor performance improvement
          secret = base64.b64decode(b(secret))
  
          super(AzureBlobsStorageDriver, self).__init__(key=key, secret=secret,
                                                        secure=secure, host=host,
                                                        port=port, **kwargs)
  
      def _ex_connection_class_kwargs(self):
          result = {}
  
          # host argument has precedence
          if not self._host_argument_set:
              result['host'] = '%s.%s' % (self.key, AZURE_STORAGE_HOST_SUFFIX)
          else:
              result['account_prefix'] = self.key
  
          return result
  
      def _xml_to_container(self, node):
          """
          Converts a container XML node to a container instance
  
          :param node: XML info of the container
          :type node: :class:`xml.etree.ElementTree.Element`
  
          :return: A container instance
          :rtype: :class:`Container`
          """
  
          name = node.findtext(fixxpath(xpath='Name'))
          props = node.find(fixxpath(xpath='Properties'))
          metadata = node.find(fixxpath(xpath='Metadata'))
  
          extra = {
              'url': node.findtext(fixxpath(xpath='Url')),
              'last_modified': node.findtext(fixxpath(xpath='Last-Modified')),
              'etag': props.findtext(fixxpath(xpath='Etag')),
              'lease': {
                  'status': props.findtext(fixxpath(xpath='LeaseStatus')),
                  'state': props.findtext(fixxpath(xpath='LeaseState')),
                  'duration': props.findtext(fixxpath(xpath='LeaseDuration')),
              },
              'meta_data': {}
          }
  
          if metadata is not None:
              for meta in list(metadata):
                  extra['meta_data'][meta.tag] = meta.text
  
          return Container(name=name, extra=extra, driver=self)
  
      def _response_to_container(self, container_name, response):
          """
          Converts a HTTP response to a container instance
  
          :param container_name: Name of the container
          :type container_name: ``str``
  
          :param response: HTTP Response
          :type node: L{}
  
          :return: A container instance
          :rtype: :class:`Container`
          """
  
          headers = response.headers
          extra = {
              'url': 'http://%s%s' % (response.connection.host,
                                      response.connection.action),
              'etag': headers['etag'],
              'last_modified': headers['last-modified'],
              'lease': {
                  'status': headers.get('x-ms-lease-status', None),
                  'state': headers.get('x-ms-lease-state', None),
                  'duration': headers.get('x-ms-lease-duration', None),
              },
              'meta_data': {}
          }
  
          for key, value in response.headers.items():
              if key.startswith('x-ms-meta-'):
                  key = key.split('x-ms-meta-')[1]
                  extra['meta_data'][key] = value
  
          return Container(name=container_name, extra=extra, driver=self)
  
      def _xml_to_object(self, container, blob):
          """
          Converts a BLOB XML node to an object instance
  
          :param container: Instance of the container holding the blob
          :type: :class:`Container`
  
          :param blob: XML info of the blob
          :type blob: L{}
  
          :return: An object instance
          :rtype: :class:`Object`
          """
  
          name = blob.findtext(fixxpath(xpath='Name'))
          props = blob.find(fixxpath(xpath='Properties'))
          metadata = blob.find(fixxpath(xpath='Metadata'))
          etag = props.findtext(fixxpath(xpath='Etag'))
          size = int(props.findtext(fixxpath(xpath='Content-Length')))
  
          extra = {
              'content_type': props.findtext(fixxpath(xpath='Content-Type')),
              'etag': etag,
              'md5_hash': props.findtext(fixxpath(xpath='Content-MD5')),
              'last_modified': props.findtext(fixxpath(xpath='Last-Modified')),
              'url': blob.findtext(fixxpath(xpath='Url')),
              'hash': props.findtext(fixxpath(xpath='Etag')),
              'lease': {
                  'status': props.findtext(fixxpath(xpath='LeaseStatus')),
                  'state': props.findtext(fixxpath(xpath='LeaseState')),
                  'duration': props.findtext(fixxpath(xpath='LeaseDuration')),
              },
              'content_encoding': props.findtext(fixxpath(
                                                 xpath='Content-Encoding')),
              'content_language': props.findtext(fixxpath(
                                                 xpath='Content-Language')),
              'blob_type': props.findtext(fixxpath(xpath='BlobType'))
          }
  
          if extra['md5_hash']:
              value = binascii.hexlify(base64.b64decode(b(extra['md5_hash'])))
              value = value.decode('ascii')
              extra['md5_hash'] = value
  
          meta_data = {}
          if metadata is not None:
              for meta in list(metadata):
                  meta_data[meta.tag] = meta.text
  
          return Object(name=name, size=size, hash=etag, meta_data=meta_data,
                        extra=extra, container=container, driver=self)
  
      def _response_to_object(self, object_name, container, response):
          """
          Converts a HTTP response to an object (from headers)
  
          :param object_name: Name of the object
          :type object_name: ``str``
  
          :param container: Instance of the container holding the blob
          :type: :class:`Container`
  
          :param response: HTTP Response
          :type node: L{}
  
          :return: An object instance
          :rtype: :class:`Object`
          """
  
          headers = response.headers
          size = int(headers['content-length'])
          etag = headers['etag']
  
          extra = {
              'url': 'http://%s%s' % (response.connection.host,
                                      response.connection.action),
              'etag': etag,
              'md5_hash': headers.get('content-md5', None),
              'content_type': headers.get('content-type', None),
              'content_language': headers.get('content-language', None),
              'content_encoding': headers.get('content-encoding', None),
              'last_modified': headers['last-modified'],
              'lease': {
                  'status': headers.get('x-ms-lease-status', None),
                  'state': headers.get('x-ms-lease-state', None),
                  'duration': headers.get('x-ms-lease-duration', None),
              },
              'blob_type': headers['x-ms-blob-type']
          }
  
          if extra['md5_hash']:
              value = binascii.hexlify(base64.b64decode(b(extra['md5_hash'])))
              value = value.decode('ascii')
              extra['md5_hash'] = value
  
          meta_data = {}
          for key, value in response.headers.items():
              if key.startswith('x-ms-meta-'):
                  key = key.split('x-ms-meta-')[1]
                  meta_data[key] = value
  
          return Object(name=object_name, size=size, hash=etag, extra=extra,
                        meta_data=meta_data, container=container, driver=self)
  
      def iterate_containers(self):
          """
          @inherits: :class:`StorageDriver.iterate_containers`
          """
          params = {'comp': 'list',
                    'maxresults': RESPONSES_PER_REQUEST,
                    'include': 'metadata'}
  
          while True:
              response = self.connection.request('/', params)
              if response.status != httplib.OK:
                  raise LibcloudError('Unexpected status code: %s' %
                                      (response.status), driver=self)
  
              body = response.parse_body()
              containers = body.find(fixxpath(xpath='Containers'))
              containers = containers.findall(fixxpath(xpath='Container'))
  
              for container in containers:
                  yield self._xml_to_container(container)
  
              params['marker'] = body.findtext('NextMarker')
              if not params['marker']:
                  break
  
      def iterate_container_objects(self, container, prefix=None,
                                    ex_prefix=None):
          """
          @inherits: :class:`StorageDriver.iterate_container_objects`
          """
          prefix = self._normalize_prefix_argument(prefix, ex_prefix)
  
          params = {'restype': 'container',
                    'comp': 'list',
                    'maxresults': RESPONSES_PER_REQUEST,
                    'include': 'metadata'}
  
          if prefix:
              params['prefix'] = prefix
  
          container_path = self._get_container_path(container)
  
          while True:
              response = self.connection.request(container_path,
                                                 params=params)
  
              if response.status == httplib.NOT_FOUND:
                  raise ContainerDoesNotExistError(value=None,
                                                   driver=self,
                                                   container_name=container.name)
  
              elif response.status != httplib.OK:
                  raise LibcloudError('Unexpected status code: %s' %
                                      (response.status), driver=self)
  
              body = response.parse_body()
              blobs = body.find(fixxpath(xpath='Blobs'))
              blobs = blobs.findall(fixxpath(xpath='Blob'))
  
              for blob in blobs:
                  yield self._xml_to_object(container, blob)
  
              params['marker'] = body.findtext('NextMarker')
              if not params['marker']:
                  break
  
      def get_container(self, container_name):
          """
          @inherits: :class:`StorageDriver.get_container`
          """
          params = {'restype': 'container'}
  
          container_path = '/%s' % (container_name)
  
          response = self.connection.request(container_path, params=params,
                                             method='HEAD')
  
          if response.status == httplib.NOT_FOUND:
              raise ContainerDoesNotExistError('Container %s does not exist' %
                                               (container_name), driver=self,
                                               container_name=container_name)
          elif response.status != httplib.OK:
              raise LibcloudError('Unexpected status code: %s' %
                                  (response.status), driver=self)
  
          return self._response_to_container(container_name, response)
  
      def get_object(self, container_name, object_name):
          """
          @inherits: :class:`StorageDriver.get_object`
          """
  
          container = self.get_container(container_name=container_name)
          object_path = self._get_object_path(container, object_name)
  
          response = self.connection.request(object_path, method='HEAD')
  
          if response.status == httplib.OK:
              obj = self._response_to_object(object_name, container, response)
              return obj
  
          raise ObjectDoesNotExistError(value=None, driver=self,
                                        object_name=object_name)
  
      def get_object_cdn_url(self, obj,
                             ex_expiry=AZURE_STORAGE_CDN_URL_EXPIRY_HOURS):
          """
          Return a SAS URL that enables reading the given object.
  
          :param obj: Object instance.
          :type  obj: :class:`Object`
  
          :param ex_expiry: The number of hours after which the URL expires.
                            Defaults to 24 hours.
          :type  ex_expiry: ``float``
  
          :return: A SAS URL for the object.
          :rtype: ``str``
          """
          object_path = self._get_object_path(obj.container, obj.name)
  
          now = datetime.utcnow()
          start = now - timedelta(minutes=AZURE_STORAGE_CDN_URL_START_MINUTES)
          expiry = now + timedelta(hours=ex_expiry)
  
          params = {
              'st': start.strftime(AZURE_STORAGE_CDN_URL_DATE_FORMAT),
              'se': expiry.strftime(AZURE_STORAGE_CDN_URL_DATE_FORMAT),
              'sp': 'r',
              'spr': 'https' if self.secure else 'http,https',
              'sv': self.connectionCls.API_VERSION,
              'sr': 'b',
          }
  
          string_to_sign = '\n'.join((
              params['sp'],
              params['st'],
              params['se'],
              '/blob/{}{}'.format(self.key, object_path),
              '',  # signedIdentifier
              '',  # signedIP
              params['spr'],
              params['sv'],
              params['sr'],
              '',  # snapshot
              '',  # rscc
              '',  # rscd
              '',  # rsce
              '',  # rscl
              '',  # rsct
          ))
  
          params['sig'] = base64.b64encode(
              hmac.new(
                  self.secret,
                  string_to_sign.encode('utf-8'),
                  hashlib.sha256
              ).digest()
          ).decode('utf-8')
  
          return '{scheme}://{host}:{port}{action}?{sas_token}'.format(
              scheme='https' if self.secure else 'http',
              host=self.connection.host,
              port=self.connection.port,
              action=self.connection.morph_action_hook(object_path),
              sas_token=urlencode(params),
          )
  
      def _get_container_path(self, container):
          """
          Return a container path
  
          :param container: Container instance
          :type  container: :class:`Container`
  
          :return: A path for this container.
          :rtype: ``str``
          """
          return '/%s' % (container.name)
  
      def _get_object_path(self, container, object_name):
          """
          Return an object's CDN path.
  
          :param container: Container instance
          :type  container: :class:`Container`
  
          :param object_name: Object name
          :type  object_name: :class:`str`
  
          :return: A  path for this object.
          :rtype: ``str``
          """
          container_url = self._get_container_path(container)
          object_name_cleaned = urlquote(object_name)
          object_path = '%s/%s' % (container_url, object_name_cleaned)
          return object_path
  
      def create_container(self, container_name):
          """
          @inherits: :class:`StorageDriver.create_container`
          """
          params = {'restype': 'container'}
  
          container_path = '/%s' % (container_name)
          response = self.connection.request(container_path, params=params,
                                             method='PUT')
  
          if response.status == httplib.CREATED:
              return self._response_to_container(container_name, response)
          elif response.status == httplib.CONFLICT:
              raise ContainerAlreadyExistsError(
                  value='Container with this name already exists. The name must '
                        'be unique among all the containers in the system',
                  container_name=container_name, driver=self)
          elif response.status == httplib.BAD_REQUEST:
              raise InvalidContainerNameError(value='Container name contains ' +
                                              'invalid characters.',
                                              container_name=container_name,
                                              driver=self)
  
          raise LibcloudError('Unexpected status code: %s' % (response.status),
                              driver=self)
  
      def delete_container(self, container):
          """
          @inherits: :class:`StorageDriver.delete_container`
          """
          # Azure does not check if the container is empty. So, we will do
          # a check to ensure that the behaviour is similar to other drivers
          for obj in container.iterate_objects():
              raise ContainerIsNotEmptyError(
                  value='Container must be empty before it can be deleted.',
                  container_name=container.name, driver=self)
  
          params = {'restype': 'container'}
          container_path = self._get_container_path(container)
  
          # Note: All the objects in the container must be deleted first
          response = self.connection.request(container_path, params=params,
                                             method='DELETE')
  
          if response.status == httplib.ACCEPTED:
              return True
          elif response.status == httplib.NOT_FOUND:
              raise ContainerDoesNotExistError(value=None,
                                               driver=self,
                                               container_name=container.name)
  
          return False
  
      def download_object(self, obj, destination_path, overwrite_existing=False,
                          delete_on_failure=True):
          """
          @inherits: :class:`StorageDriver.download_object`
          """
          obj_path = self._get_object_path(obj.container, obj.name)
          response = self.connection.request(obj_path, raw=True, data=None)
  
          return self._get_object(obj=obj, callback=self._save_object,
                                  response=response,
                                  callback_kwargs={
                                      'obj': obj,
                                      'response': response.response,
                                      'destination_path': destination_path,
                                      'overwrite_existing': overwrite_existing,
                                      'delete_on_failure': delete_on_failure},
                                  success_status_code=httplib.OK)
  
      def download_object_as_stream(self, obj, chunk_size=None):
          """
          @inherits: :class:`StorageDriver.download_object_as_stream`
          """
          obj_path = self._get_object_path(obj.container, obj.name)
          response = self.connection.request(obj_path, method='GET',
                                             stream=True, raw=True)
          iterator = response.iter_content(AZURE_DOWNLOAD_CHUNK_SIZE)
  
          return self._get_object(obj=obj,
                                  callback=read_in_chunks,
                                  response=response,
                                  callback_kwargs={'iterator': iterator,
                                                   'chunk_size': chunk_size},
                                  success_status_code=httplib.OK)
  
      def download_object_range(self, obj, destination_path, start_bytes,
                                end_bytes=None, overwrite_existing=False,
                                delete_on_failure=True):
          self._validate_start_and_end_bytes(start_bytes=start_bytes,
                                             end_bytes=end_bytes)
  
          obj_path = self._get_object_path(obj.container, obj.name)
          headers = {'Range': self._get_standard_range_str(start_bytes,
                                                           end_bytes)}
          response = self.connection.request(obj_path, headers=headers,
                                             raw=True, data=None)
  
          return self._get_object(obj=obj, callback=self._save_object,
                                  response=response,
                                  callback_kwargs={
                                      'obj': obj,
                                      'response': response.response,
                                      'destination_path': destination_path,
                                      'overwrite_existing': overwrite_existing,
                                      'delete_on_failure': delete_on_failure,
                                      'partial_download': True},
                                  success_status_code=httplib.PARTIAL_CONTENT)
  
      def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None,
                                          chunk_size=None):
          self._validate_start_and_end_bytes(start_bytes=start_bytes,
                                             end_bytes=end_bytes)
  
          obj_path = self._get_object_path(obj.container, obj.name)
  
          headers = {'Range': self._get_standard_range_str(start_bytes,
                                                           end_bytes)}
          response = self.connection.request(obj_path, method='GET',
                                             headers=headers,
                                             stream=True, raw=True)
          iterator = response.iter_content(AZURE_DOWNLOAD_CHUNK_SIZE)
  
          return self._get_object(
              obj=obj, callback=read_in_chunks,
              response=response,
              callback_kwargs={'iterator': iterator,
                               'chunk_size': chunk_size},
              success_status_code=httplib.PARTIAL_CONTENT)
  
      def _upload_in_chunks(self, stream, object_path, lease, meta_data,
                            content_type, object_name, file_path, verify_hash,
                            headers):
          """
          Uploads data from an interator in fixed sized chunks to Azure Storage
          """
  
          data_hash = None
          if verify_hash:
              data_hash = self._get_hash_function()
  
          bytes_transferred = 0
          count = 1
          chunks = []
          headers = headers or {}
  
          lease.update_headers(headers)
  
          params = {'comp': 'block'}
  
          # Read the input data in chunk sizes suitable for Azure
          for data in read_in_chunks(stream, AZURE_UPLOAD_CHUNK_SIZE,
                                     fill_size=True):
              data = b(data)
              content_length = len(data)
              bytes_transferred += content_length
  
              if verify_hash:
                  data_hash.update(data)
  
              chunk_hash = self._get_hash_function()
              chunk_hash.update(data)
              chunk_hash = base64.b64encode(b(chunk_hash.digest()))
  
              headers['Content-MD5'] = chunk_hash.decode('utf-8')
              headers['Content-Length'] = str(content_length)
  
              # Block id can be any unique string that is base64 encoded
              # A 10 digit number can hold the max value of 50000 blocks
              # that are allowed for azure
              block_id = base64.b64encode(b('%10d' % (count)))
              block_id = block_id.decode('utf-8')
              params['blockid'] = block_id
  
              # Keep this data for a later commit
              chunks.append(block_id)
  
              # Renew lease before updating
              lease.renew()
  
              resp = self.connection.request(object_path, method='PUT',
                                             data=data, headers=headers,
                                             params=params)
  
              if resp.status != httplib.CREATED:
                  resp.parse_error()
                  raise LibcloudError('Error uploading chunk %d. Code: %d' %
                                      (count, resp.status), driver=self)
  
              count += 1
  
          if verify_hash:
              data_hash = base64.b64encode(b(data_hash.digest()))
              data_hash = data_hash.decode('utf-8')
  
          response = self._commit_blocks(object_path=object_path,
                                         chunks=chunks,
                                         lease=lease,
                                         meta_data=meta_data,
                                         content_type=content_type,
                                         data_hash=data_hash,
                                         object_name=object_name,
                                         file_path=file_path)
  
          # According to the Azure docs:
          # > This header refers to the content of the request, meaning, in this
          # > case, the list of blocks, and not the content of the blob itself.
          # However, the validation code assumes that the content-md5 in the
          # server response refers to the object so we must discard the value
          response.headers['content-md5'] = None
  
          return {
              'response': response,
              'data_hash': data_hash,
              'bytes_transferred': bytes_transferred,
          }
  
      def _commit_blocks(self, object_path, chunks, lease,
                         meta_data, content_type, data_hash,
                         object_name, file_path):
          """
          Makes a final commit of the data.
          """
  
          root = ET.Element('BlockList')
  
          for block_id in chunks:
              part = ET.SubElement(root, 'Uncommitted')
              part.text = str(block_id)
  
          data = tostring(root)
          params = {'comp': 'blocklist'}
          headers = {}
  
          lease.update_headers(headers)
          lease.renew()
  
          headers['x-ms-blob-content-type'] = self._determine_content_type(
              content_type, object_name, file_path)
  
          if data_hash is not None:
              headers['x-ms-blob-content-md5'] = data_hash
  
          self._update_metadata(headers, meta_data)
  
          data_hash = self._get_hash_function()
          data_hash.update(data.encode('utf-8'))
          data_hash = base64.b64encode(b(data_hash.digest()))
          headers['Content-MD5'] = data_hash.decode('utf-8')
  
          headers['Content-Length'] = len(data)
  
          response = self.connection.request(object_path, data=data,
                                             params=params, headers=headers,
                                             method='PUT')
  
          if response.status != httplib.CREATED:
              raise LibcloudError('Error in blocklist commit', driver=self)
  
          return response
  
      def upload_object(self, file_path, container, object_name,
                        verify_hash=True, extra=None, headers=None,
                        ex_use_lease=False,
                        **deprecated_kwargs):
          """
          Upload an object currently located on a disk.
  
          @inherits: :class:`StorageDriver.upload_object`
  
          :param ex_use_lease: Indicates if we must take a lease before upload
          :type ex_use_lease: ``bool``
          """
          if deprecated_kwargs:
              raise ValueError('Support for arguments was removed: %s'
                               % ', '.join(deprecated_kwargs.keys()))
  
          blob_size = os.stat(file_path).st_size
  
          with open(file_path, 'rb') as fobj:
              return self._put_object(container=container,
                                      object_name=object_name,
                                      extra=extra, verify_hash=verify_hash,
                                      use_lease=ex_use_lease, headers=headers,
                                      blob_size=blob_size, file_path=file_path,
                                      stream=fobj)
  
      def upload_object_via_stream(self, iterator, container, object_name,
                                   verify_hash=True, extra=None, headers=None,
                                   ex_use_lease=False,
                                   **deprecated_kwargs):
          """
          @inherits: :class:`StorageDriver.upload_object_via_stream`
  
          :param ex_use_lease: Indicates if we must take a lease before upload
          :type ex_use_lease: ``bool``
          """
          if deprecated_kwargs:
              raise ValueError('Support for arguments was removed: %s'
                               % ', '.join(deprecated_kwargs.keys()))
  
          return self._put_object(container=container,
                                  object_name=object_name,
                                  extra=extra, verify_hash=verify_hash,
                                  use_lease=ex_use_lease,
                                  headers=headers,
                                  blob_size=None,
                                  stream=iterator)
  
      def delete_object(self, obj):
          """
          @inherits: :class:`StorageDriver.delete_object`
          """
          object_path = self._get_object_path(obj.container, obj.name)
          response = self.connection.request(object_path, method='DELETE')
  
          if response.status == httplib.ACCEPTED:
              return True
          elif response.status == httplib.NOT_FOUND:
              raise ObjectDoesNotExistError(value=None, driver=self,
                                            object_name=obj.name)
  
          return False
  
      def _update_metadata(self, headers, meta_data):
          """
          Update the given metadata in the headers
  
          :param headers: The headers dictionary to be updated
          :type headers: ``dict``
  
          :param meta_data: Metadata key value pairs
          :type meta_data: ``dict``
          """
          for key, value in list(meta_data.items()):
              key = 'x-ms-meta-%s' % (key)
              headers[key] = value
  
      def _put_object(self, container, object_name, stream,
                      extra=None, verify_hash=True, headers=None,
                      blob_size=None, file_path=None,
                      use_lease=False):
          """
          Control function that does the real job of uploading data to a blob
          """
          extra = extra or {}
          content_type = extra.get('content_type', None)
          meta_data = extra.get('meta_data', {})
  
          object_path = self._get_object_path(container, object_name)
  
          # Get a lease if required and do the operations
          with AzureBlobLease(self, object_path, use_lease) as lease:
              if blob_size is not None and blob_size <= AZURE_UPLOAD_CHUNK_SIZE:
                  result_dict = self._upload_directly(stream=stream,
                                                      object_path=object_path,
                                                      lease=lease,
                                                      blob_size=blob_size,
                                                      meta_data=meta_data,
                                                      headers=headers,
                                                      content_type=content_type,
                                                      object_name=object_name,
                                                      file_path=file_path)
              else:
                  result_dict = self._upload_in_chunks(stream=stream,
                                                       object_path=object_path,
                                                       lease=lease,
                                                       meta_data=meta_data,
                                                       headers=headers,
                                                       content_type=content_type,
                                                       object_name=object_name,
                                                       file_path=file_path,
                                                       verify_hash=verify_hash)
  
              response = result_dict['response']
              bytes_transferred = result_dict['bytes_transferred']
              data_hash = result_dict['data_hash']
              headers = response.headers
  
          if response.status != httplib.CREATED:
              raise LibcloudError(
                  'Unexpected status code, status_code=%s' % (response.status),
                  driver=self)
  
          server_hash = headers.get('content-md5')
  
          if server_hash:
              server_hash = binascii.hexlify(base64.b64decode(b(server_hash)))
              server_hash = server_hash.decode('utf-8')
          else:
              # TODO: HACK - We could poll the object for a while and get
              # the hash
              pass
  
          if (verify_hash and server_hash and data_hash != server_hash):
              raise ObjectHashMismatchError(
                  value='MD5 hash checksum does not match',
                  object_name=object_name, driver=self)
  
          return Object(name=object_name, size=bytes_transferred,
                        hash=headers['etag'], extra=None,
                        meta_data=meta_data, container=container,
                        driver=self)
  
      def _upload_directly(self, stream, object_path, lease, blob_size,
                           meta_data, content_type, object_name, file_path,
                           headers):
  
          headers = headers or {}
          lease.update_headers(headers)
  
          self._update_metadata(headers, meta_data)
  
          headers['Content-Length'] = str(blob_size)
          headers['x-ms-blob-type'] = 'BlockBlob'
  
          return self._upload_object(object_name=object_name,
                                     file_path=file_path,
                                     content_type=content_type,
                                     request_path=object_path,
                                     stream=stream,
                                     headers=headers)
  
      def ex_set_object_metadata(self, obj, meta_data):
          """
          Set metadata for an object
  
          :param obj: The blob object
          :type obj: :class:`Object`
  
          :param meta_data: Metadata key value pairs
          :type meta_data: ``dict``
          """
          object_path = self._get_object_path(obj.container, obj.name)
          params = {'comp': 'metadata'}
          headers = {}
  
          self._update_metadata(headers, meta_data)
  
          response = self.connection.request(object_path, method='PUT',
                                             params=params,
                                             headers=headers)
  
          if response.status != httplib.OK:
              response.parse_error('Setting metadata')
  
files: libcloud/storage/drivers/azure_blobs.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  from __future__ import with_statement
  
  import base64
  import hashlib
  import hmac
  import os
  import binascii
  from datetime import datetime, timedelta
  
  from libcloud.utils.py3 import ET
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import urlencode
  from libcloud.utils.py3 import urlquote
  from libcloud.utils.py3 import tostring
  from libcloud.utils.py3 import b
  
  from libcloud.utils.xml import fixxpath
  from libcloud.utils.files import read_in_chunks
  from libcloud.common.types import LibcloudError
  from libcloud.common.azure import AzureConnection
  
  from libcloud.storage.base import Object, Container, StorageDriver
  from libcloud.storage.types import ContainerIsNotEmptyError
  from libcloud.storage.types import ContainerAlreadyExistsError
  from libcloud.storage.types import InvalidContainerNameError
  from libcloud.storage.types import ContainerDoesNotExistError
  from libcloud.storage.types import ObjectDoesNotExistError
  from libcloud.storage.types import ObjectHashMismatchError
  
  # Desired number of items in each response inside a paginated request
  RESPONSES_PER_REQUEST = 100
  
  # According to the Azure Docs:
  # > The block must be less than or equal to 100 MB in size for version
  # > 2016-05-31 and later (4 MB for older versions).
  # However for performance reasons, using a lower upload chunk size
  # usually leads to fewer dropped requests and retries.
  AZURE_UPLOAD_CHUNK_SIZE = int(
      os.getenv('LIBCLOUD_AZURE_UPLOAD_CHUNK_SIZE_MB', '4')
  ) * 1024 * 1024
  
  AZURE_DOWNLOAD_CHUNK_SIZE = int(
      os.getenv('LIBCLOUD_AZURE_DOWNLOAD_CHUNK_SIZE_MB', '4')
  ) * 1024 * 1024
  
  # The time period (in seconds) for which a lease must be obtained.
  # If set as -1, we get an infinite lease, but that is a bad idea. If
  # after getting an infinite lease, there was an issue in releasing the
  # lease, the object will remain 'locked' forever, unless the lease is
  # released using the lease_id (which is not exposed to the user)
  AZURE_LEASE_PERIOD = int(
      os.getenv('LIBCLOUD_AZURE_LEASE_PERIOD_SECONDS', '60')
  )
  
  AZURE_STORAGE_HOST_SUFFIX = 'blob.core.windows.net'
  
  AZURE_STORAGE_CDN_URL_DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
  
  AZURE_STORAGE_CDN_URL_START_MINUTES = float(
      os.getenv('LIBCLOUD_AZURE_STORAGE_CDN_URL_START_MINUTES', '5')
  )
  
  AZURE_STORAGE_CDN_URL_EXPIRY_HOURS = float(
      os.getenv('LIBCLOUD_AZURE_STORAGE_CDN_URL_EXPIRY_HOURS', '24')
  )
  
  
  class AzureBlobLease(object):
      """
      A class to help in leasing an azure blob and renewing the lease
      """
      def __init__(self, driver, object_path, use_lease):
          """
          :param driver: The Azure storage driver that is being used
          :type driver: :class:`AzureStorageDriver`
  
          :param object_path: The path of the object we need to lease
          :type object_path: ``str``
  
          :param use_lease: Indicates if we must take a lease or not
          :type use_lease: ``bool``
          """
          self.object_path = object_path
          self.driver = driver
          self.use_lease = use_lease
          self.lease_id = None
          self.params = {'comp': 'lease'}
  
      def renew(self):
          """
          Renew the lease if it is older than a predefined time period
          """
          if self.lease_id is None:
              return
  
          headers = {'x-ms-lease-action': 'renew',
                     'x-ms-lease-id': self.lease_id,
                     'x-ms-lease-duration': '60'}
  
          response = self.driver.connection.request(self.object_path,
                                                    headers=headers,
                                                    params=self.params,
                                                    method='PUT')
  
          if response.status != httplib.OK:
              raise LibcloudError('Unable to obtain lease', driver=self)
  
      def update_headers(self, headers):
          """
          Update the lease id in the headers
          """
          if self.lease_id:
              headers['x-ms-lease-id'] = self.lease_id
  
      def __enter__(self):
          if not self.use_lease:
              return self
  
          headers = {'x-ms-lease-action': 'acquire',
                     'x-ms-lease-duration': '60'}
  
          response = self.driver.connection.request(self.object_path,
                                                    headers=headers,
                                                    params=self.params,
                                                    method='PUT')
  
          if response.status == httplib.NOT_FOUND:
              return self
          elif response.status != httplib.CREATED:
              raise LibcloudError('Unable to obtain lease', driver=self)
  
          self.lease_id = response.headers['x-ms-lease-id']
          return self
  
      def __exit__(self, type, value, traceback):
          if self.lease_id is None:
              return
  
          headers = {'x-ms-lease-action': 'release',
                     'x-ms-lease-id': self.lease_id}
          response = self.driver.connection.request(self.object_path,
                                                    headers=headers,
                                                    params=self.params,
                                                    method='PUT')
  
          if response.status != httplib.OK:
              raise LibcloudError('Unable to release lease', driver=self)
  
  
  class AzureBlobsConnection(AzureConnection):
      """
      Represents a single connection to Azure Blobs.
  
      The main Azure Blob Storage service uses a prefix in the hostname to
      distinguish between accounts, e.g. ``theaccount.blob.core.windows.net``.
      However, some custom deployments of the service, such as the Azurite
      emulator, instead use a URL prefix such as ``/theaccount``. To support
      these deployments, the parameter ``account_prefix`` must be set on the
      connection. This is done by instantiating the driver with arguments such
      as ``host='somewhere.tld'`` and ``key='theaccount'``. To specify a custom
      host without an account prefix, e.g. for use-cases where the custom host
      implements an auditing proxy or similar, the driver can be instantiated
      with ``host='theaccount.somewhere.tld'`` and ``key=''``.
  
      :param account_prefix: Optional prefix identifying the sotrage account.
                             Used when connecting to a custom deployment of the
                             storage service like Azurite or IoT Edge Storage.
      :type account_prefix: ``str``
      """
      def __init__(self, *args, **kwargs):
          self.account_prefix = kwargs.pop('account_prefix', None)
          super(AzureBlobsConnection, self).__init__(*args, **kwargs)
  
      def morph_action_hook(self, action):
          action = super(AzureBlobsConnection, self).morph_action_hook(action)
  
          if self.account_prefix is not None:
              action = '/%s%s' % (self.account_prefix, action)
  
          return action
  
      API_VERSION = '2018-11-09'
  
  
  class AzureBlobsStorageDriver(StorageDriver):
      name = 'Microsoft Azure (blobs)'
      website = 'http://windows.azure.com/'
      connectionCls = AzureBlobsConnection
      hash_type = 'md5'
      supports_chunked_encoding = False
  
      def __init__(self, key, secret=None, secure=True, host=None, port=None,
                   **kwargs):
          self._host_argument_set = bool(host)
  
          # B64decode() this key and keep it, so that we don't have to do
          # so for every request. Minor performance improvement
          secret = base64.b64decode(b(secret))
  
          super(AzureBlobsStorageDriver, self).__init__(key=key, secret=secret,
                                                        secure=secure, host=host,
                                                        port=port, **kwargs)
  
      def _ex_connection_class_kwargs(self):
          result = {}
  
          # host argument has precedence
          if not self._host_argument_set:
              result['host'] = '%s.%s' % (self.key, AZURE_STORAGE_HOST_SUFFIX)
          else:
              result['account_prefix'] = self.key
  
          return result
  
      def _xml_to_container(self, node):
          """
          Converts a container XML node to a container instance
  
          :param node: XML info of the container
          :type node: :class:`xml.etree.ElementTree.Element`
  
          :return: A container instance
          :rtype: :class:`Container`
          """
  
          name = node.findtext(fixxpath(xpath='Name'))
          props = node.find(fixxpath(xpath='Properties'))
          metadata = node.find(fixxpath(xpath='Metadata'))
  
          extra = {
              'url': node.findtext(fixxpath(xpath='Url')),
              'last_modified': node.findtext(fixxpath(xpath='Last-Modified')),
              'etag': props.findtext(fixxpath(xpath='Etag')),
              'lease': {
                  'status': props.findtext(fixxpath(xpath='LeaseStatus')),
                  'state': props.findtext(fixxpath(xpath='LeaseState')),
                  'duration': props.findtext(fixxpath(xpath='LeaseDuration')),
              },
              'meta_data': {}
          }
  
          if metadata is not None:
              for meta in list(metadata):
                  extra['meta_data'][meta.tag] = meta.text
  
          return Container(name=name, extra=extra, driver=self)
  
      def _response_to_container(self, container_name, response):
          """
          Converts a HTTP response to a container instance
  
          :param container_name: Name of the container
          :type container_name: ``str``
  
          :param response: HTTP Response
          :type node: L{}
  
          :return: A container instance
          :rtype: :class:`Container`
          """
  
          headers = response.headers
          extra = {
              'url': 'http://%s%s' % (response.connection.host,
                                      response.connection.action),
              'etag': headers['etag'],
              'last_modified': headers['last-modified'],
              'lease': {
                  'status': headers.get('x-ms-lease-status', None),
                  'state': headers.get('x-ms-lease-state', None),
                  'duration': headers.get('x-ms-lease-duration', None),
              },
              'meta_data': {}
          }
  
          for key, value in response.headers.items():
              if key.startswith('x-ms-meta-'):
                  key = key.split('x-ms-meta-')[1]
                  extra['meta_data'][key] = value
  
          return Container(name=container_name, extra=extra, driver=self)
  
      def _xml_to_object(self, container, blob):
          """
          Converts a BLOB XML node to an object instance
  
          :param container: Instance of the container holding the blob
          :type: :class:`Container`
  
          :param blob: XML info of the blob
          :type blob: L{}
  
          :return: An object instance
          :rtype: :class:`Object`
          """
  
          name = blob.findtext(fixxpath(xpath='Name'))
          props = blob.find(fixxpath(xpath='Properties'))
          metadata = blob.find(fixxpath(xpath='Metadata'))
          etag = props.findtext(fixxpath(xpath='Etag'))
          size = int(props.findtext(fixxpath(xpath='Content-Length')))
  
          extra = {
              'content_type': props.findtext(fixxpath(xpath='Content-Type')),
              'etag': etag,
              'md5_hash': props.findtext(fixxpath(xpath='Content-MD5')),
              'last_modified': props.findtext(fixxpath(xpath='Last-Modified')),
              'url': blob.findtext(fixxpath(xpath='Url')),
              'hash': props.findtext(fixxpath(xpath='Etag')),
              'lease': {
                  'status': props.findtext(fixxpath(xpath='LeaseStatus')),
                  'state': props.findtext(fixxpath(xpath='LeaseState')),
                  'duration': props.findtext(fixxpath(xpath='LeaseDuration')),
              },
              'content_encoding': props.findtext(fixxpath(
                                                 xpath='Content-Encoding')),
              'content_language': props.findtext(fixxpath(
                                                 xpath='Content-Language')),
              'blob_type': props.findtext(fixxpath(xpath='BlobType'))
          }
  
          if extra['md5_hash']:
              value = binascii.hexlify(base64.b64decode(b(extra['md5_hash'])))
              value = value.decode('ascii')
              extra['md5_hash'] = value
  
          meta_data = {}
          if metadata is not None:
              for meta in list(metadata):
                  meta_data[meta.tag] = meta.text
  
          return Object(name=name, size=size, hash=etag, meta_data=meta_data,
                        extra=extra, container=container, driver=self)
  
      def _response_to_object(self, object_name, container, response):
          """
          Converts a HTTP response to an object (from headers)
  
          :param object_name: Name of the object
          :type object_name: ``str``
  
          :param container: Instance of the container holding the blob
          :type: :class:`Container`
  
          :param response: HTTP Response
          :type node: L{}
  
          :return: An object instance
          :rtype: :class:`Object`
          """
  
          headers = response.headers
          size = int(headers['content-length'])
          etag = headers['etag']
  
          extra = {
              'url': 'http://%s%s' % (response.connection.host,
                                      response.connection.action),
              'etag': etag,
              'md5_hash': headers.get('content-md5', None),
              'content_type': headers.get('content-type', None),
              'content_language': headers.get('content-language', None),
              'content_encoding': headers.get('content-encoding', None),
              'last_modified': headers['last-modified'],
              'lease': {
                  'status': headers.get('x-ms-lease-status', None),
                  'state': headers.get('x-ms-lease-state', None),
                  'duration': headers.get('x-ms-lease-duration', None),
              },
              'blob_type': headers['x-ms-blob-type']
          }
  
          if extra['md5_hash']:
              value = binascii.hexlify(base64.b64decode(b(extra['md5_hash'])))
              value = value.decode('ascii')
              extra['md5_hash'] = value
  
          meta_data = {}
          for key, value in response.headers.items():
              if key.startswith('x-ms-meta-'):
                  key = key.split('x-ms-meta-')[1]
                  meta_data[key] = value
  
          return Object(name=object_name, size=size, hash=etag, extra=extra,
                        meta_data=meta_data, container=container, driver=self)
  
      def iterate_containers(self):
          """
          @inherits: :class:`StorageDriver.iterate_containers`
          """
          params = {'comp': 'list',
                    'maxresults': RESPONSES_PER_REQUEST,
                    'include': 'metadata'}
  
          while True:
              response = self.connection.request('/', params)
              if response.status != httplib.OK:
                  raise LibcloudError('Unexpected status code: %s' %
                                      (response.status), driver=self)
  
              body = response.parse_body()
              containers = body.find(fixxpath(xpath='Containers'))
              containers = containers.findall(fixxpath(xpath='Container'))
  
              for container in containers:
                  yield self._xml_to_container(container)
  
              params['marker'] = body.findtext('NextMarker')
              if not params['marker']:
                  break
  
      def iterate_container_objects(self, container, prefix=None,
                                    ex_prefix=None):
          """
          @inherits: :class:`StorageDriver.iterate_container_objects`
          """
          prefix = self._normalize_prefix_argument(prefix, ex_prefix)
  
          params = {'restype': 'container',
                    'comp': 'list',
                    'maxresults': RESPONSES_PER_REQUEST,
                    'include': 'metadata'}
  
          if prefix:
              params['prefix'] = prefix
  
          container_path = self._get_container_path(container)
  
          while True:
              response = self.connection.request(container_path,
                                                 params=params)
  
              if response.status == httplib.NOT_FOUND:
                  raise ContainerDoesNotExistError(value=None,
                                                   driver=self,
                                                   container_name=container.name)
  
              elif response.status != httplib.OK:
                  raise LibcloudError('Unexpected status code: %s' %
                                      (response.status), driver=self)
  
              body = response.parse_body()
              blobs = body.find(fixxpath(xpath='Blobs'))
              blobs = blobs.findall(fixxpath(xpath='Blob'))
  
              for blob in blobs:
                  yield self._xml_to_object(container, blob)
  
              params['marker'] = body.findtext('NextMarker')
              if not params['marker']:
                  break
  
      def get_container(self, container_name):
          """
          @inherits: :class:`StorageDriver.get_container`
          """
          params = {'restype': 'container'}
  
          container_path = '/%s' % (container_name)
  
          response = self.connection.request(container_path, params=params,
                                             method='HEAD')
  
          if response.status == httplib.NOT_FOUND:
              raise ContainerDoesNotExistError('Container %s does not exist' %
                                               (container_name), driver=self,
                                               container_name=container_name)
          elif response.status != httplib.OK:
              raise LibcloudError('Unexpected status code: %s' %
                                  (response.status), driver=self)
  
          return self._response_to_container(container_name, response)
  
      def get_object(self, container_name, object_name):
          """
          @inherits: :class:`StorageDriver.get_object`
          """
  
          container = self.get_container(container_name=container_name)
          object_path = self._get_object_path(container, object_name)
  
          response = self.connection.request(object_path, method='HEAD')
  
          if response.status == httplib.OK:
              obj = self._response_to_object(object_name, container, response)
              return obj
  
          raise ObjectDoesNotExistError(value=None, driver=self,
                                        object_name=object_name)
  
      def get_object_cdn_url(self, obj,
                             ex_expiry=AZURE_STORAGE_CDN_URL_EXPIRY_HOURS):
          """
          Return a SAS URL that enables reading the given object.
  
          :param obj: Object instance.
          :type  obj: :class:`Object`
  
          :param ex_expiry: The number of hours after which the URL expires.
                            Defaults to 24 hours.
          :type  ex_expiry: ``float``
  
          :return: A SAS URL for the object.
          :rtype: ``str``
          """
          object_path = self._get_object_path(obj.container, obj.name)
  
          now = datetime.utcnow()
          start = now - timedelta(minutes=AZURE_STORAGE_CDN_URL_START_MINUTES)
          expiry = now + timedelta(hours=ex_expiry)
  
          params = {
              'st': start.strftime(AZURE_STORAGE_CDN_URL_DATE_FORMAT),
              'se': expiry.strftime(AZURE_STORAGE_CDN_URL_DATE_FORMAT),
              'sp': 'r',
              'spr': 'https' if self.secure else 'http,https',
              'sv': self.connectionCls.API_VERSION,
              'sr': 'b',
          }
  
          string_to_sign = '\n'.join((
              params['sp'],
              params['st'],
              params['se'],
              '/blob/{}{}'.format(self.key, object_path),
              '',  # signedIdentifier
              '',  # signedIP
              params['spr'],
              params['sv'],
              params['sr'],
              '',  # snapshot
              '',  # rscc
              '',  # rscd
              '',  # rsce
              '',  # rscl
              '',  # rsct
          ))
  
          params['sig'] = base64.b64encode(
              hmac.new(
                  self.secret,
                  string_to_sign.encode('utf-8'),
                  hashlib.sha256
              ).digest()
          ).decode('utf-8')
  
          return '{scheme}://{host}:{port}{action}?{sas_token}'.format(
              scheme='https' if self.secure else 'http',
              host=self.connection.host,
              port=self.connection.port,
              action=self.connection.morph_action_hook(object_path),
              sas_token=urlencode(params),
          )
  
      def _get_container_path(self, container):
          """
          Return a container path
  
          :param container: Container instance
          :type  container: :class:`Container`
  
          :return: A path for this container.
          :rtype: ``str``
          """
          return '/%s' % (container.name)
  
      def _get_object_path(self, container, object_name):
          """
          Return an object's CDN path.
  
          :param container: Container instance
          :type  container: :class:`Container`
  
          :param object_name: Object name
          :type  object_name: :class:`str`
  
          :return: A  path for this object.
          :rtype: ``str``
          """
          container_url = self._get_container_path(container)
          object_name_cleaned = urlquote(object_name)
          object_path = '%s/%s' % (container_url, object_name_cleaned)
          return object_path
  
      def create_container(self, container_name):
          """
          @inherits: :class:`StorageDriver.create_container`
          """
          params = {'restype': 'container'}
  
          container_path = '/%s' % (container_name)
          response = self.connection.request(container_path, params=params,
                                             method='PUT')
  
          if response.status == httplib.CREATED:
              return self._response_to_container(container_name, response)
          elif response.status == httplib.CONFLICT:
              raise ContainerAlreadyExistsError(
                  value='Container with this name already exists. The name must '
                        'be unique among all the containers in the system',
                  container_name=container_name, driver=self)
          elif response.status == httplib.BAD_REQUEST:
              raise InvalidContainerNameError(value='Container name contains ' +
                                              'invalid characters.',
                                              container_name=container_name,
                                              driver=self)
  
          raise LibcloudError('Unexpected status code: %s' % (response.status),
                              driver=self)
  
      def delete_container(self, container):
          """
          @inherits: :class:`StorageDriver.delete_container`
          """
          # Azure does not check if the container is empty. So, we will do
          # a check to ensure that the behaviour is similar to other drivers
          for obj in container.iterate_objects():
              raise ContainerIsNotEmptyError(
                  value='Container must be empty before it can be deleted.',
                  container_name=container.name, driver=self)
  
          params = {'restype': 'container'}
          container_path = self._get_container_path(container)
  
          # Note: All the objects in the container must be deleted first
          response = self.connection.request(container_path, params=params,
                                             method='DELETE')
  
          if response.status == httplib.ACCEPTED:
              return True
          elif response.status == httplib.NOT_FOUND:
              raise ContainerDoesNotExistError(value=None,
                                               driver=self,
                                               container_name=container.name)
  
          return False
  
      def download_object(self, obj, destination_path, overwrite_existing=False,
                          delete_on_failure=True):
          """
          @inherits: :class:`StorageDriver.download_object`
          """
          obj_path = self._get_object_path(obj.container, obj.name)
          response = self.connection.request(obj_path, raw=True, data=None)
  
          return self._get_object(obj=obj, callback=self._save_object,
                                  response=response,
                                  callback_kwargs={
                                      'obj': obj,
                                      'response': response.response,
                                      'destination_path': destination_path,
                                      'overwrite_existing': overwrite_existing,
                                      'delete_on_failure': delete_on_failure},
                                  success_status_code=httplib.OK)
  
      def download_object_as_stream(self, obj, chunk_size=None):
          """
          @inherits: :class:`StorageDriver.download_object_as_stream`
          """
          obj_path = self._get_object_path(obj.container, obj.name)
          response = self.connection.request(obj_path, method='GET',
                                             stream=True, raw=True)
          iterator = response.iter_content(AZURE_DOWNLOAD_CHUNK_SIZE)
  
          return self._get_object(obj=obj,
                                  callback=read_in_chunks,
                                  response=response,
                                  callback_kwargs={'iterator': iterator,
                                                   'chunk_size': chunk_size},
                                  success_status_code=httplib.OK)
  
      def _upload_in_chunks(self, stream, object_path, lease, meta_data,
                            content_type, object_name, file_path, verify_hash,
                            headers):
          """
          Uploads data from an interator in fixed sized chunks to Azure Storage
          """
  
          data_hash = None
          if verify_hash:
              data_hash = self._get_hash_function()
  
          bytes_transferred = 0
          count = 1
          chunks = []
          headers = headers or {}
  
          lease.update_headers(headers)
  
          params = {'comp': 'block'}
  
          # Read the input data in chunk sizes suitable for Azure
          for data in read_in_chunks(stream, AZURE_UPLOAD_CHUNK_SIZE,
                                     fill_size=True):
              data = b(data)
              content_length = len(data)
              bytes_transferred += content_length
  
              if verify_hash:
                  data_hash.update(data)
  
              chunk_hash = self._get_hash_function()
              chunk_hash.update(data)
              chunk_hash = base64.b64encode(b(chunk_hash.digest()))
  
              headers['Content-MD5'] = chunk_hash.decode('utf-8')
              headers['Content-Length'] = str(content_length)
  
              # Block id can be any unique string that is base64 encoded
              # A 10 digit number can hold the max value of 50000 blocks
              # that are allowed for azure
              block_id = base64.b64encode(b('%10d' % (count)))
              block_id = block_id.decode('utf-8')
              params['blockid'] = block_id
  
              # Keep this data for a later commit
              chunks.append(block_id)
  
              # Renew lease before updating
              lease.renew()
  
              resp = self.connection.request(object_path, method='PUT',
                                             data=data, headers=headers,
                                             params=params)
  
              if resp.status != httplib.CREATED:
                  resp.parse_error()
                  raise LibcloudError('Error uploading chunk %d. Code: %d' %
                                      (count, resp.status), driver=self)
  
              count += 1
  
          if verify_hash:
              data_hash = base64.b64encode(b(data_hash.digest()))
              data_hash = data_hash.decode('utf-8')
  
          response = self._commit_blocks(object_path=object_path,
                                         chunks=chunks,
                                         lease=lease,
                                         meta_data=meta_data,
                                         content_type=content_type,
                                         data_hash=data_hash,
                                         object_name=object_name,
                                         file_path=file_path)
  
          # According to the Azure docs:
          # > This header refers to the content of the request, meaning, in this
          # > case, the list of blocks, and not the content of the blob itself.
          # However, the validation code assumes that the content-md5 in the
          # server response refers to the object so we must discard the value
          response.headers['content-md5'] = None
  
          return {
              'response': response,
              'data_hash': data_hash,
              'bytes_transferred': bytes_transferred,
          }
  
      def _commit_blocks(self, object_path, chunks, lease,
                         meta_data, content_type, data_hash,
                         object_name, file_path):
          """
          Makes a final commit of the data.
          """
  
          root = ET.Element('BlockList')
  
          for block_id in chunks:
              part = ET.SubElement(root, 'Uncommitted')
              part.text = str(block_id)
  
          data = tostring(root)
          params = {'comp': 'blocklist'}
          headers = {}
  
          lease.update_headers(headers)
          lease.renew()
  
          headers['x-ms-blob-content-type'] = self._determine_content_type(
              content_type, object_name, file_path)
  
          if data_hash is not None:
              headers['x-ms-blob-content-md5'] = data_hash
  
          self._update_metadata(headers, meta_data)
  
          data_hash = self._get_hash_function()
          data_hash.update(data.encode('utf-8'))
          data_hash = base64.b64encode(b(data_hash.digest()))
          headers['Content-MD5'] = data_hash.decode('utf-8')
  
          headers['Content-Length'] = len(data)
  
          response = self.connection.request(object_path, data=data,
                                             params=params, headers=headers,
                                             method='PUT')
  
          if response.status != httplib.CREATED:
              raise LibcloudError('Error in blocklist commit', driver=self)
  
          return response
  
      def upload_object(self, file_path, container, object_name,
                        verify_hash=True, extra=None, headers=None,
                        ex_use_lease=False,
                        **deprecated_kwargs):
          """
          Upload an object currently located on a disk.
  
          @inherits: :class:`StorageDriver.upload_object`
  
          :param ex_use_lease: Indicates if we must take a lease before upload
          :type ex_use_lease: ``bool``
          """
          if deprecated_kwargs:
              raise ValueError('Support for arguments was removed: %s'
                               % ', '.join(deprecated_kwargs.keys()))
  
          blob_size = os.stat(file_path).st_size
  
          with open(file_path, 'rb') as fobj:
              return self._put_object(container=container,
                                      object_name=object_name,
                                      extra=extra, verify_hash=verify_hash,
                                      use_lease=ex_use_lease, headers=headers,
                                      blob_size=blob_size, file_path=file_path,
                                      stream=fobj)
  
      def upload_object_via_stream(self, iterator, container, object_name,
                                   verify_hash=True, extra=None, headers=None,
                                   ex_use_lease=False,
                                   **deprecated_kwargs):
          """
          @inherits: :class:`StorageDriver.upload_object_via_stream`
  
          :param ex_use_lease: Indicates if we must take a lease before upload
          :type ex_use_lease: ``bool``
          """
          if deprecated_kwargs:
              raise ValueError('Support for arguments was removed: %s'
                               % ', '.join(deprecated_kwargs.keys()))
  
          return self._put_object(container=container,
                                  object_name=object_name,
                                  extra=extra, verify_hash=verify_hash,
                                  use_lease=ex_use_lease,
                                  headers=headers,
                                  blob_size=None,
                                  stream=iterator)
  
      def delete_object(self, obj):
          """
          @inherits: :class:`StorageDriver.delete_object`
          """
          object_path = self._get_object_path(obj.container, obj.name)
          response = self.connection.request(object_path, method='DELETE')
  
          if response.status == httplib.ACCEPTED:
              return True
          elif response.status == httplib.NOT_FOUND:
              raise ObjectDoesNotExistError(value=None, driver=self,
                                            object_name=obj.name)
  
          return False
  
      def _update_metadata(self, headers, meta_data):
          """
          Update the given metadata in the headers
  
          :param headers: The headers dictionary to be updated
          :type headers: ``dict``
  
          :param meta_data: Metadata key value pairs
          :type meta_data: ``dict``
          """
          for key, value in list(meta_data.items()):
              key = 'x-ms-meta-%s' % (key)
              headers[key] = value
  
      def _put_object(self, container, object_name, stream,
                      extra=None, verify_hash=True, headers=None,
                      blob_size=None, file_path=None,
                      use_lease=False):
          """
          Control function that does the real job of uploading data to a blob
          """
          extra = extra or {}
          content_type = extra.get('content_type', None)
          meta_data = extra.get('meta_data', {})
  
          object_path = self._get_object_path(container, object_name)
  
          # Get a lease if required and do the operations
          with AzureBlobLease(self, object_path, use_lease) as lease:
              if blob_size is not None and blob_size <= AZURE_UPLOAD_CHUNK_SIZE:
                  result_dict = self._upload_directly(stream=stream,
                                                      object_path=object_path,
                                                      lease=lease,
                                                      blob_size=blob_size,
                                                      meta_data=meta_data,
                                                      headers=headers,
                                                      content_type=content_type,
                                                      object_name=object_name,
                                                      file_path=file_path)
              else:
                  result_dict = self._upload_in_chunks(stream=stream,
                                                       object_path=object_path,
                                                       lease=lease,
                                                       meta_data=meta_data,
                                                       headers=headers,
                                                       content_type=content_type,
                                                       object_name=object_name,
                                                       file_path=file_path,
                                                       verify_hash=verify_hash)
  
              response = result_dict['response']
              bytes_transferred = result_dict['bytes_transferred']
              data_hash = result_dict['data_hash']
              headers = response.headers
  
          if response.status != httplib.CREATED:
              raise LibcloudError(
                  'Unexpected status code, status_code=%s' % (response.status),
                  driver=self)
  
          server_hash = headers.get('content-md5')
  
          if server_hash:
              server_hash = binascii.hexlify(base64.b64decode(b(server_hash)))
              server_hash = server_hash.decode('utf-8')
          else:
              # TODO: HACK - We could poll the object for a while and get
              # the hash
              pass
  
          if (verify_hash and server_hash and data_hash != server_hash):
              raise ObjectHashMismatchError(
                  value='MD5 hash checksum does not match',
                  object_name=object_name, driver=self)
  
          return Object(name=object_name, size=bytes_transferred,
                        hash=headers['etag'], extra=None,
                        meta_data=meta_data, container=container,
                        driver=self)
  
      def _upload_directly(self, stream, object_path, lease, blob_size,
                           meta_data, content_type, object_name, file_path,
                           headers):
  
          headers = headers or {}
          lease.update_headers(headers)
  
          self._update_metadata(headers, meta_data)
  
          headers['Content-Length'] = str(blob_size)
          headers['x-ms-blob-type'] = 'BlockBlob'
  
          return self._upload_object(object_name=object_name,
                                     file_path=file_path,
                                     content_type=content_type,
                                     request_path=object_path,
                                     stream=stream,
                                     headers=headers)
  
      def ex_set_object_metadata(self, obj, meta_data):
          """
          Set metadata for an object
  
          :param obj: The blob object
          :type obj: :class:`Object`
  
          :param meta_data: Metadata key value pairs
          :type meta_data: ``dict``
          """
          object_path = self._get_object_path(obj.container, obj.name)
          params = {'comp': 'metadata'}
          headers = {}
  
          self._update_metadata(headers, meta_data)
  
          response = self.connection.request(object_path, method='PUT',
                                             params=params,
                                             headers=headers)
  
          if response.status != httplib.OK:
              response.parse_error('Setting metadata')
  
files: libcloud/storage/drivers/azure_blobs.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  import base64
  import hmac
  import time
  from hashlib import sha1
  
  import libcloud.utils.py3
  
  try:
      if libcloud.utils.py3.DEFAULT_LXML:
          from lxml.etree import Element, SubElement
      else:
          from xml.etree.ElementTree import Element, SubElement
  except ImportError:
      from xml.etree.ElementTree import Element, SubElement
  
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import urlquote
  from libcloud.utils.py3 import b
  from libcloud.utils.py3 import tostring
  
  from libcloud.utils.xml import fixxpath, findtext
  from libcloud.utils.files import read_in_chunks
  from libcloud.common.types import InvalidCredsError, LibcloudError
  from libcloud.common.base import ConnectionUserAndKey, RawResponse
  from libcloud.common.aws import AWSBaseResponse, AWSDriver, \
      AWSTokenConnection, SignedAWSConnection
  
  from libcloud.storage.base import Object, Container, StorageDriver
  from libcloud.storage.types import ContainerError
  from libcloud.storage.types import ContainerIsNotEmptyError
  from libcloud.storage.types import InvalidContainerNameError
  from libcloud.storage.types import ContainerDoesNotExistError
  from libcloud.storage.types import ObjectDoesNotExistError
  from libcloud.storage.types import ObjectHashMismatchError
  
  # How long before the token expires
  EXPIRATION_SECONDS = 15 * 60
  
  S3_US_STANDARD_HOST = 's3.amazonaws.com'
  S3_US_EAST2_HOST = 's3-us-east-2.amazonaws.com'
  S3_US_WEST_HOST = 's3-us-west-1.amazonaws.com'
  S3_US_WEST_OREGON_HOST = 's3-us-west-2.amazonaws.com'
  S3_US_GOV_WEST_HOST = 's3-us-gov-west-1.amazonaws.com'
  S3_CN_NORTH_HOST = 's3.cn-north-1.amazonaws.com.cn'
  S3_CN_NORTHWEST_HOST = 's3.cn-northwest-1.amazonaws.com.cn'
  S3_EU_WEST_HOST = 's3-eu-west-1.amazonaws.com'
  S3_EU_WEST2_HOST = 's3-eu-west-2.amazonaws.com'
  S3_EU_CENTRAL_HOST = 's3-eu-central-1.amazonaws.com'
  S3_EU_NORTH1_HOST = 's3-eu-north-1.amazonaws.com'
  S3_AP_SOUTH_HOST = 's3-ap-south-1.amazonaws.com'
  S3_AP_SOUTHEAST_HOST = 's3-ap-southeast-1.amazonaws.com'
  S3_AP_SOUTHEAST2_HOST = 's3-ap-southeast-2.amazonaws.com'
  S3_AP_NORTHEAST1_HOST = 's3-ap-northeast-1.amazonaws.com'
  S3_AP_NORTHEAST2_HOST = 's3-ap-northeast-2.amazonaws.com'
  S3_AP_NORTHEAST_HOST = S3_AP_NORTHEAST1_HOST
  S3_SA_EAST_HOST = 's3-sa-east-1.amazonaws.com'
  S3_SA_SOUTHEAST2_HOST = 's3-sa-east-2.amazonaws.com'
  S3_CA_CENTRAL_HOST = 's3-ca-central-1.amazonaws.com'
  
  # Maps AWS region name to connection hostname
  REGION_TO_HOST_MAP = {
      'us-east-1': S3_US_STANDARD_HOST,
      'us-east-2': S3_US_EAST2_HOST,
      'us-west-1': S3_US_WEST_HOST,
      'us-west-2': S3_US_WEST_OREGON_HOST,
      'us-gov-west-1': S3_US_GOV_WEST_HOST,
      'cn-north-1': S3_CN_NORTH_HOST,
      'cn-northwest-1': S3_CN_NORTHWEST_HOST,
      'eu-west-1': S3_EU_WEST_HOST,
      'eu-west-2': S3_EU_WEST2_HOST,
      'eu-west-3': 's3.eu-west-3.amazonaws.com',
      'eu-north-1': 's3.eu-north-1.amazonaws.com',
      'eu-central-1': S3_EU_CENTRAL_HOST,
      'ap-south-1': S3_AP_SOUTH_HOST,
      'ap-southeast-1': S3_AP_SOUTHEAST_HOST,
      'ap-southeast-2': S3_AP_SOUTHEAST2_HOST,
      'ap-northeast-1': S3_AP_NORTHEAST1_HOST,
      'ap-northeast-2': S3_AP_NORTHEAST2_HOST,
      'ap-northeast-3': 's3.ap-northeast-3.amazonaws.com',
      'sa-east-1': S3_SA_EAST_HOST,
      'sa-east-2': S3_SA_SOUTHEAST2_HOST,
      'ca-central-1': S3_CA_CENTRAL_HOST,
      'me-south-1': 's3.me-south-1.amazonaws.com'
  }
  
  API_VERSION = '2006-03-01'
  NAMESPACE = 'http://s3.amazonaws.com/doc/%s/' % (API_VERSION)
  
  # AWS multi-part chunks must be minimum 5MB
  CHUNK_SIZE = 5 * 1024 * 1024
  
  # Desired number of items in each response inside a paginated request in
  # ex_iterate_multipart_uploads.
  RESPONSES_PER_REQUEST = 100
  
  
  class S3Response(AWSBaseResponse):
      namespace = None
      valid_response_codes = [httplib.NOT_FOUND, httplib.CONFLICT,
                              httplib.BAD_REQUEST, httplib.PARTIAL_CONTENT]
  
      def success(self):
          i = int(self.status)
          return 200 <= i <= 299 or i in self.valid_response_codes
  
      def parse_error(self):
          if self.status in [httplib.UNAUTHORIZED, httplib.FORBIDDEN]:
              raise InvalidCredsError(self.body)
          elif self.status == httplib.MOVED_PERMANENTLY:
              bucket_region = self.headers.get('x-amz-bucket-region', None)
              used_region = self.connection.driver.region
              raise LibcloudError('This bucket is located in a different '
                                  'region. Please use the correct driver. '
                                  'Bucket region "%s", used region "%s".' %
                                  (bucket_region, used_region),
                                  driver=S3StorageDriver)
          raise LibcloudError('Unknown error. Status code: %d' % (self.status),
                              driver=S3StorageDriver)
  
  
  class S3RawResponse(S3Response, RawResponse):
      pass
  
  
  class BaseS3Connection(ConnectionUserAndKey):
      """
      Represents a single connection to the S3 Endpoint
      """
  
      host = 's3.amazonaws.com'
      responseCls = S3Response
      rawResponseCls = S3RawResponse
  
      @staticmethod
      def get_auth_signature(method, headers, params, expires, secret_key, path,
                             vendor_prefix):
          """
          Signature = URL-Encode( Base64( HMAC-SHA1( YourSecretAccessKeyID,
                                      UTF-8-Encoding-Of( StringToSign ) ) ) );
  
          StringToSign = HTTP-VERB + "\n" +
              Content-MD5 + "\n" +
              Content-Type + "\n" +
              Expires + "\n" +
              CanonicalizedVendorHeaders +
              CanonicalizedResource;
          """
          special_headers = {'content-md5': '', 'content-type': '', 'date': ''}
          vendor_headers = {}
  
          for key, value in list(headers.items()):
              key_lower = key.lower()
              if key_lower in special_headers:
                  special_headers[key_lower] = value.strip()
              elif key_lower.startswith(vendor_prefix):
                  vendor_headers[key_lower] = value.strip()
  
          if expires:
              special_headers['date'] = str(expires)
  
          buf = [method]
          for _, value in sorted(special_headers.items()):
              buf.append(value)
          string_to_sign = '\n'.join(buf)
  
          buf = []
          for key, value in sorted(vendor_headers.items()):
              buf.append('%s:%s' % (key, value))
          header_string = '\n'.join(buf)
  
          values_to_sign = []
          for value in [string_to_sign, header_string, path]:
              if value:
                  values_to_sign.append(value)
  
          string_to_sign = '\n'.join(values_to_sign)
          b64_hmac = base64.b64encode(
              hmac.new(b(secret_key), b(string_to_sign), digestmod=sha1).digest()
          )
          return b64_hmac.decode('utf-8')
  
      def add_default_params(self, params):
          expires = str(int(time.time()) + EXPIRATION_SECONDS)
          params['AWSAccessKeyId'] = self.user_id
          params['Expires'] = expires
          return params
  
      def pre_connect_hook(self, params, headers):
          # pylint: disable=no-member
          params['Signature'] = self.get_auth_signature(
              method=self.method, headers=headers, params=params,
              expires=params['Expires'], secret_key=self.key, path=self.action,
              vendor_prefix=self.driver.http_vendor_prefix)
          return params, headers
  
  
  class S3Connection(AWSTokenConnection, BaseS3Connection):
      """
      Represents a single connection to the S3 endpoint, with AWS-specific
      features.
      """
      pass
  
  
  class S3SignatureV4Connection(SignedAWSConnection, BaseS3Connection):
      service_name = 's3'
      version = API_VERSION
  
      def __init__(self, user_id, key, secure=True, host=None, port=None,
                   url=None, timeout=None, proxy_url=None, token=None,
                   retry_delay=None, backoff=None):
          super(S3SignatureV4Connection, self).__init__(
              user_id, key, secure, host,
              port, url, timeout, proxy_url,
              token, retry_delay, backoff,
              4)  # force version 4
  
  
  class S3MultipartUpload(object):
      """
      Class representing an amazon s3 multipart upload
      """
  
      def __init__(self, key, id, created_at, initiator, owner):
          """
          Class representing an amazon s3 multipart upload
  
          :param key: The object/key that was being uploaded
          :type key: ``str``
  
          :param id: The upload id assigned by amazon
          :type id: ``str``
  
          :param created_at: The date/time at which the upload was started
          :type created_at: ``str``
  
          :param initiator: The AWS owner/IAM user who initiated this
          :type initiator: ``str``
  
          :param owner: The AWS owner/IAM who will own this object
          :type owner: ``str``
          """
          self.key = key
          self.id = id
          self.created_at = created_at
          self.initiator = initiator
          self.owner = owner
  
      def __repr__(self):
          return ('<S3MultipartUpload: key=%s>' % (self.key))
  
  
  class BaseS3StorageDriver(StorageDriver):
      name = 'Amazon S3 (standard)'
      website = 'http://aws.amazon.com/s3/'
      connectionCls = BaseS3Connection
      hash_type = 'md5'
      supports_chunked_encoding = False
      supports_s3_multipart_upload = True
      ex_location_name = ''
      namespace = NAMESPACE
      http_vendor_prefix = 'x-amz'
  
      def iterate_containers(self):
          response = self.connection.request('/')
          if response.status == httplib.OK:
              containers = self._to_containers(obj=response.object,
                                               xpath='Buckets/Bucket')
              return containers
  
          raise LibcloudError('Unexpected status code: %s' % (response.status),
                              driver=self)
  
      def iterate_container_objects(self, container, prefix=None,
                                    ex_prefix=None):
          """
          Return a generator of objects for the given container.
  
          :param container: Container instance
          :type container: :class:`Container`
  
          :param prefix: Only return objects starting with prefix
          :type prefix: ``str``
  
          :param ex_prefix: Only return objects starting with ex_prefix
          :type ex_prefix: ``str``
  
          :return: A generator of Object instances.
          :rtype: ``generator`` of :class:`Object`
          """
          prefix = self._normalize_prefix_argument(prefix, ex_prefix)
  
          params = {}
  
          if prefix:
              params['prefix'] = prefix
  
          last_key = None
          exhausted = False
          container_path = self._get_container_path(container)
  
          while not exhausted:
              if last_key:
                  params['marker'] = last_key
  
              response = self.connection.request(container_path,
                                                 params=params)
  
              if response.status != httplib.OK:
                  raise LibcloudError('Unexpected status code: %s' %
                                      (response.status), driver=self)
  
              objects = self._to_objs(obj=response.object,
                                      xpath='Contents', container=container)
              is_truncated = response.object.findtext(fixxpath(
                  xpath='IsTruncated', namespace=self.namespace)).lower()
              exhausted = (is_truncated == 'false')
  
              last_key = None
              for obj in objects:
                  last_key = obj.name
                  yield obj
  
      def get_container(self, container_name):
          try:
              response = self.connection.request('/%s' % container_name,
                                                 method='HEAD')
              if response.status == httplib.NOT_FOUND:
                  raise ContainerDoesNotExistError(value=None, driver=self,
                                                   container_name=container_name)
          except InvalidCredsError:
              # This just means the user doesn't have IAM permissions to do a
              # HEAD request but other requests might work.
              pass
          return Container(name=container_name, extra=None, driver=self)
  
      def get_object(self, container_name, object_name):
          container = self.get_container(container_name=container_name)
          object_path = self._get_object_path(container, object_name)
          response = self.connection.request(object_path, method='HEAD')
  
          if response.status == httplib.OK:
              obj = self._headers_to_object(object_name=object_name,
                                            container=container,
                                            headers=response.headers)
              return obj
  
          raise ObjectDoesNotExistError(value=None, driver=self,
                                        object_name=object_name)
  
      def _get_container_path(self, container):
          """
          Return a container path
  
          :param container: Container instance
          :type  container: :class:`Container`
  
          :return: A path for this container.
          :rtype: ``str``
          """
          return '/%s' % (container.name)
  
      def _get_object_path(self, container, object_name):
          """
          Return an object's CDN path.
  
          :param container: Container instance
          :type  container: :class:`Container`
  
          :param object_name: Object name
          :type  object_name: :class:`str`
  
          :return: A  path for this object.
          :rtype: ``str``
          """
          container_url = self._get_container_path(container)
          object_name_cleaned = self._clean_object_name(object_name)
          object_path = '%s/%s' % (container_url, object_name_cleaned)
          return object_path
  
      def create_container(self, container_name):
          if self.ex_location_name:
              root = Element('CreateBucketConfiguration')
              child = SubElement(root, 'LocationConstraint')
              child.text = self.ex_location_name
  
              data = tostring(root)
          else:
              data = ''
  
          response = self.connection.request('/%s' % (container_name),
                                             data=data,
                                             method='PUT')
  
          if response.status == httplib.OK:
              container = Container(name=container_name, extra=None, driver=self)
              return container
          elif response.status == httplib.CONFLICT:
              raise InvalidContainerNameError(
                  value='Container with this name already exists. The name must '
                        'be unique among all the containers in the system',
                  container_name=container_name, driver=self)
          elif response.status == httplib.BAD_REQUEST:
              raise ContainerError(
                  value='Bad request when creating container: %s' %
                        response.body,
                  container_name=container_name, driver=self)
  
          raise LibcloudError('Unexpected status code: %s' % (response.status),
                              driver=self)
  
      def delete_container(self, container):
          # Note: All the objects in the container must be deleted first
          response = self.connection.request('/%s' % (container.name),
                                             method='DELETE')
          if response.status == httplib.NO_CONTENT:
              return True
          elif response.status == httplib.CONFLICT:
              raise ContainerIsNotEmptyError(
                  value='Container must be empty before it can be deleted.',
                  container_name=container.name, driver=self)
          elif response.status == httplib.NOT_FOUND:
              raise ContainerDoesNotExistError(value=None,
                                               driver=self,
                                               container_name=container.name)
  
          return False
  
      def download_object(self, obj, destination_path, overwrite_existing=False,
                          delete_on_failure=True):
          obj_path = self._get_object_path(obj.container, obj.name)
  
          response = self.connection.request(obj_path, method='GET', raw=True)
  
          return self._get_object(obj=obj, callback=self._save_object,
                                  response=response,
                                  callback_kwargs={
                                      'obj': obj,
                                      'response': response.response,
                                      'destination_path': destination_path,
                                      'overwrite_existing': overwrite_existing,
                                      'delete_on_failure': delete_on_failure},
                                  success_status_code=httplib.OK)
  
      def download_object_as_stream(self, obj, chunk_size=None):
          obj_path = self._get_object_path(obj.container, obj.name)
          response = self.connection.request(obj_path, method='GET',
                                             stream=True, raw=True)
  
          return self._get_object(
              obj=obj, callback=read_in_chunks,
              response=response,
              callback_kwargs={'iterator': response.iter_content(CHUNK_SIZE),
                               'chunk_size': chunk_size},
              success_status_code=httplib.OK)
  
      def download_object_range(self, obj, destination_path, start_bytes,
                                end_bytes=None, overwrite_existing=False,
                                delete_on_failure=True):
          self._validate_start_and_end_bytes(start_bytes=start_bytes,
                                             end_bytes=end_bytes)
  
          obj_path = self._get_object_path(obj.container, obj.name)
  
          headers = {'Range': self._get_standard_range_str(start_bytes,
                                                           end_bytes)}
          response = self.connection.request(obj_path, method='GET',
                                             headers=headers, raw=True)
  
          return self._get_object(obj=obj, callback=self._save_object,
                                  response=response,
                                  callback_kwargs={
                                      'obj': obj,
                                      'response': response.response,
                                      'destination_path': destination_path,
                                      'overwrite_existing': overwrite_existing,
                                      'delete_on_failure': delete_on_failure,
                                      'partial_download': True},
                                  success_status_code=httplib.PARTIAL_CONTENT)
  
      def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None,
                                          chunk_size=None):
          self._validate_start_and_end_bytes(start_bytes=start_bytes,
                                             end_bytes=end_bytes)
  
          obj_path = self._get_object_path(obj.container, obj.name)
  
          headers = {'Range': self._get_standard_range_str(start_bytes,
                                                           end_bytes)}
          response = self.connection.request(obj_path, method='GET',
                                             headers=headers,
                                             stream=True, raw=True)
  
          return self._get_object(
              obj=obj, callback=read_in_chunks,
              response=response,
              callback_kwargs={'iterator': response.iter_content(CHUNK_SIZE),
                               'chunk_size': chunk_size},
              success_status_code=httplib.PARTIAL_CONTENT)
  
      def upload_object(self, file_path, container, object_name, extra=None,
                        verify_hash=True, headers=None, ex_storage_class=None):
          """
          @inherits: :class:`StorageDriver.upload_object`
  
          :param ex_storage_class: Storage class
          :type ex_storage_class: ``str``
          """
          return self._put_object(container=container, object_name=object_name,
                                  extra=extra, file_path=file_path,
                                  verify_hash=verify_hash,
                                  headers=headers,
                                  storage_class=ex_storage_class)
  
      def _initiate_multipart(self, container, object_name, headers=None):
          """
          Initiates a multipart upload to S3
  
          :param container: The destination container
          :type container: :class:`Container`
  
          :param object_name: The name of the object which we are uploading
          :type object_name: ``str``
  
          :keyword headers: Additional headers to send with the request
          :type headers: ``dict``
  
          :return: The id of the newly created multipart upload
          :rtype: ``str``
          """
          headers = headers or {}
  
          request_path = self._get_object_path(container, object_name)
          params = {'uploads': ''}
  
          response = self.connection.request(request_path, method='POST',
                                             headers=headers, params=params)
  
          if response.status != httplib.OK:
              raise LibcloudError('Error initiating multipart upload',
                                  driver=self)
  
          return findtext(element=response.object, xpath='UploadId',
                          namespace=self.namespace)
  
      def _upload_multipart_chunks(self, container, object_name, upload_id,
                                   stream, calculate_hash=True):
          """
          Uploads data from an iterator in fixed sized chunks to S3
  
          :param container: The destination container
          :type container: :class:`Container`
  
          :param object_name: The name of the object which we are uploading
          :type object_name: ``str``
  
          :param upload_id: The upload id allocated for this multipart upload
          :type upload_id: ``str``
  
          :param stream: The generator for fetching the upload data
          :type stream: ``generator``
  
          :keyword calculate_hash: Indicates if we must calculate the data hash
          :type calculate_hash: ``bool``
  
          :return: A tuple of (chunk info, checksum, bytes transferred)
          :rtype: ``tuple``
          """
          data_hash = None
          if calculate_hash:
              data_hash = self._get_hash_function()
  
          bytes_transferred = 0
          count = 1
          chunks = []
          params = {'uploadId': upload_id}
  
          request_path = self._get_object_path(container, object_name)
  
          # Read the input data in chunk sizes suitable for AWS
          for data in read_in_chunks(stream, chunk_size=CHUNK_SIZE,
                                     fill_size=True, yield_empty=True):
              bytes_transferred += len(data)
  
              if calculate_hash:
                  data_hash.update(data)
  
              chunk_hash = self._get_hash_function()
              chunk_hash.update(data)
              chunk_hash = base64.b64encode(chunk_hash.digest()).decode('utf-8')
  
              # The Content-MD5 header provides an extra level of data check and
              # is recommended by amazon
              headers = {
                  'Content-Length': len(data),
                  'Content-MD5': chunk_hash,
              }
  
              params['partNumber'] = count
  
              resp = self.connection.request(request_path, method='PUT',
                                             data=data, headers=headers,
                                             params=params)
  
              if resp.status != httplib.OK:
                  raise LibcloudError('Error uploading chunk', driver=self)
  
              server_hash = resp.headers['etag'].replace('"', '')
  
              # Keep this data for a later commit
              chunks.append((count, server_hash))
              count += 1
  
          if calculate_hash:
              data_hash = data_hash.hexdigest()
  
          return (chunks, data_hash, bytes_transferred)
  
      def _commit_multipart(self, container, object_name, upload_id, chunks):
          """
          Makes a final commit of the data.
  
          :param container: The destination container
          :type container: :class:`Container`
  
          :param object_name: The name of the object which we are uploading
          :type object_name: ``str``
  
          :param upload_id: The upload id allocated for this multipart upload
          :type upload_id: ``str``
  
          :param chunks: A list of (chunk_number, chunk_hash) tuples.
          :type chunks: ``list``
  
          :return: The server side hash of the uploaded data
          :rtype: ``str``
          """
          root = Element('CompleteMultipartUpload')
  
          for (count, etag) in chunks:
              part = SubElement(root, 'Part')
              part_no = SubElement(part, 'PartNumber')
              part_no.text = str(count)
  
              etag_id = SubElement(part, 'ETag')
              etag_id.text = str(etag)
  
          data = tostring(root)
  
          headers = {'Content-Length': len(data)}
          params = {'uploadId': upload_id}
          request_path = self._get_object_path(container, object_name)
          response = self.connection.request(request_path, headers=headers,
                                             params=params, data=data,
                                             method='POST')
  
          if response.status != httplib.OK:
              element = response.object
              # pylint: disable=maybe-no-member
              code, message = response._parse_error_details(element=element)
              msg = 'Error in multipart commit: %s (%s)' % (message, code)
              raise LibcloudError(msg, driver=self)
  
          # Get the server's etag to be passed back to the caller
          body = response.parse_body()
          server_hash = body.find(fixxpath(xpath='ETag',
                                           namespace=self.namespace)).text
          return server_hash
  
      def _abort_multipart(self, container, object_name, upload_id):
          """
          Aborts an already initiated multipart upload
  
          :param container: The destination container
          :type container: :class:`Container`
  
          :param object_name: The name of the object which we are uploading
          :type object_name: ``str``
  
          :param upload_id: The upload id allocated for this multipart upload
          :type upload_id: ``str``
          """
  
          params = {'uploadId': upload_id}
          request_path = self._get_object_path(container, object_name)
  
          resp = self.connection.request(request_path, method='DELETE',
                                         params=params)
  
          if resp.status != httplib.NO_CONTENT:
              raise LibcloudError('Error in multipart abort. status_code=%d' %
                                  (resp.status), driver=self)
  
      def upload_object_via_stream(self, iterator, container, object_name,
                                   extra=None, headers=None,
                                   ex_storage_class=None):
          """
          @inherits: :class:`StorageDriver.upload_object_via_stream`
  
          :param ex_storage_class: Storage class
          :type ex_storage_class: ``str``
          """
  
          method = 'PUT'
          params = None
  
          # This driver is used by other S3 API compatible drivers also.
          # Amazon provides a different (complex?) mechanism to do multipart
          # uploads
          if self.supports_s3_multipart_upload:
              return self._put_object_multipart(container=container,
                                                object_name=object_name,
                                                extra=extra,
                                                stream=iterator,
                                                verify_hash=False,
                                                headers=headers,
                                                storage_class=ex_storage_class)
          return self._put_object(container=container, object_name=object_name,
                                  extra=extra, method=method, query_args=params,
                                  stream=iterator, verify_hash=False,
                                  headers=headers,
                                  storage_class=ex_storage_class)
  
      def delete_object(self, obj):
          object_path = self._get_object_path(obj.container, obj.name)
          response = self.connection.request(object_path, method='DELETE')
          if response.status == httplib.NO_CONTENT:
              return True
          elif response.status == httplib.NOT_FOUND:
              raise ObjectDoesNotExistError(value=None, driver=self,
                                            object_name=obj.name)
  
          return False
  
      def ex_iterate_multipart_uploads(self, container, prefix=None,
                                       delimiter=None):
          """
          Extension method for listing all in-progress S3 multipart uploads.
  
          Each multipart upload which has not been committed or aborted is
          considered in-progress.
  
          :param container: The container holding the uploads
          :type container: :class:`Container`
  
          :keyword prefix: Print only uploads of objects with this prefix
          :type prefix: ``str``
  
          :keyword delimiter: The object/key names are grouped based on
              being split by this delimiter
          :type delimiter: ``str``
  
          :return: A generator of S3MultipartUpload instances.
          :rtype: ``generator`` of :class:`S3MultipartUpload`
          """
  
          if not self.supports_s3_multipart_upload:
              raise LibcloudError('Feature not supported', driver=self)
  
          # Get the data for a specific container
          request_path = self._get_container_path(container)
          params = {'max-uploads': RESPONSES_PER_REQUEST, 'uploads': ''}
  
          if prefix:
              params['prefix'] = prefix
  
          if delimiter:
              params['delimiter'] = delimiter
  
          def finder(node, text):
              return node.findtext(fixxpath(xpath=text,
                                            namespace=self.namespace))
  
          while True:
              response = self.connection.request(request_path, params=params)
  
              if response.status != httplib.OK:
                  raise LibcloudError('Error fetching multipart uploads. '
                                      'Got code: %s' % response.status,
                                      driver=self)
  
              body = response.parse_body()
              # pylint: disable=maybe-no-member
              for node in body.findall(fixxpath(xpath='Upload',
                                                namespace=self.namespace)):
                  initiator = node.find(fixxpath(xpath='Initiator',
                                                 namespace=self.namespace))
                  owner = node.find(fixxpath(xpath='Owner',
                                             namespace=self.namespace))
  
                  key = finder(node, 'Key')
                  upload_id = finder(node, 'UploadId')
                  created_at = finder(node, 'Initiated')
                  initiator = finder(initiator, 'DisplayName')
                  owner = finder(owner, 'DisplayName')
  
                  yield S3MultipartUpload(key, upload_id, created_at,
                                          initiator, owner)
  
              # Check if this is the last entry in the listing
              # pylint: disable=maybe-no-member
              is_truncated = body.findtext(fixxpath(xpath='IsTruncated',
                                                    namespace=self.namespace))
  
              if is_truncated.lower() == 'false':
                  break
  
              # Provide params for the next request
              upload_marker = body.findtext(fixxpath(xpath='NextUploadIdMarker',
                                                     namespace=self.namespace))
              key_marker = body.findtext(fixxpath(xpath='NextKeyMarker',
                                                  namespace=self.namespace))
  
              params['key-marker'] = key_marker
              params['upload-id-marker'] = upload_marker
  
      def ex_cleanup_all_multipart_uploads(self, container, prefix=None):
          """
          Extension method for removing all partially completed S3 multipart
          uploads.
  
          :param container: The container holding the uploads
          :type container: :class:`Container`
  
          :keyword prefix: Delete only uploads of objects with this prefix
          :type prefix: ``str``
          """
  
          # Iterate through the container and delete the upload ids
          for upload in self.ex_iterate_multipart_uploads(container, prefix,
                                                          delimiter=None):
              self._abort_multipart(container, upload.key, upload.id)
  
      def _clean_object_name(self, name):
          name = urlquote(name)
          return name
  
      def _put_object(self, container, object_name, method='PUT',
                      query_args=None, extra=None, file_path=None,
                      stream=None, verify_hash=True, storage_class=None,
                      headers=None):
          headers = headers or {}
          extra = extra or {}
  
          headers.update(self._to_storage_class_headers(storage_class))
  
          content_type = extra.get('content_type', None)
          meta_data = extra.get('meta_data', None)
          acl = extra.get('acl', None)
  
          if meta_data:
              for key, value in list(meta_data.items()):
                  key = self.http_vendor_prefix + '-meta-%s' % (key)
                  headers[key] = value
  
          if acl:
              headers[self.http_vendor_prefix + '-acl'] = acl
  
          request_path = self._get_object_path(container, object_name)
  
          if query_args:
              request_path = '?'.join((request_path, query_args))
  
          result_dict = self._upload_object(
              object_name=object_name, content_type=content_type,
              request_path=request_path, request_method=method,
              headers=headers, file_path=file_path, stream=stream)
  
          response = result_dict['response']
          bytes_transferred = result_dict['bytes_transferred']
          headers = response.headers
          response = response
          server_hash = headers.get('etag', '').replace('"', '')
          server_side_encryption = headers.get('x-amz-server-side-encryption',
                                               None)
          aws_kms_encryption = (server_side_encryption == 'aws:kms')
          hash_matches = (result_dict['data_hash'] == server_hash)
  
          # NOTE: If AWS KMS server side encryption is enabled, ETag won't
          # contain object MD5 digest so we skip the checksum check
          # See https://docs.aws.amazon.com/AmazonS3/latest/API
          # /RESTCommonResponseHeaders.html
          # and https://github.com/apache/libcloud/issues/1401
          # for details
          if verify_hash and not aws_kms_encryption and not hash_matches:
              raise ObjectHashMismatchError(
                  value='MD5 hash {0} checksum does not match {1}'.format(
                      server_hash, result_dict['data_hash']),
                  object_name=object_name, driver=self)
          elif response.status == httplib.OK:
              obj = Object(
                  name=object_name, size=bytes_transferred, hash=server_hash,
                  extra={'acl': acl}, meta_data=meta_data, container=container,
                  driver=self)
  
              return obj
          else:
              raise LibcloudError(
                  'Unexpected status code, status_code=%s' % (response.status),
                  driver=self)
  
      def _put_object_multipart(self, container, object_name, stream,
                                extra=None, verify_hash=False, headers=None,
                                storage_class=None):
          """
          Uploads an object using the S3 multipart algorithm.
  
          :param container: The destination container
          :type container: :class:`Container`
  
          :param object_name: The name of the object which we are uploading
          :type object_name: ``str``
  
          :param stream: The generator for fetching the upload data
          :type stream: ``generator``
  
          :keyword verify_hash: Indicates if we must calculate the data hash
          :type verify_hash: ``bool``
  
          :keyword extra: Additional options
          :type extra: ``dict``
  
          :keyword headers: Additional headers
          :type headers: ``dict``
  
          :keyword storage_class: The name of the S3 object's storage class
          :type extra: ``str``
  
          :return: The uploaded object
          :rtype: :class:`Object`
          """
          headers = headers or {}
          extra = extra or {}
  
          headers.update(self._to_storage_class_headers(storage_class))
  
          content_type = extra.get('content_type', None)
          meta_data = extra.get('meta_data', None)
          acl = extra.get('acl', None)
  
          headers['Content-Type'] = self._determine_content_type(
              content_type, object_name)
  
          if meta_data:
              for key, value in list(meta_data.items()):
                  key = self.http_vendor_prefix + '-meta-%s' % (key)
                  headers[key] = value
  
          if acl:
              headers[self.http_vendor_prefix + '-acl'] = acl
  
          upload_id = self._initiate_multipart(container, object_name,
                                               headers=headers)
  
          try:
              result = self._upload_multipart_chunks(container, object_name,
                                                     upload_id, stream,
                                                     calculate_hash=verify_hash)
              chunks, data_hash, bytes_transferred = result
  
              # Commit the chunk info and complete the upload
              etag = self._commit_multipart(container, object_name, upload_id,
                                            chunks)
          except Exception:
              # Amazon provides a mechanism for aborting an upload.
              self._abort_multipart(container, object_name, upload_id)
              raise
  
          return Object(
              name=object_name, size=bytes_transferred, hash=etag,
              extra={'acl': acl}, meta_data=meta_data, container=container,
              driver=self)
  
      def _to_storage_class_headers(self, storage_class):
          """
          Generates request headers given a storage class name.
  
          :keyword storage_class: The name of the S3 object's storage class
          :type extra: ``str``
  
          :return: Headers to include in a request
          :rtype: :dict:
          """
          headers = {}
          storage_class = storage_class or 'standard'
          if storage_class not in ['standard', 'reduced_redundancy']:
              raise ValueError(
                  'Invalid storage class value: %s' % (storage_class))
  
          key = self.http_vendor_prefix + '-storage-class'
          headers[key] = storage_class.upper()
          return headers
  
      def _to_containers(self, obj, xpath):
          for element in obj.findall(fixxpath(xpath=xpath,
                                              namespace=self.namespace)):
              yield self._to_container(element)
  
      def _to_objs(self, obj, xpath, container):
          return [self._to_obj(element, container) for element in
                  obj.findall(fixxpath(xpath=xpath, namespace=self.namespace))]
  
      def _to_container(self, element):
          extra = {
              'creation_date': findtext(element=element, xpath='CreationDate',
                                        namespace=self.namespace)
          }
  
          container = Container(name=findtext(element=element, xpath='Name',
                                              namespace=self.namespace),
                                extra=extra,
                                driver=self
                                )
  
          return container
  
      def _headers_to_object(self, object_name, container, headers):
          hash = headers['etag'].replace('"', '')
          extra = {'content_type': headers['content-type'],
                   'etag': headers['etag']}
          meta_data = {}
  
          if 'last-modified' in headers:
              extra['last_modified'] = headers['last-modified']
  
          for key, value in headers.items():
              if not key.lower().startswith(self.http_vendor_prefix + '-meta-'):
                  continue
  
              key = key.replace(self.http_vendor_prefix + '-meta-', '')
              meta_data[key] = value
  
          obj = Object(name=object_name, size=headers['content-length'],
                       hash=hash, extra=extra,
                       meta_data=meta_data,
                       container=container,
                       driver=self)
          return obj
  
      def _to_obj(self, element, container):
          owner_id = findtext(element=element, xpath='Owner/ID',
                              namespace=self.namespace)
          owner_display_name = findtext(element=element,
                                        xpath='Owner/DisplayName',
                                        namespace=self.namespace)
          meta_data = {'owner': {'id': owner_id,
                                 'display_name': owner_display_name}}
          last_modified = findtext(element=element,
                                   xpath='LastModified',
                                   namespace=self.namespace)
          extra = {'last_modified': last_modified}
  
          obj = Object(name=findtext(element=element, xpath='Key',
                                     namespace=self.namespace),
                       size=int(findtext(element=element, xpath='Size',
                                         namespace=self.namespace)),
                       hash=findtext(element=element, xpath='ETag',
                                     namespace=self.namespace).replace('"', ''),
                       extra=extra,
                       meta_data=meta_data,
                       container=container,
                       driver=self
                       )
  
          return obj
  
  
  class S3StorageDriver(AWSDriver, BaseS3StorageDriver):
      name = 'Amazon S3'
      connectionCls = S3SignatureV4Connection
      region_name = 'us-east-1'
  
      def __init__(self, key, secret=None, secure=True, host=None, port=None,
                   region=None, token=None, **kwargs):
          # Here for backward compatibility for old and deprecated driver class
          # per region approach
          if hasattr(self, 'region_name') and not region:
              region = self.region_name  # pylint: disable=no-member
  
          self.region_name = region
  
          if region and region not in REGION_TO_HOST_MAP.keys():
              raise ValueError('Invalid or unsupported region: %s' % (region))
  
          self.name = 'Amazon S3 (%s)' % (region)
  
          if host is None:
              host = REGION_TO_HOST_MAP[region]
  
          super(S3StorageDriver, self).__init__(key=key, secret=secret,
                                                secure=secure, host=host,
                                                port=port,
                                                region=region, token=token,
                                                **kwargs)
  
      @classmethod
      def list_regions(self):
          return REGION_TO_HOST_MAP.keys()
  
  
  class S3USEast2Connection(S3SignatureV4Connection):
      host = S3_US_EAST2_HOST
  
  
  class S3USEast2StorageDriver(S3StorageDriver):
      name = 'Amazon S3 (us-east-2)'
      connectionCls = S3USEast2Connection
      ex_location_name = 'us-east-2'
      region_name = 'us-east-2'
  
  
  class S3USWestConnection(S3SignatureV4Connection):
      host = S3_US_WEST_HOST
  
  
  class S3USWestStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (us-west-1)'
      connectionCls = S3USWestConnection
      ex_location_name = 'us-west-1'
      region_name = 'us-west-1'
  
  
  class S3USWestOregonConnection(S3SignatureV4Connection):
      host = S3_US_WEST_OREGON_HOST
  
  
  class S3USWestOregonStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (us-west-2)'
      connectionCls = S3USWestOregonConnection
      ex_location_name = 'us-west-2'
      region_name = 'us-west-2'
  
  
  class S3USGovWestConnection(S3SignatureV4Connection):
      host = S3_US_GOV_WEST_HOST
  
  
  class S3USGovWestStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (us-gov-west-1)'
      connectionCls = S3USGovWestConnection
      ex_location_name = 'us-gov-west-1'
      region_name = 'us-gov-west-1'
  
  
  class S3CNNorthWestConnection(S3SignatureV4Connection):
      host = S3_CN_NORTHWEST_HOST
  
  
  class S3CNNorthWestStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (cn-northwest-1)'
      connectionCls = S3CNNorthWestConnection
      ex_location_name = 'cn-northwest-1'
      region_name = 'cn-northwest-1'
  
  
  class S3CNNorthConnection(S3SignatureV4Connection):
      host = S3_CN_NORTH_HOST
  
  
  class S3CNNorthStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (cn-north-1)'
      connectionCls = S3CNNorthConnection
      ex_location_name = 'cn-north-1'
      region_name = 'cn-north-1'
  
  
  class S3EUWestConnection(S3SignatureV4Connection):
      host = S3_EU_WEST_HOST
  
  
  class S3EUWestStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (eu-west-1)'
      connectionCls = S3EUWestConnection
      ex_location_name = 'EU'
      region_name = 'eu-west-1'
  
  
  class S3EUWest2Connection(S3SignatureV4Connection):
      host = S3_EU_WEST2_HOST
  
  
  class S3EUWest2StorageDriver(S3StorageDriver):
      name = 'Amazon S3 (eu-west-2)'
      connectionCls = S3EUWest2Connection
      ex_location_name = 'eu-west-2'
      region_name = 'eu-west-2'
  
  
  class S3EUCentralConnection(S3SignatureV4Connection):
      host = S3_EU_CENTRAL_HOST
  
  
  class S3EUCentralStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (eu-central-1)'
      connectionCls = S3EUCentralConnection
      ex_location_name = 'eu-central-1'
      region_name = 'eu-central-1'
  
  
  class S3APSEConnection(S3SignatureV4Connection):
      host = S3_AP_SOUTHEAST_HOST
  
  
  class S3EUNorth1Connection(S3SignatureV4Connection):
      host = S3_EU_NORTH1_HOST
  
  
  class S3EUNorth1StorageDriver(S3StorageDriver):
      name = 'Amazon S3 (eu-north-1)'
      connectionCls = S3EUNorth1Connection
      ex_location_name = 'eu-north-1'
      region_name = 'eu-north-1'
  
  
  class S3APSEStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (ap-southeast-1)'
      connectionCls = S3APSEConnection
      ex_location_name = 'ap-southeast-1'
      region_name = 'ap-southeast-1'
  
  
  class S3APSE2Connection(S3SignatureV4Connection):
      host = S3_AP_SOUTHEAST2_HOST
  
  
  class S3APSE2StorageDriver(S3StorageDriver):
      name = 'Amazon S3 (ap-southeast-2)'
      connectionCls = S3APSE2Connection
      ex_location_name = 'ap-southeast-2'
      region_name = 'ap-southeast-2'
  
  
  class S3APNE1Connection(S3SignatureV4Connection):
      host = S3_AP_NORTHEAST1_HOST
  
  
  S3APNEConnection = S3APNE1Connection
  
  
  class S3APNE1StorageDriver(S3StorageDriver):
      name = 'Amazon S3 (ap-northeast-1)'
      connectionCls = S3APNEConnection
      ex_location_name = 'ap-northeast-1'
      region_name = 'ap-northeast-1'
  
  
  S3APNEStorageDriver = S3APNE1StorageDriver
  
  
  class S3APNE2Connection(S3SignatureV4Connection):
      host = S3_AP_NORTHEAST2_HOST
  
  
  class S3APNE2StorageDriver(S3StorageDriver):
      name = 'Amazon S3 (ap-northeast-2)'
      connectionCls = S3APNE2Connection
      ex_location_name = 'ap-northeast-2'
      region_name = 'ap-northeast-2'
  
  
  class S3APSouthConnection(S3SignatureV4Connection):
      host = S3_AP_SOUTH_HOST
  
  
  class S3APSouthStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (ap-south-1)'
      connectionCls = S3APSouthConnection
      ex_location_name = 'ap-south-1'
      region_name = 'ap-south-1'
  
  
  class S3SAEastConnection(S3SignatureV4Connection):
      host = S3_SA_EAST_HOST
  
  
  class S3SAEastStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (sa-east-1)'
      connectionCls = S3SAEastConnection
      ex_location_name = 'sa-east-1'
      region_name = 'sa-east-1'
  
  
  class S3CACentralConnection(S3SignatureV4Connection):
      host = S3_CA_CENTRAL_HOST
  
  
  class S3CACentralStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (ca-central-1)'
      connectionCls = S3CACentralConnection
      ex_location_name = 'ca-central-1'
      region_name = 'ca-central-1'
  
files: libcloud/storage/drivers/s3.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  from typing import Optional
  
  import base64
  import hmac
  import time
  from hashlib import sha1
  
  import libcloud.utils.py3
  
  try:
      if libcloud.utils.py3.DEFAULT_LXML:
          from lxml.etree import Element, SubElement
      else:
          from xml.etree.ElementTree import Element, SubElement
  except ImportError:
      from xml.etree.ElementTree import Element, SubElement
  
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import urlquote
  from libcloud.utils.py3 import b
  from libcloud.utils.py3 import tostring
  
  from libcloud.utils.xml import fixxpath, findtext
  from libcloud.utils.files import read_in_chunks
  from libcloud.common.types import InvalidCredsError, LibcloudError
  from libcloud.common.base import ConnectionUserAndKey, RawResponse
  from libcloud.common.aws import AWSBaseResponse, AWSDriver, \
      AWSTokenConnection, SignedAWSConnection
  
  from libcloud.storage.base import Object, Container, StorageDriver
  from libcloud.storage.types import ContainerError
  from libcloud.storage.types import ContainerIsNotEmptyError
  from libcloud.storage.types import InvalidContainerNameError
  from libcloud.storage.types import ContainerDoesNotExistError
  from libcloud.storage.types import ObjectDoesNotExistError
  from libcloud.storage.types import ObjectHashMismatchError
  
  # How long before the token expires
  EXPIRATION_SECONDS = 15 * 60
  
  S3_US_STANDARD_HOST = 's3.amazonaws.com'
  S3_US_EAST2_HOST = 's3-us-east-2.amazonaws.com'
  S3_US_WEST_HOST = 's3-us-west-1.amazonaws.com'
  S3_US_WEST_OREGON_HOST = 's3-us-west-2.amazonaws.com'
  S3_US_GOV_WEST_HOST = 's3-us-gov-west-1.amazonaws.com'
  S3_CN_NORTH_HOST = 's3.cn-north-1.amazonaws.com.cn'
  S3_CN_NORTHWEST_HOST = 's3.cn-northwest-1.amazonaws.com.cn'
  S3_EU_WEST_HOST = 's3-eu-west-1.amazonaws.com'
  S3_EU_WEST2_HOST = 's3-eu-west-2.amazonaws.com'
  S3_EU_CENTRAL_HOST = 's3-eu-central-1.amazonaws.com'
  S3_EU_NORTH1_HOST = 's3-eu-north-1.amazonaws.com'
  S3_AP_SOUTH_HOST = 's3-ap-south-1.amazonaws.com'
  S3_AP_SOUTHEAST_HOST = 's3-ap-southeast-1.amazonaws.com'
  S3_AP_SOUTHEAST2_HOST = 's3-ap-southeast-2.amazonaws.com'
  S3_AP_NORTHEAST1_HOST = 's3-ap-northeast-1.amazonaws.com'
  S3_AP_NORTHEAST2_HOST = 's3-ap-northeast-2.amazonaws.com'
  S3_AP_NORTHEAST_HOST = S3_AP_NORTHEAST1_HOST
  S3_SA_EAST_HOST = 's3-sa-east-1.amazonaws.com'
  S3_SA_SOUTHEAST2_HOST = 's3-sa-east-2.amazonaws.com'
  S3_CA_CENTRAL_HOST = 's3-ca-central-1.amazonaws.com'
  
  # Maps AWS region name to connection hostname
  REGION_TO_HOST_MAP = {
      'us-east-1': S3_US_STANDARD_HOST,
      'us-east-2': S3_US_EAST2_HOST,
      'us-west-1': S3_US_WEST_HOST,
      'us-west-2': S3_US_WEST_OREGON_HOST,
      'us-gov-west-1': S3_US_GOV_WEST_HOST,
      'cn-north-1': S3_CN_NORTH_HOST,
      'cn-northwest-1': S3_CN_NORTHWEST_HOST,
      'eu-west-1': S3_EU_WEST_HOST,
      'eu-west-2': S3_EU_WEST2_HOST,
      'eu-west-3': 's3.eu-west-3.amazonaws.com',
      'eu-north-1': 's3.eu-north-1.amazonaws.com',
      'eu-central-1': S3_EU_CENTRAL_HOST,
      'ap-south-1': S3_AP_SOUTH_HOST,
      'ap-southeast-1': S3_AP_SOUTHEAST_HOST,
      'ap-southeast-2': S3_AP_SOUTHEAST2_HOST,
      'ap-northeast-1': S3_AP_NORTHEAST1_HOST,
      'ap-northeast-2': S3_AP_NORTHEAST2_HOST,
      'ap-northeast-3': 's3.ap-northeast-3.amazonaws.com',
      'sa-east-1': S3_SA_EAST_HOST,
      'sa-east-2': S3_SA_SOUTHEAST2_HOST,
      'ca-central-1': S3_CA_CENTRAL_HOST,
      'me-south-1': 's3.me-south-1.amazonaws.com'
  }
  
  API_VERSION = '2006-03-01'
  NAMESPACE = 'http://s3.amazonaws.com/doc/%s/' % (API_VERSION)
  
  # AWS multi-part chunks must be minimum 5MB
  CHUNK_SIZE = 5 * 1024 * 1024
  
  # Desired number of items in each response inside a paginated request in
  # ex_iterate_multipart_uploads.
  RESPONSES_PER_REQUEST = 100
  
  
  class S3Response(AWSBaseResponse):
      namespace = None
      valid_response_codes = [httplib.NOT_FOUND, httplib.CONFLICT,
                              httplib.BAD_REQUEST, httplib.PARTIAL_CONTENT]
  
      def success(self):
          i = int(self.status)
          return 200 <= i <= 299 or i in self.valid_response_codes
  
      def parse_error(self):
          if self.status in [httplib.UNAUTHORIZED, httplib.FORBIDDEN]:
              raise InvalidCredsError(self.body)
          elif self.status == httplib.MOVED_PERMANENTLY:
              bucket_region = self.headers.get('x-amz-bucket-region', None)
              used_region = self.connection.driver.region
              raise LibcloudError('This bucket is located in a different '
                                  'region. Please use the correct driver. '
                                  'Bucket region "%s", used region "%s".' %
                                  (bucket_region, used_region),
                                  driver=S3StorageDriver)
          raise LibcloudError('Unknown error. Status code: %d' % (self.status),
                              driver=S3StorageDriver)
  
  
  class S3RawResponse(S3Response, RawResponse):
      pass
  
  
  class BaseS3Connection(ConnectionUserAndKey):
      """
      Represents a single connection to the S3 Endpoint
      """
  
      host = 's3.amazonaws.com'
      responseCls = S3Response
      rawResponseCls = S3RawResponse
  
      @staticmethod
      def get_auth_signature(method, headers, params, expires, secret_key, path,
                             vendor_prefix):
          """
          Signature = URL-Encode( Base64( HMAC-SHA1( YourSecretAccessKeyID,
                                      UTF-8-Encoding-Of( StringToSign ) ) ) );
  
          StringToSign = HTTP-VERB + "\n" +
              Content-MD5 + "\n" +
              Content-Type + "\n" +
              Expires + "\n" +
              CanonicalizedVendorHeaders +
              CanonicalizedResource;
          """
          special_headers = {'content-md5': '', 'content-type': '', 'date': ''}
          vendor_headers = {}
  
          for key, value in list(headers.items()):
              key_lower = key.lower()
              if key_lower in special_headers:
                  special_headers[key_lower] = value.strip()
              elif key_lower.startswith(vendor_prefix):
                  vendor_headers[key_lower] = value.strip()
  
          if expires:
              special_headers['date'] = str(expires)
  
          buf = [method]
          for _, value in sorted(special_headers.items()):
              buf.append(value)
          string_to_sign = '\n'.join(buf)
  
          buf = []
          for key, value in sorted(vendor_headers.items()):
              buf.append('%s:%s' % (key, value))
          header_string = '\n'.join(buf)
  
          values_to_sign = []
          for value in [string_to_sign, header_string, path]:
              if value:
                  values_to_sign.append(value)
  
          string_to_sign = '\n'.join(values_to_sign)
          b64_hmac = base64.b64encode(
              hmac.new(b(secret_key), b(string_to_sign), digestmod=sha1).digest()
          )
          return b64_hmac.decode('utf-8')
  
      def add_default_params(self, params):
          expires = str(int(time.time()) + EXPIRATION_SECONDS)
          params['AWSAccessKeyId'] = self.user_id
          params['Expires'] = expires
          return params
  
      def pre_connect_hook(self, params, headers):
          # pylint: disable=no-member
          params['Signature'] = self.get_auth_signature(
              method=self.method, headers=headers, params=params,
              expires=params['Expires'], secret_key=self.key, path=self.action,
              vendor_prefix=self.driver.http_vendor_prefix)
          return params, headers
  
  
  class S3Connection(AWSTokenConnection, BaseS3Connection):
      """
      Represents a single connection to the S3 endpoint, with AWS-specific
      features.
      """
      pass
  
  
  class S3SignatureV4Connection(SignedAWSConnection, BaseS3Connection):
      service_name = 's3'
      version = API_VERSION
  
      def __init__(self, user_id, key, secure=True, host=None, port=None,
                   url=None, timeout=None, proxy_url=None, token=None,
                   retry_delay=None, backoff=None):
          super(S3SignatureV4Connection, self).__init__(
              user_id, key, secure, host,
              port, url, timeout, proxy_url,
              token, retry_delay, backoff,
              4)  # force version 4
  
  
  class S3MultipartUpload(object):
      """
      Class representing an amazon s3 multipart upload
      """
  
      def __init__(self, key, id, created_at, initiator, owner):
          """
          Class representing an amazon s3 multipart upload
  
          :param key: The object/key that was being uploaded
          :type key: ``str``
  
          :param id: The upload id assigned by amazon
          :type id: ``str``
  
          :param created_at: The date/time at which the upload was started
          :type created_at: ``str``
  
          :param initiator: The AWS owner/IAM user who initiated this
          :type initiator: ``str``
  
          :param owner: The AWS owner/IAM who will own this object
          :type owner: ``str``
          """
          self.key = key
          self.id = id
          self.created_at = created_at
          self.initiator = initiator
          self.owner = owner
  
      def __repr__(self):
          return ('<S3MultipartUpload: key=%s>' % (self.key))
  
  
  class BaseS3StorageDriver(StorageDriver):
      name = 'Amazon S3 (standard)'
      website = 'http://aws.amazon.com/s3/'
      connectionCls = BaseS3Connection
      hash_type = 'md5'
      supports_chunked_encoding = False
      supports_s3_multipart_upload = True
      ex_location_name = ''
      namespace = NAMESPACE
      http_vendor_prefix = 'x-amz'
  
      def iterate_containers(self):
          response = self.connection.request('/')
          if response.status == httplib.OK:
              containers = self._to_containers(obj=response.object,
                                               xpath='Buckets/Bucket')
              return containers
  
          raise LibcloudError('Unexpected status code: %s' % (response.status),
                              driver=self)
  
      def iterate_container_objects(self, container, prefix=None,
                                    ex_prefix=None):
          """
          Return a generator of objects for the given container.
  
          :param container: Container instance
          :type container: :class:`Container`
  
          :param prefix: Only return objects starting with prefix
          :type prefix: ``str``
  
          :param ex_prefix: Only return objects starting with ex_prefix
          :type ex_prefix: ``str``
  
          :return: A generator of Object instances.
          :rtype: ``generator`` of :class:`Object`
          """
          prefix = self._normalize_prefix_argument(prefix, ex_prefix)
  
          params = {}
  
          if prefix:
              params['prefix'] = prefix
  
          last_key = None
          exhausted = False
          container_path = self._get_container_path(container)
  
          while not exhausted:
              if last_key:
                  params['marker'] = last_key
  
              response = self.connection.request(container_path,
                                                 params=params)
  
              if response.status != httplib.OK:
                  raise LibcloudError('Unexpected status code: %s' %
                                      (response.status), driver=self)
  
              objects = self._to_objs(obj=response.object,
                                      xpath='Contents', container=container)
              is_truncated = response.object.findtext(fixxpath(
                  xpath='IsTruncated', namespace=self.namespace)).lower()
              exhausted = (is_truncated == 'false')
  
              last_key = None
              for obj in objects:
                  last_key = obj.name
                  yield obj
  
      def get_container(self, container_name):
          try:
              response = self.connection.request('/%s' % container_name,
                                                 method='HEAD')
              if response.status == httplib.NOT_FOUND:
                  raise ContainerDoesNotExistError(value=None, driver=self,
                                                   container_name=container_name)
          except InvalidCredsError:
              # This just means the user doesn't have IAM permissions to do a
              # HEAD request but other requests might work.
              pass
          return Container(name=container_name, extra=None, driver=self)
  
      def get_object(self, container_name, object_name):
          container = self.get_container(container_name=container_name)
          object_path = self._get_object_path(container, object_name)
          response = self.connection.request(object_path, method='HEAD')
  
          if response.status == httplib.OK:
              obj = self._headers_to_object(object_name=object_name,
                                            container=container,
                                            headers=response.headers)
              return obj
  
          raise ObjectDoesNotExistError(value=None, driver=self,
                                        object_name=object_name)
  
      def _get_container_path(self, container):
          """
          Return a container path
  
          :param container: Container instance
          :type  container: :class:`Container`
  
          :return: A path for this container.
          :rtype: ``str``
          """
          return '/%s' % (container.name)
  
      def _get_object_path(self, container, object_name):
          """
          Return an object's CDN path.
  
          :param container: Container instance
          :type  container: :class:`Container`
  
          :param object_name: Object name
          :type  object_name: :class:`str`
  
          :return: A  path for this object.
          :rtype: ``str``
          """
          container_url = self._get_container_path(container)
          object_name_cleaned = self._clean_object_name(object_name)
          object_path = '%s/%s' % (container_url, object_name_cleaned)
          return object_path
  
      def create_container(self, container_name):
          if self.ex_location_name:
              root = Element('CreateBucketConfiguration')
              child = SubElement(root, 'LocationConstraint')
              child.text = self.ex_location_name
  
              data = tostring(root)
          else:
              data = ''
  
          response = self.connection.request('/%s' % (container_name),
                                             data=data,
                                             method='PUT')
  
          if response.status == httplib.OK:
              container = Container(name=container_name, extra=None, driver=self)
              return container
          elif response.status == httplib.CONFLICT:
              raise InvalidContainerNameError(
                  value='Container with this name already exists. The name must '
                        'be unique among all the containers in the system',
                  container_name=container_name, driver=self)
          elif response.status == httplib.BAD_REQUEST:
              raise ContainerError(
                  value='Bad request when creating container: %s' %
                        response.body,
                  container_name=container_name, driver=self)
  
          raise LibcloudError('Unexpected status code: %s' % (response.status),
                              driver=self)
  
      def delete_container(self, container):
          # Note: All the objects in the container must be deleted first
          response = self.connection.request('/%s' % (container.name),
                                             method='DELETE')
          if response.status == httplib.NO_CONTENT:
              return True
          elif response.status == httplib.CONFLICT:
              raise ContainerIsNotEmptyError(
                  value='Container must be empty before it can be deleted.',
                  container_name=container.name, driver=self)
          elif response.status == httplib.NOT_FOUND:
              raise ContainerDoesNotExistError(value=None,
                                               driver=self,
                                               container_name=container.name)
  
          return False
  
      def download_object(self, obj, destination_path, overwrite_existing=False,
                          delete_on_failure=True):
          obj_path = self._get_object_path(obj.container, obj.name)
  
          response = self.connection.request(obj_path, method='GET', raw=True)
  
          return self._get_object(obj=obj, callback=self._save_object,
                                  response=response,
                                  callback_kwargs={
                                      'obj': obj,
                                      'response': response.response,
                                      'destination_path': destination_path,
                                      'overwrite_existing': overwrite_existing,
                                      'delete_on_failure': delete_on_failure},
                                  success_status_code=httplib.OK)
  
      def download_object_as_stream(self, obj, chunk_size=None):
          obj_path = self._get_object_path(obj.container, obj.name)
          response = self.connection.request(obj_path, method='GET',
                                             stream=True, raw=True)
  
          return self._get_object(
              obj=obj, callback=read_in_chunks,
              response=response,
              callback_kwargs={'iterator': response.iter_content(CHUNK_SIZE),
                               'chunk_size': chunk_size},
              success_status_code=httplib.OK)
  
      def download_object_range(self, obj, destination_path, start_bytes,
                                end_bytes=None, overwrite_existing=False,
                                delete_on_failure=True):
          self._validate_start_and_end_bytes(start_bytes=start_bytes,
                                             end_bytes=end_bytes)
  
          obj_path = self._get_object_path(obj.container, obj.name)
  
          headers = {'Range': self._get_range_str(start_bytes, end_bytes)}
          response = self.connection.request(obj_path, method='GET',
                                             headers=headers, raw=True)
  
          return self._get_object(obj=obj, callback=self._save_object,
                                  response=response,
                                  callback_kwargs={
                                      'obj': obj,
                                      'response': response.response,
                                      'destination_path': destination_path,
                                      'overwrite_existing': overwrite_existing,
                                      'delete_on_failure': delete_on_failure,
                                      'partial_download': True},
                                  success_status_code=httplib.PARTIAL_CONTENT)
  
      def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None,
                                          chunk_size=None):
          self._validate_start_and_end_bytes(start_bytes=start_bytes,
                                             end_bytes=end_bytes)
  
          obj_path = self._get_object_path(obj.container, obj.name)
  
          headers = {'Range': self._get_range_str(start_bytes, end_bytes)}
          response = self.connection.request(obj_path, method='GET',
                                             headers=headers,
                                             stream=True, raw=True)
  
          return self._get_object(
              obj=obj, callback=read_in_chunks,
              response=response,
              callback_kwargs={'iterator': response.iter_content(CHUNK_SIZE),
                               'chunk_size': chunk_size},
              success_status_code=httplib.PARTIAL_CONTENT)
  
      def upload_object(self, file_path, container, object_name, extra=None,
                        verify_hash=True, headers=None, ex_storage_class=None):
          """
          @inherits: :class:`StorageDriver.upload_object`
  
          :param ex_storage_class: Storage class
          :type ex_storage_class: ``str``
          """
          return self._put_object(container=container, object_name=object_name,
                                  extra=extra, file_path=file_path,
                                  verify_hash=verify_hash,
                                  headers=headers,
                                  storage_class=ex_storage_class)
  
      def _initiate_multipart(self, container, object_name, headers=None):
          """
          Initiates a multipart upload to S3
  
          :param container: The destination container
          :type container: :class:`Container`
  
          :param object_name: The name of the object which we are uploading
          :type object_name: ``str``
  
          :keyword headers: Additional headers to send with the request
          :type headers: ``dict``
  
          :return: The id of the newly created multipart upload
          :rtype: ``str``
          """
          headers = headers or {}
  
          request_path = self._get_object_path(container, object_name)
          params = {'uploads': ''}
  
          response = self.connection.request(request_path, method='POST',
                                             headers=headers, params=params)
  
          if response.status != httplib.OK:
              raise LibcloudError('Error initiating multipart upload',
                                  driver=self)
  
          return findtext(element=response.object, xpath='UploadId',
                          namespace=self.namespace)
  
      def _upload_multipart_chunks(self, container, object_name, upload_id,
                                   stream, calculate_hash=True):
          """
          Uploads data from an iterator in fixed sized chunks to S3
  
          :param container: The destination container
          :type container: :class:`Container`
  
          :param object_name: The name of the object which we are uploading
          :type object_name: ``str``
  
          :param upload_id: The upload id allocated for this multipart upload
          :type upload_id: ``str``
  
          :param stream: The generator for fetching the upload data
          :type stream: ``generator``
  
          :keyword calculate_hash: Indicates if we must calculate the data hash
          :type calculate_hash: ``bool``
  
          :return: A tuple of (chunk info, checksum, bytes transferred)
          :rtype: ``tuple``
          """
          data_hash = None
          if calculate_hash:
              data_hash = self._get_hash_function()
  
          bytes_transferred = 0
          count = 1
          chunks = []
          params = {'uploadId': upload_id}
  
          request_path = self._get_object_path(container, object_name)
  
          # Read the input data in chunk sizes suitable for AWS
          for data in read_in_chunks(stream, chunk_size=CHUNK_SIZE,
                                     fill_size=True, yield_empty=True):
              bytes_transferred += len(data)
  
              if calculate_hash:
                  data_hash.update(data)
  
              chunk_hash = self._get_hash_function()
              chunk_hash.update(data)
              chunk_hash = base64.b64encode(chunk_hash.digest()).decode('utf-8')
  
              # The Content-MD5 header provides an extra level of data check and
              # is recommended by amazon
              headers = {
                  'Content-Length': len(data),
                  'Content-MD5': chunk_hash,
              }
  
              params['partNumber'] = count
  
              resp = self.connection.request(request_path, method='PUT',
                                             data=data, headers=headers,
                                             params=params)
  
              if resp.status != httplib.OK:
                  raise LibcloudError('Error uploading chunk', driver=self)
  
              server_hash = resp.headers['etag'].replace('"', '')
  
              # Keep this data for a later commit
              chunks.append((count, server_hash))
              count += 1
  
          if calculate_hash:
              data_hash = data_hash.hexdigest()
  
          return (chunks, data_hash, bytes_transferred)
  
      def _commit_multipart(self, container, object_name, upload_id, chunks):
          """
          Makes a final commit of the data.
  
          :param container: The destination container
          :type container: :class:`Container`
  
          :param object_name: The name of the object which we are uploading
          :type object_name: ``str``
  
          :param upload_id: The upload id allocated for this multipart upload
          :type upload_id: ``str``
  
          :param chunks: A list of (chunk_number, chunk_hash) tuples.
          :type chunks: ``list``
  
          :return: The server side hash of the uploaded data
          :rtype: ``str``
          """
          root = Element('CompleteMultipartUpload')
  
          for (count, etag) in chunks:
              part = SubElement(root, 'Part')
              part_no = SubElement(part, 'PartNumber')
              part_no.text = str(count)
  
              etag_id = SubElement(part, 'ETag')
              etag_id.text = str(etag)
  
          data = tostring(root)
  
          headers = {'Content-Length': len(data)}
          params = {'uploadId': upload_id}
          request_path = self._get_object_path(container, object_name)
          response = self.connection.request(request_path, headers=headers,
                                             params=params, data=data,
                                             method='POST')
  
          if response.status != httplib.OK:
              element = response.object
              # pylint: disable=maybe-no-member
              code, message = response._parse_error_details(element=element)
              msg = 'Error in multipart commit: %s (%s)' % (message, code)
              raise LibcloudError(msg, driver=self)
  
          # Get the server's etag to be passed back to the caller
          body = response.parse_body()
          server_hash = body.find(fixxpath(xpath='ETag',
                                           namespace=self.namespace)).text
          return server_hash
  
      def _abort_multipart(self, container, object_name, upload_id):
          """
          Aborts an already initiated multipart upload
  
          :param container: The destination container
          :type container: :class:`Container`
  
          :param object_name: The name of the object which we are uploading
          :type object_name: ``str``
  
          :param upload_id: The upload id allocated for this multipart upload
          :type upload_id: ``str``
          """
  
          params = {'uploadId': upload_id}
          request_path = self._get_object_path(container, object_name)
  
          resp = self.connection.request(request_path, method='DELETE',
                                         params=params)
  
          if resp.status != httplib.NO_CONTENT:
              raise LibcloudError('Error in multipart abort. status_code=%d' %
                                  (resp.status), driver=self)
  
      def upload_object_via_stream(self, iterator, container, object_name,
                                   extra=None, headers=None,
                                   ex_storage_class=None):
          """
          @inherits: :class:`StorageDriver.upload_object_via_stream`
  
          :param ex_storage_class: Storage class
          :type ex_storage_class: ``str``
          """
  
          method = 'PUT'
          params = None
  
          # This driver is used by other S3 API compatible drivers also.
          # Amazon provides a different (complex?) mechanism to do multipart
          # uploads
          if self.supports_s3_multipart_upload:
              return self._put_object_multipart(container=container,
                                                object_name=object_name,
                                                extra=extra,
                                                stream=iterator,
                                                verify_hash=False,
                                                headers=headers,
                                                storage_class=ex_storage_class)
          return self._put_object(container=container, object_name=object_name,
                                  extra=extra, method=method, query_args=params,
                                  stream=iterator, verify_hash=False,
                                  headers=headers,
                                  storage_class=ex_storage_class)
  
      def delete_object(self, obj):
          object_path = self._get_object_path(obj.container, obj.name)
          response = self.connection.request(object_path, method='DELETE')
          if response.status == httplib.NO_CONTENT:
              return True
          elif response.status == httplib.NOT_FOUND:
              raise ObjectDoesNotExistError(value=None, driver=self,
                                            object_name=obj.name)
  
          return False
  
      def ex_iterate_multipart_uploads(self, container, prefix=None,
                                       delimiter=None):
          """
          Extension method for listing all in-progress S3 multipart uploads.
  
          Each multipart upload which has not been committed or aborted is
          considered in-progress.
  
          :param container: The container holding the uploads
          :type container: :class:`Container`
  
          :keyword prefix: Print only uploads of objects with this prefix
          :type prefix: ``str``
  
          :keyword delimiter: The object/key names are grouped based on
              being split by this delimiter
          :type delimiter: ``str``
  
          :return: A generator of S3MultipartUpload instances.
          :rtype: ``generator`` of :class:`S3MultipartUpload`
          """
  
          if not self.supports_s3_multipart_upload:
              raise LibcloudError('Feature not supported', driver=self)
  
          # Get the data for a specific container
          request_path = self._get_container_path(container)
          params = {'max-uploads': RESPONSES_PER_REQUEST, 'uploads': ''}
  
          if prefix:
              params['prefix'] = prefix
  
          if delimiter:
              params['delimiter'] = delimiter
  
          def finder(node, text):
              return node.findtext(fixxpath(xpath=text,
                                            namespace=self.namespace))
  
          while True:
              response = self.connection.request(request_path, params=params)
  
              if response.status != httplib.OK:
                  raise LibcloudError('Error fetching multipart uploads. '
                                      'Got code: %s' % response.status,
                                      driver=self)
  
              body = response.parse_body()
              # pylint: disable=maybe-no-member
              for node in body.findall(fixxpath(xpath='Upload',
                                                namespace=self.namespace)):
                  initiator = node.find(fixxpath(xpath='Initiator',
                                                 namespace=self.namespace))
                  owner = node.find(fixxpath(xpath='Owner',
                                             namespace=self.namespace))
  
                  key = finder(node, 'Key')
                  upload_id = finder(node, 'UploadId')
                  created_at = finder(node, 'Initiated')
                  initiator = finder(initiator, 'DisplayName')
                  owner = finder(owner, 'DisplayName')
  
                  yield S3MultipartUpload(key, upload_id, created_at,
                                          initiator, owner)
  
              # Check if this is the last entry in the listing
              # pylint: disable=maybe-no-member
              is_truncated = body.findtext(fixxpath(xpath='IsTruncated',
                                                    namespace=self.namespace))
  
              if is_truncated.lower() == 'false':
                  break
  
              # Provide params for the next request
              upload_marker = body.findtext(fixxpath(xpath='NextUploadIdMarker',
                                                     namespace=self.namespace))
              key_marker = body.findtext(fixxpath(xpath='NextKeyMarker',
                                                  namespace=self.namespace))
  
              params['key-marker'] = key_marker
              params['upload-id-marker'] = upload_marker
  
      def ex_cleanup_all_multipart_uploads(self, container, prefix=None):
          """
          Extension method for removing all partially completed S3 multipart
          uploads.
  
          :param container: The container holding the uploads
          :type container: :class:`Container`
  
          :keyword prefix: Delete only uploads of objects with this prefix
          :type prefix: ``str``
          """
  
          # Iterate through the container and delete the upload ids
          for upload in self.ex_iterate_multipart_uploads(container, prefix,
                                                          delimiter=None):
              self._abort_multipart(container, upload.key, upload.id)
  
      def _get_range_str(self, start_bytes, end_bytes=None):
          # type: (int, Optional[int]) -> str
          """
          Return range string which is used as a Range header value for range
          requests.
          """
          range_str = 'bytes=%s-' % (start_bytes)
  
          if end_bytes:
              range_str += str(end_bytes)
  
          return range_str
  
      def _clean_object_name(self, name):
          name = urlquote(name)
          return name
  
      def _put_object(self, container, object_name, method='PUT',
                      query_args=None, extra=None, file_path=None,
                      stream=None, verify_hash=True, storage_class=None,
                      headers=None):
          headers = headers or {}
          extra = extra or {}
  
          headers.update(self._to_storage_class_headers(storage_class))
  
          content_type = extra.get('content_type', None)
          meta_data = extra.get('meta_data', None)
          acl = extra.get('acl', None)
  
          if meta_data:
              for key, value in list(meta_data.items()):
                  key = self.http_vendor_prefix + '-meta-%s' % (key)
                  headers[key] = value
  
          if acl:
              headers[self.http_vendor_prefix + '-acl'] = acl
  
          request_path = self._get_object_path(container, object_name)
  
          if query_args:
              request_path = '?'.join((request_path, query_args))
  
          result_dict = self._upload_object(
              object_name=object_name, content_type=content_type,
              request_path=request_path, request_method=method,
              headers=headers, file_path=file_path, stream=stream)
  
          response = result_dict['response']
          bytes_transferred = result_dict['bytes_transferred']
          headers = response.headers
          response = response
          server_hash = headers.get('etag', '').replace('"', '')
          server_side_encryption = headers.get('x-amz-server-side-encryption',
                                               None)
          aws_kms_encryption = (server_side_encryption == 'aws:kms')
          hash_matches = (result_dict['data_hash'] == server_hash)
  
          # NOTE: If AWS KMS server side encryption is enabled, ETag won't
          # contain object MD5 digest so we skip the checksum check
          # See https://docs.aws.amazon.com/AmazonS3/latest/API
          # /RESTCommonResponseHeaders.html
          # and https://github.com/apache/libcloud/issues/1401
          # for details
          if verify_hash and not aws_kms_encryption and not hash_matches:
              raise ObjectHashMismatchError(
                  value='MD5 hash {0} checksum does not match {1}'.format(
                      server_hash, result_dict['data_hash']),
                  object_name=object_name, driver=self)
          elif response.status == httplib.OK:
              obj = Object(
                  name=object_name, size=bytes_transferred, hash=server_hash,
                  extra={'acl': acl}, meta_data=meta_data, container=container,
                  driver=self)
  
              return obj
          else:
              raise LibcloudError(
                  'Unexpected status code, status_code=%s' % (response.status),
                  driver=self)
  
      def _put_object_multipart(self, container, object_name, stream,
                                extra=None, verify_hash=False, headers=None,
                                storage_class=None):
          """
          Uploads an object using the S3 multipart algorithm.
  
          :param container: The destination container
          :type container: :class:`Container`
  
          :param object_name: The name of the object which we are uploading
          :type object_name: ``str``
  
          :param stream: The generator for fetching the upload data
          :type stream: ``generator``
  
          :keyword verify_hash: Indicates if we must calculate the data hash
          :type verify_hash: ``bool``
  
          :keyword extra: Additional options
          :type extra: ``dict``
  
          :keyword headers: Additional headers
          :type headers: ``dict``
  
          :keyword storage_class: The name of the S3 object's storage class
          :type extra: ``str``
  
          :return: The uploaded object
          :rtype: :class:`Object`
          """
          headers = headers or {}
          extra = extra or {}
  
          headers.update(self._to_storage_class_headers(storage_class))
  
          content_type = extra.get('content_type', None)
          meta_data = extra.get('meta_data', None)
          acl = extra.get('acl', None)
  
          headers['Content-Type'] = self._determine_content_type(
              content_type, object_name)
  
          if meta_data:
              for key, value in list(meta_data.items()):
                  key = self.http_vendor_prefix + '-meta-%s' % (key)
                  headers[key] = value
  
          if acl:
              headers[self.http_vendor_prefix + '-acl'] = acl
  
          upload_id = self._initiate_multipart(container, object_name,
                                               headers=headers)
  
          try:
              result = self._upload_multipart_chunks(container, object_name,
                                                     upload_id, stream,
                                                     calculate_hash=verify_hash)
              chunks, data_hash, bytes_transferred = result
  
              # Commit the chunk info and complete the upload
              etag = self._commit_multipart(container, object_name, upload_id,
                                            chunks)
          except Exception:
              # Amazon provides a mechanism for aborting an upload.
              self._abort_multipart(container, object_name, upload_id)
              raise
  
          return Object(
              name=object_name, size=bytes_transferred, hash=etag,
              extra={'acl': acl}, meta_data=meta_data, container=container,
              driver=self)
  
      def _to_storage_class_headers(self, storage_class):
          """
          Generates request headers given a storage class name.
  
          :keyword storage_class: The name of the S3 object's storage class
          :type extra: ``str``
  
          :return: Headers to include in a request
          :rtype: :dict:
          """
          headers = {}
          storage_class = storage_class or 'standard'
          if storage_class not in ['standard', 'reduced_redundancy']:
              raise ValueError(
                  'Invalid storage class value: %s' % (storage_class))
  
          key = self.http_vendor_prefix + '-storage-class'
          headers[key] = storage_class.upper()
          return headers
  
      def _to_containers(self, obj, xpath):
          for element in obj.findall(fixxpath(xpath=xpath,
                                              namespace=self.namespace)):
              yield self._to_container(element)
  
      def _to_objs(self, obj, xpath, container):
          return [self._to_obj(element, container) for element in
                  obj.findall(fixxpath(xpath=xpath, namespace=self.namespace))]
  
      def _to_container(self, element):
          extra = {
              'creation_date': findtext(element=element, xpath='CreationDate',
                                        namespace=self.namespace)
          }
  
          container = Container(name=findtext(element=element, xpath='Name',
                                              namespace=self.namespace),
                                extra=extra,
                                driver=self
                                )
  
          return container
  
      def _headers_to_object(self, object_name, container, headers):
          hash = headers['etag'].replace('"', '')
          extra = {'content_type': headers['content-type'],
                   'etag': headers['etag']}
          meta_data = {}
  
          if 'last-modified' in headers:
              extra['last_modified'] = headers['last-modified']
  
          for key, value in headers.items():
              if not key.lower().startswith(self.http_vendor_prefix + '-meta-'):
                  continue
  
              key = key.replace(self.http_vendor_prefix + '-meta-', '')
              meta_data[key] = value
  
          obj = Object(name=object_name, size=headers['content-length'],
                       hash=hash, extra=extra,
                       meta_data=meta_data,
                       container=container,
                       driver=self)
          return obj
  
      def _to_obj(self, element, container):
          owner_id = findtext(element=element, xpath='Owner/ID',
                              namespace=self.namespace)
          owner_display_name = findtext(element=element,
                                        xpath='Owner/DisplayName',
                                        namespace=self.namespace)
          meta_data = {'owner': {'id': owner_id,
                                 'display_name': owner_display_name}}
          last_modified = findtext(element=element,
                                   xpath='LastModified',
                                   namespace=self.namespace)
          extra = {'last_modified': last_modified}
  
          obj = Object(name=findtext(element=element, xpath='Key',
                                     namespace=self.namespace),
                       size=int(findtext(element=element, xpath='Size',
                                         namespace=self.namespace)),
                       hash=findtext(element=element, xpath='ETag',
                                     namespace=self.namespace).replace('"', ''),
                       extra=extra,
                       meta_data=meta_data,
                       container=container,
                       driver=self
                       )
  
          return obj
  
  
  class S3StorageDriver(AWSDriver, BaseS3StorageDriver):
      name = 'Amazon S3'
      connectionCls = S3SignatureV4Connection
      region_name = 'us-east-1'
  
      def __init__(self, key, secret=None, secure=True, host=None, port=None,
                   region=None, token=None, **kwargs):
          # Here for backward compatibility for old and deprecated driver class
          # per region approach
          if hasattr(self, 'region_name') and not region:
              region = self.region_name  # pylint: disable=no-member
  
          self.region_name = region
  
          if region and region not in REGION_TO_HOST_MAP.keys():
              raise ValueError('Invalid or unsupported region: %s' % (region))
  
          self.name = 'Amazon S3 (%s)' % (region)
  
          if host is None:
              host = REGION_TO_HOST_MAP[region]
  
          super(S3StorageDriver, self).__init__(key=key, secret=secret,
                                                secure=secure, host=host,
                                                port=port,
                                                region=region, token=token,
                                                **kwargs)
  
      @classmethod
      def list_regions(self):
          return REGION_TO_HOST_MAP.keys()
  
  
  class S3USEast2Connection(S3SignatureV4Connection):
      host = S3_US_EAST2_HOST
  
  
  class S3USEast2StorageDriver(S3StorageDriver):
      name = 'Amazon S3 (us-east-2)'
      connectionCls = S3USEast2Connection
      ex_location_name = 'us-east-2'
      region_name = 'us-east-2'
  
  
  class S3USWestConnection(S3SignatureV4Connection):
      host = S3_US_WEST_HOST
  
  
  class S3USWestStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (us-west-1)'
      connectionCls = S3USWestConnection
      ex_location_name = 'us-west-1'
      region_name = 'us-west-1'
  
  
  class S3USWestOregonConnection(S3SignatureV4Connection):
      host = S3_US_WEST_OREGON_HOST
  
  
  class S3USWestOregonStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (us-west-2)'
      connectionCls = S3USWestOregonConnection
      ex_location_name = 'us-west-2'
      region_name = 'us-west-2'
  
  
  class S3USGovWestConnection(S3SignatureV4Connection):
      host = S3_US_GOV_WEST_HOST
  
  
  class S3USGovWestStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (us-gov-west-1)'
      connectionCls = S3USGovWestConnection
      ex_location_name = 'us-gov-west-1'
      region_name = 'us-gov-west-1'
  
  
  class S3CNNorthWestConnection(S3SignatureV4Connection):
      host = S3_CN_NORTHWEST_HOST
  
  
  class S3CNNorthWestStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (cn-northwest-1)'
      connectionCls = S3CNNorthWestConnection
      ex_location_name = 'cn-northwest-1'
      region_name = 'cn-northwest-1'
  
  
  class S3CNNorthConnection(S3SignatureV4Connection):
      host = S3_CN_NORTH_HOST
  
  
  class S3CNNorthStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (cn-north-1)'
      connectionCls = S3CNNorthConnection
      ex_location_name = 'cn-north-1'
      region_name = 'cn-north-1'
  
  
  class S3EUWestConnection(S3SignatureV4Connection):
      host = S3_EU_WEST_HOST
  
  
  class S3EUWestStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (eu-west-1)'
      connectionCls = S3EUWestConnection
      ex_location_name = 'EU'
      region_name = 'eu-west-1'
  
  
  class S3EUWest2Connection(S3SignatureV4Connection):
      host = S3_EU_WEST2_HOST
  
  
  class S3EUWest2StorageDriver(S3StorageDriver):
      name = 'Amazon S3 (eu-west-2)'
      connectionCls = S3EUWest2Connection
      ex_location_name = 'eu-west-2'
      region_name = 'eu-west-2'
  
  
  class S3EUCentralConnection(S3SignatureV4Connection):
      host = S3_EU_CENTRAL_HOST
  
  
  class S3EUCentralStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (eu-central-1)'
      connectionCls = S3EUCentralConnection
      ex_location_name = 'eu-central-1'
      region_name = 'eu-central-1'
  
  
  class S3APSEConnection(S3SignatureV4Connection):
      host = S3_AP_SOUTHEAST_HOST
  
  
  class S3EUNorth1Connection(S3SignatureV4Connection):
      host = S3_EU_NORTH1_HOST
  
  
  class S3EUNorth1StorageDriver(S3StorageDriver):
      name = 'Amazon S3 (eu-north-1)'
      connectionCls = S3EUNorth1Connection
      ex_location_name = 'eu-north-1'
      region_name = 'eu-north-1'
  
  
  class S3APSEStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (ap-southeast-1)'
      connectionCls = S3APSEConnection
      ex_location_name = 'ap-southeast-1'
      region_name = 'ap-southeast-1'
  
  
  class S3APSE2Connection(S3SignatureV4Connection):
      host = S3_AP_SOUTHEAST2_HOST
  
  
  class S3APSE2StorageDriver(S3StorageDriver):
      name = 'Amazon S3 (ap-southeast-2)'
      connectionCls = S3APSE2Connection
      ex_location_name = 'ap-southeast-2'
      region_name = 'ap-southeast-2'
  
  
  class S3APNE1Connection(S3SignatureV4Connection):
      host = S3_AP_NORTHEAST1_HOST
  
  
  S3APNEConnection = S3APNE1Connection
  
  
  class S3APNE1StorageDriver(S3StorageDriver):
      name = 'Amazon S3 (ap-northeast-1)'
      connectionCls = S3APNEConnection
      ex_location_name = 'ap-northeast-1'
      region_name = 'ap-northeast-1'
  
  
  S3APNEStorageDriver = S3APNE1StorageDriver
  
  
  class S3APNE2Connection(S3SignatureV4Connection):
      host = S3_AP_NORTHEAST2_HOST
  
  
  class S3APNE2StorageDriver(S3StorageDriver):
      name = 'Amazon S3 (ap-northeast-2)'
      connectionCls = S3APNE2Connection
      ex_location_name = 'ap-northeast-2'
      region_name = 'ap-northeast-2'
  
  
  class S3APSouthConnection(S3SignatureV4Connection):
      host = S3_AP_SOUTH_HOST
  
  
  class S3APSouthStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (ap-south-1)'
      connectionCls = S3APSouthConnection
      ex_location_name = 'ap-south-1'
      region_name = 'ap-south-1'
  
  
  class S3SAEastConnection(S3SignatureV4Connection):
      host = S3_SA_EAST_HOST
  
  
  class S3SAEastStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (sa-east-1)'
      connectionCls = S3SAEastConnection
      ex_location_name = 'sa-east-1'
      region_name = 'sa-east-1'
  
  
  class S3CACentralConnection(S3SignatureV4Connection):
      host = S3_CA_CENTRAL_HOST
  
  
  class S3CACentralStorageDriver(S3StorageDriver):
      name = 'Amazon S3 (ca-central-1)'
      connectionCls = S3CACentralConnection
      ex_location_name = 'ca-central-1'
      region_name = 'ca-central-1'
  
files: libcloud/storage/drivers/s3.py