files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  import copy
  import os
  import time
  import base64
  import hmac
  
  from hashlib import sha256
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import b
  from libcloud.utils.xml import fixxpath
  
  from libcloud.utils.py3 import ET
  from libcloud.common.types import InvalidCredsError
  from libcloud.common.types import LibcloudError, MalformedResponseError
  from libcloud.common.base import ConnectionUserAndKey, RawResponse
  from libcloud.common.base import CertificateConnection
  from libcloud.common.base import XmlResponse
  from libcloud.common.base import BaseDriver
  
  # The time format for headers in Azure requests
  AZURE_TIME_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
  
  
  class AzureRedirectException(Exception):
  
      def __init__(self, response):
          self.location = response.headers['location']
  
  
  class AzureResponse(XmlResponse):
  
      valid_response_codes = [
          httplib.NOT_FOUND,
          httplib.CONFLICT,
          httplib.BAD_REQUEST,
          # added TEMPORARY_REDIRECT as this can sometimes be
          # sent by azure instead of a success or fail response
          httplib.TEMPORARY_REDIRECT,
          # Used by Azure Blobs range downloads
          httplib.PARTIAL_CONTENT
      ]
  
      def success(self):
          i = int(self.status)
          return 200 <= i <= 299 or i in self.valid_response_codes
  
      def parse_error(self, msg=None):
          error_msg = 'Unknown error'
  
          try:
              # Azure does give some meaningful errors, but is inconsistent
              # Some APIs respond with an XML error. Others just dump HTML
              body = self.parse_body()
  
              # pylint: disable=no-member
              if type(body) == ET.Element:
                  code = body.findtext(fixxpath(xpath='Code'))
                  message = body.findtext(fixxpath(xpath='Message'))
                  message = message.split('\n')[0]
                  error_msg = '%s: %s' % (code, message)
  
          except MalformedResponseError:
              pass
  
          if msg:
              error_msg = '%s - %s' % (msg, error_msg)
  
          if self.status in [httplib.UNAUTHORIZED, httplib.FORBIDDEN]:
              raise InvalidCredsError(error_msg)
  
          raise LibcloudError(
              '%s Status code: %d.' % (error_msg, self.status),
              driver=self
          )
  
      def parse_body(self):
          is_redirect = int(self.status) == httplib.TEMPORARY_REDIRECT
  
          if is_redirect and self.connection.driver.follow_redirects:
              raise AzureRedirectException(self)
          else:
              return super(AzureResponse, self).parse_body()
  
  
  class AzureRawResponse(RawResponse):
      pass
  
  
  class AzureConnection(ConnectionUserAndKey):
      """
      Represents a single connection to Azure
      """
  
      responseCls = AzureResponse
      rawResponseCls = AzureRawResponse
  
      API_VERSION = '2012-02-12'
  
      def add_default_params(self, params):
          return params
  
      def pre_connect_hook(self, params, headers):
          headers = copy.deepcopy(headers)
  
          # We have to add a date header in GMT
          headers['x-ms-date'] = time.strftime(AZURE_TIME_FORMAT, time.gmtime())
          headers['x-ms-version'] = self.API_VERSION
  
          # Add the authorization header
          headers['Authorization'] = self._get_azure_auth_signature(
              method=self.method,
              headers=headers,
              params=params,
              account=self.user_id,
              secret_key=self.key,
              path=self.action
          )
  
          # Azure cribs about this in 'raw' connections
          headers.pop('Host', None)
  
          return params, headers
  
      def _get_azure_auth_signature(self,
                                    method,
                                    headers,
                                    params,
                                    account,
                                    secret_key,
                                    path='/'):
          """
          Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID,
                              UTF-8-Encoding-Of( StringToSign ) ) ) );
  
          StringToSign = HTTP-VERB + "\n" +
              Content-Encoding + "\n" +
              Content-Language + "\n" +
              Content-Length + "\n" +
              Content-MD5 + "\n" +
              Content-Type + "\n" +
              Date + "\n" +
              If-Modified-Since + "\n" +
              If-Match + "\n" +
              If-None-Match + "\n" +
              If-Unmodified-Since + "\n" +
              Range + "\n" +
              CanonicalizedHeaders +
              CanonicalizedResource;
          """
          xms_header_values = []
          param_list = []
  
          # Split the x-ms headers and normal headers and make everything
          # lower case
          headers_copy = {}
          for header, value in headers.items():
              header = header.lower()
              value = str(value).strip()
              if header.startswith('x-ms-'):
                  xms_header_values.append((header, value))
              else:
                  headers_copy[header] = value
  
          # Get the values for the headers in the specific order
          special_header_values = self._format_special_header_values(
              headers_copy, method)
  
          # Prepare the first section of the string to be signed
          values_to_sign = [method] + special_header_values
          # string_to_sign = '\n'.join([method] + special_header_values)
  
          # The x-ms-* headers have to be in lower case and sorted
          xms_header_values.sort()
  
          for header, value in xms_header_values:
              values_to_sign.append('%s:%s' % (header, value))
  
          # Add the canonicalized path
          values_to_sign.append('/%s%s' % (account, path))
  
          # URL query parameters (sorted and lower case)
          for key, value in params.items():
              param_list.append((key.lower(), str(value).strip()))
  
          param_list.sort()
  
          for key, value in param_list:
              values_to_sign.append('%s:%s' % (key, value))
  
          string_to_sign = b('\n'.join(values_to_sign))
          secret_key = b(secret_key)
          b64_hmac = base64.b64encode(
              hmac.new(secret_key, string_to_sign, digestmod=sha256).digest()
          )
  
          return 'SharedKey %s:%s' % (self.user_id, b64_hmac.decode('utf-8'))
  
      def _format_special_header_values(self, headers, method):
          is_change = method not in ('GET', 'HEAD')
          is_old_api = self.API_VERSION <= '2014-02-14'
  
          special_header_keys = [
              'content-encoding',
              'content-language',
              'content-length',
              'content-md5',
              'content-type',
              'date',
              'if-modified-since',
              'if-match',
              'if-none-match',
              'if-unmodified-since',
              'range'
          ]
  
          special_header_values = []
  
          for header in special_header_keys:
              header = header.lower()  # Just for safety
              if header in headers:
                  special_header_values.append(headers[header])
              elif header == 'content-length' and is_change and is_old_api:
                  # For old API versions, the Content-Length header must be '0'
                  # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key#content-length-header-in-version-2014-02-14-and-earlier
                  special_header_values.append('0')
              else:
                  special_header_values.append('')
  
          return special_header_values
  
  
  class AzureBaseDriver(BaseDriver):
      name = "Microsoft Azure Service Management API"
  
  
  class AzureServiceManagementConnection(CertificateConnection):
      # This needs the following approach -
      # 1. Make request using LibcloudHTTPSConnection which is a overloaded
      # class which takes in a client certificate
      # 2. Depending on the type of operation use a PollingConnection
      # when the response id is returned
      # 3. The Response can be used in an AzureServiceManagementResponse
  
      """
      Authentication class for "Service Account" authentication.
      """
  
      driver = AzureBaseDriver
      response
  
  [... the rest is omitted ...]
files: libcloud/common/azure.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  import copy
  import os
  import time
  import base64
  import hmac
  
  from hashlib import sha256
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import b
  from libcloud.utils.xml import fixxpath
  
  from libcloud.utils.py3 import ET
  from libcloud.common.types import InvalidCredsError
  from libcloud.common.types import LibcloudError, MalformedResponseError
  from libcloud.common.base import ConnectionUserAndKey, RawResponse
  from libcloud.common.base import CertificateConnection
  from libcloud.common.base import XmlResponse
  from libcloud.common.base import BaseDriver
  
  # The time format for headers in Azure requests
  AZURE_TIME_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
  
  
  class AzureRedirectException(Exception):
  
      def __init__(self, response):
          self.location = response.headers['location']
  
  
  class AzureResponse(XmlResponse):
  
      valid_response_codes = [
          httplib.NOT_FOUND,
          httplib.CONFLICT,
          httplib.BAD_REQUEST,
          httplib.TEMPORARY_REDIRECT
          # added TEMPORARY_REDIRECT as this can sometimes be
          # sent by azure instead of a success or fail response
      ]
  
      def success(self):
          i = int(self.status)
          return 200 <= i <= 299 or i in self.valid_response_codes
  
      def parse_error(self, msg=None):
          error_msg = 'Unknown error'
  
          try:
              # Azure does give some meaningful errors, but is inconsistent
              # Some APIs respond with an XML error. Others just dump HTML
              body = self.parse_body()
  
              # pylint: disable=no-member
              if type(body) == ET.Element:
                  code = body.findtext(fixxpath(xpath='Code'))
                  message = body.findtext(fixxpath(xpath='Message'))
                  message = message.split('\n')[0]
                  error_msg = '%s: %s' % (code, message)
  
          except MalformedResponseError:
              pass
  
          if msg:
              error_msg = '%s - %s' % (msg, error_msg)
  
          if self.status in [httplib.UNAUTHORIZED, httplib.FORBIDDEN]:
              raise InvalidCredsError(error_msg)
  
          raise LibcloudError(
              '%s Status code: %d.' % (error_msg, self.status),
              driver=self
          )
  
      def parse_body(self):
          is_redirect = int(self.status) == httplib.TEMPORARY_REDIRECT
  
          if is_redirect and self.connection.driver.follow_redirects:
              raise AzureRedirectException(self)
          else:
              return super(AzureResponse, self).parse_body()
  
  
  class AzureRawResponse(RawResponse):
      pass
  
  
  class AzureConnection(ConnectionUserAndKey):
      """
      Represents a single connection to Azure
      """
  
      responseCls = AzureResponse
      rawResponseCls = AzureRawResponse
  
      API_VERSION = '2012-02-12'
  
      def add_default_params(self, params):
          return params
  
      def pre_connect_hook(self, params, headers):
          headers = copy.deepcopy(headers)
  
          # We have to add a date header in GMT
          headers['x-ms-date'] = time.strftime(AZURE_TIME_FORMAT, time.gmtime())
          headers['x-ms-version'] = self.API_VERSION
  
          # Add the authorization header
          headers['Authorization'] = self._get_azure_auth_signature(
              method=self.method,
              headers=headers,
              params=params,
              account=self.user_id,
              secret_key=self.key,
              path=self.action
          )
  
          # Azure cribs about this in 'raw' connections
          headers.pop('Host', None)
  
          return params, headers
  
      def _get_azure_auth_signature(self,
                                    method,
                                    headers,
                                    params,
                                    account,
                                    secret_key,
                                    path='/'):
          """
          Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID,
                              UTF-8-Encoding-Of( StringToSign ) ) ) );
  
          StringToSign = HTTP-VERB + "\n" +
              Content-Encoding + "\n" +
              Content-Language + "\n" +
              Content-Length + "\n" +
              Content-MD5 + "\n" +
              Content-Type + "\n" +
              Date + "\n" +
              If-Modified-Since + "\n" +
              If-Match + "\n" +
              If-None-Match + "\n" +
              If-Unmodified-Since + "\n" +
              Range + "\n" +
              CanonicalizedHeaders +
              CanonicalizedResource;
          """
          xms_header_values = []
          param_list = []
  
          # Split the x-ms headers and normal headers and make everything
          # lower case
          headers_copy = {}
          for header, value in headers.items():
              header = header.lower()
              value = str(value).strip()
              if header.startswith('x-ms-'):
                  xms_header_values.append((header, value))
              else:
                  headers_copy[header] = value
  
          # Get the values for the headers in the specific order
          special_header_values = self._format_special_header_values(
              headers_copy, method)
  
          # Prepare the first section of the string to be signed
          values_to_sign = [method] + special_header_values
          # string_to_sign = '\n'.join([method] + special_header_values)
  
          # The x-ms-* headers have to be in lower case and sorted
          xms_header_values.sort()
  
          for header, value in xms_header_values:
              values_to_sign.append('%s:%s' % (header, value))
  
          # Add the canonicalized path
          values_to_sign.append('/%s%s' % (account, path))
  
          # URL query parameters (sorted and lower case)
          for key, value in params.items():
              param_list.append((key.lower(), str(value).strip()))
  
          param_list.sort()
  
          for key, value in param_list:
              values_to_sign.append('%s:%s' % (key, value))
  
          string_to_sign = b('\n'.join(values_to_sign))
          secret_key = b(secret_key)
          b64_hmac = base64.b64encode(
              hmac.new(secret_key, string_to_sign, digestmod=sha256).digest()
          )
  
          return 'SharedKey %s:%s' % (self.user_id, b64_hmac.decode('utf-8'))
  
      def _format_special_header_values(self, headers, method):
          is_change = method not in ('GET', 'HEAD')
          is_old_api = self.API_VERSION <= '2014-02-14'
  
          special_header_keys = [
              'content-encoding',
              'content-language',
              'content-length',
              'content-md5',
              'content-type',
              'date',
              'if-modified-since',
              'if-match',
              'if-none-match',
              'if-unmodified-since',
              'range'
          ]
  
          special_header_values = []
  
          for header in special_header_keys:
              header = header.lower()  # Just for safety
              if header in headers:
                  special_header_values.append(headers[header])
              elif header == 'content-length' and is_change and is_old_api:
                  # For old API versions, the Content-Length header must be '0'
                  # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key#content-length-header-in-version-2014-02-14-and-earlier
                  special_header_values.append('0')
              else:
                  special_header_values.append('')
  
          return special_header_values
  
  
  class AzureBaseDriver(BaseDriver):
      name = "Microsoft Azure Service Management API"
  
  
  class AzureServiceManagementConnection(CertificateConnection):
      # This needs the following approach -
      # 1. Make request using LibcloudHTTPSConnection which is a overloaded
      # class which takes in a client certificate
      # 2. Depending on the type of operation use a PollingConnection
      # when the response id is returned
      # 3. The Response can be used in an AzureServiceManagementResponse
  
      """
      Authentication class for "Service Account" authentication.
      """
  
      driver = AzureBaseDriver
      responseCls = AzureResponse
      rawResponseCls = AzureRawResponse
      name = '
  
  [... the rest is omitted ...]
files: libcloud/common/azure.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  """
  Provides base classes for working with storage
  """
  
  # Backward compatibility for Python 2.5
  from __future__ import with_statement
  
  from typing import Dict
  from typing import Iterator
  from typing import List
  from typing import Optional
  from typing import Type
  
  import os.path                          # pylint: disable-msg=W0404
  import hashlib
  import warnings
  import errno
  from os.path import join as pjoin
  
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import b
  
  import libcloud.utils.files
  from libcloud.common.types import LibcloudError
  from libcloud.common.base import Connection
  from libcloud.common.base import ConnectionUserAndKey, BaseDriver
  from libcloud.storage.types import ObjectDoesNotExistError
  
  __all__ = [
      'Object',
      'Container',
      'StorageDriver',
  
      'CHUNK_SIZE',
      'DEFAULT_CONTENT_TYPE'
  ]
  
  CHUNK_SIZE = 8096
  
  # Default Content-Type which is sent when uploading an object if one is not
  # supplied and can't be detected when using non-strict mode.
  DEFAULT_CONTENT_TYPE = 'application/octet-stream'
  
  
  class Object(object):
      """
      Represents an object (BLOB).
      """
  
      def __init__(self,
                   name,  # type: str
                   size,  # type: int
                   hash,  # type: str
                   extra,  # type: dict
                   meta_data,  # type: dict
                   container,  # type: Container
                   driver,  # type: StorageDriver
                   ):
          """
          :param name: Object name (must be unique per container).
          :type  name: ``str``
  
          :param size: Object size in bytes.
          :type  size: ``int``
  
          :param hash: Object hash.
          :type  hash: ``str``
  
          :param container: Object container.
          :type  container: :class:`libcloud.storage.base.Container`
  
          :param extra: Extra attributes.
          :type  extra: ``dict``
  
          :param meta_data: Optional object meta data.
          :type  meta_data: ``dict``
  
          :param driver: StorageDriver instance.
          :type  driver: :class:`libcloud.storage.base.StorageDriver`
          """
  
          self.name = name
          self.size = size
          self.hash = hash
          self.container = container
          self.extra = extra or {}
          self.meta_data = meta_data or {}
          self.driver = driver
  
      def get_cdn_url(self):
          # type: () -> str
          return self.driver.get_object_cdn_url(obj=self)
  
      def enable_cdn(self):
          # type: () -> bool
          return self.driver.enable_object_cdn(obj=self)
  
      def download(self, destination_path, overwrite_existing=False,
                   delete_on_failure=True):
          # type: (str, bool, bool) -> bool
          return self.driver.download_object(
              obj=self,
              destination_path=destination_path,
              overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def as_stream(self, chunk_size=None):
          # type: (Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_as_stream(obj=self,
                                                       chunk_size=chunk_size)
  
      def download_range(self, destination_path, start_bytes, end_bytes=None,
                         overwrite_existing=False,
                         delete_on_failure=True):
          # type: (str, int, Optional[int], bool, bool) -> bool
          return self.driver.download_object_range(
              obj=self,
              destination_path=destination_path,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def range_as_stream(self, start_bytes, end_bytes=None, chunk_size=None):
          # type: (int, Optional[int], Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_range_as_stream(
              obj=self,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              chunk_size=chunk_size)
  
      def delete(self):
          # type: () -> bool
          return self.driver.delete_object(self)
  
      def __repr__(self):
          return ('<Object: name=%s, size=%s, hash=%s, provider=%s ...>' %
                  (self.name, self.size, self.hash, self.driver.name))
  
  
  class Container(object):
      """
      Represents a container (bucket) which can hold multiple objects.
      """
  
      def __init__(self,
                   name,  # type: str
                   extra,  # type: dict
                   driver,  # type: StorageDriver
                   ):
          """
          :param name: Container name (must be unique).
          :type name: ``str``
  
          :param extra: Extra attributes.
          :type extra: ``dict``
  
          :param driver: StorageDriver instance.
          :type driver: :class:`libcloud.storage.base.StorageDriver`
          """
  
          self.name = name
          self.extra = extra or {}
          self.driver = driver
  
      def iterate_objects(self, prefix=None, ex_prefix=None):
          # type: (Optional[str], Optional[str]) -> Iterator[Object]
          return self.driver.iterate_container_objects(container=self,
                                                       prefix=prefix,
                                                       ex_prefix=ex_prefix)
  
      def list_objects(self, prefix=None, ex_prefix=None):
          # type: (Optional[str], Optional[str]) -> List[Object]
          return self.driver.list_container_objects(container=self,
                                                    prefix=prefix,
                                                    ex_prefix=ex_prefix)
  
      def get_cdn_url(self):
          # type: () -> str
          return self.driver.get_container_cdn_url(container=self)
  
      def enable_cdn(self):
          # type: () -> bool
          return self.driver.enable_container_cdn(container=self)
  
      def get_object(self, object_name):
          # type: (str) -> Object
          return self.driver.get_object(container_name=self.name,
                                        object_name=object_name)
  
      def upload_object(self, file_path, object_name, extra=None,
                        verify_hash=True, headers=None):
          # type: (str, str, Optional[dict], bool, Optional[Dict[str, str]]) -> Object  # noqa: E501
          return self.driver.upload_object(
              file_path, self, object_name, extra=extra,
              verify_hash=verify_hash, headers=headers)
  
      def upload_object_via_stream(self, iterator, object_name, extra=None,
                                   headers=None):
          # type: (Iterator[bytes], str, Optional[dict], Optional[Dict[str, str]]) -> Object  # noqa: E501
          return self.driver.upload_object_via_stream(
              iterator, self, object_name, extra=extra, headers=headers)
  
      def download_object(self, obj, destination_path, overwrite_existing=False,
                          delete_on_failure=True):
          # type: (Object, str, bool, bool) -> bool
          return self.driver.download_object(
              obj, destination_path, overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def download_object_as_stream(self, obj, chunk_size=None):
          # type: (Object, Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_as_stream(obj, chunk_size)
  
      def download_object_range(self, obj, destination_path, start_bytes,
                                end_bytes=None, overwrite_existing=False,
                                delete_on_failure=True):
          # type: (Object, str, int, Optional[int], bool, bool) -> bool
          return self.driver.download_object_range(
              obj=obj,
              destination_path=destination_path,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None,
                                          chunk_size=None):
          # type: (Object, int, Optional[int], Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_range_as_stream(
              obj=obj,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              chunk_size=chunk_size)
  
      def delete_object(self, obj):
          # type: (
  
  [... the rest is omitted ...]
files: libcloud/storage/base.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  """
  Provides base classes for working with storage
  """
  
  # Backward compatibility for Python 2.5
  from __future__ import with_statement
  
  from typing import Dict
  from typing import Iterator
  from typing import List
  from typing import Optional
  from typing import Type
  
  import os.path                          # pylint: disable-msg=W0404
  import hashlib
  import warnings
  import errno
  from os.path import join as pjoin
  
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import b
  
  import libcloud.utils.files
  from libcloud.common.types import LibcloudError
  from libcloud.common.base import Connection
  from libcloud.common.base import ConnectionUserAndKey, BaseDriver
  from libcloud.storage.types import ObjectDoesNotExistError
  
  __all__ = [
      'Object',
      'Container',
      'StorageDriver',
  
      'CHUNK_SIZE',
      'DEFAULT_CONTENT_TYPE'
  ]
  
  CHUNK_SIZE = 8096
  
  # Default Content-Type which is sent when uploading an object if one is not
  # supplied and can't be detected when using non-strict mode.
  DEFAULT_CONTENT_TYPE = 'application/octet-stream'
  
  
  class Object(object):
      """
      Represents an object (BLOB).
      """
  
      def __init__(self,
                   name,  # type: str
                   size,  # type: int
                   hash,  # type: str
                   extra,  # type: dict
                   meta_data,  # type: dict
                   container,  # type: Container
                   driver,  # type: StorageDriver
                   ):
          """
          :param name: Object name (must be unique per container).
          :type  name: ``str``
  
          :param size: Object size in bytes.
          :type  size: ``int``
  
          :param hash: Object hash.
          :type  hash: ``str``
  
          :param container: Object container.
          :type  container: :class:`libcloud.storage.base.Container`
  
          :param extra: Extra attributes.
          :type  extra: ``dict``
  
          :param meta_data: Optional object meta data.
          :type  meta_data: ``dict``
  
          :param driver: StorageDriver instance.
          :type  driver: :class:`libcloud.storage.base.StorageDriver`
          """
  
          self.name = name
          self.size = size
          self.hash = hash
          self.container = container
          self.extra = extra or {}
          self.meta_data = meta_data or {}
          self.driver = driver
  
      def get_cdn_url(self):
          # type: () -> str
          return self.driver.get_object_cdn_url(obj=self)
  
      def enable_cdn(self):
          # type: () -> bool
          return self.driver.enable_object_cdn(obj=self)
  
      def download(self, destination_path, overwrite_existing=False,
                   delete_on_failure=True):
          # type: (str, bool, bool) -> bool
          return self.driver.download_object(
              obj=self,
              destination_path=destination_path,
              overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def as_stream(self, chunk_size=None):
          # type: (Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_as_stream(obj=self,
                                                       chunk_size=chunk_size)
  
      def download_range(self, destination_path, start_bytes, end_bytes=None,
                         overwrite_existing=False,
                         delete_on_failure=True):
          # type: (str, int, Optional[int], bool, bool) -> bool
          return self.driver.download_object_range(
              obj=self,
              destination_path=destination_path,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def range_as_stream(self, start_bytes, end_bytes=None, chunk_size=None):
          # type: (int, Optional[int], Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_range_as_stream(
              obj=self,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              chunk_size=chunk_size)
  
      def delete(self):
          # type: () -> bool
          return self.driver.delete_object(self)
  
      def __repr__(self):
          return ('<Object: name=%s, size=%s, hash=%s, provider=%s ...>' %
                  (self.name, self.size, self.hash, self.driver.name))
  
  
  class Container(object):
      """
      Represents a container (bucket) which can hold multiple objects.
      """
  
      def __init__(self,
                   name,  # type: str
                   extra,  # type: dict
                   driver,  # type: StorageDriver
                   ):
          """
          :param name: Container name (must be unique).
          :type name: ``str``
  
          :param extra: Extra attributes.
          :type extra: ``dict``
  
          :param driver: StorageDriver instance.
          :type driver: :class:`libcloud.storage.base.StorageDriver`
          """
  
          self.name = name
          self.extra = extra or {}
          self.driver = driver
  
      def iterate_objects(self, prefix=None, ex_prefix=None):
          # type: (Optional[str], Optional[str]) -> Iterator[Object]
          return self.driver.iterate_container_objects(container=self,
                                                       prefix=prefix,
                                                       ex_prefix=ex_prefix)
  
      def list_objects(self, prefix=None, ex_prefix=None):
          # type: (Optional[str], Optional[str]) -> List[Object]
          return self.driver.list_container_objects(container=self,
                                                    prefix=prefix,
                                                    ex_prefix=ex_prefix)
  
      def get_cdn_url(self):
          # type: () -> str
          return self.driver.get_container_cdn_url(container=self)
  
      def enable_cdn(self):
          # type: () -> bool
          return self.driver.enable_container_cdn(container=self)
  
      def get_object(self, object_name):
          # type: (str) -> Object
          return self.driver.get_object(container_name=self.name,
                                        object_name=object_name)
  
      def upload_object(self, file_path, object_name, extra=None,
                        verify_hash=True, headers=None):
          # type: (str, str, Optional[dict], bool, Optional[Dict[str, str]]) -> Object  # noqa: E501
          return self.driver.upload_object(
              file_path, self, object_name, extra=extra,
              verify_hash=verify_hash, headers=headers)
  
      def upload_object_via_stream(self, iterator, object_name, extra=None,
                                   headers=None):
          # type: (Iterator[bytes], str, Optional[dict], Optional[Dict[str, str]]) -> Object  # noqa: E501
          return self.driver.upload_object_via_stream(
              iterator, self, object_name, extra=extra, headers=headers)
  
      def download_object(self, obj, destination_path, overwrite_existing=False,
                          delete_on_failure=True):
          # type: (Object, str, bool, bool) -> bool
          return self.driver.download_object(
              obj, destination_path, overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def download_object_as_stream(self, obj, chunk_size=None):
          # type: (Object, Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_as_stream(obj, chunk_size)
  
      def download_object_range(self, obj, destination_path, start_bytes,
                                end_bytes=None, overwrite_existing=False,
                                delete_on_failure=True):
          # type: (Object, str, int, Optional[int], bool, bool) -> bool
          return self.driver.download_object_range(
              obj=obj,
              destination_path=destination_path,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              overwrite_existing=overwrite_existing,
              delete_on_failure=delete_on_failure)
  
      def download_object_range_as_stream(self, obj, start_bytes, end_bytes=None,
                                          chunk_size=None):
          # type: (Object, int, Optional[int], Optional[int]) -> Iterator[bytes]
          return self.driver.download_object_range_as_stream(
              obj=obj,
              start_bytes=start_bytes,
              end_bytes=end_bytes,
              chunk_size=chunk_size)
  
      def delete_object(self, obj):
          # type: (
  
  [... the rest is omitted ...]
files: libcloud/storage/base.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  from __future__ import with_statement
  
  import base64
  import hashlib
  import hmac
  import os
  import binascii
  from datetime import datetime, timedelta
  
  from libcloud.utils.py3 import ET
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import urlencode
  from libcloud.utils.py3 import urlquote
  from libcloud.utils.py3 import tostring
  from libcloud.utils.py3 import b
  
  from libcloud.utils.xml import fixxpath
  from libcloud.utils.files import read_in_chunks
  from libcloud.common.types import LibcloudError
  from libcloud.common.azure import AzureConnection
  
  from libcloud.storage.base import Object, Container, StorageDriver
  from libcloud.storage.types import ContainerIsNotEmptyError
  from libcloud.storage.types import ContainerAlreadyExistsError
  from libcloud.storage.types import InvalidContainerNameError
  from libcloud.storage.types import ContainerDoesNotExistError
  from libcloud.storage.types import ObjectDoesNotExistError
  from libcloud.storage.types import ObjectHashMismatchError
  
  # Desired number of items in each response inside a paginated request
  RESPONSES_PER_REQUEST = 100
  
  # According to the Azure Docs:
  # > The block must be less than or equal to 100 MB in size for version
  # > 2016-05-31 and later (4 MB for older versions).
  # However for performance reasons, using a lower upload chunk size
  # usually leads to fewer dropped requests and retries.
  AZURE_UPLOAD_CHUNK_SIZE = int(
      os.getenv('LIBCLOUD_AZURE_UPLOAD_CHUNK_SIZE_MB', '4')
  ) * 1024 * 1024
  
  AZURE_DOWNLOAD_CHUNK_SIZE = int(
      os.getenv('LIBCLOUD_AZURE_DOWNLOAD_CHUNK_SIZE_MB', '4')
  ) * 1024 * 1024
  
  # The time period (in seconds) for which a lease must be obtained.
  # If set as -1, we get an infinite lease, but that is a bad idea. If
  # after getting an infinite lease, there was an issue in releasing the
  # lease, the object will remain 'locked' forever, unless the lease is
  # released using the lease_id (which is not exposed to the user)
  AZURE_LEASE_PERIOD = int(
      os.getenv('LIBCLOUD_AZURE_LEASE_PERIOD_SECONDS', '60')
  )
  
  AZURE_STORAGE_HOST_SUFFIX = 'blob.core.windows.net'
  
  AZURE_STORAGE_CDN_URL_DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
  
  AZURE_STORAGE_CDN_URL_START_MINUTES = float(
      os.getenv('LIBCLOUD_AZURE_STORAGE_CDN_URL_START_MINUTES', '5')
  )
  
  AZURE_STORAGE_CDN_URL_EXPIRY_HOURS = float(
      os.getenv('LIBCLOUD_AZURE_STORAGE_CDN_URL_EXPIRY_HOURS', '24')
  )
  
  
  class AzureBlobLease(object):
      """
      A class to help in leasing an azure blob and renewing the lease
      """
      def __init__(self, driver, object_path, use_lease):
          """
          :param driver: The Azure storage driver that is being used
          :type driver: :class:`AzureStorageDriver`
  
          :param object_path: The path of the object we need to lease
          :type object_path: ``str``
  
          :param use_lease: Indicates if we must take a lease or not
          :type use_lease: ``bool``
          """
          self.object_path = object_path
          self.driver = driver
          self.use_lease = use_lease
          self.lease_id = None
          self.params = {'comp': 'lease'}
  
      def renew(self):
          """
          Renew the lease if it is older than a predefined time period
          """
          if self.lease_id is None:
              return
  
          headers = {'x-ms-lease-action': 'renew',
                     'x-ms-lease-id': self.lease_id,
                     'x-ms-lease-duration': '60'}
  
          response = self.driver.connection.request(self.object_path,
                                                    headers=headers,
                                                    params=self.params,
                                                    method='PUT')
  
          if response.status != httplib.OK:
              raise LibcloudError('Unable to obtain lease', driver=self)
  
      def update_headers(self, headers):
          """
          Update the lease id in the headers
          """
          if self.lease_id:
              headers['x-ms-lease-id'] = self.lease_id
  
      def __enter__(self):
          if not self.use_lease:
              return self
  
          headers = {'x-ms-lease-action': 'acquire',
                     'x-ms-lease-duration': '60'}
  
          response = self.driver.connection.request(self.object_path,
                                                    headers=headers,
                                                    params=self.params,
                                                    method='PUT')
  
          if response.status == httplib.NOT_FOUND:
              return self
          elif response.status != httplib.CREATED:
              raise LibcloudError('Unable to obtain lease', driver=self)
  
          self.lease_id = response.headers['x-ms-lease-id']
          return self
  
      def __exit__(self, type, value, traceback):
          if self.lease_id is None:
              return
  
          headers = {'x-ms-lease-action': 'release',
                     'x-ms-lease-id': self.lease_id}
          response = self.driver.connection.request(self.object_path,
                                                    headers=headers,
                                                    params=self.params,
                                                    method='PUT')
  
          if response.status != httplib.OK:
              raise LibcloudError('Unable to release lease', driver=self)
  
  
  class AzureBlobsConnection(AzureConnection):
      """
      Represents a single connection to Azure Blobs.
  
      The main Azure Blob Storage service uses a prefix in the hostname to
      distinguish between accounts, e.g. ``theaccount.blob.core.windows.net``.
      However, some custom deployments of the service, such as the Azurite
      emulator, instead use a URL prefix such as ``/theaccount``. To support
      these deployments, the parameter ``account_prefix`` must be set on the
      connection. This is done by instantiating the driver with arguments such
      as ``host='somewhere.tld'`` and ``key='theaccount'``. To specify a custom
      host without an account prefix, e.g. for use-cases where the custom host
      implements an auditing proxy or similar, the driver can be instantiated
      with ``host='theaccount.somewhere.tld'`` and ``key=''``.
  
      :param account_prefix: Optional prefix identifying the sotrage account.
                             Used when connecting to a custom deployment of the
                             storage service like Azurite or IoT Edge Storage.
      :type account_prefix: ``str``
      """
      def __init__(self, *args, **kwargs):
          self.account_prefix = kwargs.pop('account_prefix', None)
          super(AzureBlobsConnection, self).__init__(*args, **kwargs)
  
      def morph_action_hook(self, action):
          action = super(AzureBlobsConnection, self).morph_action_hook(action)
  
          if self.account_prefix is not None:
              action = '/%s%s' % (self.account_prefix, action)
  
          return action
  
      API_VERSION = '2018-11-09'
  
  
  class AzureBlobsStorageDriver(StorageDriver):
      name = 'Microsoft Azure (blobs)'
      website = 'http://windows.azure.com/'
      connectionCls = AzureBlobsConnection
      hash_type = 'md5'
      supports_chunked_encoding = False
  
      def __init__(self, key, secret=None, secure=True, host=None, port=None,
                   **kwargs):
          self._host_argument_set = bool(host)
  
          # B64decode() this key and keep it, so that we don't have to do
          # so for every request. Minor performance improvement
          secret = base64.b64decode(b(secret))
  
          super(AzureBlobsStorageDriver, self).__init__(key=key, secret=secret,
                                                        secure=secure, host=host,
                                                        port=port, **kwargs)
  
      def
  
  [... the rest is omitted ...]
files: libcloud/storage/drivers/azure_blobs.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  from __future__ import with_statement
  
  import base64
  import hashlib
  import hmac
  import os
  import binascii
  from datetime import datetime, timedelta
  
  from libcloud.utils.py3 import ET
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import urlencode
  from libcloud.utils.py3 import urlquote
  from libcloud.utils.py3 import tostring
  from libcloud.utils.py3 import b
  
  from libcloud.utils.xml import fixxpath
  from libcloud.utils.files import read_in_chunks
  from libcloud.common.types import LibcloudError
  from libcloud.common.azure import AzureConnection
  
  from libcloud.storage.base import Object, Container, StorageDriver
  from libcloud.storage.types import ContainerIsNotEmptyError
  from libcloud.storage.types import ContainerAlreadyExistsError
  from libcloud.storage.types import InvalidContainerNameError
  from libcloud.storage.types import ContainerDoesNotExistError
  from libcloud.storage.types import ObjectDoesNotExistError
  from libcloud.storage.types import ObjectHashMismatchError
  
  # Desired number of items in each response inside a paginated request
  RESPONSES_PER_REQUEST = 100
  
  # According to the Azure Docs:
  # > The block must be less than or equal to 100 MB in size for version
  # > 2016-05-31 and later (4 MB for older versions).
  # However for performance reasons, using a lower upload chunk size
  # usually leads to fewer dropped requests and retries.
  AZURE_UPLOAD_CHUNK_SIZE = int(
      os.getenv('LIBCLOUD_AZURE_UPLOAD_CHUNK_SIZE_MB', '4')
  ) * 1024 * 1024
  
  AZURE_DOWNLOAD_CHUNK_SIZE = int(
      os.getenv('LIBCLOUD_AZURE_DOWNLOAD_CHUNK_SIZE_MB', '4')
  ) * 1024 * 1024
  
  # The time period (in seconds) for which a lease must be obtained.
  # If set as -1, we get an infinite lease, but that is a bad idea. If
  # after getting an infinite lease, there was an issue in releasing the
  # lease, the object will remain 'locked' forever, unless the lease is
  # released using the lease_id (which is not exposed to the user)
  AZURE_LEASE_PERIOD = int(
      os.getenv('LIBCLOUD_AZURE_LEASE_PERIOD_SECONDS', '60')
  )
  
  AZURE_STORAGE_HOST_SUFFIX = 'blob.core.windows.net'
  
  AZURE_STORAGE_CDN_URL_DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
  
  AZURE_STORAGE_CDN_URL_START_MINUTES = float(
      os.getenv('LIBCLOUD_AZURE_STORAGE_CDN_URL_START_MINUTES', '5')
  )
  
  AZURE_STORAGE_CDN_URL_EXPIRY_HOURS = float(
      os.getenv('LIBCLOUD_AZURE_STORAGE_CDN_URL_EXPIRY_HOURS', '24')
  )
  
  
  class AzureBlobLease(object):
      """
      A class to help in leasing an azure blob and renewing the lease
      """
      def __init__(self, driver, object_path, use_lease):
          """
          :param driver: The Azure storage driver that is being used
          :type driver: :class:`AzureStorageDriver`
  
          :param object_path: The path of the object we need to lease
          :type object_path: ``str``
  
          :param use_lease: Indicates if we must take a lease or not
          :type use_lease: ``bool``
          """
          self.object_path = object_path
          self.driver = driver
          self.use_lease = use_lease
          self.lease_id = None
          self.params = {'comp': 'lease'}
  
      def renew(self):
          """
          Renew the lease if it is older than a predefined time period
          """
          if self.lease_id is None:
              return
  
          headers = {'x-ms-lease-action': 'renew',
                     'x-ms-lease-id': self.lease_id,
                     'x-ms-lease-duration': '60'}
  
          response = self.driver.connection.request(self.object_path,
                                                    headers=headers,
                                                    params=self.params,
                                                    method='PUT')
  
          if response.status != httplib.OK:
              raise LibcloudError('Unable to obtain lease', driver=self)
  
      def update_headers(self, headers):
          """
          Update the lease id in the headers
          """
          if self.lease_id:
              headers['x-ms-lease-id'] = self.lease_id
  
      def __enter__(self):
          if not self.use_lease:
              return self
  
          headers = {'x-ms-lease-action': 'acquire',
                     'x-ms-lease-duration': '60'}
  
          response = self.driver.connection.request(self.object_path,
                                                    headers=headers,
                                                    params=self.params,
                                                    method='PUT')
  
          if response.status == httplib.NOT_FOUND:
              return self
          elif response.status != httplib.CREATED:
              raise LibcloudError('Unable to obtain lease', driver=self)
  
          self.lease_id = response.headers['x-ms-lease-id']
          return self
  
      def __exit__(self, type, value, traceback):
          if self.lease_id is None:
              return
  
          headers = {'x-ms-lease-action': 'release',
                     'x-ms-lease-id': self.lease_id}
          response = self.driver.connection.request(self.object_path,
                                                    headers=headers,
                                                    params=self.params,
                                                    method='PUT')
  
          if response.status != httplib.OK:
              raise LibcloudError('Unable to release lease', driver=self)
  
  
  class AzureBlobsConnection(AzureConnection):
      """
      Represents a single connection to Azure Blobs.
  
      The main Azure Blob Storage service uses a prefix in the hostname to
      distinguish between accounts, e.g. ``theaccount.blob.core.windows.net``.
      However, some custom deployments of the service, such as the Azurite
      emulator, instead use a URL prefix such as ``/theaccount``. To support
      these deployments, the parameter ``account_prefix`` must be set on the
      connection. This is done by instantiating the driver with arguments such
      as ``host='somewhere.tld'`` and ``key='theaccount'``. To specify a custom
      host without an account prefix, e.g. for use-cases where the custom host
      implements an auditing proxy or similar, the driver can be instantiated
      with ``host='theaccount.somewhere.tld'`` and ``key=''``.
  
      :param account_prefix: Optional prefix identifying the sotrage account.
                             Used when connecting to a custom deployment of the
                             storage service like Azurite or IoT Edge Storage.
      :type account_prefix: ``str``
      """
      def __init__(self, *args, **kwargs):
          self.account_prefix = kwargs.pop('account_prefix', None)
          super(AzureBlobsConnection, self).__init__(*args, **kwargs)
  
      def morph_action_hook(self, action):
          action = super(AzureBlobsConnection, self).morph_action_hook(action)
  
          if self.account_prefix is not None:
              action = '/%s%s' % (self.account_prefix, action)
  
          return action
  
      API_VERSION = '2018-11-09'
  
  
  class AzureBlobsStorageDriver(StorageDriver):
      name = 'Microsoft Azure (blobs)'
      website = 'http://windows.azure.com/'
      connectionCls = AzureBlobsConnection
      hash_type = 'md5'
      supports_chunked_encoding = False
  
      def __init__(self, key, secret=None, secure=True, host=None, port=None,
                   **kwargs):
          self._host_argument_set = bool(host)
  
          # B64decode() this key and keep it, so that we don't have to do
          # so for every request. Minor performance improvement
          secret = base64.b64decode(b(secret))
  
          super(AzureBlobsStorageDriver, self).__init__(key=key, secret=secret,
                                                        secure=secure, host=host,
                                                        port=port, **kwargs)
  
      def
  
  [... the rest is omitted ...]
files: libcloud/storage/drivers/azure_blobs.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  import base64
  import hmac
  import time
  from hashlib import sha1
  
  import libcloud.utils.py3
  
  try:
      if libcloud.utils.py3.DEFAULT_LXML:
          from lxml.etree import Element, SubElement
      else:
          from xml.etree.ElementTree import Element, SubElement
  except ImportError:
      from xml.etree.ElementTree import Element, SubElement
  
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import urlquote
  from libcloud.utils.py3 import b
  from libcloud.utils.py3 import tostring
  
  from libcloud.utils.xml import fixxpath, findtext
  from libcloud.utils.files import read_in_chunks
  from libcloud.common.types import InvalidCredsError, LibcloudError
  from libcloud.common.base import ConnectionUserAndKey, RawResponse
  from libcloud.common.aws import AWSBaseResponse, AWSDriver, \
      AWSTokenConnection, SignedAWSConnection
  
  from libcloud.storage.base import Object, Container, StorageDriver
  from libcloud.storage.types import ContainerError
  from libcloud.storage.types import ContainerIsNotEmptyError
  from libcloud.storage.types import InvalidContainerNameError
  from libcloud.storage.types import ContainerDoesNotExistError
  from libcloud.storage.types import ObjectDoesNotExistError
  from libcloud.storage.types import ObjectHashMismatchError
  
  # How long before the token expires
  EXPIRATION_SECONDS = 15 * 60
  
  S3_US_STANDARD_HOST = 's3.amazonaws.com'
  S3_US_EAST2_HOST = 's3-us-east-2.amazonaws.com'
  S3_US_WEST_HOST = 's3-us-west-1.amazonaws.com'
  S3_US_WEST_OREGON_HOST = 's3-us-west-2.amazonaws.com'
  S3_US_GOV_WEST_HOST = 's3-us-gov-west-1.amazonaws.com'
  S3_CN_NORTH_HOST = 's3.cn-north-1.amazonaws.com.cn'
  S3_CN_NORTHWEST_HOST = 's3.cn-northwest-1.amazonaws.com.cn'
  S3_EU_WEST_HOST = 's3-eu-west-1.amazonaws.com'
  S3_EU_WEST2_HOST = 's3-eu-west-2.amazonaws.com'
  S3_EU_CENTRAL_HOST = 's3-eu-central-1.amazonaws.com'
  S3_EU_NORTH1_HOST = 's3-eu-north-1.amazonaws.com'
  S3_AP_SOUTH_HOST = 's3-ap-south-1.amazonaws.com'
  S3_AP_SOUTHEAST_HOST = 's3-ap-southeast-1.amazonaws.com'
  S3_AP_SOUTHEAST2_HOST = 's3-ap-southeast-2.amazonaws.com'
  S3_AP_NORTHEAST1_HOST = 's3-ap-northeast-1.amazonaws.com'
  S3_AP_NORTHEAST2_HOST = 's3-ap-northeast-2.amazonaws.com'
  S3_AP_NORTHEAST_HOST = S3_AP_NORTHEAST1_HOST
  S3_SA_EAST_HOST = 's3-sa-east-1.amazonaws.com'
  S3_SA_SOUTHEAST2_HOST = 's3-sa-east-2.amazonaws.com'
  S3_CA_CENTRAL_HOST = 's3-ca-central-1.amazonaws.com'
  
  # Maps AWS region name to connection hostname
  REGION_TO_HOST_MAP = {
      'us-east-1': S3_US_STANDARD_HOST,
      'us-east-2': S3_US_EAST2_HOST,
      'us-west-1': S3_US_WEST_HOST,
      'us-west-2': S3_US_WEST_OREGON_HOST,
      'us-gov-west-1': S3_US_GOV_WEST_HOST,
      'cn-north-1': S3_CN_NORTH_HOST,
      'cn-northwest-1': S3_CN_NORTHWEST_HOST,
      'eu-west-1': S3_EU_WEST_HOST,
      'eu-west-2': S3_EU_WEST2_HOST,
      'eu-west-3': 's3.eu-west-3.amazonaws.com',
      'eu-north-1': 's3.eu-north-1.amazonaws.com',
      'eu-central-1': S3_EU_CENTRAL_HOST,
      'ap-south-1': S3_AP_SOUTH_HOST,
      'ap-southeast-1': S3_AP_SOUTHEAST_HOST,
      'ap-southeast-2': S3_AP_SOUTHEAST2_HOST,
      'ap-northeast-1': S3_AP_NORTHEAST1_HOST,
      'ap-northeast-2': S3_AP_NORTHEAST2_HOST,
      'ap-northeast-3': 's3.ap-northeast-3.amazonaws.com',
      'sa-east-1': S3_SA_EAST_HOST,
      'sa-east-2': S3_SA_SOUTHEAST2_HOST,
      'ca-central-1': S3_CA_CENTRAL_HOST,
      'me-south-1': 's3.me-south-1.amazonaws.com'
  }
  
  API_VERSION = '2006-03-01'
  NAMESPACE = 'http://s3.amazonaws.com/doc/%s/' % (API_VERSION)
  
  # AWS multi-part chunks must be minimum 5MB
  CHUNK_SIZE = 5 * 1024 * 1024
  
  # Desired number of items in each response inside a paginated request in
  # ex_iterate_multipart_uploads.
  RESPONSES_PER_REQUEST = 100
  
  
  class S3Response(AWSBaseResponse):
      namespace = None
      valid_response_codes = [httplib.NOT_FOUND, httplib.CONFLICT,
                              httplib.BAD_REQUEST, httplib.PARTIAL_CONTENT]
  
      def success(self):
          i = int(self.status)
          return 200 <= i <= 299 or i in self.valid_response_codes
  
      def parse_error(self):
          if self.status in [httplib.UNAUTHORIZED, httplib.FORBIDDEN]:
              raise InvalidCredsError(self.body)
          elif self.status == httplib.MOVED_PERMANENTLY:
              bucket_region = self.headers.get('x-amz-bucket-region', None)
              used_region = self.connection.driver.region
              raise LibcloudError('This bucket is located in a different '
                                  'region. Please use the correct driver. '
                                  'Bucket region "%s", used region "%s".' %
                                  (bucket_region, used_region),
                                  driver=S3StorageDriver)
          raise LibcloudError('Unknown error. Status code: %d' % (self.status),
                              driver=S3StorageDriver)
  
  
  class S3RawResponse(S3Response, RawResponse):
      pass
  
  
  class BaseS3Connection(ConnectionUserAndKey):
      """
      Represents a single connection to the S3 Endpoint
      """
  
      host = 's3.amazonaws.com'
      responseCls = S3Response
      rawResponseCls = S3RawResponse
  
      @staticmethod
      def get_auth_signature(method, headers, params, expires, secret_key, path,
                             vendor_prefix):
          """
          Signature = URL-Encode( Base64( HMAC-SHA1( YourSecretAccessKeyID,
                                      UTF-8-Encoding-Of( StringToSign ) ) ) );
  
          StringToSign = HTTP-VERB + "\n" +
              Content-MD5 + "\n" +
              Content-Type + "\n" +
              Expires + "\n" +
              CanonicalizedVendorHeaders +
              CanonicalizedResource;
          """
          special_headers = {'content-md5': '', 'content-type': '', 'date': ''}
          vendor_headers = {}
  
          for key, value in list(headers.items()):
              key_lower = key.lower()
              if key_lower in special_headers:
                  special_headers[key_lower] = value.strip()
              elif key_lower.startswith(vendor_prefix):
                  vendor_headers[key_lower] = value.strip()
  
          if expires:
              special_headers['date'] = str(expires)
  
          buf = [method]
          for _, value in sorted(special
  
  [... the rest is omitted ...]
files: libcloud/storage/drivers/s3.py
files: # Licensed to the Apache Software Foundation (ASF) under one or more
  # contributor license agreements.  See the NOTICE file distributed with
  # this work for additional information regarding copyright ownership.
  # The ASF licenses this file to You under the Apache License, Version 2.0
  # (the "License"); you may not use this file except in compliance with
  # the License.  You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  from typing import Optional
  
  import base64
  import hmac
  import time
  from hashlib import sha1
  
  import libcloud.utils.py3
  
  try:
      if libcloud.utils.py3.DEFAULT_LXML:
          from lxml.etree import Element, SubElement
      else:
          from xml.etree.ElementTree import Element, SubElement
  except ImportError:
      from xml.etree.ElementTree import Element, SubElement
  
  from libcloud.utils.py3 import httplib
  from libcloud.utils.py3 import urlquote
  from libcloud.utils.py3 import b
  from libcloud.utils.py3 import tostring
  
  from libcloud.utils.xml import fixxpath, findtext
  from libcloud.utils.files import read_in_chunks
  from libcloud.common.types import InvalidCredsError, LibcloudError
  from libcloud.common.base import ConnectionUserAndKey, RawResponse
  from libcloud.common.aws import AWSBaseResponse, AWSDriver, \
      AWSTokenConnection, SignedAWSConnection
  
  from libcloud.storage.base import Object, Container, StorageDriver
  from libcloud.storage.types import ContainerError
  from libcloud.storage.types import ContainerIsNotEmptyError
  from libcloud.storage.types import InvalidContainerNameError
  from libcloud.storage.types import ContainerDoesNotExistError
  from libcloud.storage.types import ObjectDoesNotExistError
  from libcloud.storage.types import ObjectHashMismatchError
  
  # How long before the token expires
  EXPIRATION_SECONDS = 15 * 60
  
  S3_US_STANDARD_HOST = 's3.amazonaws.com'
  S3_US_EAST2_HOST = 's3-us-east-2.amazonaws.com'
  S3_US_WEST_HOST = 's3-us-west-1.amazonaws.com'
  S3_US_WEST_OREGON_HOST = 's3-us-west-2.amazonaws.com'
  S3_US_GOV_WEST_HOST = 's3-us-gov-west-1.amazonaws.com'
  S3_CN_NORTH_HOST = 's3.cn-north-1.amazonaws.com.cn'
  S3_CN_NORTHWEST_HOST = 's3.cn-northwest-1.amazonaws.com.cn'
  S3_EU_WEST_HOST = 's3-eu-west-1.amazonaws.com'
  S3_EU_WEST2_HOST = 's3-eu-west-2.amazonaws.com'
  S3_EU_CENTRAL_HOST = 's3-eu-central-1.amazonaws.com'
  S3_EU_NORTH1_HOST = 's3-eu-north-1.amazonaws.com'
  S3_AP_SOUTH_HOST = 's3-ap-south-1.amazonaws.com'
  S3_AP_SOUTHEAST_HOST = 's3-ap-southeast-1.amazonaws.com'
  S3_AP_SOUTHEAST2_HOST = 's3-ap-southeast-2.amazonaws.com'
  S3_AP_NORTHEAST1_HOST = 's3-ap-northeast-1.amazonaws.com'
  S3_AP_NORTHEAST2_HOST = 's3-ap-northeast-2.amazonaws.com'
  S3_AP_NORTHEAST_HOST = S3_AP_NORTHEAST1_HOST
  S3_SA_EAST_HOST = 's3-sa-east-1.amazonaws.com'
  S3_SA_SOUTHEAST2_HOST = 's3-sa-east-2.amazonaws.com'
  S3_CA_CENTRAL_HOST = 's3-ca-central-1.amazonaws.com'
  
  # Maps AWS region name to connection hostname
  REGION_TO_HOST_MAP = {
      'us-east-1': S3_US_STANDARD_HOST,
      'us-east-2': S3_US_EAST2_HOST,
      'us-west-1': S3_US_WEST_HOST,
      'us-west-2': S3_US_WEST_OREGON_HOST,
      'us-gov-west-1': S3_US_GOV_WEST_HOST,
      'cn-north-1': S3_CN_NORTH_HOST,
      'cn-northwest-1': S3_CN_NORTHWEST_HOST,
      'eu-west-1': S3_EU_WEST_HOST,
      'eu-west-2': S3_EU_WEST2_HOST,
      'eu-west-3': 's3.eu-west-3.amazonaws.com',
      'eu-north-1': 's3.eu-north-1.amazonaws.com',
      'eu-central-1': S3_EU_CENTRAL_HOST,
      'ap-south-1': S3_AP_SOUTH_HOST,
      'ap-southeast-1': S3_AP_SOUTHEAST_HOST,
      'ap-southeast-2': S3_AP_SOUTHEAST2_HOST,
      'ap-northeast-1': S3_AP_NORTHEAST1_HOST,
      'ap-northeast-2': S3_AP_NORTHEAST2_HOST,
      'ap-northeast-3': 's3.ap-northeast-3.amazonaws.com',
      'sa-east-1': S3_SA_EAST_HOST,
      'sa-east-2': S3_SA_SOUTHEAST2_HOST,
      'ca-central-1': S3_CA_CENTRAL_HOST,
      'me-south-1': 's3.me-south-1.amazonaws.com'
  }
  
  API_VERSION = '2006-03-01'
  NAMESPACE = 'http://s3.amazonaws.com/doc/%s/' % (API_VERSION)
  
  # AWS multi-part chunks must be minimum 5MB
  CHUNK_SIZE = 5 * 1024 * 1024
  
  # Desired number of items in each response inside a paginated request in
  # ex_iterate_multipart_uploads.
  RESPONSES_PER_REQUEST = 100
  
  
  class S3Response(AWSBaseResponse):
      namespace = None
      valid_response_codes = [httplib.NOT_FOUND, httplib.CONFLICT,
                              httplib.BAD_REQUEST, httplib.PARTIAL_CONTENT]
  
      def success(self):
          i = int(self.status)
          return 200 <= i <= 299 or i in self.valid_response_codes
  
      def parse_error(self):
          if self.status in [httplib.UNAUTHORIZED, httplib.FORBIDDEN]:
              raise InvalidCredsError(self.body)
          elif self.status == httplib.MOVED_PERMANENTLY:
              bucket_region = self.headers.get('x-amz-bucket-region', None)
              used_region = self.connection.driver.region
              raise LibcloudError('This bucket is located in a different '
                                  'region. Please use the correct driver. '
                                  'Bucket region "%s", used region "%s".' %
                                  (bucket_region, used_region),
                                  driver=S3StorageDriver)
          raise LibcloudError('Unknown error. Status code: %d' % (self.status),
                              driver=S3StorageDriver)
  
  
  class S3RawResponse(S3Response, RawResponse):
      pass
  
  
  class BaseS3Connection(ConnectionUserAndKey):
      """
      Represents a single connection to the S3 Endpoint
      """
  
      host = 's3.amazonaws.com'
      responseCls = S3Response
      rawResponseCls = S3RawResponse
  
      @staticmethod
      def get_auth_signature(method, headers, params, expires, secret_key, path,
                             vendor_prefix):
          """
          Signature = URL-Encode( Base64( HMAC-SHA1( YourSecretAccessKeyID,
                                      UTF-8-Encoding-Of( StringToSign ) ) ) );
  
          StringToSign = HTTP-VERB + "\n" +
              Content-MD5 + "\n" +
              Content-Type + "\n" +
              Expires + "\n" +
              CanonicalizedVendorHeaders +
              CanonicalizedResource;
          """
          special_headers = {'content-md5': '', 'content-type': '', 'date': ''}
          vendor_headers = {}
  
          for key, value in list(headers.items()):
              key_lower = key.lower()
              if key_lower in special_headers:
                  special_headers[key_lower] = value.strip()
              elif key_lower.startswith(vendor_prefix):
                  vendor_headers[key_lower] = value.strip()
  
          if expires:
              special_headers['date'] = str(expires)
  
          buf = [method]
          for _,
  
  [... the rest is omitted ...]
files: libcloud/storage/drivers/s3.py