text: # Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License"). You
  # may not use this file except in compliance with the License. A copy of
  # the License is located at
  #
  # http://aws.amazon.com/apache2.0/
  #
  # or in the "license" file accompanying this file. This file is
  # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  # ANY KIND, either express or implied. See the License for the specific
  # language governing permissions and limitations under the License.
  from tests import unittest, RawResponse
  import datetime
  
  from dateutil.tz import tzutc
  from nose.tools import assert_equal
  
  from botocore import parsers
  from botocore import model
  from botocore.compat import json, MutableMapping
  
  
  # HTTP responses will typically return a custom HTTP
  # dict.  We want to ensure we're able to work with any
  # kind of mutable mapping implementation.
  class CustomHeaderDict(MutableMapping):
      def __init__(self, original_dict):
          self._d = original_dict
  
      def __getitem__(self, item):
          return self._d[item]
  
      def __setitem__(self, item, value):
          self._d[item] = value
  
      def __delitem__(self, item):
          del self._d[item]
  
      def __iter__(self):
          return iter(self._d)
  
      def __len__(self):
          return len(self._d)
  
  
  # These tests contain botocore specific tests that either
  # don't make sense in the protocol tests or haven't been added
  # yet.
  class TestResponseMetadataParsed(unittest.TestCase):
      def test_response_metadata_parsed_for_query_service(self):
          parser = parsers.QueryParser()
          response = (
              '<OperationNameResponse>'
              '  <OperationNameResult><Str>myname</Str></OperationNameResult>'
              '  <ResponseMetadata>'
              '    <RequestId>request-id</RequestId>'
              '  </ResponseMetadata>'
              '</OperationNameResponse>').encode('utf-8')
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'resultWrapper': 'OperationNameResult',
                  'members': {
                      'Str': {
                          'shape': 'StringType',
                      },
                      'Num': {
                          'shape': 'IntegerType',
                      }
                  }
              },
              model.ShapeResolver({
                  'StringType': {
                      'type': 'string',
                  },
                  'IntegerType': {
                      'type': 'integer',
                  }
              })
          )
          parsed = parser.parse(
              {'body': response,
               'headers': {},
               'status_code': 200}, output_shape)
          self.assertEqual(
              parsed, {'Str': 'myname',
                       'ResponseMetadata': {'RequestId': 'request-id',
                                            'HTTPStatusCode': 200,
                                            'HTTPHeaders': {}}})
  
      def test_metadata_always_exists_for_query(self):
          # ResponseMetadata is used for more than just the request id. It
          # should always get populated, even if the request doesn't seem to
          # have an id.
          parser = parsers.QueryParser()
          response = (
              '<OperationNameResponse>'
              '  <OperationNameResult><Str>myname</Str></OperationNameResult>'
              '</OperationNameResponse>').encode('utf-8')
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'resultWrapper': 'OperationNameResult',
                  'members': {
                      'Str': {
                          'shape': 'StringType',
                      },
                      'Num': {
                          'shape': 'IntegerType',
                      }
                  }
              },
              model.ShapeResolver({
                  'StringType': {
                      'type': 'string',
                  },
                  'IntegerType': {
                      'type': 'integer',
                  }
              })
          )
          parsed = parser.parse(
              {'body': response, 'headers': {}, 'status_code': 200},
              output_shape)
          expected = {
              'Str': 'myname',
              'ResponseMetadata': {
                  'HTTPStatusCode': 200,
                  'HTTPHeaders': {}
              }
          }
          self.assertEqual(parsed, expected)
  
      def test_response_metadata_parsed_for_ec2(self):
          parser = parsers.EC2QueryParser()
          response = (
              '<OperationNameResponse>'
              '  <Str>myname</Str>'
              '  <requestId>request-id</requestId>'
              '</OperationNameResponse>').encode('utf-8')
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'members': {
                      'Str': {
                          'shape': 'StringType',
                      }
                  }
              },
              model.ShapeResolver({'StringType': {'type': 'string'}})
          )
          parsed = parser.parse({'headers': {},
                                 'body': response,
                                 'status_code': 200}, output_shape)
          # Note that the response metadata is normalized to match the query
          # protocol, even though this is not how it appears in the output.
          self.assertEqual(
              parsed, {'Str': 'myname',
                       'ResponseMetadata': {'RequestId': 'request-id',
                                            'HTTPStatusCode': 200,
                                            'HTTPHeaders': {}}})
  
      def test_metadata_always_exists_for_ec2(self):
          # ResponseMetadata is used for more than just the request id. It
          # should always get populated, even if the request doesn't seem to
          # have an id.
          parser = parsers.EC2QueryParser()
          response = (
              '<OperationNameResponse>'
              '  <Str>myname</Str>'
              '</OperationNameResponse>').encode('utf-8')
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'members': {
                      'Str': {
                          'shape': 'StringType',
                      }
                  }
              },
              model.ShapeResolver({'StringType': {'type': 'string'}})
          )
          parsed = parser.parse(
              {'headers': {}, 'body': response, 'status_code': 200},
              output_shape)
          expected = {
              'Str': 'myname',
              'ResponseMetadata': {
                  'HTTPStatusCode': 200,
                  'HTTPHeaders': {}
              }
          }
          self.assertEqual(
              parsed, expected)
  
      def test_response_metadata_on_json_request(self):
          parser = parsers.JSONParser()
          response = b'{"Str": "mystring"}'
          headers = {'x-amzn-requestid': 'request-id'}
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'members': {
                      'Str': {
                          'shape': 'StringType',
                      }
                  }
              },
              model.ShapeResolver({'StringType': {'type': 'string'}})
          )
          parsed = parser.parse({'body': response, 'headers': headers,
                                 'status_code': 200}, output_shape)
          # Note that the response metadata is normalized to match the query
          # protocol, even though this is not how it appears in the output.
          self.assertEqual(
              parsed, {'Str': 'mystring',
                       'ResponseMetadata': {'RequestId': 'request-id',
                                            'HTTPStatusCode': 200,
                                            'HTTPHeaders': headers}})
  
      def test_response_no_initial_event_stream(self):
          parser = parsers.JSONParser()
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'members': {
                      'Payload': {'shape': 'Payload'}
                  }
              },
              model.ShapeResolver({
                  'Payload': {
                      'type': 'structure',
                      'members': [],
                      'eventstream': True
                  }
              })
          )
          with self.assertRaises(parsers.ResponseParserError):
              response_dict = {
                  'status_code': 200,
                  'headers': {},
                  'body': RawResponse(b''),
                  'context': {
                      'operation_name': 'TestOperation'
                  }
              }
              parser.parse(response_dict, output_shape)
  
      def test_metadata_always_exists_for_json(self):
          # ResponseMetadata is used for more than just the request id. It
          # should always get populated, even if the request doesn't seem to
          # have an id.
          parser = parsers.JSONParser()
          response = b'{"Str": "mystring"}'
          headers = {}
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'members': {
                      'Str': {
                          'shape': 'StringType',
                      }
                  }
              },
              model.ShapeResolver({'StringType': {'type': 'string'}})
          )
          parsed = parser.parse(
              {'body': response, 'headers': headers, 'status_code': 200},
              output_shape)
          expected = {
              'Str': 'mystring',
              'ResponseMetadata': {
                  'HTTPStatusCode': 200,
                  'HTTPHeaders': headers
              }
          }
          self.assertEqual(parsed, expected)
  
      def test_response_metadata_on_rest_json_response(self):
          parser = parsers.RestJSONParser()
          response = b'{"Str": "mystring"}'
          headers = {'x-amzn-requestid': 'request-id'}
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'members': {
                      'Str': {
                          'shape': 'StringType',
                      }
                  }
              },
              model.ShapeResolver({'StringType': {'type': 'string'}})
          )
          parsed = parser.parse({'body': response, 'headers': headers,
                                 'status_code': 200}, output_shape)
          # Note that the response metadata is normalized to match the query
          # protocol, even though this is not how it appears in the output.
          self.assertEqual(
              parsed, {'Str': 'mystring',
                       'ResponseMetadata': {'RequestId': 'request-id',
                                            'HTTPStatusCode': 200,
                                            'HTTPHeaders': headers}})
  
      def test_metadata_always_exists_on_rest_json_response(self):
          # ResponseMetadata is used for more than just the request id. It
          # should always get populated, even if the request doesn't seem to
          # have an id.
          parser = parsers.RestJSONParser()
          response = b'{"Str": "mystring"}'
          headers = {}
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'members': {
                      'Str': {
                          'shape': 'StringType',
                      }
                  }
              },
              model.ShapeResolver({'StringType': {'type': 'string'}})
          )
          parsed = parser.parse(
              {'body': response, 'headers': headers, 'status_code': 200},
              output_shape)
          expected = {
              'Str': 'mystring',
              'ResponseMetadata': {
                  'HTTPStatusCode': 200,
                  'HTTPHeaders': headers
              }
          }
          self.assertEqual(parsed, expected)
  
      def test_response_metadata_from_s3_response(self):
          # Even though s3 is a rest-xml service, it's response metadata
          # is slightly different.  It has two request ids, both come from
          # the response headers, are both are named differently from other
          # rest-xml responses.
          headers = {
              'x-amz-id-2': 'second-id',
              'x-amz-request-id': 'request-id'
          }
          parser = parsers.RestXMLParser()
          parsed = parser.parse(
              {'body': '', 'headers': headers, 'status_code': 200}, None)
          self.assertEqual(
              parsed,
              {'ResponseMetadata': {'RequestId': 'request-id',
                                    'HostId': 'second-id',
                                    'HTTPStatusCode': 200,
                                    'HTTPHeaders': headers}})
  
      def test_metadata_always_exists_on_rest_xml_response(self):
          # ResponseMetadata is used for more than just the request id. It
          # should always get populated, even if the request doesn't seem to
          # have an id.
          headers = {}
          parser = parsers.RestXMLParser()
          parsed = parser.parse(
              {'body': '', 'headers': headers, 'status_code': 200}, None)
          expected = {
              'ResponseMetadata': {
                  'HTTPStatusCode': 200,
                  'HTTPHeaders': headers
              }
          }
          self.assertEqual(parsed, expected)
  
  
  class TestTaggedUnions(unittest.TestCase):
      def assert_tagged_union_response_with_unknown_member(self,
                                                           parser,
                                                           response,
                                                           output_shape,
                                                           expected_parsed_response,
                                                           expected_log):
          with self.assertLogs() as captured_log:
              parsed = parser.parse(response, output_shape)
              self.assertEqual(parsed, expected_parsed_response)
              self.assertEqual(len(captured_log.records), 1)
              self.assertIn(('Received a tagged union response with member '
                             'unknown to client'),
                            captured_log.records[0].getMessage())
  
      def test_base_json_parser_handles_unknwon_member(self):
          parser = parsers.JSONParser()
          response = b'{"Foo": "mystring"}'
          headers = {'x-amzn-requestid': 'request-id'}
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'union':True,
                  'members': {
                      'Str': {
                          'shape': 'StringType',
                      }
                  }
              },
              model.ShapeResolver({'StringType': {'type': 'string'}})
          )
          response = {'body': response, 'headers': headers, 'status_code': 200}
          # Parsed response omits data from service since it is not
          # modeled in the client
          expected_parsed_response = {
              'SDK_UNKNOWN_MEMBER': {
                  'name': 'Foo'
              },
              'ResponseMetadata':
              {'RequestId': 'request-id',
               'HTTPStatusCode': 200,
               'HTTPHeaders': {
                   'x-amzn-requestid': 'request-id'}
               }
          }
          expected_log = "Received a response with an unknown member Foo set"
          self.assert_tagged_union_response_with_unknown_member(
              parser, response, output_shape, expected_parsed_response,
              expected_log
          )
  
      def test_base_xml_parser_handles_unknown_member(self):
          parser = parsers.QueryParser()
          response = (
              '<OperationNameResponse>'
              '  <OperationNameResult><Foo>mystring</Foo></OperationNameResult>'
              '  <ResponseMetadata>'
              '    <RequestId>request-id</RequestId>'
              '  </ResponseMetadata>'
              '</OperationNameResponse>').encode('utf-8')
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'union':True,
                  'resultWrapper': 'OperationNameResult',
                  'members': {
                      'Str': {
                          'shape': 'StringType',
                      },
                  }
              },
              model.ShapeResolver({
                  'StringType': {
                      'type': 'string',
                  },
              })
          )
          response = {'body': response, 'headers': {}, 'status_code': 200}
          # Parsed response omits data from service since it is not
          # modeled in the client
          expected_parsed_response = {
              'SDK_UNKNOWN_MEMBER': {
                  'name': 'Foo'
              },
              'ResponseMetadata':{
                  'RequestId': 'request-id',
                  'HTTPStatusCode': 200,
                  'HTTPHeaders': {}
              }
          }
          expected_log = "Received a response with an unknown member Foo set"
          self.assert_tagged_union_response_with_unknown_member(
              parser, response, output_shape, expected_parsed_response,
              expected_log
          )
  
      def test_parser_errors_out_when_multiple_members_set(self):
          parser = parsers.JSONParser()
          response = b'{"Foo": "mystring", "Bar": "mystring2"}'
          headers = {'x-amzn-requestid': 'request-id'}
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'union':True,
                  'members': {
                      'Foo': {
                          'shape': 'StringType',
                      },
                      'Bar': {
                          'shape': 'StringType',
                      }
                  }
              },
              model.ShapeResolver({'StringType': {'type': 'string'}})
          )
  
          response = {'body': response, 'headers': headers, 'status_code': 200}
          with self.assertRaises(parsers.ResponseParserError):
              parser.parse(response, output_shape)
  
  
  class TestHeaderResponseInclusion(unittest.TestCase):
      def create_parser(self):
          return parsers.JSONParser()
  
      def create_arbitary_output_shape(self):
          output_shape = model.StructureShape(
              'OutputShape',
              {
                  'type': 'structure',
                  'members': {
                      'Str': {
                          'shape': 'StringType',
                      }
                  }
              },
              model.ShapeResolver({'StringType': {'type': 'string'}})
          )
          return output_shape
  
      def test_can_add_errors_into_response(self):
          parser = self.create_parser()
          headers = {
              'x-amzn-requestid': 'request-id',
              'Header1': 'foo',
              'Header2': 'bar',
          }
          output_shape = self.create_arbitary_output_shape()
          parsed = parser.parse(
              {'body': b'{}', 'headers': headers,
               'status_code': 200}, output_shape)
          # The mapped header's keys should all be lower cased
          parsed_headers = {
              'x-amzn-requestid': 'request-id',
              'header1': 'foo',
              'header2': 'bar',
          }
          # Response headers should be mapped as HTTPHeaders.
          self.assertEqual(
              parsed['ResponseMetadata']['HTTPHeaders'], parsed_headers)
  
      def test_can_always_json_serialize_headers(self):
          parser = self.create_parser()
          original_headers = {
              'x-amzn-requestid': 'request-id',
              'Header1': 'foo',
          }
          headers = CustomHeaderDict(original_headers)
          output_shape = self.create_arbitary_output_shape()
          parsed = parser.parse(
              {'body': b'{}', 'headers': headers,
               'status_code': 200}, output_shape)
          metadata = parsed['ResponseMetadata']
          # We've had the contract that you can json serialize a
          # response.  So we want to ensure that despite using a CustomHeaderDict
          # we can always JSON dumps the response metadata.
          self.assertEqual(
              json.loads(json.dumps(metadata))['HTTPHeaders']['header1'], 'foo')
  
  
  class TestResponseParsingDatetimes(unittest.TestCase):
      def test_can_parse_float_timestamps(self):
          # The type "timestamp" can come back as both an integer or as a float.
          # We need to make sure we handle the case where the timestamp comes
          # back as a float.  It might make sense to move this to protocol tests.
          output_shape = model.Shape(shape_name='datetime',
                                     shape_model={'type': 'timestamp'})
          parser = parsers.JSONParser()
          timestamp_as_float = b'1407538750.49'
          expected_parsed = datetime.datetime(
              2014, 8, 8, 22, 59, 10, 490000, tzinfo=tzutc())
          parsed = parser.parse(
              {'body': timestamp_as_float,
               'headers': [],
               'status_code': 200}, output_shape)
          self.assertEqual(parsed, expected_parsed)
  
  
  class TestResponseParserFactory(unittest.TestCase):
      def setUp(self):
          self.factory = parsers.ResponseParserFactory()
  
      def test_rest_parser(self):
          parser = self.factory.create_parser('rest-xml')
          self.assertTrue(isinstance(parser, parsers.BaseRestParser))
          self.assertTrue(isinstance(parser, parsers.BaseXMLResponseParser))
  
      def test_json_parser(self):
          parser = self.factory.create_parser('json')
          self.assertTrue(isinstance(parser, parsers.BaseJSONParser))
  
  
  class TestCanDecorateResponseParsing(unittest.TestCase):
      def setUp(self):
          self.factory = parsers.ResponseParserFactory()
  
      def create_request_dict(self, with_body):
          return {
              'body': with_body, 'headers': [], 'status_code': 200
          }
  
      def test_normal_blob_parsing(self):
          output_shape = model.Shape(shape_name='BlobType',
                                     shape_model={'type': 'blob'})
          parser = self.factory.create_parser('json')
  
          hello_world_b64 = b'"aGVsbG8gd29ybGQ="'
          expected_parsed = b'hello world'
          parsed = parser.parse(
              self.create_request_dict(with_body=hello_world_b64),
              output_shape)
          self.assertEqual(parsed, expected_parsed)
  
      def test_can_decorate_scalar_parsing(self):
          output_shape = model.Shape(shape_name='BlobType',
                                     shape_model={'type': 'blob'})
          # Here we're overriding the blob parser so that
          # we can change it to a noop parser.
          self.factory.set_parser_defaults(
              blob_parser=lambda x: x)
          parser = self.factory.create_parser('json')
  
          hello_world_b64 = b'"aGVsbG8gd29ybGQ="'
          expected_parsed = "aGVsbG8gd29ybGQ="
          parsed = parser.parse(
              self.create_request_dict(with_body=hello_world_b64),
              output_shape)
          self.assertEqual(parsed, expected_parsed)
  
      def test_can_decorate_timestamp_parser(self):
          output_shape = model.Shape(shape_name='datetime',
                                     shape_model={'type': 'timestamp'})
          # Here we're overriding the timestamp parser so that
          # we can change it to just convert a string to an integer
          # instead of converting to a datetime.
          self.factory.set_parser_defaults(
              timestamp_parser=lambda x: int(x))
          parser = self.factory.create_parser('json')
  
          timestamp_as_int = b'1407538750'
          expected_parsed = int(timestamp_as_int)
          parsed = parser.parse(
              self.create_request_dict(with_body=timestamp_as_int),
              output_shape)
          self.assertEqual(parsed, expected_parsed)
  
  
  class TestHandlesNoOutputShape(unittest.TestCase):
      """Verify that each protocol handles no output shape properly."""
  
      def test_empty_rest_json_response(self):
          headers = {'x-amzn-requestid': 'request-id'}
          parser = parsers.RestJSONParser()
          output_shape = None
          parsed = parser.parse(
              {'body': b'', 'headers': headers, 'status_code': 200},
              output_shape)
          self.assertEqual(
              parsed,
              {'ResponseMetadata': {'RequestId': 'request-id',
                                    'HTTPStatusCode': 200,
                                    'HTTPHeaders': headers}})
  
      def test_empty_rest_xml_response(self):
          # This is the format used by cloudfront, route53.
          headers = {'x-amzn-requestid': 'request-id'}
          parser = parsers.RestXMLParser()
          output_shape = None
          parsed = parser.parse(
              {'body': b'', 'headers': headers, 'status_code': 200},
              output_shape)
          self.assertEqual(
              parsed,
              {'ResponseMetadata': {'RequestId': 'request-id',
                                    'HTTPStatusCode': 200,
                                    'HTTPHeaders': headers}})
  
      def test_empty_query_response(self):
          body = (
              b'<DeleteTagsResponse xmlns="http://autoscaling.amazonaws.com/">'
              b'  <ResponseMetadata>'
              b'    <RequestId>request-id</RequestId>'
              b'  </ResponseMetadata>'
              b'</DeleteTagsResponse>'
          )
          parser = parsers.QueryParser()
          output_shape = None
          parsed = parser.parse(
              {'body': body, 'headers': {}, 'status_code': 200},
              output_shape)
          self.assertEqual(
              parsed,
              {'ResponseMetadata': {'RequestId': 'request-id',
                                    'HTTPStatusCode': 200,
                                    'HTTPHeaders': {}}})
  
      def test_empty_json_response(self):
          headers = {'x-amzn-requestid': 'request-id'}
          # Output shape of None represents no output shape in the model.
          output_shape = None
          parser = parsers.JSONParser()
          parsed = parser.parse(
              {'body': b'', 'headers': headers, 'status_code': 200},
              output_shape)
          self.assertEqual(
              parsed,
              {'ResponseMetadata': {'RequestId': 'request-id',
                                    'HTTPStatusCode': 200,
                                    'HTTPHeaders': headers}})
  
  
  class TestHandlesInvalidXMLResponses(unittest.TestCase):
      def test_invalid_xml_shown_in_error_message(self):
          # Missing the closing XML tags.
          invalid_xml = (
              b'<DeleteTagsResponse xmlns="http://autoscaling.amazonaws.com/">'
              b'  <ResponseMetadata>'
          )
          parser = parsers.QueryParser()
          output_shape = None
          # The XML body should be in the error message.
          with self.assertRaisesRegex(parsers.ResponseParserError,
                                      '<DeleteTagsResponse'):
              parser.parse(
                  {'body': invalid_xml, 'headers': {}, 'status_code': 200},
                  output_shape)
  
  
  class TestRESTXMLResponses(unittest.TestCase):
      def test_multiple_structures_list_returns_struture(self):
          # This is to handle the scenario when something is modeled
          # as a structure and instead a list of structures is returned.
          # For this case, a single element from the list should be parsed
          # For botocore, this will be the first element.
          # Currently, this logic may happen in s3's GetBucketLifecycle
          # operation.
          headers = {}
          parser = parsers.RestXMLParser()
          body = (
              '<?xml version="1.0" ?>'
              '<OperationName xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
              '	<Foo><Bar>first_value</Bar></Foo>'
              '	<Foo><Bar>middle_value</Bar></Foo>'
              '	<Foo><Bar>last_value</Bar></Foo>'
              '</OperationName>'
          )
          builder = model.DenormalizedStructureBuilder()
          output_shape = builder.with_members({
              'Foo': {
                  'type': 'structure',
                  'members': {
                      'Bar': {
                          'type': 'string',
                      }
                  }
              }
          }).build_model()
          parsed = parser.parse(
              {'body': body, 'headers': headers, 'status_code': 200},
              output_shape)
          # Ensure the first element is used out of the list.
          self.assertEqual(parsed['Foo'], {'Bar': 'first_value'})
  
  
  class TestEventStreamParsers(unittest.TestCase):
  
      def setUp(self):
          self.parser = parsers.EventStreamXMLParser()
          self.output_shape = model.StructureShape(
              'EventStream',
              {
                  'eventstream': True,
                  'type': 'structure',
                  'members': {
                      'EventA': {'shape': 'EventAStructure'},
                      'EventB': {'shape': 'EventBStructure'},
                      'EventC': {'shape': 'EventCStructure'},
                      'EventD': {'shape': 'EventDStructure'},
                      'EventException': {'shape': 'ExceptionShape'},
                  }
              },
              model.ShapeResolver({
                  'EventAStructure': {
                      'event': True,
                      'type': 'structure',
                      'members': {
                          'Stats': {
                              'shape': 'StatsStructure',
                              'eventpayload': True
                          },
                          'Header': {
                              'shape': 'IntShape',
                              'eventheader': True
                          }
                      }
                  },
                  'EventBStructure': {
                      'event': True,
                      'type': 'structure',
                      'members': {
                          'Body': {
                              'shape': 'BlobShape',
                              'eventpayload': True
                          }
                      }
                  },
                  'EventCStructure': {
                      'event': True,
                      'type': 'structure',
                      'members': {
                          'Body': {
                              'shape': 'StringShape',
                              'eventpayload': True
                          }
                      }
                  },
                  'EventDStructure': {
                      'event': True,
                      'type': 'structure',
                      'members': {
                          'StringField': {'shape': 'StringShape'},
                          'IntField': {'shape': 'IntShape'},
                          'Header': {
                              'shape': 'IntShape',
                              'eventheader': True
                          }
                      }
                  },
                  'StatsStructure': {
                      'type': 'structure',
                      'members': {
                          'StringField': {'shape': 'StringShape'},
                          'IntField': {'shape': 'IntShape'}
                      }
                  },
                  'BlobShape': {'type': 'blob'},
                  'StringShape': {'type': 'string'},
                  'IntShape': {'type': 'integer'},
                  'ExceptionShape': {
                      'exception': True,
                      'type': 'structure',
                      'members': {
                          'message': {'shape': 'StringShape'}
                      }
                  },
              })
          )
  
      def parse_event(self, headers=None, body=None, status_code=200):
          response_dict = {
              'body': body,
              'headers': headers,
              'status_code': status_code
          }
          return self.parser.parse(response_dict, self.output_shape)
  
      def test_parses_event_xml(self):
          headers = {
              'Header': 123,
              ':event-type': 'EventA'
          }
          body = (
              b'<Stats xmlns="">'
              b'  <StringField>abcde</StringField>'
              b'  <IntField>1234</IntField>'
              b'</Stats>'
          )
          parsed = self.parse_event(headers, body)
          expected = {
              'EventA': {
                  'Header': 123,
                  'Stats': {
                      'StringField': 'abcde',
                      'IntField': 1234
                  }
              }
          }
          self.assertEqual(parsed, expected)
  
      def test_parses_event_bad_xml(self):
          headers = {
              'Header': 123,
              ':event-type': 'EventA'
          }
          parsed = self.parse_event(headers, b'')
          expected = {
              'EventA': {
                  'Header': 123,
                  'Stats': {}
              }
          }
          self.assertEqual(parsed, expected)
  
      def test_parses_event_blob(self):
          headers = {':event-type': 'EventB'}
          parsed = self.parse_event(headers, b'blob')
          expected = {'EventB': {'Body': b'blob'}}
          self.assertEqual(parsed, expected)
  
      def test_parses_event_string(self):
          headers = {':event-type': 'EventC'}
          parsed = self.parse_event(headers, b'blob')
          expected = {'EventC': {'Body': u'blob'}}
          self.assertEqual(parsed, expected)
  
      def test_parses_payload_implicit(self):
          headers = {
              'Header': 123,
              ':event-type': 'EventD'
          }
          body = (
              b'<EventD xmlns="">'
              b'  <StringField>abcde</StringField>'
              b'  <IntField>1234</IntField>'
              b'</EventD>'
          )
          parsed = self.parse_event(headers, body)
          expected = {
              'EventD': {
                  'Header': 123,
                  'StringField': 'abcde',
                  'IntField': 1234
              }
          }
          self.assertEqual(parsed, expected)
  
      def test_parses_error_event(self):
          error_code = 'client/SomeError'
          error_message = 'You did something wrong'
          headers = {
              ':message-type': 'error',
              ':error-code': error_code,
              ':error-message': error_message
          }
          body = b''
          parsed = self.parse_event(headers, body, status_code=400)
          expected = {
              'Error': {
                  'Code': error_code,
                  'Message': error_message
              }
          }
          self.assertEqual(parsed, expected)
  
      def test_parses_exception_event(self):
          self.parser = parsers.EventStreamJSONParser()
          error_code = 'EventException'
          headers = {
              ':message-type': 'exception',
              ':exception-type': error_code,
          }
          body = b'{"message": "You did something wrong"}'
          parsed = self.parse_event(headers, body, status_code=400)
          expected = {
              'Error': {
                  'Code': error_code,
                  'Message': 'You did something wrong'
              }
          }
          self.assertEqual(parsed, expected)
  
      def test_parses_event_json(self):
          self.parser = parsers.EventStreamJSONParser()
          headers = {':event-type': 'EventD'}
          body = (
              b'{'
              b'  "StringField": "abcde",'
              b'  "IntField": 1234'
              b'}'
          )
          parsed = self.parse_event(headers, body)
          expected = {
              'EventD': {
                  'StringField': 'abcde',
                  'IntField': 1234
              }
          }
          self.assertEqual(parsed, expected)
  
  
  class TestParseErrorResponses(unittest.TestCase):
      # This class consolidates all the error parsing tests
      # across all the protocols.  We may potentially pull
      # this into the shared protocol tests in the future,
      # so consolidating them into a single class will make
      # this easier.
      def setUp(self):
          self.error_shape = model.StructureShape(
              'ErrorShape',
              {
                  'type': 'structure',
                  'exception': True,
                  'members': {
                      'ModeledField': {
                          'shape': 'StringType',
                      }
                  }
              },
              model.ShapeResolver({'StringType': {'type': 'string'}})
          )
  
      def test_response_metadata_errors_for_json_protocol(self):
          parser = parsers.JSONParser()
          response = {
              "body": b"""
                  {"__type":"amazon.foo.validate#ValidationException",
                   "message":"this is a message"}
                  """,
              "status_code": 400,
              "headers": {
                  "x-amzn-requestid": "request-id"
              }
          }
          parsed = parser.parse(response, None)
          # Even (especially) on an error condition, the
          # ResponseMetadata should be populated.
          self.assertIn('ResponseMetadata', parsed)
          self.assertEqual(parsed['ResponseMetadata']['RequestId'], 'request-id')
  
          self.assertIn('Error', parsed)
          self.assertEqual(parsed['Error']['Message'], 'this is a message')
          self.assertEqual(parsed['Error']['Code'], 'ValidationException')
  
      def test_response_metadata_errors_alternate_form_json_protocol(self):
          # Sometimes there is no '#' in the __type.  We need to be
          # able to parse this error message as well.
          parser = parsers.JSONParser()
          response = {
              "body": b"""
                  {"__type":"ValidationException",
                   "message":"this is a message"}
                  """,
              "status_code": 400,
              "headers": {
                  "x-amzn-requestid": "request-id"
              }
          }
          parsed = parser.parse(response, None)
          self.assertIn('Error', parsed)
          self.assertEqual(parsed['Error']['Message'], 'this is a message')
          self.assertEqual(parsed['Error']['Code'], 'ValidationException')
  
      def test_parse_error_response_for_query_protocol(self):
          body = (
              '<ErrorResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/">'
              '  <Error>'
              '    <Type>Sender</Type>'
              '    <Code>InvalidInput</Code>'
              '    <Message>ARN asdf is not valid.</Message>'
              '  </Error>'
              '  <RequestId>request-id</RequestId>'
              '</ErrorResponse>'
          ).encode('utf-8')
          parser = parsers.QueryParser()
          parsed = parser.parse({
              'body': body, 'headers': {}, 'status_code': 400}, None)
          self.assertIn('Error', parsed)
          self.assertEqual(parsed['Error'], {
              'Code': 'InvalidInput',
              'Message': 'ARN asdf is not valid.',
              'Type': 'Sender',
          })
  
      def test_can_parse_sdb_error_response_query_protocol(self):
          body = (
              '<OperationNameResponse>'
              '    <Errors>'
              '        <Error>'
              '            <Code>1</Code>'
              '            <Message>msg</Message>'
              '        </Error>'
              '    </Errors>'
              '    <RequestId>abc-123</RequestId>'
              '</OperationNameResponse>'
          ).encode('utf-8')
          parser = parsers.QueryParser()
          parsed = parser.parse({
              'body': body, 'headers': {}, 'status_code': 500}, None)
          self.assertIn('Error', parsed)
          self.assertEqual(parsed['Error'], {
              'Code': '1',
              'Message': 'msg'
          })
          self.assertEqual(parsed['ResponseMetadata'], {
              'RequestId': 'abc-123',
              'HTTPStatusCode': 500,
              'HTTPHeaders': {}
          })
  
      def test_can_parser_ec2_errors(self):
          body = (
              '<Response>'
              '  <Errors>'
              '    <Error>'
              '      <Code>InvalidInstanceID.NotFound</Code>'
              '      <Message>The instance ID i-12345 does not exist</Message>'
              '    </Error>'
              '  </Errors>'
              '  <RequestID>06f382b0-d521-4bb6-988c-ca49d5ae6070</RequestID>'
              '</Response>'
          ).encode('utf-8')
          parser = parsers.EC2QueryParser()
          parsed = parser.parse({
              'body': body, 'headers': {}, 'status_code': 400}, None)
          self.assertIn('Error', parsed)
          self.assertEqual(parsed['Error'], {
              'Code': 'InvalidInstanceID.NotFound',
              'Message': 'The instance ID i-12345 does not exist',
          })
  
      def test_can_parse_rest_xml_errors(self):
          body = (
              '<ErrorResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/">'
              '  <Error>'
              '    <Type>Sender</Type>'
              '    <Code>NoSuchHostedZone</Code>'
              '    <Message>No hosted zone found with ID: foobar</Message>'
              '  </Error>'
              '  <RequestId>bc269cf3-d44f-11e5-8779-2d21c30eb3f1</RequestId>'
              '</ErrorResponse>'
          ).encode('utf-8')
          parser = parsers.RestXMLParser()
          parsed = parser.parse({
              'body': body, 'headers': {}, 'status_code': 400}, None)
          self.assertIn('Error', parsed)
          self.assertEqual(parsed['Error'], {
              'Code': 'NoSuchHostedZone',
              'Message': 'No hosted zone found with ID: foobar',
              'Type': 'Sender',
          })
  
      def test_can_parse_rest_json_errors(self):
          body = (
              '{"Message":"Function not found: foo","Type":"User"}'
          ).encode('utf-8')
          headers = {
              'x-amzn-requestid': 'request-id',
              'x-amzn-errortype': 'ResourceNotFoundException:http://url/',
          }
          parser = parsers.RestJSONParser()
          parsed = parser.parse({
              'body': body, 'headers': headers, 'status_code': 400}, None)
          self.assertIn('Error', parsed)
          self.assertEqual(parsed['Error'], {
              'Code': 'ResourceNotFoundException',
              'Message': 'Function not found: foo',
          })
  
      def test_error_response_with_no_body_rest_json(self):
          parser = parsers.RestJSONParser()
          response = b''
          headers = {'content-length': '0', 'connection': 'keep-alive'}
          output_shape = None
          parsed = parser.parse({'body': response, 'headers': headers,
                                 'status_code': 504}, output_shape)
  
          self.assertIn('Error', parsed)
          self.assertEqual(parsed['Error'], {
              'Code': '504',
              'Message': 'Gateway Timeout'
          })
          self.assertEqual(parsed['ResponseMetadata'], {
              'HTTPStatusCode': 504,
              'HTTPHeaders': headers
          })
  
      def test_error_response_with_string_body_rest_json(self):
          parser = parsers.RestJSONParser()
          response = b'HTTP content length exceeded 1049600 bytes.'
          headers = {'content-length': '0', 'connection': 'keep-alive'}
          output_shape = None
          parsed = parser.parse({'body': response, 'headers': headers,
                                 'status_code': 413}, output_shape)
  
          self.assertIn('Error', parsed)
          self.assertEqual(parsed['Error'], {
              'Code': '413',
              'Message': response.decode('utf-8')
          })
          self.assertEqual(parsed['ResponseMetadata'], {
              'HTTPStatusCode': 413,
              'HTTPHeaders': headers
          })
  
      def test_error_response_with_xml_body_rest_json(self):
          parser = parsers.RestJSONParser()
          response = (
              '<AccessDeniedException>'
              '   <Message>Unable to determine service/operation name to be authorized</Message>'
              '</AccessDeniedException>'
          ).encode('utf-8')
          headers = {'content-length': '0', 'connection': 'keep-alive'}
          output_shape = None
          parsed = parser.parse({'body': response, 'headers': headers,
                                 'status_code': 403}, output_shape)
  
          self.assertIn('Error', parsed)
          self.assertEqual(parsed['Error'], {
              'Code': '403',
              'Message': response.decode('utf-8')
          })
          self.assertEqual(parsed['ResponseMetadata'], {
              'HTTPStatusCode': 403,
              'HTTPHeaders': headers
          })
  
      def test_s3_error_response(self):
          body = (
              '<Error>'
              '  <Code>NoSuchBucket</Code>'
              '  <Message>error message</Message>'
              '  <BucketName>asdf</BucketName>'
              '  <RequestId>EF1EF43A74415102</RequestId>'
              '  <HostId>hostid</HostId>'
              '</Error>'
          ).encode('utf-8')
          headers = {
              'x-amz-id-2': 'second-id',
              'x-amz-request-id': 'request-id'
          }
          parser = parsers.RestXMLParser()
          parsed = parser.parse(
              {'body': body, 'headers': headers, 'status_code': 400}, None)
          self.assertIn('Error', parsed)
          self.assertEqual(parsed['Error'], {
              'Code': 'NoSuchBucket',
              'Message': 'error message',
              'BucketName': 'asdf',
              # We don't want the RequestId/HostId because they're already
              # present in the ResponseMetadata key.
          })
          self.assertEqual(parsed['ResponseMetadata'], {
              'RequestId': 'request-id',
              'HostId': 'second-id',
              'HTTPStatusCode': 400,
              'HTTPHeaders': headers
          })
  
      def test_s3_error_response_with_no_body(self):
          # If you try to HeadObject a key that does not exist,
          # you will get an empty body.  When this happens
          # we expect that we will use Code/Message from the
          # HTTP status code.
          body = ''
          headers = {
              'x-amz-id-2': 'second-id',
              'x-amz-request-id': 'request-id'
          }
          parser = parsers.RestXMLParser()
          parsed = parser.parse(
              {'body': body, 'headers': headers, 'status_code': 404}, None)
          self.assertIn('Error', parsed)
          self.assertEqual(parsed['Error'], {
              'Code': '404',
              'Message': 'Not Found',
          })
          self.assertEqual(parsed['ResponseMetadata'], {
              'RequestId': 'request-id',
              'HostId': 'second-id',
              'HTTPStatusCode': 404,
              'HTTPHeaders': headers
          })
  
      def test_can_parse_glacier_error_response(self):
          body = (b'{"code":"AccessDeniedException","type":"Client","message":'
                  b'"Access denied"}')
          headers = {
               'x-amzn-requestid': 'request-id'
          }
          parser = parsers.RestJSONParser()
          parsed = parser.parse(
              {'body': body, 'headers': headers, 'status_code': 400}, None)
          self.assertEqual(parsed['Error'], {'Message': 'Access denied',
                                             'Code': 'AccessDeniedException'})
  
      def test_can_parse_restjson_error_code(self):
          body = b'''{
              "status": "error",
              "errors": [{"message": "[*Deprecated*: blah"}],
              "adds": 0,
              "__type": "#WasUnableToParseThis",
              "message": "blah",
              "deletes": 0}'''
          headers = {
               'x-amzn-requestid': 'request-id'
          }
          parser = parsers.RestJSONParser()
          parsed = parser.parse(
              {'body': body, 'headers': headers, 'status_code': 400}, None)
          self.assertEqual(parsed['Error'], {'Message': 'blah',
                                             'Code': 'WasUnableToParseThis'})
  
      def test_can_parse_with_case_insensitive_keys(self):
          body = (b'{"Code":"AccessDeniedException","type":"Client","Message":'
                  b'"Access denied"}')
          headers = {
               'x-amzn-requestid': 'request-id'
          }
          parser = parsers.RestJSONParser()
          parsed = parser.parse(
              {'body': body, 'headers': headers, 'status_code': 400}, None)
          self.assertEqual(parsed['Error'], {'Message': 'Access denied',
                                             'Code': 'AccessDeniedException'})
  
      def test_can_parse_rest_json_modeled_fields(self):
          body = (
              b'{"ModeledField":"Some modeled field",'
              b'"Message":"Some message"}'
          )
          parser = parsers.RestJSONParser()
          response_dict = {
              'status_code': 400,
              'headers': {},
              'body': body,
          }
          parsed = parser.parse(response_dict, self.error_shape)
          expected_parsed = {
              'ModeledField': 'Some modeled field',
          }
          self.assertEqual(parsed, expected_parsed)
  
      def test_can_parse_rest_xml_modeled_fields(self):
          parser = parsers.RestXMLParser()
          body = (
              b'<?xml version="1.0"?>\n<ErrorResponse xmlns="http://foo.bar">'
              b'<Error><Type>Sender</Type><Code>NoSuchDistribution</Code>'
              b'<Message>The specified distribution does not exist.</Message>'
              b'<ModeledField>Some modeled field</ModeledField>'
              b'</Error>'
              b'</ErrorResponse>'
          )
          response_dict = {
              'status_code': 400,
              'headers': {},
              'body': body,
          }
          parsed = parser.parse(response_dict, self.error_shape)
          expected_parsed = {
              'ModeledField': 'Some modeled field',
          }
          self.assertEqual(parsed, expected_parsed)
  
      def test_can_parse_ec2_modeled_fields(self):
          body = (
              b'<Response><Errors><Error>'
              b'<Code>ExceptionShape</Code>'
              b'<Message>Foo message</Message>'
              b'<ModeledField>Some modeled field</ModeledField>'
              b'</Error></Errors></Response>'
          )
          parser = parsers.EC2QueryParser()
          response_dict = {
              'status_code': 400,
              'headers': {},
              'body': body,
          }
          parsed = parser.parse(response_dict, self.error_shape)
          expected_parsed = {
              'ModeledField': 'Some modeled field',
          }
          self.assertEqual(parsed, expected_parsed)
  
      def test_can_parse_query_modeled_fields(self):
          parser = parsers.QueryParser()
          body = (
              b'<?xml version="1.0"?>\n<ErrorResponse xmlns="http://foo.bar">'
              b'<Error><Type>Sender</Type><Code>SomeCode</Code>'
              b'<Message>A message</Message>'
              b'<ModeledField>Some modeled field</ModeledField>'
              b'</Error>'
              b'</ErrorResponse>'
          )
          response_dict = {
              'status_code': 400,
              'headers': {},
              'body': body,
          }
          parsed = parser.parse(response_dict, self.error_shape)
          expected_parsed = {
              'ModeledField': 'Some modeled field',
          }
          self.assertEqual(parsed, expected_parsed)
  
      def test_can_parse_json_modeled_fields(self):
          body = (
              b'{"ModeledField":"Some modeled field",'
              b'"Message":"Some message",'
              b'"__type": "Prefix#SomeError"}'
          )
          parser = parsers.JSONParser()
          response_dict = {
              'status_code': 400,
              'headers': {},
              'body': body,
          }
          parsed = parser.parse(response_dict, self.error_shape)
          expected_parsed = {
              'ModeledField': 'Some modeled field',
          }
          self.assertEqual(parsed, expected_parsed)
  
      def test_can_parse_route53_with_missing_message(self):
          # The message isn't always in the XML response (or even the headers).
          # We should be able to handle this gracefully and still at least
          # populate a "Message" key so that consumers don't have to
          # conditionally check for this.
          body =  (
              '<ErrorResponse>'
              '  <Error>'
              '    <Type>Sender</Type>'
              '    <Code>InvalidInput</Code>'
              '  </Error>'
              '  <RequestId>id</RequestId>'
              '</ErrorResponse>'
          ).encode('utf-8')
          parser = parsers.RestXMLParser()
          parsed = parser.parse({
              'body': body, 'headers': {}, 'status_code': 400}, None)
          error = parsed['Error']
          self.assertEqual(error['Code'], 'InvalidInput')
          # Even though there's no <Message /> we should
          # still populate an empty string.
          self.assertEqual(error['Message'], '')
  
  
  def test_can_handle_generic_error_message():
      # There are times when you can get a service to respond with a generic
      # html error page.  We should be able to handle this case.
      for parser_cls in parsers.PROTOCOL_PARSERS.values():
          generic_html_body =  (
              '<html><body><b>Http/1.1 Service Unavailable</b></body></html>'
          ).encode('utf-8')
          empty_body = b''
          none_body = None
          yield _assert_parses_generic_error, parser_cls(), generic_html_body
          yield _assert_parses_generic_error, parser_cls(), empty_body
          yield _assert_parses_generic_error, parser_cls(), none_body
  
  
  def _assert_parses_generic_error(parser, body):
      # There are times when you can get a service to respond with a generic
      # html error page.  We should be able to handle this case.
      parsed = parser.parse({
          'body': body, 'headers': {}, 'status_code': 503}, None)
      assert_equal(
          parsed['Error'],
          {'Code': '503', 'Message': 'Service Unavailable'})
      assert_equal(parsed['ResponseMetadata']['HTTPStatusCode'], 503)
  