Problems overriding _iterencode method on JSONEncoder that employs a generator
Question: I am trying to create my own custom encoder that mostly reuses the JSONEncoder
behaviour:

    
    
    BinaryJSONEncoder(JSONEncoder):
        def _iterencode(self, o, markers):
            sys.stderr.write("Calling custom _iterencode\n")
            try:
                 return JSONEncoder._iterencode(self, o, markers)
            except UnicodeDecodeError:
                 sys.stderr.write("Got exception\n")
                 return ""
    

However, I still get an unhandled UnicodeDecodeError, because once the
_iterencode method is called, it cycles through its generators before
returning. What's strange is that the exception originates from the
_iterencode method, but my method is nowhere in the return stack! But my
function is being called, because the "Calling custom _iterencode" messages
are coming out in the Apache error_log. How can I work around this issue
without implementing the entire method from scratch?

Here is the stack:

    
    
    Traceback (most recent call last):
      File "/var/www/radiator/cgi-bin/ldapsearch.py", line 108, in <module>
        res.body(json.dumps(res_data, cls=BinaryJSONEncoder))
      File "/usr/lib64/python2.6/json/__init__.py", line 237, in dumps
        **kw).encode(obj)
      File "/usr/lib64/python2.6/json/encoder.py", line 367, in encode
        chunks = list(self.iterencode(o))
      File "/usr/lib64/python2.6/json/encoder.py", line 306, in _iterencode
        for chunk in self._iterencode_list(o, markers):
      File "/usr/lib64/python2.6/json/encoder.py", line 204, in _iterencode_list
        for chunk in self._iterencode(value, markers):
      File "/usr/lib64/python2.6/json/encoder.py", line 306, in _iterencode
        for chunk in self._iterencode_list(o, markers):
      File "/usr/lib64/python2.6/json/encoder.py", line 204, in _iterencode_list
        for chunk in self._iterencode(value, markers):
      File "/usr/lib64/python2.6/json/encoder.py", line 309, in _iterencode
        for chunk in self._iterencode_dict(o, markers):
      File "/usr/lib64/python2.6/json/encoder.py", line 275, in _iterencode_dict
        for chunk in self._iterencode(value, markers):
      File "/usr/lib64/python2.6/json/encoder.py", line 306, in _iterencode
        for chunk in self._iterencode_list(o, markers):
      File "/usr/lib64/python2.6/json/encoder.py", line 204, in _iterencode_list
        for chunk in self._iterencode(value, markers):
      File "/usr/lib64/python2.6/json/encoder.py", line 294, in _iterencode
        yield encoder(o)
    UnicodeDecodeError: 'utf8' codec can't decode byte 0xc3 in position 2: invalid continuation byte
    


Answer: It seems the only viable way to overcome this problem is to implement the
whole function again, with handlers for the binary data:

from json.encoder import encode_basestring_ascii, encode_basestring

    
    
    class BinaryJSONEncoder(JSONEncoder):
        def _iterencode(self, o, markers=None):
            if isinstance(o, str):
                try:
                    o = unicode(o, "utf8")
                except UnicodeDecodeError:
                    o = base64.b64encode(o)
    
            if isinstance(o, basestring):
                if self.ensure_ascii:
                    encoder = encode_basestring_ascii
                else:
                    encoder = encode_basestring
                _encoding = self.encoding
                if (_encoding is not None and isinstance(o, str)
                        and not (_encoding == 'utf-8')):
                    o = o.decode(_encoding)
                yield encoder(o)
            elif o is None:
                yield 'null'
            elif o is True:
                yield 'true'
            elif o is False:
                yield 'false'
            elif isinstance(o, (int, long)):
                yield str(o)
            elif isinstance(o, float):
                yield floatstr(o, self.allow_nan)
            elif isinstance(o, (list, tuple)):
                for chunk in self._iterencode_list(o, markers):
                    yield chunk
            elif isinstance(o, dict):
                for chunk in self._iterencode_dict(o, markers):
                    yield chunk
            else:
                if markers is not None:
                    markerid = id(o)
                    if markerid in markers:
                        raise ValueError("Circular reference detected")
                    markers[markerid] = o
                for chunk in self._iterencode_default(o, markers):
                    yield chunk
                if markers is not None:
                    del markers[markerid]
    

I understand the issue with the yields exiting when an exception is thrown,
but why can't the superclass method be called there after without it then only
calling the superclass methods?

