TypeError when passing bytearray to C++ extension
Question: Python code:

    
    
    image = urllib2.urlopen('http://localhost/test.png').read()
    bytes = bytearray(image)
    print [myext.do_stuff(bytes, mode=1)]
    

C++ code:

    
    
    static PyObject * 
    do_stuff(PyObject *self, PyObject *args, PyObject *kwargs)
    {
       PyByteArrayObject *imgdata;
       char *image;
       int mode;
       char *keywords[] = { "image", "mode", NULL };
    
       if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i", keywords, &imgdata, &mode))
          return NULL;
    
       image = PyByteArray_AsString((PyObject*) imgdata);
       char *result = do_something_more(image, mode);
       return Py_BuildValue("s", result);
    }
    

Added:

    
    
    char * do_something_more(char imagebuffer[], int mode)
    {
      vector<char> vec(imagebuffer, imagebuffer + sizeof(imagebuffer));
      Mat input = imdecode(vec, 1);
    }
    


Answer: The typeerror is simply due to the fact that the `Y` format specifier does
_not_ exist in python2 but only in python3. If you want to pass a `bytearray`
in python2 you must use the `O` format specifier.

The fact that the result string is just the first few bytes of the actual
contect is pretty simple:

  * `strlen` is a C function that deals with C _null terminated_ strings. Your image data contains some null bytes and hence the function does _not_ return the _actual_ size. 
  * [`PyBuild_Value`](http://docs.python.org/2/c-api/arg.html#Py_BuildValue)'s `s` format specifier takes a C _null terminated_ string and returns a python string object. Since your data contains null bytes not all the content is put in the result.

In your C++ code the `char *image` pointer _does_ point to _all_ the data, but
you should _not_ rely on C's string functions if your strings contain null
bytes. You must always keep track of the length of the string.

* * *

To make clearer what I mean. Here's a self-contained C extension that can be
used to demonstrate your problem:

    
    
    #include <Python.h>
    
    
    static PyObject * 
    do_stuff(PyObject *self, PyObject *args, PyObject *kwargs)
    {
       PyByteArrayObject *imgdata;
       char *image;
       int mode;
       char *keywords[] = { "image", "mode", NULL };
    
       if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i", keywords, &imgdata, &mode))
          return NULL;
    
       image = PyByteArray_AsString((PyObject*) imgdata);
       return Py_BuildValue("s", image);
    }
    
    static PyMethodDef noddy_methods[] = {
        {"do_stuff",  do_stuff, METH_VARARGS | METH_KEYWORDS, "Does stuff"},
        {NULL, NULL, 0, NULL}  /* Sentinel */
    };
    
    void
    initdemo(void) 
    {
       (void) Py_InitModule("demo", noddy_methods);
    }
    

With `setup.py`:

    
    
    from distutils.core import setup, Extension
    
    module1 = Extension('demo',
                        sources = ['demo_ext.c'])
    
    setup (name = 'Demo',
           version = '1.0',
           description = 'This is a demo package',
           ext_modules = [module1])
    

Used as:

    
    
    >>> import demo
    >>> with open('/Path/to/A/PNG/Image.png', 'rb') as f:
    ...     contents = f.read()
    ... 
    >>> byt = bytearray(contents)
    >>> byt[:20]
    bytearray(b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x02V')
    >>> demo.do_stuff(byt)   # "truncates" the data
    '\x89PNG\r\n\x1a\n'
    

Now if you change the `do_stuff` function to:

    
    
    static PyObject * 
    do_stuff(PyObject *self, PyObject *args, PyObject *kwargs)
    {
       PyObject *imgdata;
       char *image;
       int mode;
       Py_ssize_t length;
       char *keywords[] = { "image", "mode", NULL };
    
       if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i", keywords, &imgdata, &mode))
          return NULL;
    
       image = PyByteArray_AsString(imgdata);
       length = PyObject_Length(imgdata);
       PyObject *res = PyString_FromStringAndSize(image, length);
       return res;
    }
    

You get:

    
    
    >>> import demo
    >>> with open('/home/giacomo/Immagini/bad_grouping.png', 'rb') as f:
    ...     contents = f.read()
    ... 
    >>> byt = bytearray(contents)
    >>> byt[:20]
    bytearray(b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x02V')
    >>> demo.do_stuff(byt)[:20]
    '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x02V'
    

As you can see `do_stuff` does _not_ truncate the data anymore.

All functions such as `strlen` assume no null byte is in the string and will
misbehave when this is not true (as in this case). Also some python's API
calls assume C strings, such as `Py_BuildValue`. As you can see the `char
*image` **does** contain _all_ the data. The problem is that you are using
functions that don't handle it properly.

