jQuery serializeObject function
Here's what I came up with:

var addNestedPropToObj = function (obj, name, value) {
    var path = name.split('.'),
    current = obj,
    len = path.length - 1,
    i = 0;
    for (; i &lt; len; i++) {
        current[path[i]] = current[path[i]] || {};
        current = current[path[i]];
    }
    if ( 0 &lt; path[i].indexOf( "[]" ) ) {
        name = path[i].replace('[]', '');
        current[name] = current[name] || [];
        current[name].push(value);
    } else {
        current[path[i]] = value;
    }
    return obj;
};
jQuery.fn.serializeObject = function () {
    var o = {},
    a = this.serializeArray(),
    i = 0,
    len = a.length;
    for (; i &lt; len; i++) {
        o = addNestedPropToObj(o, a[i].name, a[i].value);
    }
    return o;
};


Demo and test cases

Tips:


Always try to separate most of the logic from the plugin into small testable functions.
Use a regular loop instead of the $.each().
String.prototype.split() returns an array of the entire string if a match isn't found. So the else condition isn't required.

// not needed
else {
    o[this.name] = this.value;
}

Instead of checking to see if you're at the end of the loop, iterate to path.length - 1 then afterwards perform the statement, current[path[i]] = this.value;.


As for the array format contained within the name property, for simplicity I suggest that the array notation should be an endpoint where the value is the push to the current property. So for a value to a added to an array, then [] must be at the end of the name.

Here's an example:

var func = function( str ){
    return JSON.stringify( addNestedPropToObj( {}, str, 1));
};
func("a[].b") === '{"a[]":{"b":1}}';
func("a.b[]") ===  '{"a":{"b":[1]}}';


Seen as an array: a[] or a.b[]

Seen as a string: a[b], a[b.c]