Converting any CSS colour to RGB(a)
You are right that returning an object where a is a function would not work, but it would work to return the result of a call to a function!

You can also restructure things a bit to reduce duplicated code. By having a starting string or a starting object, and optionally adding to that string or object if some conditions are fulfilled.

This is untested code, but I think it should do the same as your original.

function result(addAlpha, rgb, asObject) {
    if (asObject) {
        var obj = { r: rgb[0], g: rgb[1], b: rgb[2] };
        if (addAlpha !== false) {
            obj.a = addAlpha;
        }
        else if (rgb.length == 4) {
            obj.a = rgb[3];
        }
        return obj;
    }

    var a = (addAlpha !== false) || (rgb.length == 4) ? "a" : "";
    var alpha = "";
    if (addAlpha !== false) alpha = "," + addAlpha;
    else if (rgb.length == 4) alpha = "," + rgb[3];
    return "rgb" + a + "(" + rgb[0] + "," + rgb[1] + "," + rgb[2] + alpha + ")";
}