Checking for balanced brackets in JavaScript
The function must not operate on brackets inside strings and comments.
If that's the case then why not just compare the number of opened vs closed symbols?
Example:
var haveSameLength = function(str, a, b){
    return (str.match(a) || [] ).length === (str.match(b) || [] ).length;
};
var isBalanced = function(str){
    var arr = [ 
        [ /\(/gm, /\)/gm ], [ /\{/gm, /\}/gm ], [ /\[/gm, /\]/gm ] 
    ], i = arr.length, isClean = true;
    
    while( i-- &amp;&amp; isClean ){
        isClean = haveSameLength( str, arr[i][0], arr[i][1] );
    }
    return isClean;
};

Simple Testcases.
console.log( isBalanced( &quot;var a = function(){return 'b';}&quot; ) === true ); 
console.log( isBalanced( &quot;var a = function(){return 'b';&quot; ) === false ); 
console.log( isBalanced( &quot;/*Comment*/var a = function(){ \n // coment again \n return 'b';&quot; ) === false ); 
console.log( isBalanced( &quot;var a = function(){return 'b';&quot; ) === false ); 

Here's a demo:
http://jsfiddle.net/9esyk/
Update
Your code is optimal if performance is the main consideration, but the complexity is too high.
Here are a few tips.
1)
Split up your function into smaller methods to reduce the complexity. One way to do this would be to have functions to filter your string so that you only analyze the meaningful characters.
2)
Avoid using the keyword char since it's a java reserved keyword.
Final Result:
var removeComments = function(str){
    var re_comment = /(\/[*][^*]*[*]\/)|(\/\/[^\n]*)/gm;
    return (&quot;&quot;+str).replace( re_comment, &quot;&quot; );
};
var getOnlyBrackets = function(str){
    var re = /[^()\[\]{}]/g;
    return (&quot;&quot;+str).replace(re, &quot;&quot;);
};
var areBracketsInOrder = function(str){
    str = &quot;&quot;+str;
    var bracket = {
            &quot;]&quot;: &quot;[&quot;,
            &quot;}&quot;: &quot;{&quot;,
            &quot;)&quot;: &quot;(&quot;
        },
        openBrackets = [], 
        isClean = true,
        i = 0,
        len = str.length;
        
    for(; isClean &amp;&amp; i&lt;len; i++ ){
        if( bracket[ str[ i ] ] ){
            isClean = ( openBrackets.pop() === bracket[ str[ i ] ] );
        }else{
            openBrackets.push( str[i] );
        }
    }
    return isClean &amp;&amp; !openBrackets.length;
};
var isBalanced = function(str){
    str = removeComments(str);
    str = getOnlyBrackets(str);
    return areBracketsInOrder(str);
};

Testcases
test(&quot;test isBalanced for good values&quot;, function(){
    var func = isBalanced;
    ok(func( &quot;&quot; ));
    ok(func( &quot;(function(){return [new Bears()]}());&quot; ));
    ok(func( &quot;var a = function(){return 'b';}&quot; ));
    ok(func( &quot;/*Comment: a = [} is bad */var a = function(){return 'b';}&quot; ));
    ok(func( &quot;/*[[[ */ function(){return {b:(function(x){ return x+1; })('c')}} /*_)(([}*/&quot; ));
    ok(func( &quot;//Complex object;\n a = [{a:1,b:2,c:[ new Car( 1, 'black' ) ]}]&quot; ));
});
test(&quot;test isBalanced for bad values&quot;, function(){
    var func = isBalanced;
    ok(!func( &quot;{&quot; ));
    ok(!func( &quot;{]&quot; ));
    ok(!func( &quot;{}(&quot; ));
    ok(!func( &quot;({)()()[][][}]&quot; ));
    ok(!func( &quot;[//]&quot; ));
    ok(!func( &quot;[/*]*/&quot; ));
    ok(!func( &quot;(function(){return [new Bears()}())];&quot; ));
    ok(!func( &quot;var a = [function(){return 'b';]}&quot; ));
    ok(!func( &quot;/*Comment: a = [} is bad */var a = function({)return 'b';}&quot; ));
    ok(!func( &quot;/*[[[ */ function(){return {b:(function(x){ return x+1; })'c')}} /*_)(([}*/&quot; ));
    ok(!func( &quot;//Complex object;\n a = [{a:1,b:2,c:[ new Car( 1, 'black' ) ]]&quot; ));
});

Demo: http://jsfiddle.net/9esyk/3/