whole_func_string: function(rules, flag) {
      flag = flag ? "g" + flag : "g";
      this.rules = rules;
  
      this.regExps = {};
      this.matchMappings = {};
      for ( var key in this.rules) {
          var rule = this.rules[key];
          var state = rule;
          var ruleRegExps = [];
          var matchTotal = 0;
          var mapping = this.matchMappings[key] = {};
  
          for ( var i = 0; i < state.length; i++) {
  
              if (state[i].regex instanceof RegExp)
                  state[i].regex = state[i].regex.toString().slice(1, -1);
  
              // Count number of matching groups. 2 extra groups from the full match
              // And the catch-all on the end (used to force a match);
              var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;
  
              // Replace any backreferences and offset appropriately.
              var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
                  return "\\" + (parseInt(digit, 10) + matchTotal + 1);
              });
  
              if (matchcount > 1 && state[i].token.length !== matchcount-1)
                  throw new Error("Matching groups and length of the token array don't match in rule #" + i + " of state " + key);
  
              mapping[matchTotal] = {
                  rule: i,
                  len: matchcount
              };
              matchTotal += matchcount;
  
              ruleRegExps.push(adjustedregex);
          }
  
          this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag);
      }
  }
func_code_string: function(rules, flag) {
      flag = flag ? "g" + flag : "g";
      this.rules = rules;
  
      this.regExps = {};
      this.matchMappings = {};
      for ( var key in this.rules) {
          var rule = this.rules[key];
          var state = rule;
          var ruleRegExps = [];
          var matchTotal = 0;
          var mapping = this.matchMappings[key] = {};
  
          for ( var i = 0; i < state.length; i++) {
  
              if (state[i].regex instanceof RegExp)
                  state[i].regex = state[i].regex.toString().slice(1, -1);
  
              // Count number of matching groups. 2 extra groups from the full match
              // And the catch-all on the end (used to force a match);
              var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;
  
              // Replace any backreferences and offset appropriately.
              var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
                  return "\\" + (parseInt(digit, 10) + matchTotal + 1);
              });
  
              if (matchcount > 1 && state[i].token.length !== matchcount-1)
                  throw new Error("Matching groups and length of the token array don't match in rule #" + i + " of state " + key);
  
              mapping[matchTotal] = {
                  rule: i,
                  len: matchcount
              };
              matchTotal += matchcount;
  
              ruleRegExps.push(adjustedregex);
          }
  
          this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag);
      }
  }
func_documentation_string: class Tokenizer
  
  This class takes a set of highlighting rules, and creates a tokenizer out of them. For more information, see [the wiki on extending highlighters](https://github.com/ajaxorg/ace/wiki/Creating-or-Extending-an-Edit-Mode#wiki-extendingTheHighlighter).
  
   
  new Tokenizer(rules, flag)
  - rules (Object): The highlighting rules
  - flag (String): Any additional regular expression flags to pass (like "i" for case insensitive)
  
  Constructs a new tokenizer based on the given rules and flags.