code: def _token_counts_from_generator(
      generator, max_chars, reserved_tokens
  ):
      """Builds token counts from generator."""
      reserved_tokens = list(reserved_tokens) + [
          _UNDERSCORE_REPLACEMENT
      ]
      tokenizer = text_encoder.Tokenizer(
          alphanum_only=False, reserved_tokens=reserved_tokens
      )
      num_chars = 0
      token_counts = collections.defaultdict(int)
      for s in generator:
          s = tf.compat.as_text(s)
          if max_chars and (num_chars + len(s)) >= max_chars:
              s = s[: (max_chars - num_chars)]
          tokens = tokenizer.tokenize(s)
          tokens = _prepare_tokens_for_encode(tokens)
          for t in tokens:
              token_counts[t] += 1
          if max_chars:
              num_chars += len(s)
              if num_chars > max_chars:
                  break
      return token_counts
  
imports: from __future__ import unicode_literals
  import collections
  from absl import logging
  from tensorflow_datasets.core.deprecated.text import text_encoder
  from tensorflow_datasets.core.utils.lazy_imports_utils import tensorflow
label_desc: Use image node_id 1 for calling a global function with example usage: _token_counts_from_generator(generator, max_chars, reserved_tokens) and returns: token_counts