docstr: Tests generator calculation of token statistics.
docstr_tok: ["Tests","generator","calculation","of","token","statistics","."]
code: def test_nl_generator_token_stats(self):
      """Tests generator calculation of token statistics."""
      with tempfile.NamedTemporaryFile() as vocab_file:
          vocab_file.write(b"Foo\nBar\n")
          vocab_file.flush()
          input_batches = [pa.array([[0, 1, 0], [1, 0, 0]])]
          generator = nlsg.NLStatsGenerator(
              schema=self._schema,
              vocab_paths={"my_vocab": vocab_file.name},
              num_quantiles_histogram_buckets=0,
              num_rank_histogram_buckets=0,
              num_histogram_buckets=3,
          )
          expected_reported_sequences = [
              ["Foo", "Bar", "Foo"],
              ["Bar", "Foo", "Foo"],
          ] * 2
          position_histogram_1 = statistics_pb2.Histogram()
          position_histogram_1.buckets.add(
              low_value=0, high_value=float(1) / 3, sample_count=1
          )
          position_histogram_1.buckets.add(
              low_value=float(1) / 3,
              high_value=float(2) / 3,
              sample_count=1,
          )
          position_histogram_foo = statistics_pb2.Histogram()
          position_histogram_foo.buckets.add(
              low_value=0, high_value=float(1) / 3, sample_count=1
          )
          position_histogram_foo.buckets.add(
              low_value=float(1) / 3,
              high_value=float(2) / 3,
              sample_count=1,
          )
          position_histogram_foo.buckets.add(
              low_value=float(2) / 3, high_value=1, sample_count=2
          )
          expected_token_stats = {
              1: (2, 1.0, 1, 1, 1, position_histogram_1),
              "Foo": (4, 1.0, 2, 2, 2, position_histogram_foo),
          }
          self.assertCombinerOutputEqual(
              input_batches,
              generator,
              self._create_expected_feature_name_statistics(
                  feature_coverage=1.0,
                  avg_token_length=3,
                  min_sequence_length=3,
                  max_sequence_length=3,
                  reported_sequences=expected_reported_sequences,
                  token_statistics=expected_token_stats,
              ),
              self._int_nlp_feature_with_vocab_and_token_constraints_path,
          )
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import tempfile
  from absl.testing import absltest
  import pyarrow
  from tensorflow_data_validation import types
  from tensorflow_data_validation.statistics.generators import natural_language_stats_generator
  from tensorflow_data_validation.utils import test_util
  from google.protobuf import text_format
  from tensorflow_metadata.proto.v0 import schema_pb2
  from tensorflow_metadata.proto.v0 import statistics_pb2
label_desc: Use image node_id 13 for calling the NaturalLanguageStatsGeneratorTest obj's underlying member method code with example usage: obj.test_nl_generator_token_stats() without return types