docstr: Tests generator calculation of token and sequence histograms.
docstr_tok: ["Tests","generator","calculation","of","token","and","sequence","histograms","."]
code: def test_nl_generator_token_and_sequence_histograms(self):
      """Tests generator calculation of token and sequence histograms."""
      with tempfile.NamedTemporaryFile() as vocab_file:
          vocab_file.write(b"Foo\nBar\nBaz\nBazz\nCar\nRazzz\n")
          vocab_file.flush()
          input_batches = [pa.array([[0, 1, 2, 4, 4], [3, 3, 3, 5]])]
          generator = nlsg.NLStatsGenerator(
              schema=self._schema,
              vocab_paths={"my_vocab": vocab_file.name},
              num_quantiles_histogram_buckets=2,
              num_rank_histogram_buckets=2,
              num_histogram_buckets=2,
          )
          expected_reported_sequences = [
              ["Foo", "Bar", "Baz", "Car", "Car"],
              ["Bazz", "Bazz", "Bazz", "Razzz"],
          ] * 2
          self.assertCombinerOutputEqual(
              input_batches,
              generator,
              self._create_expected_feature_name_statistics(
                  feature_coverage=0.8571428571428571,
                  avg_token_length=(3 + 3 + 4 + 4 + 4 + 5) / 6,
                  min_sequence_length=3,
                  max_sequence_length=5,
                  token_len_quantiles=[(3, 4, 5), (4, 5, 1)],
                  sequence_len_quantiles=[(3, 5, 1.5), (5, 5, 0.5)],
                  sorted_token_names_and_counts=[
                      ("Bazz", 3),
                      ("Car", 2),
                  ],
                  reported_sequences=expected_reported_sequences,
              ),
              self._int_nlp_feature_with_vocab_path,
          )
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import tempfile
  from absl.testing import absltest
  import pyarrow
  from tensorflow_data_validation import types
  from tensorflow_data_validation.statistics.generators import natural_language_stats_generator
  from tensorflow_data_validation.utils import test_util
  from google.protobuf import text_format
  from tensorflow_metadata.proto.v0 import schema_pb2
  from tensorflow_metadata.proto.v0 import statistics_pb2
label_desc: Use image node_id 12 for calling the NaturalLanguageStatsGeneratorTest obj's underlying member method code with example usage: obj.test_nl_generator_token_and_sequence_histograms() without return types