code: def test_c4_preprocess(self):
      def load_c4_dataset(split="train"):
          dataset = _c4_dataset(split=split)
          return dataset.map(lambda example: (example, example["text"]))
  
      def examine_processed_dataset(proc_dataset):
          count = 0
          lengths = []
          for example in tfds.as_numpy(proc_dataset):
              count += 1
              ex = example[0]
              # Targets are in the example.
              self.assertIn("targets", ex)
              self.assertEqual(ex["targets"].dtype, np.int64)
              lengths.append(len(ex["targets"]))
          return count, lengths
  
      unfiltered_count = 0
      for example in tfds.as_numpy(load_c4_dataset()):
          unfiltered_count += 1
          # Targets are NOT in the example.
          self.assertNotIn("targets", example[0])
  
      proc_dataset = tf_inputs.c4_preprocess(
          load_c4_dataset(), False, 2048
      )
  
      # `examine_processed_dataset` has some asserts in it.
      proc_count, char_lengths = examine_processed_dataset(proc_dataset)
  
      # Both the original and filtered datasets have examples.
      self.assertGreater(unfiltered_count, 0)
      self.assertGreater(proc_count, 0)
  
      # Because we filter out some entries on length.
      self.assertLess(proc_count, unfiltered_count)
  
      # Preprocess using the sentencepiece model in testdata.
      spc_proc_dataset = tf_inputs.c4_preprocess(
          load_c4_dataset(),
          False,
          2048,
          tokenization="spc",
          spm_path=_spm_path(),
      )
  
      spc_proc_count, spc_lengths = examine_processed_dataset(
          spc_proc_dataset
      )
  
      # spc shortens the target sequence a lot, should be almost equal to
      # unfiltered
      self.assertLessEqual(proc_count, spc_proc_count)
      self.assertEqual(unfiltered_count, spc_proc_count)
  
      # Assert all spc_lengths are lesser than their char counterparts.
      for spc_len, char_len in zip(spc_lengths, char_lengths):
          self.assertLessEqual(spc_len, char_len)
  
imports: import collections
  import os
  from unittest import mock
  import gin
  import numpy
  from t5.data import assert_dataset
  from t5.data import preprocessors
  import tensorflow
  import tensorflow_datasets
  from trax.data import inputs
  from trax.data import tf_inputs
label_desc: Use image node_id 12 for calling the TFInputsTest obj's underlying member method code with example usage: obj.test_c4_preprocess() and returns: dataset, count,  lengths