class_docstr: Generates and loads the Charades data set.
class_docstr_tok: ["Generates","and","loads","the","Charades","data","set","."]
docstr: Returns Charades as a tf.data.Dataset.
  
  After running this function, calling padded_batch() on the Dataset object
  will produce batches of data, but additional preprocessing may be desired.
  If using padded_batch, the indicator_matrix output distinguishes valid
  from padded frames.
  
  Args:
    split: either "train" or "test"
    shuffle: if true, shuffles both files and examples.
    repeat: if true, repeats the data set forever.
    serialized_prefetch_size: the buffer size for reading from disk.
    decoded_prefetch_size: the buffer size after decoding.
  Returns:
    A tf.data.Dataset object with the following structure: {
      "images": uint8 tensor, shape [time, height, width, channels]
      "segment_matrix": binary tensor of segments, shape [time, num_segments].
        See one_hot_segments() for details.
      "indicator_matrix": binary tensor indicating valid frames,
        shape [time, 1]. If padded with zeros to align sizes, the indicator
        marks where segments is valid.
      "classification_target": binary tensor of classification targets,
        shape [time, 158 classes]. More than one value in a row can be 1.0 if
        segments overlap.
      "example_id": a unique string id for each example, shape [].
      "sampling_rate": the frame rate for each sequence, shape [].
      "gt_segment_seconds": the start and end time of each segment,
        shape [num_segments, 2].
      "gt_segment_classes": the class labels for each segment,
        shape [num_segments].
      "num_segments": the number of segments in the example, shape [].
      "num_timesteps": the number of timesteps in the example, shape [].
docstr_tok: ["Returns","Charades","as","a","tf.data.Dataset",".","After","running","this","function",",","calling","padded_batch","(",")","on","the","Dataset","object","will","produce","batches","of","data",",","but","additional","preprocessing","may","be","desired",".","If","using","padded_batch",",","the","indicator_matrix","output","distinguishes","valid","from","padded","frames",".","Args",":","split",":","either","``","train","''","or","``","test","''","shuffle",":","if","true",",","shuffles","both","files","and","examples",".","repeat",":","if","true",",","repeats","the","data","set","forever",".","serialized_prefetch_size",":","the","buffer","size","for","reading","from","disk",".","decoded_prefetch_size",":","the","buffer","size","after","decoding",".","Returns",":","A","tf.data.Dataset","object","with","the","following","structure",":","{","``","images","''",":","uint8","tensor",",","shape","[","time",",","height",",","width",",","channels","]","``","segment_matrix","''",":","binary","tensor","of","segments",",","shape","[","time",",","num_segments","]",".","See","one_hot_segments","(",")","for","details",".","``","indicator_matrix","''",":","binary","tensor","indicating","valid","frames",",","shape","[","time",",","1","]",".","If","padded","with","zeros","to","align","sizes",",","the","indicator","marks","where","segments","is","valid",".","``","classification_target","''",":","binary","tensor","of","classification","targets",",","shape","[","time",",","158","classes","]",".","More","than","one","value","in","a","row","can","be","1.0","if","segments","overlap",".","``","example_id","''",":","a","unique","string","id","for","each","example",",","shape","[","]",".","``","sampling_rate","''",":","the","frame","rate","for","each","sequence",",","shape","[","]",".","``","gt_segment_seconds","''",":","the","start","and","end","time","of","each","segment",",","shape","[","num_segments",",","2","]",".","``","gt_segment_classes","''",":","the","class","labels","for","each","segment",",","shape","[","num_segments","]",".","``","num_segments","''",":","the","number","of","segments","in","the","example",",","shape","[","]",".","``","num_timesteps","''",":","the","number","of","timesteps","in","the","example",",","shape","[","]","."]
code: def as_dataset(
      self,
      split,
      shuffle=False,
      repeat=False,
      serialized_prefetch_size=32,
      decoded_prefetch_size=32,
  ):
      """Returns Charades as a tf.data.Dataset.
  
      After running this function, calling padded_batch() on the Dataset object
      will produce batches of data, but additional preprocessing may be desired.
      If using padded_batch, the indicator_matrix output distinguishes valid
      from padded frames.
  
      Args:
        split: either "train" or "test"
        shuffle: if true, shuffles both files and examples.
        repeat: if true, repeats the data set forever.
        serialized_prefetch_size: the buffer size for reading from disk.
        decoded_prefetch_size: the buffer size after decoding.
      Returns:
        A tf.data.Dataset object with the following structure: {
          "images": uint8 tensor, shape [time, height, width, channels]
          "segment_matrix": binary tensor of segments, shape [time, num_segments].
            See one_hot_segments() for details.
          "indicator_matrix": binary tensor indicating valid frames,
            shape [time, 1]. If padded with zeros to align sizes, the indicator
            marks where segments is valid.
          "classification_target": binary tensor of classification targets,
            shape [time, 158 classes]. More than one value in a row can be 1.0 if
            segments overlap.
          "example_id": a unique string id for each example, shape [].
          "sampling_rate": the frame rate for each sequence, shape [].
          "gt_segment_seconds": the start and end time of each segment,
            shape [num_segments, 2].
          "gt_segment_classes": the class labels for each segment,
            shape [num_segments].
          "num_segments": the number of segments in the example, shape [].
          "num_timesteps": the number of timesteps in the example, shape [].
      """
  
      def parse_fn(sequence_example):
          """Parses a Charades example."""
          context_features = {
              ms.get_example_id_key(): ms.get_example_id_default_parser(),
              ms.get_segment_start_index_key(): (
                  ms.get_segment_start_index_default_parser()
              ),
              ms.get_segment_end_index_key(): (
                  ms.get_segment_end_index_default_parser()
              ),
              ms.get_segment_label_index_key(): (
                  ms.get_segment_label_index_default_parser()
              ),
              ms.get_segment_label_string_key(): (
                  ms.get_segment_label_string_default_parser()
              ),
              ms.get_segment_start_timestamp_key(): (
                  ms.get_segment_start_timestamp_default_parser()
              ),
              ms.get_segment_end_timestamp_key(): (
                  ms.get_segment_end_timestamp_default_parser()
              ),
              ms.get_image_frame_rate_key(): (
                  ms.get_image_frame_rate_default_parser()
              ),
          }
  
          sequence_features = {
              ms.get_image_encoded_key(): ms.get_image_encoded_default_parser()
          }
          (
              parsed_context,
              parsed_sequence,
          ) = tf.io.parse_single_sequence_example(
              sequence_example, context_features, sequence_features
          )
  
          sequence_length = tf.shape(
              parsed_sequence[ms.get_image_encoded_key()]
          )[0]
          num_segments = tf.shape(
              parsed_context[ms.get_segment_label_index_key()]
          )[0]
          # segments matrix and targets for training.
          segments_matrix, indicator = one_hot_segments(
              tf.sparse_tensor_to_dense(
                  parsed_context[ms.get_segment_start_index_key()]
              ),
              tf.sparse_tensor_to_dense(
                  parsed_context[ms.get_segment_end_index_key()]
              ),
              sequence_length,
          )
  
          classification_target = timepoint_classification_target(
              segments_matrix,
              tf.sparse_tensor_to_dense(
                  parsed_context[ms.get_segment_label_index_key()]
              )
              + CLASS_LABEL_OFFSET,
              NUM_CLASSES + CLASS_LABEL_OFFSET,
          )
  
          # [segments, 2] start and end time in seconds.
          gt_segment_seconds = tf.to_float(
              tf.concat(
                  [
                      tf.expand_dims(
                          tf.sparse_tensor_to_dense(
                              parsed_context[
                                  ms.get_segment_start_timestamp_key()
                              ]
                          ),
                          1,
                      ),
                      tf.expand_dims(
                          tf.sparse_tensor_to_dense(
                              parsed_context[
                                  ms.get_segment_end_timestamp_key()
                              ]
                          ),
                          1,
                      ),
                  ],
                  1,
              )
          ) / float(SECONDS_TO_MICROSECONDS)
          gt_segment_classes = (
              tf.sparse_tensor_to_dense(
                  parsed_context[ms.get_segment_label_index_key()]
              )
              + CLASS_LABEL_OFFSET
          )
          example_id = parsed_context[ms.get_example_id_key()]
          sampling_rate = parsed_context[ms.get_image_frame_rate_key()]
  
          images = tf.map_fn(
              tf.image.decode_jpeg,
              parsed_sequence[ms.get_image_encoded_key()],
              back_prop=False,
              dtype=tf.uint8,
          )
  
          output_dict = {
              "segment_matrix": segments_matrix,
              "indicator_matrix": indicator,
              "classification_target": classification_target,
              "example_id": example_id,
              "sampling_rate": sampling_rate,
              "gt_segment_seconds": gt_segment_seconds,
              "gt_segment_classes": gt_segment_classes,
              "num_segments": num_segments,
              "num_timesteps": sequence_length,
              "images": images,
          }
          return output_dict
  
      if split not in SPLITS:
          raise ValueError(
              "Split %s not in %s" % split, str(list(SPLITS.keys()))
          )
      all_shards = tf.io.gfile.glob(
          os.path.join(self.path_to_data, SPLITS[split][0] + "-*-of-*")
      )
      random.shuffle(all_shards)
      all_shards_dataset = tf.data.Dataset.from_tensor_slices(
          all_shards
      )
      cycle_length = min(16, len(all_shards))
      dataset = all_shards_dataset.apply(
          tf.contrib.data.parallel_interleave(
              tf.data.TFRecordDataset,
              cycle_length=cycle_length,
              block_length=1,
              sloppy=True,
              buffer_output_elements=serialized_prefetch_size,
          )
      )
      dataset = dataset.prefetch(serialized_prefetch_size)
      if shuffle:
          dataset = dataset.shuffle(serialized_prefetch_size)
      if repeat:
          dataset = dataset.repeat()
      dataset = dataset.map(parse_fn)
      dataset = dataset.prefetch(decoded_prefetch_size)
      return dataset
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import contextlib
  import csv
  import os
  import random
  import subprocess
  import sys
  import tempfile
  import zipfile
  from absl import app
  from absl import flags
  from absl import logging
  from six.moves import range
  from six.moves import urllib
  import tensorflow.compat.v1
  from mediapipe.util.sequence import media_sequence
label_desc: Use image node_id 2 for calling the Charades obj's underlying member method code with example usage: obj.as_dataset(split, shuffle, repeat, serialized_prefetch_size, decoded_prefetch_size) and returns: dataset, output_dict