class_docstr: The WebVid dataset builder.
class_docstr_tok: ["The","WebVid","dataset","builder","."]
docstr: Yields examples.
docstr_tok: ["Yields","examples","."]
code: def _generate_examples(self, captions_csv_path, image_base_path):
      """Yields examples."""
      beam = tfds.core.lazy_imports.apache_beam
      pd = tfds.core.lazy_imports.pandas
  
      ns = "ProcessExample"
      counter = beam.metrics.Metrics.counter
      total_counter = counter(ns, "Total")
      process_error_counter = counter(ns, "Processing error")
      empty_video_counter = counter(ns, "Empty video")
      timeout_counter = counter(ns, "Timed out")
      success_counter = counter(ns, "Success")
      text_cropped_counter = counter(ns, "Text cropped")
  
      distribution = beam.metrics.Metrics.distribution
      initial_caption_len_dist = distribution(
          ns, "Original caption length"
      )
      final_caption_len_dist = distribution(ns, "Final caption length")
      frame_count_dist = distribution(ns, "FrameCount")
  
      def _load_csv(path):
          # Read all columns as strings, to avoid buggily converting ids with
          # leading zeros into ints.
          return pd.read_csv(
              io.StringIO(epath.Path(path).read_text()),
              sep=",",
              encoding="utf-8",
              dtype="str",
          )
  
      def _process_example(data_row):
          """Load each video."""
          total_counter.inc()
  
          file_path = data_row[_VIDEO_PATH_KEY]
          new_video_id = data_row[_NEW_VIDEO_ID_KEY]
          caption = data_row[_CAPTION_KEY]
          url = data_row[_URL_KEY]
  
          # Read out videos and skip those that don't exist or fail to process.
          height, width = _IMG_SIZE
          extra_input_args = [
              # transform to fps and scale+crop (we must set fps because the input
              # videos don't have consistent fps).
              "-vf",
              f"fps={_FPS},scale={width}:{height}:force_original_aspect_ratio=increase,crop={width}:{height}",  # pylint: disable=line-too-long
              "-sws_flags",
              "lanczos+full_chroma_int+full_chroma_inp+accurate_rnd",
          ]
          extra_output_args = [
              "-vcodec",
              "mjpeg",
              "-qmin",
              "1",
              "-q:v",
              "2",
          ]
  
          try:
              logging.info("Processing video %s", file_path)
  
              frames = _ffmpeg_decode_diskless(
                  str(file_path),
                  extra_ffmpeg_input_args=extra_input_args,
                  extra_ffmpeg_output_args=extra_output_args,
                  num_target_frames=_MAX_FRAMES,
                  timeout=_FFMPEG_TIMEOUT_SECONDS,
              )
          except subprocess.TimeoutExpired as e:
              timeout_counter.inc()
              logging.warning(
                  "Timed out while processing %s with exception: %s",
                  file_path,
                  e,
              )
              return
          except Exception:  # pylint: disable=broad-except
              process_error_counter.inc()
              logging.exception(
                  "Failed to process video %s.", file_path
              )
              return
  
          if frames is None:
              empty_video_counter.inc()
              logging.warning("Empty video %s", file_path)
              return
  
          frame_count_dist.update(len(frames))
          logging.info(
              "Successfully processed %s with original frame count %d",
              file_path,
              len(frames),
          )
  
          initial_caption_len_dist.update(len(caption))
          if len(caption) > _MAX_TEXT_LENGTH:
              text_cropped_counter.inc()
  
          features = {
              "id": new_video_id,
              "video": frames[:_MAX_FRAMES],
              "caption": caption[:_MAX_TEXT_LENGTH],
              "url": url,
          }
  
          final_caption_len_dist.update(len(features["caption"]))
          success_counter.inc()
          yield new_video_id, features
  
      # Get list of videos in file system.
      files = epath.Path(image_base_path).glob(
          os.path.join("*", "*_*", "*.mp4")
      )
      df_files = pd.DataFrame(list(files), columns=[_VIDEO_PATH_KEY])
      logging.info("df_files.shape %s", df_files.shape)
  
      # Read info CSV.
      df_info = _load_csv(captions_csv_path)
      logging.info("df_info.shape %s", df_info.shape)
  
      def path_to_id(path):
          return os.path.splitext(
              "__".join(pathlib.Path(path).parts[-2:])
          )[0]
  
      # Create base names to join
      df_info[_NEW_VIDEO_ID_KEY] = (
          df_info[_PAGE_DIR_KEY] + "__" + df_info[_VIDEO_ID_KEY]
      )
      df_files[_NEW_VIDEO_ID_KEY] = df_files[_VIDEO_PATH_KEY].map(
          path_to_id
      )
  
      # Merge the two
      df = df_files.merge(df_info, on=_NEW_VIDEO_ID_KEY)
  
      logging.info("df.shape %s", df.shape)
      logging.info("Number of rows %s", df.shape)
      df = df.to_dict("records")
  
      return beam.Create(df) | beam.FlatMap(_process_example)
  
file_path: datasets/tensorflow_datasets/datasets/webvid/webvid_dataset_builder.py
imports: import io
  import os
  import pathlib
  import subprocess
  from typing import List
  from absl import logging
  from etils import epath
  import tensorflow_datasets.public_api
label_desc: Use image node_id 3 for calling the Builder obj's underlying member method code with example usage: obj._generate_examples(captions_csv_path, image_base_path) and returns: unknown, pd, os