class_docstr: DatasetBuilder for LAION-400M dataset.
class_docstr_tok: ["DatasetBuilder","for","LAION-400M","dataset","."]
docstr: Yields examples from a single shard.
docstr_tok: ["Yields","examples","from","a","single","shard","."]
code: def _generate_examples_one_shard(
      self,
      dl_manager: tfds.download.DownloadManager,
      file_name_to_dl_path: Dict[str, epath.Path],
      shard_idx: int,
  ):
      """Yields examples from a single shard."""
      pd = tfds.core.lazy_imports.pandas
  
      if self.builder_config.name == LAION400M_IMAGES_CONFIG.name:
          img_archive_path = (
              dl_manager.manual_dir / f"{shard_idx:05d}.tar"
          )
          metadata_path = (
              dl_manager.manual_dir / f"{shard_idx:05d}.parquet"
          )
  
          with metadata_path.open("rb") as f:
              metadata_df = pd.read_parquet(f)
  
          for file_name, file_obj in dl_manager.iter_archive(
              img_archive_path
          ):
              file_path = epath.Path(file_name)
              if file_path.suffix in (".json", ".txt"):
                  continue
  
              row_idx = int(file_path.stem)
  
              key = f"{shard_idx}_{row_idx}"
              example = {
                  "image": file_obj.read(),
                  **_get_example_metadata(metadata_df.iloc[row_idx]),
              }
              yield (key, example)
      else:
          (
              img_emb_file_name,
              text_emb_file_name,
              metadata_file_name,
          ) = _get_embeddings_file_names(shard_idx)
  
          with file_name_to_dl_path[img_emb_file_name].open("rb") as f:
              img_emb_arr = np.load(f)
  
          with file_name_to_dl_path[text_emb_file_name].open("rb") as f:
              text_emb_arr = np.load(f)
  
          with file_name_to_dl_path[metadata_file_name].open("rb") as f:
              metadata_df = pd.read_parquet(f)
  
          for row_idx, row in metadata_df.iterrows():
              key = f"{shard_idx}_{row_idx}"
              example = {
                  "image_embedding": img_emb_arr[row_idx],
                  "text_embedding": text_emb_arr[row_idx],
                  **_get_example_metadata(row),
              }
              yield (key, example)
  
file_path: datasets/tensorflow_datasets/vision_language/laion400m/laion400m.py
imports: import functools
  from typing import Dict, Tuple
  from etils import epath
  import numpy
  from tensorflow_datasets.core.utils.lazy_imports_utils import tensorflow
  import tensorflow_datasets.public_api
label_desc: Use image node_id 5 for calling the Laion400m obj's underlying member method code with example usage: obj._generate_examples_one_shard(dl_manager, file_name_to_dl_path, shard_idx) without return types