class_docstr: Process embeddings in blocks & save to disk.
class_docstr_tok: ["Process","embeddings","in","blocks","&","save","to","disk","."]
docstr: Embed a text dataset.
docstr_tok: ["Embed","a","text","dataset","."]
code: def embed_text_dataset(self, name, workdir, text_dataset):
      """Embed a text dataset."""
  
      # Dataset workdir.
      os.makedirs(workdir, exist_ok=True)
  
      # Missing embedding blocks (stored on disk).
      def validate(f):
          assert f["data"].shape[1] == 1024
  
      (
          n_missing_world,
          missing_embedding_blocks,
      ) = get_missing_blocks_by_rank(
          workdir, len(text_dataset), self.block_size, validate=validate
      )
  
      # Prevent missing file race condition.
      torch.distributed.barrier()
  
      # Embed batches.
      self.embed_text_blocks(
          name, workdir, text_dataset, missing_embedding_blocks
      )
  
imports: from functools import partial
  import numpy
  import os
  import time
  import torch
  from torch.utils.data import BatchSampler, DataLoader, SequentialSampler, Subset
  from torch.utils.data._utils.collate import default_collate
  from tqdm import tqdm
  from megatron import get_args, get_tokenizer, print_rank_0
  from megatron import core
  from megatron.arguments import core_transformer_config_from_args
  from megatron.core.enums import ModelType
  from megatron.core.pipeline_parallel import get_forward_backward_func
  from megatron.model import BertModel
  from megatron.training import setup_model_and_optimizer
  from .dataset import BertEmbeddingDataset
  from .external_libs import h5py
  from .huggingface import HuggingfaceEmbedder
  from .utils import get_missing_blocks_by_rank
label_desc: Use image node_id 3 for calling the DiskDataParallelBertEmbedder obj's underlying member method code with example usage: obj.embed_text_dataset(name, workdir, text_dataset) without return types