docstr: Add to index (rank 0's method).
docstr_tok: ["Add","to","index","(","rank","0","'s","method",")","."]
code: def _add(self, text_dataset):
      """Add to index (rank 0's method)."""
  
      assert torch.distributed.get_rank() == 0
  
      args = get_retro_args()
  
      dataset_sample_ranges = num_samples_to_block_ranges(
          len(text_dataset)
      )
  
      # Set num threads (torch.distributed reset it to 1).
      faiss.omp_set_num_threads(64)
  
      # Bert embedder.
      embedder = BertEmbedder(
          args.retro_bert_batch_size,
          args.retro_bert_max_chunk_length,
          args.bert_embedder_type,
      )
  
      # Empty/added index paths.
      empty_index_path = self.get_empty_index_path()
      added_index_path = self.get_added_index_path()
  
      # Skip adding, if index exists.
      if os.path.isfile(added_index_path):
          return
  
      # Read trained index.
      index = faiss.read_index(empty_index_path)
  
      # Iterate data blocks & add.
      for sample_range in tqdm(dataset_sample_ranges, "faiss_base.add"):
          # Embed text.
          embeds = self.embed_text_dataset_block(
              embedder, text_dataset, sample_range
          )
  
          # Add to index.
          index.add(embeds)
  
      # Write index.
      faiss.write_index(index, added_index_path)
  
file_path: megatron-lm/tools/retro/index/indexes/faiss_base.py
imports: from datetime import timedelta
  import numpy
  import os
  import torch
  from tqdm import tqdm
  from megatron import get_retro_args, print_rank_0
  from tools.bert_embedding import BertEmbedder
  from tools.retro.external_libs import faiss
  from tools.retro.index.index import Index
  from tools.retro.index.utils import get_training_data_merged_path, num_samples_to_block_ranges
label_desc: Use image node_id 3 for calling the FaissBaseIndex obj's underlying member method code with example usage: obj._add(text_dataset) without return types