code: def add_index(shard_dir, index_path):
      data_shard_list = []
  
      for shard_address in glob(str(shard_dir) + "/*/"):
          data_shard_list.append(load_from_disk(shard_address))
  
      concat = concatenate_datasets(data_shard_list)
      faiss.omp_set_num_threads(96)
  
      index = faiss.IndexHNSWFlat(768, 128, faiss.METRIC_INNER_PRODUCT)
      concat.add_faiss_index("embeddings", custom_index=index)
      concat.get_index("embeddings").save(index_path)
  
file_path: transformers/examples/research_projects/rag-end2end-retriever/kb_encode_utils.py
imports: import os
  from functools import partial
  from glob import glob
  import faiss
  from datasets import Features, Sequence, Value, concatenate_datasets, load_dataset, load_from_disk
  from transformers import DPRContextEncoder, DPRContextEncoderTokenizerFast
label_desc: Use image node_id 4 for calling a global function with example usage: add_index(shard_dir, index_path) without return types