code: def dedup(
      chunk_id_to_range, I, tmp_neighbors, chunk_id_start, offset
  ):
      """
      deduplicate the KNN who are from the same document as the data chunks.
      chunk_id_to_range is calculated by build_map function, which maps chunk_id - offset to range of ids of the same document
      I is original KNN search result from Faiss.
      chunk_id_start is the chunk_id offset.
      offset is the map offset
  
      filtered KNN will be stored in the tmp_neighbors
  
      """
      if chunk_id_start < offset or chunk_id_start + len(
          I
      ) - offset > len(chunk_id_to_range):
          raise ValueError("chunk_id_start out side the range")
      _dedup(
          chunk_id_to_range, I, tmp_neighbors, chunk_id_start, offset
      )
  
imports: import argparse
  import multiprocessing
  import pathlib
  import sys
  import time
  from multiprocessing import Pool
  import faiss
  import numpy
  import torch
  from numba import njit, prange
  from sentence_transformers import SentenceTransformer
  from nemo.collections.nlp.data.language_modeling.megatron.indexed_retrieval_dataset import KNNIndex, MMapRetrievalIndexedDataset, merge_knn_files
  from nemo.collections.nlp.modules.common.tokenizer_utils import get_nmt_tokenizer
  from nemo.utils import logging
label_desc: Use image node_id 3 for calling a global function with example usage: dedup(chunk_id_to_range, I, tmp_neighbors, chunk_id_start, offset) without return types