code: def test_dedup(self):
      total = 1000
      id_start = np.array([0, 100, 200, 300, 500, 900])
      beg = 30
      end = 210
      chunk_id_to_doc_id_map = np.zeros((end - beg, 2), dtype=np.int64)
      build_map(id_start, chunk_id_to_doc_id_map, total, beg, end)
      for i in range(30, 100):
          assert_array_equal(
              chunk_id_to_doc_id_map[i - beg], id_start[0:2]
          )
      for i in range(100, 200):
          assert_array_equal(
              chunk_id_to_doc_id_map[i - beg], id_start[1:3]
          )
      for i in range(200, 210):
          assert_array_equal(
              chunk_id_to_doc_id_map[i - beg], id_start[2:4]
          )
      beg = 5
      end = 100
      chunk_id_to_doc_id_map = np.zeros((end - beg, 2), dtype=np.int64)
      build_map(id_start, chunk_id_to_doc_id_map, total, beg, end)
      for i in range(beg, end):
          assert_array_equal(
              chunk_id_to_doc_id_map[i - beg], id_start[0:2]
          )
      beg = 100
      end = 200
      chunk_id_to_doc_id_map = np.zeros((end - beg, 2), dtype=np.int64)
      build_map(id_start, chunk_id_to_doc_id_map, total, beg, end)
      for i in range(beg, end):
          assert_array_equal(
              chunk_id_to_doc_id_map[i - beg], id_start[1:3]
          )
      beg = 900
      end = 1000
      chunk_id_to_doc_id_map = np.zeros((end - beg, 2), dtype=np.int64)
      build_map(id_start, chunk_id_to_doc_id_map, total, beg, end)
      for i in range(beg, end):
          assert_array_equal(
              chunk_id_to_doc_id_map[i - beg], np.array([900, 1000])
          )
      beg = 150
      end = 250
      chunk_id_to_doc_id_map = np.zeros((end - beg, 2), dtype=np.int64)
      build_map(id_start, chunk_id_to_doc_id_map, total, beg, end)
      for i in range(beg, 200):
          assert_array_equal(
              chunk_id_to_doc_id_map[i - beg], id_start[1:3]
          )
      for i in range(200, end):
          assert_array_equal(
              chunk_id_to_doc_id_map[i - beg], id_start[2:4]
          )
  
      I = np.arange(1000)[None, :]
      tmp_neighbors = np.ones_like(I) * -1
      with pytest.raises(ValueError):
          dedup(chunk_id_to_doc_id_map, I, tmp_neighbors, 0, beg)
  
      I = np.arange(1000)[None, :]
      tmp_neighbors = np.ones_like(I) * -1
      with pytest.raises(ValueError):
          dedup(chunk_id_to_doc_id_map, I, tmp_neighbors, 250, beg)
  
      for i in range(beg, 200):
          I = np.arange(1000)[None, :]
          tmp_neighbors = np.ones_like(I) * -1
          dedup(chunk_id_to_doc_id_map, I, tmp_neighbors, i, beg)
          gt = np.array(
              list(range(100)) + list(range(200, 1000)) + ([-1] * 100)
          )
          assert_array_equal(tmp_neighbors[0], gt)
  
      for i in range(200, 250):
          I = np.arange(1000)[None, :]
          tmp_neighbors = np.ones_like(I) * -1
          dedup(chunk_id_to_doc_id_map, I, tmp_neighbors, i, beg)
          gt = np.array(
              list(range(200)) + list(range(300, 1000)) + ([-1] * 100)
          )
          assert_array_equal(tmp_neighbors[0], gt)
  
      I = np.arange(1000)[None, :]
      I = np.repeat(I, 70, axis=0)
      tmp_neighbors = np.ones_like(I) * -1
      dedup(chunk_id_to_doc_id_map, I, tmp_neighbors, 180, beg)
      gt0 = np.array(
          list(range(100)) + list(range(200, 1000)) + ([-1] * 100)
      )
      gt1 = np.array(
          list(range(200)) + list(range(300, 1000)) + ([-1] * 100)
      )
      for i in range(20):
          assert_array_equal(tmp_neighbors[i], gt0)
      for i in range(20, 70):
          assert_array_equal(tmp_neighbors[i], gt1)
  
imports: import os
  import numpy
  import pytest
  import torch
  from numpy.testing import assert_array_equal
  from omegaconf import OmegaConf
  from scripts.nlp_language_modeling.build_knn_map_index import build_map, dedup
  from nemo.collections.nlp.data.language_modeling.megatron.indexed_retrieval_dataset import KNNIndex, MMapRetrievalIndexedDataset, MMapRetrievalIndexedDatasetBuilder, merge_knn_files
  from nemo.collections.nlp.data.language_modeling.megatron.retro_dataset import RETRODataset
label_desc: Use image node_id 10 for calling the TestRetrievalIndexFiles obj's underlying member method code with example usage: obj.test_dedup() without return types