class_docstr: Parent class for entity linking encoder training and index
  datasets
  
  Args:
      tokenizer (obj): huggingface tokenizer,
      data_file (str): path to tab separated column file where data 
          pairs apear in the format 
          concept_ID  concept_synonym1        concept_synonym2
  
      newline_idx_file (str): path to pickle file containing location
          of data_file newline characters
      max_seq_length (int): maximum length of a concept in tokens
      is_index_data (bool): Whether dataset will be used for building
                          a nearest neighbors index
class_docstr_tok: ["Parent","class","for","entity","linking","encoder","training","and","index","datasets","Args",":","tokenizer","(","obj",")",":","huggingface","tokenizer",",","data_file","(","str",")",":","path","to","tab","separated","column","file","where","data","pairs","apear","in","the","format","concept_ID","concept_synonym1","concept_synonym2","newline_idx_file","(","str",")",":","path","to","pickle","file","containing","location","of","data_file","newline","characters","max_seq_length","(","int",")",":","maximum","length","of","a","concept","in","tokens","is_index_data","(","bool",")",":","Whether","dataset","will","be","used","for","building","a","nearest","neighbors","index"]
code: def __init__(
      self,
      tokenizer: object,
      data_file: str,
      newline_idx_file: Optional[str] = None,
      max_seq_length: Optional[int] = 512,
      is_index_data: bool = False,
  ):
      self.tokenizer = tokenizer
  
      # Try and load pair indices file if already exists
      newline_indices, newline_idx_file, _ = load_data_indices(
          newline_idx_file, data_file, "newline_indices"
      )
  
      # If pair indices file doesn't exists, generate and store them
      if newline_indices is None:
          logging.info("Getting datafile newline indices")
  
          with open(data_file, "rb") as f:
              contents = f.read()
              newline_indices = find_newlines(contents)
              newline_indices = array.array("I", newline_indices)
  
          # Store data file indicies to avoid generating them again
          with open(newline_idx_file, "wb") as f:
              pkl.dump(newline_indices, f)
  
      self.newline_indices = newline_indices
      self.data_file = data_file
      self.num_lines = len(newline_indices)
      self.max_seq_length = max_seq_length
      self.is_index_data = is_index_data
  
      logging.info(f"Loaded dataset with {self.num_lines} examples")
  
file_path: NeMo/nemo/collections/nlp/data/entity_linking/entity_linking_dataset.py
imports: import array
  import pickle
  from typing import Optional
  import torch
  from nemo.collections.nlp.data.data_utils.data_preprocessing import find_newlines, load_data_indices
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new EntityLinkingDataset object from inherited base classes: Dataset with example: obj = EntityLinkingDataset(tokenizer, data_file, newline_idx_file, max_seq_length, is_index_data)