class_docstr: Creates dataset to use during training for token classification tasks with a pretrained model.
  
  Converts from raw data to an instance that can be used by Dataloader.
  For dataset to use during inference without labels, see BertTokenClassificationInferDataset.
  
  Args:
      text_file: file to sequences, each line should a sentence, no header.
      label_file: file to labels, each line corresponds to word labels for a sentence in the text_file. No header.
      max_seq_length: max sequence length minus 2 for [CLS] and [SEP]
      tokenizer: such as AutoTokenizer
      num_samples: number of samples you want to use for the dataset.
          If -1, use all dataset. Useful for testing.
      pad_label: pad value use for labels. By default, it's the neutral label.
      label_ids: label_ids (dict): dict to map labels to label ids.
          Starts with pad_label->0 and then increases in alphabetical order
          For dev set use label_ids generated during training to support
          cases when not all labels are present in the dev set.
          For training set label_ids should be None.
      ignore_extra_tokens: whether to ignore extra tokens in the loss_mask
      ignore_start_end: whether to ignore bos and eos tokens in the loss_mask
      use_cache: whether to use processed data cache or not
docstr: Initializes BertTokenClassificationDataset. 
docstr_tok: ["Initializes","BertTokenClassificationDataset","."]
code: def __init__(
      self,
      text_file: str,
      label_file: str,
      max_seq_length: int,
      tokenizer: TokenizerSpec,
      num_samples: int = -1,
      pad_label: str = "O",
      label_ids: Dict[str, int] = None,
      ignore_extra_tokens: bool = False,
      ignore_start_end: bool = False,
      use_cache: bool = True,
  ):
      """Initializes BertTokenClassificationDataset."""
  
      data_dir = os.path.dirname(text_file)
      text_filename = os.path.basename(text_file)
      lbl_filename = os.path.basename(label_file)
  
      if not text_filename.endswith(".txt"):
          raise ValueError("{text_file} should have extension .txt")
  
      vocab_size = getattr(tokenizer, "vocab_size", 0)
      features_pkl = os.path.join(
          data_dir,
          f"cached__{text_filename}__{lbl_filename}__{tokenizer.name}_{max_seq_length}_{vocab_size}_{num_samples}",
      )
  
      master_device = is_global_rank_zero()
      features = None
      if master_device and (
          not use_cache or not os.path.exists(features_pkl)
      ):
          if num_samples == 0:
              raise ValueError(
                  "num_samples has to be positive", num_samples
              )
  
          with open(text_file, "r") as f:
              text_lines = f.readlines()
  
          labels_lines = []
          with open(label_file, "r") as f:
              for line in f:
                  line = line.strip().split()
                  labels_lines.append(line)
  
          if len(labels_lines) != len(text_lines):
              raise ValueError(
                  "Labels file should contain labels for every word"
              )
  
          if num_samples > 0:
              dataset = list(zip(text_lines, labels_lines))
              dataset = dataset[:num_samples]
  
              dataset = list(zip(*dataset))
              text_lines = dataset[0]
              labels_lines = dataset[1]
  
          features = get_features(
              queries=text_lines,
              max_seq_length=max_seq_length,
              tokenizer=tokenizer,
              pad_label=pad_label,
              raw_labels=labels_lines,
              label_ids=label_ids,
              ignore_extra_tokens=ignore_extra_tokens,
              ignore_start_end=ignore_start_end,
          )
  
          # save features to a temp file first to make sure that non-master processes don't start reading the file
          # until the master process is done with writing
          ofd, tmp_features_pkl = tempfile.mkstemp(
              suffix=".pkl",
              prefix=os.path.basename(features_pkl),
              dir=os.path.dirname(features_pkl),
          )
          with os.fdopen(ofd, "wb") as temp_f:
              pickle.dump(features, temp_f)
  
          os.rename(tmp_features_pkl, features_pkl)
          logging.info(f"features saved to {features_pkl}")
  
      # wait until the master process writes to the processed data files
      if not master_device:
          while features is None and not os.path.exists(features_pkl):
              time.sleep(10)
  
      if features is None:
          features = pickle.load(open(features_pkl, "rb"))
          logging.info(f"features restored from {features_pkl}")
  
      self.all_input_ids = features[0]
      self.all_segment_ids = features[1]
      self.all_input_mask = features[2]
      self.all_subtokens_mask = features[3]
      self.all_loss_mask = features[4]
      self.all_labels = features[5]
  
imports: import os
  import pickle
  import tempfile
  import time
  from typing import Dict, List, Optional
  import numpy
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.data_utils.data_preprocessing import get_stats
  from nemo.core.classes import Dataset
  from nemo.core.neural_types import ChannelType, LabelsType, MaskType, NeuralType
  from nemo.utils import logging
  from nemo.utils.get_rank import is_global_rank_zero
label_desc: Use image node_id 2 to create a new BertTokenClassificationDataset object from inherited base classes: Dataset with example: obj = BertTokenClassificationDataset(text_file, label_file, max_seq_length, tokenizer, num_samples, pad_label, label_ids, ignore_extra_tokens, ignore_start_end, use_cache)