class_docstr: Dataset for bert pretraining when using data preprocessing including tokenization
class_docstr_tok: ["Dataset","for","bert","pretraining","when","using","data","preprocessing","including","tokenization"]
code: def __getitem__(self, idx: int, min_doc_length: Optional[int] = 16):
      # Each sequence has three special tokens, as follows:
      # tokenizer.cls_token <document a> tokenizer.sep_token <document b> tokenizer.eos_token
      num_special_tokens = 3
  
      max_num_tokens = self.max_seq_length - num_special_tokens
      target_seq_length = max_num_tokens
      if random.random() < self.short_seq_prob:
          # TODO: maybe introduce an argument to control this.
          target_seq_length = random.randint(2, max_num_tokens)
  
      # prefer the seq_a to be slightly longer than seq_b, 0.6 by default
      target_seq_length_a = int(
          round(target_seq_length * self.seq_a_ratio)
      )
      target_seq_length_b = target_seq_length - target_seq_length_a
  
      def get_document(filepath, offset):
          # Retrieve a specific line from a file and return as a document
          if os.path.isdir(self.dataset):
              filepath = os.path.join(self.dataset, filepath)
  
          with open(filepath, "rb") as f:
              f.seek(offset)
              doc_text = f.readline()[:-1].decode(
                  "utf-8", errors="ignore"
              )
              document = self.tokenizer.text_to_ids(doc_text)
  
          return document
  
      def match_target_seq_length(
          document: str,
          target_seq_length: int,
          filename: str,
          line_idx: int,
          sentence_indices: Dict[str, dict],
      ):
          # If document is shorter than target sequence length,
          # append the next line or take a random line as replacement.
          num_lines = len(sentence_indices[filename])
  
          while len(document) < target_seq_length:
              if line_idx < (num_lines - 1):
                  # append the next line
                  line_idx += 1
              else:
                  # current line is the last line, take a random one
                  line_idx = random.randrange(num_lines)
                  document = []
  
              offset = sentence_indices[filename][line_idx]
              document += get_document(filename, offset)
  
          return document, line_idx
  
      # Take sequence A from a random file and a random line
      a_filename = random.choice(self.filenames)
      a_line_idx = random.randrange(
          len(self.sentence_indices[a_filename])
      )
      a_line_offset = self.sentence_indices[a_filename][a_line_idx]
      a_document = get_document(a_filename, a_line_offset)
      a_document, a_line_idx = match_target_seq_length(
          a_document,
          target_seq_length_a,
          a_filename,
          a_line_idx,
          self.sentence_indices,
      )
  
      is_last_line = a_line_idx >= (
          len(self.sentence_indices[a_filename]) - 1
      )
      # About 50% of the time, B is a random sentence from the corpus
      take_random_b = (random.random() < 0.5) or is_last_line
  
      if take_random_b:
          # This should rarely go for more than one iteration for large
          # corpora. However, just to be careful, we try to make sure that
          # the random document is not the same as the document
          # we're processing.
          for _ in range(10):
              b_filename = random.choice(self.filenames)
              b_line_idx = random.choice(
                  range(len(self.sentence_indices[b_filename]))
              )
              if b_filename != a_filename:
                  break
              else:
                  # Take another line from the same file
                  b_line_pos = self.sentence_indices[b_filename][
                      b_line_idx
                  ]
                  a_line_pos = self.sentence_indices[a_filename][
                      a_line_idx
                  ]
                  # TODO unclear about the following check
                  if abs(b_line_pos - a_line_pos) > max_num_tokens:
                      break
                  else:
                      pass
      else:
          b_filename = a_filename
          b_line_idx = a_line_idx + 1
  
      is_next = int(not take_random_b)
      b_line_pos = self.sentence_indices[b_filename][b_line_idx]
      b_document = get_document(b_filename, b_line_pos)
      b_document, b_line_idx = match_target_seq_length(
          b_document,
          target_seq_length_b,
          b_filename,
          b_line_idx,
          self.sentence_indices,
      )
  
      def truncate_seq_pair(a, b, max_num_tokens):
          # Truncates a pair of sequences to a maximum sequence length
          while (len(a) + len(b)) > max_num_tokens:
              # Truncate the longer sequence
              if len(a) > len(b):
                  trunc_document = a
              else:
                  trunc_document = b
  
              if len(trunc_document) <= 1:
                  raise ValueError(
                      "Input text corpora probably too small. "
                      "Failed to truncate sequence pair to "
                      "maximum sequence legnth."
                  )
  
              # Randomly truncate from the front or the back
              if random.random() < 0.5:
                  del trunc_document[0]
              else:
                  trunc_document.pop()
  
      truncate_seq_pair(a_document, b_document, max_num_tokens)
  
      output_ids = (
          [self.tokenizer.cls_id]
          + a_document
          + [self.tokenizer.sep_id]
          + b_document
          + [self.tokenizer.eos_id]
      )
  
      input_ids, output_mask = self.mask_ids(output_ids)
  
      input_mask = np.zeros(self.max_seq_length, dtype=np.longlong)
      input_mask[: len(input_ids)] = 1
  
      input_type_ids = np.zeros(self.max_seq_length, dtype=np.int64)
      input_type_ids[len(a_document) + 2 : len(output_ids) + 1] = 1
  
      padding_length = max(0, self.max_seq_length - len(input_ids))
      if padding_length > 0:
          input_ids.extend([self.tokenizer.pad_id] * padding_length)
          output_ids.extend([self.tokenizer.pad_id] * padding_length)
          output_mask.extend([0] * padding_length)
  
      # TODO: wrap the return value with () for consistent style.
      return (
          np.array(input_ids),
          input_type_ids,
          np.array(input_mask, dtype=np.longlong),
          np.array(output_ids),
          np.array(output_mask, dtype=np.float32),
          is_next,
      )
  
imports: import array
  import os
  import pickle
  import random
  from typing import Dict, List, Optional
  import h5py
  import numpy
  from torch.utils.data import DataLoader, DistributedSampler
  from tqdm import tqdm
  from nemo.collections.nlp.data.data_utils.data_preprocessing import find_newlines, load_data_indices
  from nemo.core.classes import Dataset
label_desc: Use image node_id 3 for calling the BertPretrainingDataset obj's underlying member method code with example usage: obj.__getitem__(idx, min_doc_length) and returns: np,  input_type_ids,  np,  np,  np,  is_next, document, document,  line_idx