code: def __init__(
      self,
      tokens,
      seq_len,
      pad_idx,
      num_original_tokens,
      num_tokenized_tokens,
      overalapping_eval=None,
  ):
      self.tokens = tokens
      self.seq_len = seq_len
      self.pad_idx = pad_idx
      self.overalapping_eval = overalapping_eval
      if self.overalapping_eval is None:
          self.overalapping_eval = self.seq_len
      self.overalapping_eval = max(1, self.overalapping_eval)
      self.num_original_tokens = num_original_tokens
      self.num_tokenized_tokens = num_tokenized_tokens
      self.total_targets = len(self.tokens) - 1
      # remove first sequence tokens
      targets = max(self.total_targets - self.overalapping_eval, 0)
      self.total_sequences = max(
          math.ceil(targets / self.overalapping_eval) + 1, 1
      )
  
imports: import json
  import math
  import numpy
  import torch
  from megatron import get_args
  from megatron import print_rank_0
  from megatron import get_tokenizer
  from .detokenizer import get_detokenizer
label_desc: Use image node_id 1 to create a new _LMDataset object from inherited base classes: torch.utils.data with example: obj = _LMDataset(tokens, seq_len, pad_idx, num_original_tokens, num_tokenized_tokens, overalapping_eval)