docstr: tokens should be a numpy array
          
docstr_tok: ["tokens","should","be","a","numpy","array"]
code: def __init__(self, tokens, seq_len, drop_last=True):
      """tokens should be a numpy array"""
      self.seq_len = seq_len
      ntokens = len(tokens)
      if drop_last:
          ntokens = ((ntokens - 1) // seq_len) * seq_len + 1
      self.ntokens = ntokens
      # We're careful not to slice tokens, since it could be a memmap'ed array or H5 dataset,
      # and slicing would load it to memory.
      self.tokens = tokens
      self.total_sequences = math.ceil(
          (self.ntokens - 1) / self.seq_len
      )
  
file_path: flash-attention/training/src/datamodules/datasets/lm_dataset.py
imports: import math
  import numpy
  import torch
label_desc: Use image node_id 1 to create a new LMDataset object from inherited base classes: torch.utils.data with example: obj = LMDataset(tokens, seq_len, drop_last)