class_docstr: ## Dataset for fine-tuning GPT-NeoX
  
  This is not optimized to very large datasets.
class_docstr_tok: ["#","#","Dataset","for","fine-tuning","GPT-NeoX","This","is","not","optimized","to","very","large","datasets","."]
docstr: :param tokens: is the list of token ids
  :param seq_len: is the sequence length of a single training sample
docstr_tok: [":","param","tokens",":","is","the","list","of","token","ids",":","param","seq_len",":","is","the","sequence","length","of","a","single","training","sample"]
code: def __init__(self, tokens: List[int], seq_len: int):
      """
      :param tokens: is the list of token ids
      :param seq_len: is the sequence length of a single training sample
      """
  
      self.seq_len = seq_len
      # Number of samples
      n_samples = len(tokens) // seq_len
      self.n_samples = n_samples
      # Truncate
      tokens = tokens[: n_samples * seq_len + 1]
      # Create a PyTorch tensor
      self.tokens = torch.tensor(tokens)
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/neox/utils/text_dataset.py
imports: from pathlib import PurePath, Path
  from typing import Optional, List
  import torch
  import torch.utils.data
  from labml import lab
  from labml import monit
  from labml.logger import inspect
  from labml.utils.download import download_file
  from labml_nn.neox.tokenizer import get_tokenizer
label_desc: Use image node_id 1 to create a new NeoXDataset object from inherited base classes: torch.utils.data with example: obj = NeoXDataset(tokens, seq_len)