docstr: The Tokens Loader is an optimizer item loader for NLP.
  
  Arguments:
      block_size: The context length to use during training.
docstr_tok: ["The","Tokens","Loader","is","an","optimizer","item","loader","for","NLP",".","Arguments",":","block_size",":","The","context","length","to","use","during","training","."]
code: def __init__(self, block_size: int):
      """The Tokens Loader is an optimizer item loader for NLP.
  
      Arguments:
          block_size: The context length to use during training.
  
      """
  
      super().__init__()
      self._block_size = block_size
      self._mmaps: Dict[int, np.memmap] = {}
      self._buffers: Dict[int, bytes] = {}
      self._dtype: Optional[torch.dtype] = None
      self._chunk_filepaths: Dict[str, bool] = {}
  
imports: import os
  from abc import ABC, abstractmethod
  from time import sleep
  from typing import Any, Dict, List, Optional, Tuple
  import numpy
  import torch
  from lightning.data.streaming.constants import _TORCH_DTYPES_MAPPING, _TORCH_GREATER_EQUAL_2_1_0
  from lightning.data.streaming.serializers import Serializer
label_desc: Use image node_id 1 to create a new TokensLoader object from inherited base classes: BaseItemLoader with example: obj = TokensLoader(block_size)