class_docstr: PyTorch dataset class for storing tokenized sentences and other properties used for BERT score calculation.
class_docstr_tok: ["PyTorch","dataset","class","for","storing","tokenized","sentences","and","other","properties","used","for","BERT","score","calculation","."]
docstr: Initialize text dataset class.
  
  Args:
      text: An iterable of sentences.
      tokenizer: `AutoTokenizer` instance from `transformers` package.
      max_length: A maximum sequence length.
      preprocess_text_fn: A function used for processing the input sentences.
      idf: An indication of whether calculate token inverse document frequencies to weight the model embeddings.
      tokens_idf: Inverse document frequencies (these should be calculated on reference sentences).
docstr_tok: ["Initialize","text","dataset","class",".","Args",":","text",":","An","iterable","of","sentences",".","tokenizer",":","`","AutoTokenizer","`","instance","from","`","transformers","`","package",".","max_length",":","A","maximum","sequence","length",".","preprocess_text_fn",":","A","function","used","for","processing","the","input","sentences",".","idf",":","An","indication","of","whether","calculate","token","inverse","document","frequencies","to","weight","the","model","embeddings",".","tokens_idf",":","Inverse","document","frequencies","(","these","should","be","calculated","on","reference","sentences",")","."]
code: def __init__(
      self,
      text: List[str],
      tokenizer: Any,
      max_length: int = 512,
      preprocess_text_fn: Callable[
          [List[str], Any, int],
          Union[
              Dict[str, Tensor],
              Tuple[Dict[str, Tensor], Optional[Tensor]],
          ],
      ] = _preprocess_text,
      idf: bool = False,
      tokens_idf: Optional[Dict[int, float]] = None,
  ) -> None:
      """Initialize text dataset class.
  
      Args:
          text: An iterable of sentences.
          tokenizer: `AutoTokenizer` instance from `transformers` package.
          max_length: A maximum sequence length.
          preprocess_text_fn: A function used for processing the input sentences.
          idf: An indication of whether calculate token inverse document frequencies to weight the model embeddings.
          tokens_idf: Inverse document frequencies (these should be calculated on reference sentences).
  
      """
      _text = preprocess_text_fn(text, tokenizer, max_length)
      if isinstance(_text, tuple):
          self.text, self.sorting_indices = _text
      else:
          self.text = _text
      self.max_length = self.text["input_ids"].shape[1]
      self.num_sentences = len(text)
      self.idf = idf
      self.tokens_idf = {}
      if idf:
          self.tokens_idf = (
              tokens_idf
              if tokens_idf is not None
              else self._get_tokens_idf()
          )
  
imports: import math
  import os
  from collections import Counter, defaultdict
  from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, Union
  import torch
  from torch import Tensor
  from torch.utils.data import DataLoader, Dataset
  from torchmetrics.utilities.data import _cumsum
  from torchmetrics.utilities.imports import _TQDM_AVAILABLE, _TRANSFORMERS_GREATER_EQUAL_4_4
label_desc: Use image node_id 1 to create a new TextDataset object from inherited base classes: Dataset with example: obj = TextDataset(text, tokenizer, max_length, preprocess_text_fn, idf, tokens_idf)