class_docstr: Collate instances for supervised dataset.
  Each instance is a tokenized dictionary with fields
  `input_ids`(List[int]), `labels`(List[int]) and `sequence`(str).
class_docstr_tok: ["Collate","instances","for","supervised","dataset",".","Each","instance","is","a","tokenized","dictionary","with","fields","`","input_ids","`","(","List","[","int","]",")",",","`","labels","`","(","List","[","int","]",")","and","`","sequence","`","(","str",")","."]
docstr: Args:
      instances (`Sequence[Dict[str, List[int]]]`):
          Mini-batch samples, each sample is stored in an individual dictionary.
  
  Returns:
      (`Dict[str, torch.Tensor]`): Contains the following `torch.Tensor`:
          `input_ids`: `torch.Tensor` of shape (bsz, max_len);
          `attention_mask`: `torch.BoolTensor` of shape (bsz, max_len);
          `labels`: `torch.Tensor` of shape (bsz, max_len), which contains `IGNORE_INDEX`.
docstr_tok: ["Args",":","instances","(","`","Sequence","[","Dict","[","str",",","List","[","int","]","]","]","`",")",":","Mini-batch","samples",",","each","sample","is","stored","in","an","individual","dictionary",".","Returns",":","(","`","Dict","[","str",",","torch.Tensor","]","`",")",":","Contains","the","following","`","torch.Tensor","`",":","`","input_ids","`",":","`","torch.Tensor","`","of","shape","(","bsz",",","max_len",")",";","`","attention_mask","`",":","`","torch.BoolTensor","`","of","shape","(","bsz",",","max_len",")",";","`","labels","`",":","`","torch.Tensor","`","of","shape","(","bsz",",","max_len",")",",","which","contains","`","IGNORE_INDEX","`","."]
code: def __call__(
      self, instances: Sequence[Dict[str, List[int]]]
  ) -> Dict[str, torch.Tensor]:
      """
  
      Args:
          instances (`Sequence[Dict[str, List[int]]]`):
              Mini-batch samples, each sample is stored in an individual dictionary.
  
      Returns:
          (`Dict[str, torch.Tensor]`): Contains the following `torch.Tensor`:
              `input_ids`: `torch.Tensor` of shape (bsz, max_len);
              `attention_mask`: `torch.BoolTensor` of shape (bsz, max_len);
              `labels`: `torch.Tensor` of shape (bsz, max_len), which contains `IGNORE_INDEX`.
      """
      assert (
          isinstance(self.tokenizer.pad_token_id, int)
          and self.tokenizer.pad_token_id >= 0
      ), (
          f"`{self.tokenizer.__class__.__name__}.pad_token_id` must be a valid non-negative integer index value, "
          f"but now `{self.tokenizer.pad_token_id}`"
      )
  
      # `List[torch.Tensor]`
      batch_input_ids = [
          torch.LongTensor(instance["input_ids"][: self.max_length])
          if len(instance["input_ids"]) > self.max_length
          else torch.LongTensor(instance["input_ids"])
          for instance in instances
      ]
      batch_labels = [
          torch.LongTensor(instance["labels"][: self.max_length])
          if len(instance["labels"]) > self.max_length
          else torch.LongTensor(instance["labels"])
          for instance in instances
      ]
  
      if self.tokenizer.padding_side == "right":
          input_ids = torch.nn.utils.rnn.pad_sequence(
              sequences=batch_input_ids,
              batch_first=True,
              padding_value=self.tokenizer.pad_token_id,
          )  # (bsz, max_len)
          labels = torch.nn.utils.rnn.pad_sequence(
              sequences=batch_labels,
              batch_first=True,
              padding_value=self.ignore_index,
          )  # (bsz, max_len)
          # pad to max
          to_pad = self.max_length - input_ids.size(1)
          input_ids = F.pad(
              input_ids, (0, to_pad), value=self.tokenizer.pad_token_id
          )
          labels = F.pad(labels, (0, to_pad), value=self.ignore_index)
      elif self.tokenizer.padding_side == "left":
          reversed_input_ids = [
              seq.flip(dims=(0,)) for seq in batch_input_ids
          ]
          reversed_input_ids = torch.nn.utils.rnn.pad_sequence(
              sequences=reversed_input_ids,
              batch_first=True,
              padding_value=self.tokenizer.pad_token_id,
          )  # (bsz, max_len)
          input_ids = torch.flip(
              reversed_input_ids, dims=(1,)
          )  # (bsz, max_len)
          reversed_labels = [
              seq.flip(dims=(0,)) for seq in batch_labels
          ]
          reversed_labels = torch.nn.utils.rnn.pad_sequence(
              sequences=reversed_labels,
              batch_first=True,
              padding_value=self.ignore_index,
          )  # (bsz, max_len)
          labels = torch.flip(
              reversed_labels, dims=(1,)
          )  # (bsz, max_len)
      else:
          raise RuntimeError(
              f"`{self.tokenizer.__class__.__name__}.padding_side` can only be `left` or `right`, "
              f"but now `{self.tokenizer.padding_side}`"
          )
  
      attention_mask = input_ids.ne(
          self.tokenizer.pad_token_id
      )  # `torch.BoolTensor`, (bsz, max_len)
  
      return dict(
          input_ids=input_ids,
          attention_mask=attention_mask,
          labels=labels,
      )
  
imports: import numpy
  import os
  import random
  from dataclasses import dataclass
  from typing import Dict, List, Union, Sequence, Optional, Iterator, Callable
  import torch
  from datasets import dataset_dict, load_from_disk
  from datasets import Dataset
  from torch.distributed import ProcessGroup
  from torch.distributed.distributed_c10d import _get_default_group
  from torch.utils.data import ConcatDataset, Dataset, DataLoader, DistributedSampler
  from transformers.tokenization_utils import PreTrainedTokenizer
  import torch.nn.functional
label_desc: Use image node_id 1 for calling the DataCollatorForSupervisedDataset obj's underlying member method code with example usage: obj.__call__(instances) and returns: dict