class_docstr: Data collator that will dynamically pad the inputs received, as well as the labels.
  
  Args:
      tokenizer (:class:`~transformers.PreTrainedTokenizer` or :class:`~transformers.PreTrainedTokenizerFast`):
          The tokenizer used for encoding the data.
      model (:class:`~transformers.PreTrainedModel`):
          The model that is being trained. If set and has the `prepare_decoder_input_ids_from_labels`, use it to
          prepare the `decoder_input_ids`
  
          This is useful when using `label_smoothing` to avoid calculating loss twice.
      padding (:obj:`bool`, :obj:`str` or :class:`~transformers.file_utils.PaddingStrategy`, `optional`, defaults to :obj:`True`):
          Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
          among:
  
          * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
            sequence is provided).
          * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
            maximum acceptable input length for the model if that argument is not provided.
          * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
            different lengths).
      max_length (:obj:`int`, `optional`):
          Maximum length of the returned list and optionally padding length (see above).
      max_target_length (:obj:`int`, `optional`):
          Maximum length of target sequence length.
      pad_to_multiple_of (:obj:`int`, `optional`):
          If set will pad the sequence to a multiple of the provided value.
  
          This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >=
          7.5 (Volta).
      label_pad_token_id (:obj:`int`, `optional`, defaults to -100):
          The id to use when padding the labels (-100 will be automatically ignored by PyTorch loss functions).
class_docstr_tok: ["Data","collator","that","will","dynamically","pad","the","inputs","received",",","as","well","as","the","labels",".","Args",":","tokenizer","(",":","class",":","`","~transformers.PreTrainedTokenizer","`","or",":","class",":","`","~transformers.PreTrainedTokenizerFast","`",")",":","The","tokenizer","used","for","encoding","the","data",".","model","(",":","class",":","`","~transformers.PreTrainedModel","`",")",":","The","model","that","is","being","trained",".","If","set","and","has","the","`","prepare_decoder_input_ids_from_labels","`",",","use","it","to","prepare","the","`","decoder_input_ids","`","This","is","useful","when","using","`","label_smoothing","`","to","avoid","calculating","loss","twice",".","padding","(",":","obj",":","`","bool","`",",",":","obj",":","`","str","`","or",":","class",":","`","~transformers.file_utils.PaddingStrategy","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","Select","a","strategy","to","pad","the","returned","sequences","(","according","to","the","model","'s","padding","side","and","padding","index",")","among",":","*",":","obj",":","`","True","`","or",":","obj",":","`","'longest","'","`",":","Pad","to","the","longest","sequence","in","the","batch","(","or","no","padding","if","only","a","single","sequence","is","provided",")",".","*",":","obj",":","`","'max_length","'","`",":","Pad","to","a","maximum","length","specified","with","the","argument",":","obj",":","`","max_length","`","or","to","the","maximum","acceptable","input","length","for","the","model","if","that","argument","is","not","provided",".","*",":","obj",":","`","False","`","or",":","obj",":","`","'do_not_pad","'","`","(","default",")",":","No","padding","(","i.e.",",","can","output","a","batch","with","sequences","of","different","lengths",")",".","max_length","(",":","obj",":","`","int","`",",","`","optional","`",")",":","Maximum","length","of","the","returned","list","and","optionally","padding","length","(","see","above",")",".","max_target_length","(",":","obj",":","`","int","`",",","`","optional","`",")",":","Maximum","length","of","target","sequence","length",".","pad_to_multiple_of","(",":","obj",":","`","int","`",",","`","optional","`",")",":","If","set","will","pad","the","sequence","to","a","multiple","of","the","provided","value",".","This","is","especially","useful","to","enable","the","use","of","Tensor","Cores","on","NVIDIA","hardware","with","compute","capability",">","=","7.5","(","Volta",")",".","label_pad_token_id","(",":","obj",":","`","int","`",",","`","optional","`",",","defaults","to","-100",")",":","The","id","to","use","when","padding","the","labels","(","-100","will","be","automatically","ignored","by","PyTorch","loss","functions",")","."]
docstr: Make Meta Schema Batch
  
  Args:
      features (Dict): [description]
          - sample_prompt: indicates sample_prompt example, need pop after call
          - spots (List[str]): List of spots in this sentence, need pop after call
          - asocs (List[str]): List of asocs in this sentence, need pop after call
          - input_ids
          - attention_mask
          - labels
  
  Returns:
docstr_tok: ["Make","Meta","Schema","Batch","Args",":","features","(","Dict",")",":","[","description","]","-","sample_prompt",":","indicates","sample_prompt","example",",","need","pop","after","call","-","spots","(","List","[","str","]",")",":","List","of","spots","in","this","sentence",",","need","pop","after","call","-","asocs","(","List","[","str","]",")",":","List","of","asocs","in","this","sentence",",","need","pop","after","call","-","input_ids","-","attention_mask","-","labels","Returns",":"]
code: def __call__(self, features):
      """Make Meta Schema Batch
  
      Args:
          features (Dict): [description]
              - sample_prompt: indicates sample_prompt example, need pop after call
              - spots (List[str]): List of spots in this sentence, need pop after call
              - asocs (List[str]): List of asocs in this sentence, need pop after call
              - input_ids
              - attention_mask
              - labels
  
      Returns:
      """
      for feature in features:
          sample_prompt = feature["sample_prompt"]
  
          if not sample_prompt:
              # Evaluation using Ordered SSI
              converted_spot_prefix = self.negative_sampler.full_spot(
                  shuffle=self.model.training
              )
              converted_asoc_prefix = self.negative_sampler.full_asoc(
                  shuffle=self.model.training
              )
          else:
              # Sample SSI
              (
                  converted_spot_prefix,
                  positive_spot,
                  negative_spot,
              ) = self.negative_sampler.sample_spot(
                  positive=feature.get("spots", [])
              )
              (
                  converted_asoc_prefix,
                  negative_asoc,
              ) = self.negative_sampler.sample_asoc(
                  positive=feature.get("asocs", [])
              )
  
              # Dynamic generating spot-asoc during training
              if "spot_asoc" in feature:
                  # Deleted positive example Spot in Target that was not sampled by Prefix
                  feature["spot_asoc"] = [
                      spot_asoc
                      for spot_asoc in feature["spot_asoc"]
                      if spot_asoc["label"] in positive_spot
                  ]
  
                  # Inject rejection noise
                  if self.spot_asoc_nosier is not None:
                      if isinstance(
                          self.spot_asoc_nosier, SpotAsocNoiser
                      ):
                          feature[
                              "spot_asoc"
                          ] = self.spot_asoc_nosier.add_noise(
                              feature["spot_asoc"],
                              spot_label_list=negative_spot,
                              asoc_label_list=negative_asoc,
                          )
                      else:
                          raise NotImplementedError(
                              f"{self.spot_asoc_nosier} is not implemented."
                          )
  
                  # Generate new record
                  record = convert_to_record_function[
                      self.decoding_format
                  ](
                      feature["spot_asoc"],
                      structure_maker=BaseStructureMarker(),
                  )
                  feature["labels"] = self.tokenizer.encode(record)
  
          feature.pop(
              "sample_prompt"
          ) if "sample_prompt" in feature else None
          feature.pop("spot_asoc") if "spot_asoc" in feature else None
          feature.pop("spots") if "spots" in feature else None
          feature.pop("asocs") if "asocs" in feature else None
          # prefix = converted_spot_prefix + converted_asoc_prefix
          prefix = converted_asoc_prefix
          # truncate `prefix` to max length
          if (
              self.max_prefix_length is not None
              and self.max_prefix_length >= 0
          ):
              prefix = prefix[: self.max_prefix_length]
  
          feature["input_ids"] = (
              prefix
              + [self.negative_sampler.text_start]
              + feature["input_ids"]
          )
  
          # truncate `input_ids` to max length
          if self.max_length:
              feature["input_ids"] = feature["input_ids"][
                  : self.max_length
              ]
          if self.max_target_length and "labels" in feature:
              feature["labels"] = feature["labels"][
                  : self.max_target_length
              ]
  
          feature["attention_mask"] = [1] * len(feature["input_ids"])
  
      labels = (
          [feature["labels"] for feature in features]
          if "labels" in features[0].keys()
          else None
      )
      # We have to pad the labels before calling `tokenizer.pad` as this method won't pad them and needs them of the
      # same length to return tensors.
      if labels is not None:
          max_label_length = max(len(_label) for _label in labels)
          padding_side = self.tokenizer.padding_side
          for feature in features:
              remainder = [self.label_pad_token_id] * (
                  max_label_length - len(feature["labels"])
              )
              feature["labels"] = (
                  feature["labels"] + remainder
                  if padding_side == "right"
                  else remainder + feature["labels"]
              )
  
      features = self.tokenizer.pad(
          features,
          padding=self.padding,
          max_length=self.max_length,
          pad_to_multiple_of=self.pad_to_multiple_of,
          return_tensors="pt",
      )
  
      # prepare decoder_input_ids
      if self.model is not None and hasattr(
          self.model, "prepare_decoder_input_ids_from_labels"
      ):
          decoder_input_ids = (
              self.model.prepare_decoder_input_ids_from_labels(
                  labels=features["labels"]
              )
          )
          features["decoder_input_ids"] = decoder_input_ids
  
      return features
  
imports: import os
  from dataclasses import dataclass
  import torch
  import logging
  import random
  import math
  from typing import Optional, Union
  from collections import OrderedDict
  from transformers import PreTrainedTokenizerBase, PreTrainedModel
  from transformers.file_utils import PaddingStrategy
  from ...extraction.record_schema import RecordSchema
  from ...extraction.dataset_processer import spot_prompt, asoc_prompt
  from ...extraction.constants import BaseStructureMarker, text_start
  from ...extraction.utils import convert_to_record_function
  from ...extraction.noiser.spot_asoc_noiser import SpotAsocNoiser
label_desc: Use image node_id 1 for calling the DataCollatorForMetaSeq2Seq obj's underlying member method code with example usage: obj.__call__(features) and returns: features