code: def preprocess_datasets(self):
      from transformers import (
          AutoModelForQuestionAnswering,
          DataCollatorForTokenClassification,
          DataCollatorWithPadding,
          DefaultDataCollator,
      )
  
      def text_classification_preprocess_function(examples):
          return self.trainer_tokenizer(
              examples["text"], truncation=True, max_length=None
          )
  
      def token_classification_preprocess_function(examples):
          tokenized_inputs = self.trainer_tokenizer(
              examples["tokens"],
              truncation=True,
              is_split_into_words=True,
          )
  
          labels = []
          for i, label in enumerate(examples["ner_tags"]):
              word_ids = tokenized_inputs.word_ids(batch_index=i)
              previous_word_idx = None
              label_ids = []
              for word_idx in word_ids:
                  if word_idx is None:
                      label_ids.append(-100)
                  elif word_idx != previous_word_idx:
                      label_ids.append(label[word_idx])
                  else:
                      label_ids.append(-100)
                  previous_word_idx = word_idx
              labels.append(label_ids)
  
          tokenized_inputs["labels"] = labels
          return tokenized_inputs
  
      def question_answering_preprocess_function(examples):
          questions = [q.strip() for q in examples["question"]]
          inputs = self.trainer_tokenizer(
              questions,
              examples["context"],
              truncation="only_second",
              return_offsets_mapping=True,
              padding="max_length",
          )
  
          offset_mapping = inputs.pop("offset_mapping")
          answers = examples["answer"]
          start_positions = []
          end_positions = []
  
          for i, offset in enumerate(offset_mapping):
              answer = answers[i]
              start_char = answer["answer_start"][0]
              end_char = answer["answer_start"][0] + len(
                  answer["text"][0]
              )
              sequence_ids = inputs.sequence_ids(i)
  
              # Find the start and end of the context
              idx = 0
              while sequence_ids[idx] != 1:
                  idx += 1
              context_start = idx
              while sequence_ids[idx] == 1:
                  idx += 1
              context_end = idx - 1
  
              # If the answer is not fully inside the context, label it (0, 0)
              if (
                  offset[context_start][0] > end_char
                  or offset[context_end][1] < start_char
              ):
                  start_positions.append(0)
                  end_positions.append(0)
              else:
                  # Otherwise it's the start and end token positions
                  idx = context_start
                  while (
                      idx <= context_end
                      and offset[idx][0] <= start_char
                  ):
                      idx += 1
                  start_positions.append(idx - 1)
  
                  idx = context_end
                  while (
                      idx >= context_start
                      and offset[idx][1] >= end_char
                  ):
                      idx -= 1
                  end_positions.append(idx + 1)
  
          inputs["start_positions"] = start_positions
          inputs["end_positions"] = end_positions
          return inputs
  
      def question_answering_preprocess_function_validation(examples):
          questions = [q.strip() for q in examples["question"]]
          inputs = self.trainer_tokenizer(
              questions,
              examples["context"],
              truncation="only_second",
              return_overflowing_tokens=True,
              return_offsets_mapping=True,
              padding="max_length",
          )
  
          sample_map = inputs.pop("overflow_to_sample_mapping")
          example_ids = []
  
          for i in range(len(inputs["input_ids"])):
              sample_idx = sample_map[i]
              example_ids.append(examples["id"][sample_idx])
  
              sequence_ids = inputs.sequence_ids(i)
              offset = inputs["offset_mapping"][i]
              inputs["offset_mapping"][i] = [
                  o if sequence_ids[k] == 1 else None
                  for k, o in enumerate(offset)
              ]
  
          inputs["example_id"] = example_ids
          return inputs
  
      # set correct tokenization
      if self._record_class == TextClassificationRecord:
          preprocess_function = text_classification_preprocess_function
          self._data_collator = DataCollatorWithPadding(
              tokenizer=self.trainer_tokenizer
          )
          if self._multi_label:
              remove_columns = ["feat_id", "text", "feat_label"]
          else:
              remove_columns = ["id", "text"]
          replace_labels = True
      elif self._record_class == TokenClassificationRecord:
          preprocess_function = token_classification_preprocess_function
          self._data_collator = DataCollatorForTokenClassification(
              tokenizer=self.trainer_tokenizer
          )
          remove_columns = ["id", "tokens", "ner_tags"]
          replace_labels = False
      elif self._model_class == AutoModelForQuestionAnswering:
          preprocess_function = question_answering_preprocess_function
          self._data_collator = DefaultDataCollator()
          remove_columns = self._train_dataset.column_names
          replace_labels = False
      else:
          raise NotImplementedError(
              "`Text2TextRecord` is not supported yet."
          )
  
      self._tokenized_train_dataset = self._train_dataset.map(
          preprocess_function,
          batched=True,
          remove_columns=remove_columns,
      )
      if replace_labels:
          self._tokenized_train_dataset = (
              self._tokenized_train_dataset.rename_column(
                  "label", "labels"
              )
          )
  
      if self._eval_dataset is not None:
          if self._model_class == AutoModelForQuestionAnswering:
              # We need to preprocess the validation dataset separately, because we need to return the example_id
              self._tokenized_eval_dataset = self._eval_dataset.map(
                  question_answering_preprocess_function_validation,
                  batched=True,
                  remove_columns=remove_columns,
              )
          else:
              self._tokenized_eval_dataset = self._eval_dataset.map(
                  preprocess_function,
                  batched=True,
                  remove_columns=remove_columns,
              )
  
          if replace_labels:
              self._tokenized_eval_dataset = (
                  self._tokenized_eval_dataset.rename_column(
                      "label", "labels"
                  )
              )
      else:
          self._tokenized_eval_dataset = None
  
file_path: argilla/src/argilla/training/transformers.py
imports: import logging
  from typing import List, Union
  from datasets import DatasetDict
  from argilla.client.models import TextClassificationRecord, TokenClassificationRecord
  from argilla.training.base import ArgillaTrainerSkeleton
  from argilla.training.utils import _apply_column_mapping, filter_allowed_args, get_default_args
  from argilla.utils.dependency import require_dependencies
label_desc: Use image node_id 7 for calling the ArgillaTransformersTrainer obj's underlying member method code with example usage: obj.preprocess_datasets() and returns: self, tokenized_inputs, inputs, inputs