class_docstr: Main TabTransformer model that inherits from AbstractModel.
  
  This model includes the full torch pipeline (TabNet) and the internal Transformer embeddings (TabTransformer).
  This file serves as the connection of all these internal models and architectures to AutoGluon.
  
  TabTransformer uses modifications to the typical Transformer architecture and the pretraining in BERT
  and applies them to the use case of tabular data. Specifically, this makes TabTransformer suitable for unsupervised
  training of Tabular data with a subsequent fine-tuning step on labeled data.
docstr: Main training function for TabTransformer
  "state" must be one of [None, 'pretrain', 'finetune']
  None: corresponds to purely supervised learning
  pretrain: discriminative task will be a pretext task
  finetune: same as supervised learning except that the model base has
            exponentially decaying learning rate.
docstr_tok: ["Main","training","function","for","TabTransformer","''","state","''","must","be","one","of","[","None",",","'pretrain","'",",","'finetune","'","]","None",":","corresponds","to","purely","supervised","learning","pretrain",":","discriminative","task","will","be","a","pretext","task","finetune",":","same","as","supervised","learning","except","that","the","model","base","has","exponentially","decaying","learning","rate","."]
code: def tt_fit(
      self,
      loader_train,
      loader_val=None,
      y_val=None,
      state=None,
      time_limit=None,
      reporter=None,
  ):
      """
      Main training function for TabTransformer
      "state" must be one of [None, 'pretrain', 'finetune']
      None: corresponds to purely supervised learning
      pretrain: discriminative task will be a pretext task
      finetune: same as supervised learning except that the model base has
                exponentially decaying learning rate.
      """
      import torch
      import torch.nn as nn
      import torch.optim as optim
  
      from . import pretexts
  
      start_time = time.time()
      pretext_tasks = pretexts.__dict__
      optimizers = []
      lr = self.params["lr"]
      weight_decay = self.params["weight_decay"]
      epochs = (
          self.params["pretrain_epochs"]
          if state == "pretrain"
          else self.params["epochs"]
      )
      epochs_wo_improve = self.params["epochs_wo_improve"]
  
      if state is None:
          optimizers = [
              optim.Adam(
                  self.model.parameters(),
                  lr=lr,
                  weight_decay=weight_decay,
              )
          ]
          pretext = pretext_tasks["SupervisedPretext"](
              self.problem_type, self.device
          )
      elif state == "pretrain":
          optimizers = [
              optim.Adam(
                  self.model.parameters(),
                  lr=lr,
                  weight_decay=weight_decay,
              )
          ]
          pretext = pretext_tasks["BERTPretext"](
              self.cat_feat_origin_cards,
              self.device,
              self.params["hidden_dim"],
          )
      elif state == "finetune":
          base_exp_decay = self.params["base_exp_decay"]
          optimizer_fc = [
              optim.Adam(
                  fc_layer.parameters(),
                  lr=lr,
                  weight_decay=weight_decay,
              )
              for fc_layer in self.model.fc
          ]
          optimizer_embeds = optim.Adam(
              self.model.embed.parameters(),
              lr=lr,
              weight_decay=weight_decay,
          )
          scheduler = optim.lr_scheduler.ExponentialLR(
              optimizer_embeds, gamma=base_exp_decay
          )  # TODO: Should we be using this in _epoch()?
          optimizers.extend(optimizer_fc)
          optimizers.append(optimizer_embeds)
  
          pretext = pretext_tasks["SupervisedPretext"](
              self.problem_type, self.device
          )
  
      else:
          raise NotImplementedError(
              "state must be one of [None, 'pretrain', 'finetune']"
          )
  
      if self.problem_type == REGRESSION:
          loss_criterion = nn.MSELoss()
      else:
          loss_criterion = nn.CrossEntropyLoss()
  
      best_val_metric = -np.inf  # higher = better
      best_val_epoch = 0
      best_loss = np.inf
  
      if self._verbosity <= 1:
          verbose_eval = -1
      elif self._verbosity == 2:
          verbose_eval = 50
      elif self._verbosity == 3:
          verbose_eval = 10
      else:
          verbose_eval = 1
  
      if verbose_eval <= 0:
          databar_disable = True  # Whether or not we want to suppress output based on our verbosity
      else:
          databar_disable = False
  
      for e in range(epochs):
          if e == 0:
              logger.log(15, "TabTransformer architecture:")
              logger.log(15, str(self.model))
  
          train_loss, val_metric = self._epoch(
              net=self.model,
              loader_train=loader_train,
              loader_val=loader_val,
              y_val=y_val,
              optimizers=optimizers,
              loss_criterion=loss_criterion,
              pretext=pretext,
              state=state,
              scheduler=None,
              epoch=e,
              epochs=epochs,
              databar_disable=databar_disable,
              reporter=reporter,
              params=self.params,
          )
  
          # Early stopping for pretrain'ing based on loss.
          if state == "pretrain":
              if train_loss < best_loss or e == 0:
                  if train_loss < best_loss:
                      best_loss = train_loss
                  best_val_epoch = e
          else:
              if val_metric >= best_val_metric or e == 0:
                  if loader_val is not None:
                      if not np.isnan(val_metric):
                          best_val_metric = val_metric
  
                  best_val_epoch = e
                  os.makedirs(os.path.dirname(self.path), exist_ok=True)
                  torch.save(
                      self.model,
                      os.path.join(self.path, self._temp_file_name),
                  )
  
          # If time limit has exceeded or we haven't improved in some number of epochs, stop early.
          if e - best_val_epoch > epochs_wo_improve:
              break
          if time_limit:
              time_elapsed = time.time() - start_time
              time_left = time_limit - time_elapsed
              if time_left <= 0:
                  logger.log(
                      20, "\tRan out of time, stopping training early."
                  )
                  break
  
      if loader_val is not None:
          try:
              self.model = torch.load(
                  os.path.join(self.path, self._temp_file_name)
              )
              os.remove(os.path.join(self.path, self._temp_file_name))
          except:
              pass
          logger.log(
              15, "Best model found in epoch %d" % best_val_epoch
          )
  
imports: import logging
  import os
  import time
  import numpy
  import pandas
  from tqdm import tqdm
  from autogluon.common.features.types import R_OBJECT, S_TEXT_AS_CATEGORY, S_TEXT_NGRAM
  from autogluon.common.utils.try_import import try_import_torch
  from autogluon.core.constants import BINARY, MULTICLASS, REGRESSION
  from autogluon.core.models.abstract.abstract_nn_model import AbstractNeuralNetworkModel
  from autogluon.core.utils.loaders import load_pkl
  from .hyperparameters.parameters import get_default_param
  from .hyperparameters.searchspaces import get_default_searchspace
label_desc: Use image node_id 9 for calling the TabTransformerModel obj's underlying member method code with example usage: obj.tt_fit(loader_train, loader_val, y_val, state, time_limit, reporter) without return types