docstr: General data processing steps used for all models.
docstr_tok: ["General","data","processing","steps","used","for","all","models","."]
code: def general_data_processing(
      self,
      X: DataFrame,
      X_val: DataFrame,
      X_unlabeled: DataFrame,
      holdout_frac: float,
      num_bag_folds: int,
  ):
      """General data processing steps used for all models."""
      X = copy.deepcopy(X)
      # treat None, NaN, INF, NINF as NA
      X[self.label].replace([np.inf, -np.inf], np.nan, inplace=True)
      invalid_labels = X[self.label].isna()
      if invalid_labels.any():
          first_invalid_label_idx = invalid_labels.idxmax()
          raise ValueError(
              f"Label column cannot contain non-finite values (NaN, Inf, Ninf). First invalid label at idx: {first_invalid_label_idx}"
          )
  
      holdout_frac_og = holdout_frac
      if X_val is not None and self.label in X_val.columns:
          # treat None, NaN, INF, NINF as NA
          X_val[self.label].replace(
              [np.inf, -np.inf], np.nan, inplace=True
          )
          invalid_tuning_labels = X_val[self.label].isna()
          if invalid_tuning_labels.any():
              first_invalid_label_idx = invalid_tuning_labels.idxmax()
              raise ValueError(
                  f"Label column cannot contain non-finite values (NaN, Inf, Ninf). First invalid label at idx: {first_invalid_label_idx}"
              )
  
          holdout_frac = 1
  
      if (
          (self.eval_metric is not None)
          and (self.eval_metric.name in ["log_loss", "pac_score"])
          and (self.problem_type == MULTICLASS)
      ):
          if num_bag_folds > 0:
              self.threshold = 2
              if self.groups is None:
                  X = augment_rare_classes(X, self.label, threshold=2)
          else:
              self.threshold = 1
  
      (
          self.threshold,
          holdout_frac,
          num_bag_folds,
      ) = self.adjust_threshold_if_necessary(
          X[self.label],
          threshold=self.threshold,
          holdout_frac=holdout_frac,
          num_bag_folds=num_bag_folds,
      )
  
      # Gets labels prior to removal of infrequent classes
      y_uncleaned = X[self.label].copy()
  
      self.cleaner = Cleaner.construct(
          problem_type=self.problem_type,
          label=self.label,
          threshold=self.threshold,
      )
      X = self.cleaner.fit_transform(
          X
      )  # TODO: Consider merging cleaner into label_cleaner
      X, y = self.extract_label(X)
      self.label_cleaner = LabelCleaner.construct(
          problem_type=self.problem_type,
          y=y,
          y_uncleaned=y_uncleaned,
          positive_class=self._positive_class,
      )
      y = self.label_cleaner.transform(y)
      X = self.set_predefined_weights(X, y)
      X, w = extract_column(X, self.sample_weight)
      X, groups = extract_column(X, self.groups)
      if (
          self.label_cleaner.num_classes is not None
          and self.problem_type != BINARY
      ):
          logger.log(
              20,
              f"Train Data Class Count: {self.label_cleaner.num_classes}",
          )
  
      if X_val is not None and self.label in X_val.columns:
          y_val_og = X_val[self.label]
          X_val = self.cleaner.transform(X_val)
          if len(X_val) == 0:
              logger.warning(
                  "############################################################################################################\n"
                  "WARNING: All X_val data contained low frequency classes, ignoring X_val and generating from subset of X\n"
                  "\tYour input validation data or training data labels might be corrupted, please manually inspect them for correctness!"
              )
              if self.problem_type in [BINARY, MULTICLASS]:
                  train_classes = sorted(list(y_uncleaned.unique()))
                  val_classes = sorted(list(y_val_og.unique()))
                  logger.warning(f"\tTraining Classes: {train_classes}")
                  logger.warning(f"\tTuning   Classes: {val_classes}")
                  logger.warning(
                      f"\tTraining Class Dtype: {y_uncleaned.dtype}"
                  )
                  logger.warning(
                      f"\tTuning   Class Dtype: {y_val_og.dtype}"
                  )
                  missing_classes = [
                      c for c in val_classes if c not in train_classes
                  ]
                  logger.warning(
                      f"\tClasses missing from Training Data: {missing_classes}"
                  )
              logger.warning(
                  "############################################################################################################"
              )
  
              X_val = None
              y_val = None
              w_val = None
              holdout_frac = holdout_frac_og
          else:
              X_val, y_val = self.extract_label(X_val)
              y_val = self.label_cleaner.transform(y_val)
              X_val = self.set_predefined_weights(X_val, y_val)
              X_val, w_val = extract_column(X_val, self.sample_weight)
      else:
          y_val = None
          w_val = None
  
      self._original_features = list(X.columns)
      # TODO: Move this up to top of data before removing data, this way our feature generator is better
      logger.log(
          20, f"Using Feature Generators to preprocess the data ..."
      )
      if X_val is not None:
          # Do this if working with SKLearn models, otherwise categorical features may perform very badly on the test set
          logger.log(
              15,
              "Performing general data preprocessing with merged train & validation data, so validation performance may not accurately reflect performance on new test data",
          )
          X_super = pd.concat(
              [X, X_val, X_unlabeled], ignore_index=True
          )
          if self.feature_generator.is_fit():
              logger.log(
                  20,
                  f"{self.feature_generator.__class__.__name__} is already fit, so the training data will be processed via .transform() instead of .fit_transform().",
              )
              X_super = self.feature_generator.transform(X_super)
              self.feature_generator.print_feature_metadata_info()
          else:
              if X_unlabeled is None:
                  y_super = pd.concat([y, y_val], ignore_index=True)
              else:
                  y_unlabeled = pd.Series(
                      np.nan, index=X_unlabeled.index
                  )
                  y_super = pd.concat(
                      [y, y_val, y_unlabeled], ignore_index=True
                  )
              X_super = self.fit_transform_features(
                  X_super,
                  y_super,
                  problem_type=self.label_cleaner.problem_type_transform,
                  eval_metric=self.eval_metric,
              )
          X = X_super.head(len(X)).set_index(X.index)
  
          X_val = (
              X_super.head(len(X) + len(X_val))
              .tail(len(X_val))
              .set_index(X_val.index)
          )
  
          if X_unlabeled is not None:
              X_unlabeled = X_super.tail(len(X_unlabeled)).set_index(
                  X_unlabeled.index
              )
          del X_super
      else:
          X_super = pd.concat([X, X_unlabeled], ignore_index=True)
          if self.feature_generator.is_fit():
              logger.log(
                  20,
                  f"{self.feature_generator.__class__.__name__} is already fit, so the training data will be processed via .transform() instead of .fit_transform().",
              )
              X_super = self.feature_generator.transform(X_super)
              self.feature_generator.print_feature_metadata_info()
          else:
              if X_unlabeled is None:
                  y_super = y.reset_index(drop=True)
              else:
                  y_unlabeled = pd.Series(
                      np.nan, index=X_unlabeled.index
                  )
                  y_super = pd.concat(
                      [y, y_unlabeled], ignore_index=True
                  )
              X_super = self.fit_transform_features(
                  X_super,
                  y_super,
                  problem_type=self.label_cleaner.problem_type_transform,
                  eval_metric=self.eval_metric,
              )
  
          X = X_super.head(len(X)).set_index(X.index)
          if X_unlabeled is not None:
              X_unlabeled = X_super.tail(len(X_unlabeled)).set_index(
                  X_unlabeled.index
              )
          del X_super
      X, X_val = self.bundle_weights(
          X, w, X_val, w_val
      )  # TODO: consider not bundling sample-weights inside X, X_val
      return (
          X,
          y,
          X_val,
          y_val,
          X_unlabeled,
          holdout_frac,
          num_bag_folds,
          groups,
      )
  
imports: import copy
  import logging
  import math
  import time
  import numpy
  import pandas
  from pandas import DataFrame
  from autogluon.common.utils.log_utils import convert_time_in_s_to_log_friendly
  from autogluon.common.utils.system_info import get_ag_system_info
  from autogluon.core.constants import AUTO_WEIGHT, BALANCE_WEIGHT, BINARY, MULTICLASS, QUANTILE, REGRESSION
  from autogluon.core.data import LabelCleaner
  from autogluon.core.data.cleaner import Cleaner
  from autogluon.core.utils.time import sample_df_for_time_func, time_func
  from autogluon.core.utils.utils import augment_rare_classes, extract_column
  from ..trainer import AutoTrainer
  from .abstract_learner import AbstractTabularLearner
label_desc: Use image node_id 4 for calling the DefaultLearner obj's underlying member method code with example usage: obj.general_data_processing(X, X_val, X_unlabeled, holdout_frac, num_bag_folds) and returns: X,  y,  X_val,  y_val,  X_unlabeled,  holdout_frac,  num_bag_folds,  groups