code: def test_make_leaderboard_custom_metric():
      custom_mae = h2o.upload_custom_metric(
          CustomMaeFunc, func_name="mae", func_file="mm_mae.py"
      )
      custom_mae2 = h2o.upload_custom_metric(
          CustomMaeFunc, func_name="mae2", func_file="mm_mae.py"
      )
  
      ftrain, fvalid, _ = dataset_prostate()
      nfolds = 5
  
      model_mae = H2OGradientBoostingEstimator(
          model_id="prostate",
          ntrees=1000,
          max_depth=5,
          score_each_iteration=True,
          stopping_metric="mae",
          stopping_tolerance=0.1,
          stopping_rounds=3,
          nfolds=nfolds,
          seed=123,
      )
      model_mae.train(
          y="AGE",
          x=ftrain.names,
          training_frame=ftrain,
          validation_frame=fvalid,
      )
  
      model_custom1 = H2OGradientBoostingEstimator(
          model_id="prostate_custom1",
          ntrees=1000,
          max_depth=5,
          score_each_iteration=True,
          custom_metric_func=custom_mae,
          stopping_metric="custom",
          stopping_tolerance=0.1,
          stopping_rounds=3,
          nfolds=nfolds,
          keep_cross_validation_predictions=True,
          seed=123,
      )
      model_custom1.train(
          y="AGE",
          x=ftrain.names,
          training_frame=ftrain,
          validation_frame=fvalid,
      )
  
      model_custom2 = H2OGradientBoostingEstimator(
          model_id="prostate_custom2",
          ntrees=1000,
          max_depth=2,
          score_each_iteration=True,
          custom_metric_func=custom_mae,
          stopping_metric="custom",
          stopping_tolerance=0.1,
          stopping_rounds=3,
          nfolds=nfolds,
          keep_cross_validation_predictions=True,
          seed=123,
      )
      model_custom2.train(
          y="AGE",
          x=ftrain.names,
          training_frame=ftrain,
          validation_frame=fvalid,
      )
  
      model_custom3 = H2OGradientBoostingEstimator(
          model_id="prostate_custom3",
          ntrees=2,
          max_depth=2,
          score_each_iteration=True,
          custom_metric_func=custom_mae,
          stopping_metric="custom",
          stopping_tolerance=0.1,
          stopping_rounds=3,
          nfolds=nfolds,
          seed=123,
      )
      model_custom3.train(
          y="AGE",
          x=ftrain.names,
          training_frame=ftrain,
          validation_frame=fvalid,
      )
  
      model_custom_alt = H2OGradientBoostingEstimator(
          model_id="prostate_custom_alt",
          ntrees=2,
          max_depth=2,
          score_each_iteration=True,
          custom_metric_func=custom_mae2,
          stopping_metric="custom",
          stopping_tolerance=0.1,
          stopping_rounds=3,
          nfolds=nfolds,
          seed=123,
      )
      model_custom_alt.train(
          y="AGE",
          x=ftrain.names,
          training_frame=ftrain,
          validation_frame=fvalid,
      )
  
      model_se = H2OStackedEnsembleEstimator(
          base_models=[model_custom1, model_custom2],
          metalearner_algorithm="gbm",
          custom_metric_func=custom_mae,
      )
      model_se.train(
          y="AGE",
          x=ftrain.names,
          training_frame=ftrain,
          validation_frame=fvalid,
      )
  
      assert (
          "custom"
          in h2o.make_leaderboard(
              [model_custom1, model_custom2, model_custom3, model_se],
              fvalid,
          ).columns
      )
      ldb = h2o.make_leaderboard(
          [model_custom1, model_custom2, model_custom3, model_se],
          fvalid,
      ).as_data_frame()
      print(ldb)
      assert (ldb["mae"] == ldb["custom"]).all()
  
      ldb_custom = h2o.make_leaderboard(
          [model_custom1, model_custom2, model_custom3, model_se],
          fvalid,
          sort_metric="custom",
      ).as_data_frame()
      ldb_mae = h2o.make_leaderboard(
          [model_custom1, model_custom2, model_custom3, model_se],
          fvalid,
          sort_metric="mae",
      ).as_data_frame()
  
      assert (ldb_mae["model_id"] == ldb_custom["model_id"]).all()
  
      for scoring_data in ["train", "valid", "xval", "AUTO"]:
          print(scoring_data)
          ldb_custom = h2o.make_leaderboard(
              [model_custom1, model_custom2, model_custom3, model_se],
              sort_metric="custom",
              scoring_data=scoring_data,
          ).as_data_frame()
          ldb_mae = h2o.make_leaderboard(
              [model_custom1, model_custom2, model_custom3, model_se],
              sort_metric="mae",
              scoring_data=scoring_data,
          ).as_data_frame()
  
          print(ldb_custom)
          print(ldb_mae)
          assert (ldb_mae["model_id"] == ldb_custom["model_id"]).all()
          assert (ldb_mae["mae"] == ldb_mae["custom"]).all()
          assert (ldb_custom["mae"] == ldb_custom["custom"]).all()
  
      try:
          print(
              h2o.make_leaderboard([model_custom1, model_mae], fvalid)
          )
          assert False, "Should fail - different metrics present."
      except Exception:
          pass
  
      try:
          print(
              h2o.make_leaderboard(
                  [model_custom1, model_custom_alt], fvalid
              )
          )
          assert (
              False
          ), "Should fail - different custom metrics present."
      except Exception:
          pass
  
file_path: h2o-3/h2o-py/tests/testdir_misc/explain/pyunit_make_leaderboard.py
imports: import os
  import sys
  import h2o
  import h2o.exceptions
  from tests import pyunit_utils, dataset_prostate, CustomMaeFunc
  from h2o.automl import H2OAutoML
  from h2o.estimators import *
  from h2o.grid import H2OGridSearch
label_desc: Use image node_id 6 for calling a global function with example usage: test_make_leaderboard_custom_metric() without return types