code: def test_suite_remove_automl():
      def contains_leaderboard(project_name, keys):
          return (
              "Leaderboard_{}".format(project_name)
              in keys["all"].values
          )
  
      def contains_event_log(project_name, keys):
          return "Events_{}".format(project_name) in keys["all"].values
  
      def frame_in_cluster(frame):
          # reload the first row of the frame to verify that no vec has been removed
          return (
              frame.key is not None
              and H2OFrame.get_frame(frame.key, rows=1) is not None
          )
  
      def test_remove_automl_with_xval():
          ds = import_dataset()
          project_name = "aml_with_xval_remove_test"
          max_models = 5
          nfolds = 5
          aml = H2OAutoML(
              project_name=project_name,
              nfolds=nfolds,
              max_models=max_models,
              seed=1,
          )
          aml.train(
              y=ds.target,
              training_frame=ds.train,
              validation_frame=ds.valid,
              leaderboard_frame=ds.test,
          )
  
          keys = list_keys_in_memory()
          assert aml.key.startswith(project_name)
          assert contains_leaderboard(aml.key, keys)
          assert contains_event_log(aml.key, keys)
          num_SEs = len(keys["metalearners"])
          print({k: len(v) for k, v in keys.items()})
          expectations = dict(
              models_base=max_models + num_SEs,
              cv_models=0,
              predictions=0,
              metrics=(
                  max_models
                  * 3  # for each non-SE model, 1 on training_frame, 1 on validation_frame, 1 on leaderboard_frame
                  + (
                      num_SEs * 2
                  )  # for each SE model, 1 on training frame, 1 on leaderboard frame
                  + (
                      num_SEs * 2
                  )  # for each SE metalearner, 1+1 on levelone training+validation
                  + (
                      1
                      if any(
                          ("DeepLearning" in x for x in keys["metrics"])
                      )
                      else 0
                  )  # DeepLearning has 2 training metrics (IDK why)
              ),
          )
          for k, v in expectations.items():
              assert (
                  len(keys[k]) == v
              ), "expected {} {}, but got {}".format(v, k, len(keys[k]))
  
          h2o.remove(aml)
          clean = list_keys_in_memory()
          print(clean["all"].values)
          assert not contains_leaderboard(aml.key, clean)
          assert not contains_event_log(aml.key, clean)
          assert len(clean["models_base"]) == 0
          assert len(clean["cv_models"]) == 0
          assert len(clean["models_all"]) == 0
          assert len(clean["predictions"]) == 0
          assert len(clean["metrics"]) == 0
          assert len(clean["automl"]) == 0
          for frame in [ds.train, ds.valid, ds.test]:
              assert frame_in_cluster(
                  frame
              ), "frame {} has been removed from cluster".format(
                  frame.frame_id
              )
  
      def test_remove_automl_with_xval_when_keeping_all_cv_details():
          ds = import_dataset()
          project_name = "aml_with_xval_remove_test"
          max_models = 5
          nfolds = 5
          aml = H2OAutoML(
              project_name=project_name,
              nfolds=nfolds,
              max_models=max_models,
              seed=1,
              keep_cross_validation_predictions=True,
              keep_cross_validation_fold_assignment=True,
              keep_cross_validation_models=True,
          )
          aml.train(y=ds.target, training_frame=ds.train)
  
          keys = list_keys_in_memory()
          # print(keys['all'].values)
          assert aml.key.startswith(project_name)
          assert contains_leaderboard(aml.key, keys)
          assert contains_event_log(aml.key, keys)
          num_SEs = len(keys["metalearners"]) / (
              nfolds + 1
          )  # keeping cv models, so metalearners include cv models
          print({k: len(v) for k, v in keys.items()})
          expectations = dict(
              models_base=max_models + num_SEs,
              cv_models=(max_models + num_SEs)
              * nfolds,  # 1 cv model per fold for all models, incl. SEs
              predictions=(
                  len(keys["cv_models"])  # cv predictions
                  + len(keys["models_base"])  # cv holdout predictions
              ),
              metrics=(
                  len(keys["cv_models"])
                  * 3  # for each cv model, 1 on training frame, 1 on validation frame (=training for cv), one on adapted frame (to be removed with PUBDEV-6638)
                  + len(
                      keys["models_base"]
                  )  # for each model, 1 on training_frame
                  + (num_SEs * 1)  # for each SE, 1 on levelone training
                  + (
                      1
                      if any(
                          ("DeepLearning" in x for x in keys["metrics"])
                      )
                      else 0
                  )  # DeepLearning has 2 training metrics (IDK why)
              ),
          )
          for k, v in expectations.items():
              assert (
                  len(keys[k]) == v
              ), "expected {} {}, but got {}".format(v, k, len(keys[k]))
  
          h2o.remove(aml)
          clean = list_keys_in_memory()
          print(clean["all"].values)
          assert not contains_leaderboard(aml.key, clean)
          assert not contains_event_log(aml.key, clean)
          assert len(clean["models_base"]) == 0
          assert len(clean["cv_models"]) == 0
          assert len(clean["models_all"]) == 0
          assert len(clean["predictions"]) == 0
          assert len(clean["metrics"]) == 0
          assert len(clean["automl"]) == 0
          for frame in [ds.train, ds.valid, ds.test]:
              assert frame_in_cluster(
                  frame
              ), "frame {} has been removed from cluster".format(
                  frame.frame_id
              )
  
      def test_remove_automl_no_xval():
          ds = import_dataset()
          project_name = "aml_no_xval_remove_test"
          max_models = 5
          aml = H2OAutoML(
              project_name=project_name,
              nfolds=0,
              max_models=max_models,
              seed=1,
          )
          aml.train(
              y=ds.target,
              training_frame=ds.train,
              blending_frame=ds.valid,
          )
  
          keys = list_keys_in_memory()
          # print(keys['all'].values)
          assert aml.key.startswith(project_name)
          assert contains_leaderboard(aml.key, keys)
          assert contains_event_log(aml.key, keys)
          num_SEs = len(keys["metalearners"])
          print({k: len(v) for k, v in keys.items()})
          expectations = dict(
              models_base=max_models + num_SEs,
              cv_models=0,
              predictions=0,
              metrics=(
                  2 * len(keys["models_all"])
              ),  # for each model, 1 on training_frame, 1 on validation frame which is also the leaderboard frame
          )
          for k, v in expectations.items():
              assert (
                  len(keys[k]) == v
              ), "expected {} {}, but got {}".format(v, k, len(keys[k]))
  
          h2o.remove(aml)
          clean = list_keys_in_memory()
          print(clean["all"].values)
          assert not contains_leaderboard(aml.key, clean)
          assert not contains_event_log(aml.key, clean)
          assert len(clean["models_base"]) == 0
          assert len(clean["cv_models"]) == 0
          assert len(clean["models_all"]) == 0
          assert len(clean["metrics"]) == 0
          assert len(clean["predictions"]) == 0
          assert len(clean["automl"]) == 0
          for frame in [ds.train, ds.valid, ds.test]:
              assert frame_in_cluster(
                  frame
              ), "frame {} has been removed from cluster".format(
                  frame.frame_id
              )
  
      def test_remove_automl_after_individual_manual_deletions():
          ds = import_dataset()
          project_name = "aml_no_xval_remove_test"
          max_models = 3
          aml = H2OAutoML(
              project_name=project_name,
              nfolds=0,
              max_models=max_models,
              seed=1,
          )
          aml.train(
              y=ds.target,
              training_frame=ds.train,
              blending_frame=ds.valid,
          )
  
          keys = list_keys_in_memory()
          # manually remove the first item for each category to verify robustness of global automl deletion
          # for example, to verify that exceptions (if any) are handled correctly when automl is trying to remove a base model that was already removed
          for k, v in keys.items():
              if k == "all":
                  continue
              if len(v) > 0:
                  h2o.remove(v[0])
  
          h2o.remove(aml)
          clean = list_keys_in_memory()
          print(clean["all"].values)
          assert aml.key.startswith(project_name)
          assert not contains_leaderboard(aml.key, clean)
          assert not contains_event_log(aml.key, clean)
          assert len(clean["models_base"]) == 0
          assert len(clean["cv_models"]) == 0
          assert len(clean["models_all"]) == 0
          assert len(clean["metrics"]) == 0
          assert len(clean["predictions"]) == 0
          assert len(clean["automl"]) == 0
          for frame in [ds.train, ds.valid, ds.test]:
              assert frame_in_cluster(
                  frame
              ), "frame {} has been removed from cluster".format(
                  frame.frame_id
              )
  
      return [
          test_remove_automl_with_xval,
          test_remove_automl_with_xval_when_keeping_all_cv_details,
          test_remove_automl_no_xval,
          test_remove_automl_after_individual_manual_deletions,
      ]
  
file_path: h2o-3/h2o-py/tests/testdir_algos/automl/pyunit_automl_memory_cleanup.py
imports: import sys, os
  import re
  import h2o
  from h2o import H2OFrame
  from h2o.automl import H2OAutoML
  from tests import pyunit_utils
  from _automl_utils import import_dataset, get_partitioned_model_names
label_desc: Use image node_id 5 for calling a global function with example usage: test_suite_remove_automl() and returns: list, unknown, unknown, unknown