code: def test_resume_with_recovery():
      export_dir = tempfile.mkdtemp()
      grid_id = "resume_with_recovery_gbm"
      print("Using directory %s" % export_dir)
      hyper_parameters = {
          "learn_rate": [0.01, 0.05],
          "ntrees": [100, 110, 120, 130],
      }
      grid_size = 1
      for p in hyper_parameters:
          grid_size *= len(hyper_parameters[p])
      print("Grid size %d" % grid_size)
      print("Starting baseline grid")
  
      df = h2o.import_file(
          path=pyunit_utils.locate(
              "smalldata/gbm_test/ecology_model.csv"
          )
      )
      df["Angaus"] = df["Angaus"].asfactor()
      df["Weights"] = h2o.H2OFrame.from_python(
          abs(np.random.randn(df.nrow, 1)).tolist()
      )[0]
      train, calib = df.split_frame(
          ratios=[0.8],
          destination_frames=["eco_train", "eco_calib"],
          seed=42,
      )
      params = {
          "distribution": "bernoulli",
          "min_rows": 10,
          "max_depth": 5,
          "weights_column": "Weights",
          "calibrate_model": True,
          "calibration_frame": calib,
      }
      recovery_dir_1 = export_dir + "/recovery_1"
      grid = H2OGridSearch(
          H2OGradientBoostingEstimator,
          grid_id=grid_id,
          hyper_params=hyper_parameters,
          recovery_dir=recovery_dir_1,
      )
      grid.start(
          x=list(range(2, train.ncol)),
          y="Angaus",
          training_frame=train,
          **params,
      )
      grid_1_model_count = _wait_for_grid_models(
          grid, grid_id, 1, grid_size
      )
  
      loaded = h2o.load_grid(
          "%s/%s" % (recovery_dir_1, grid_id),
          load_params_references=True,
      )
      _check_grid_loaded_properly(loaded, train, grid_1_model_count)
      print("Resuming grid")
      recovery_dir_2 = export_dir + "/recovery_2"
      loaded.resume(detach=True, recovery_dir=recovery_dir_2)
      grid_2_model_count = _wait_for_grid_models(
          loaded, grid_id, len(loaded.model_ids) + 1, grid_size
      )
  
      loaded_2 = h2o.load_grid(
          "%s/%s" % (recovery_dir_2, grid_id),
          load_params_references=True,
      )
      _check_grid_loaded_properly(loaded_2, train, grid_2_model_count)
      print("Resuming grid to finish")
      loaded_2.resume()
      print("Finished grid has %d models" % len(loaded_2.model_ids))
      assert grid_size == len(loaded_2.model_ids)
  
file_path: h2o-3/h2o-py/tests/testdir_algos/grid/pyunit_grid_resume_with_recovery.py
imports: import sys
  import os
  import tempfile
  import time
  import numpy
  import h2o
  from tests import pyunit_utils
  from h2o.grid.grid_search import H2OGridSearch
  from h2o.estimators.gbm import H2OGradientBoostingEstimator
label_desc: Use image node_id 3 for calling a global function with example usage: test_resume_with_recovery() without return types