code: def test_checkpoints_to_keep(ray_start_4_cpus, mode):
      """
      Test that ``CheckpointConfig`` is respected.
  
      - Report 4 times with different metrics.
      - Assert that the kept checkpoints match the expectation.
      """
  
      def train_func():
          with create_dict_checkpoint({"idx": 0}) as checkpoint:
              train.report(
                  dict(loss=float("nan")), checkpoint=checkpoint
              )  # nan, deleted
          with create_dict_checkpoint({"idx": 1}) as checkpoint:
              train.report(
                  dict(loss=3), checkpoint=checkpoint
              )  # best for min, worst for max (del)
          with create_dict_checkpoint({"idx": 2}) as checkpoint:
              train.report(
                  dict(loss=7), checkpoint=checkpoint
              )  # worst for min (del), best for max
          with create_dict_checkpoint({"idx": 3}) as checkpoint:
              train.report(dict(loss=5), checkpoint=checkpoint)
  
      checkpoint_config = CheckpointConfig(
          num_to_keep=2,
          checkpoint_score_attribute="loss",
          checkpoint_score_order=mode,
      )
  
      trainer = DataParallelTrainer(
          train_func,
          scaling_config=scale_config,
          run_config=RunConfig(checkpoint_config=checkpoint_config),
      )
      result = trainer.fit()
      assert len(result.best_checkpoints) == 2
  
      # Last checkpoint
      assert load_dict_checkpoint(result.checkpoint)["idx"] == 3
  
      if mode == "min":
          indices = [3, 1]
          losses = [5, 3]
      else:
          indices = [3, 2]
          losses = [5, 7]
  
      assert (
          load_dict_checkpoint(result.best_checkpoints[0][0])["idx"]
          == indices[0]
      )
      assert (
          load_dict_checkpoint(result.best_checkpoints[1][0])["idx"]
          == indices[1]
      )
      assert result.best_checkpoints[0][1]["loss"] == losses[0]
      assert result.best_checkpoints[1][1]["loss"] == losses[1]
  
file_path: ray/python/ray/train/tests/test_data_parallel_trainer_checkpointing.py
imports: import pytest
  import ray
  from ray import train
  from ray.train import CheckpointConfig, RunConfig, ScalingConfig
  from ray.train.data_parallel_trainer import DataParallelTrainer
  from ray.train.tests.util import create_dict_checkpoint, load_dict_checkpoint
label_desc: Use image node_id 5 for calling a global function with example usage: test_checkpoints_to_keep(ray_start_4_cpus, mode) without return types