def benchmark(synthesizers, datasets=DEFAULT_DATASETS, iterations=3, add_leaderboard=True, leaderboard_path=LEADERBOARD_PATH, replace_existing=True):
    'Compute the benchmark scores for the synthesizers and return a leaderboard.\n\n    The ``synthesizers`` object can either be a single synthesizer or, an iterable of\n    synthesizers or a dict containing synthesizer names as keys and synthesizers as values.\n\n    If ``add_leaderboard`` is ``True``, append the obtained scores to the leaderboard\n    stored in the ``lederboard_path``. By default, the leaderboard used is the one which\n    is included in the package, which contains the scores obtained by the SDGym Synthesizers.\n\n    If ``replace_existing`` is ``True`` and any of the given synthesizers already existed\n    in the leaderboard, the old rows are dropped.\n\n    Args:\n        synthesizers (function, class, list, tuple or dict):\n            The synthesizer or synthesizers to evaluate. It can be a single synthesizer\n            (function or method or class), or an iterable of synthesizers, or a dict\n            containing synthesizer names as keys and synthesizers as values. If the input\n            is not a dict, synthesizer names will be extracted from the given object.\n        datasets (list[str]):\n            Names of the datasets to use for the benchmark. Defaults to all the ones available.\n        iterations (int):\n            Number of iterations to perform over each dataset and synthesizer. Defaults to 3.\n        add_leaderboard (bool):\n            Whether to append the obtained scores to the previous leaderboard or not. Defaults\n            to ``True``.\n        leaderboard_path (str):\n            Path to where the leaderboard is stored. Defaults to the leaderboard included\n            with the package, which contains the scores obtained by the SDGym synthesizers.\n        replace_existing (bool):\n            Whether to replace old scores or keep them in the returned leaderboard. Defaults\n            to ``True``.\n\n    Returns:\n        pandas.DataFrame:\n            Table containing one row per synthesizer and one column for each dataset and metric.\n    '
    synthesizers = _get_synthesizers(synthesizers)
    scores = list()
    for (synthesizer_name, synthesizer) in synthesizers.items():
        synthesizer_scores = compute_benchmark(synthesizer, datasets, iterations)
        summary_row = _summarize_scores(synthesizer_scores)
        summary_row.name = synthesizer_name
        scores.append(summary_row)
    leaderboard = pd.DataFrame(scores)
    leaderboard['timestamp'] = datetime.utcnow()
    if add_leaderboard:
        old_leaderboard = pd.read_csv(leaderboard_path, index_col=0, parse_dates=['timestamp'])[leaderboard.columns]
        if replace_existing:
            old_leaderboard.drop(labels=[leaderboard.index], errors='ignore', inplace=True)
        leaderboard = old_leaderboard.append(leaderboard, sort=False)
    return leaderboard
Compute the benchmark scores for the synthesizers and return a leaderboard.

The ``synthesizers`` object can either be a single synthesizer or, an iterable of
synthesizers or a dict containing synthesizer names as keys and synthesizers as values.

If ``add_leaderboard`` is ``True``, append the obtained scores to the leaderboard
stored in the ``lederboard_path``. By default, the leaderboard used is the one which
is included in the package, which contains the scores obtained by the SDGym Synthesizers.

If ``replace_existing`` is ``True`` and any of the given synthesizers already existed
in the leaderboard, the old rows are dropped.

Args:
    synthesizers (function, class, list, tuple or dict):
        The synthesizer or synthesizers to evaluate. It can be a single synthesizer
        (function or method or class), or an iterable of synthesizers, or a dict
        containing synthesizer names as keys and synthesizers as values. If the input
        is not a dict, synthesizer names will be extracted from the given object.
    datasets (list[str]):
        Names of the datasets to use for the benchmark. Defaults to all the ones available.
    iterations (int):
        Number of iterations to perform over each dataset and synthesizer. Defaults to 3.
    add_leaderboard (bool):
        Whether to append the obtained scores to the previous leaderboard or not. Defaults
        to ``True``.
    leaderboard_path (str):
        Path to where the leaderboard is stored. Defaults to the leaderboard included
        with the package, which contains the scores obtained by the SDGym synthesizers.
    replace_existing (bool):
        Whether to replace old scores or keep them in the returned leaderboard. Defaults
        to ``True``.

Returns:
    pandas.DataFrame:
        Table containing one row per synthesizer and one column for each dataset and metric.
def benchmark(synthesizers, datasets=DEFAULT_DATASETS, iterations=3, add_leaderboard=True, leaderboard_path=LEADERBOARD_PATH, replace_existing=True):
    'Compute the benchmark scores for the synthesizers and return a leaderboard.\n\n    The ``synthesizers`` object can either be a single synthesizer or, an iterable of\n    synthesizers or a dict containing synthesizer names as keys and synthesizers as values.\n\n    If ``add_leaderboard`` is ``True``, append the obtained scores to the leaderboard\n    stored in the ``lederboard_path``. By default, the leaderboard used is the one which\n    is included in the package, which contains the scores obtained by the SDGym Synthesizers.\n\n    If ``replace_existing`` is ``True`` and any of the given synthesizers already existed\n    in the leaderboard, the old rows are dropped.\n\n    Args:\n        synthesizers (function, class, list, tuple or dict):\n            The synthesizer or synthesizers to evaluate. It can be a single synthesizer\n            (function or method or class), or an iterable of synthesizers, or a dict\n            containing synthesizer names as keys and synthesizers as values. If the input\n            is not a dict, synthesizer names will be extracted from the given object.\n        datasets (list[str]):\n            Names of the datasets to use for the benchmark. Defaults to all the ones available.\n        iterations (int):\n            Number of iterations to perform over each dataset and synthesizer. Defaults to 3.\n        add_leaderboard (bool):\n            Whether to append the obtained scores to the previous leaderboard or not. Defaults\n            to ``True``.\n        leaderboard_path (str):\n            Path to where the leaderboard is stored. Defaults to the leaderboard included\n            with the package, which contains the scores obtained by the SDGym synthesizers.\n        replace_existing (bool):\n            Whether to replace old scores or keep them in the returned leaderboard. Defaults\n            to ``True``.\n\n    Returns:\n        pandas.DataFrame:\n            Table containing one row per synthesizer and one column for each dataset and metric.\n    '
    synthesizers = _get_synthesizers(synthesizers)
    scores = list()
    for (synthesizer_name, synthesizer) in synthesizers.items():
        synthesizer_scores = compute_benchmark(synthesizer, datasets, iterations)
        summary_row = _summarize_scores(synthesizer_scores)
        summary_row.name = synthesizer_name
        scores.append(summary_row)
    leaderboard = pd.DataFrame(scores)
    leaderboard['timestamp'] = datetime.utcnow()
    if add_leaderboard:
        old_leaderboard = pd.read_csv(leaderboard_path, index_col=0, parse_dates=['timestamp'])[leaderboard.columns]
        if replace_existing:
            old_leaderboard.drop(labels=[leaderboard.index], errors='ignore', inplace=True)
        leaderboard = old_leaderboard.append(leaderboard, sort=False)
    return leaderboard