"""Collection of tests for :mod:`orion.plotting.backend_plotly`."""
import copy

import numpy
import pandas
import plotly
import pytest

import orion.client
from orion.analysis.partial_dependency_utils import partial_dependency_grid
from orion.core.worker.experiment import Experiment
from orion.plotting.base import (
    lpi,
    parallel_coordinates,
    partial_dependencies,
    rankings,
    regret,
    regrets,
)
from orion.testing import create_experiment
from orion.testing.plotting import (
    assert_lpi_plot,
    assert_parallel_coordinates_plot,
    assert_partial_dependencies_plot,
    assert_rankings_plot,
    assert_regret_plot,
    assert_regrets_plot,
)

config = dict(
    name="experiment-name",
    space={"x": "uniform(0, 200)"},
    metadata={
        "user": "test-user",
        "orion_version": "XYZ",
        "VCS": {
            "type": "git",
            "is_dirty": False,
            "HEAD_sha": "test",
            "active_branch": None,
            "diff_sha": "diff",
        },
    },
    version=1,
    pool_size=1,
    max_trials=10,
    working_dir="",
    algorithms={"random": {"seed": 1}},
    producer={"strategy": "NoParallelStrategy"},
)

trial_config = {
    "experiment": 0,
    "status": "completed",
    "worker": None,
    "start_time": None,
    "end_time": None,
    "heartbeat": None,
    "results": [],
    "params": [],
}


def mock_space(x="uniform(0, 6)", y="uniform(0, 3)", **kwargs):
    """Build a mocked space"""
    mocked_config = copy.deepcopy(config)
    mocked_config["space"] = {"x": x}
    if y is not None:
        mocked_config["space"]["y"] = y
    mocked_config["space"].update(kwargs)
    return mocked_config


def mock_experiment(
    monkeypatch, ids=None, x=None, y=None, z=None, objectives=None, status=None
):
    """Mock experiment to_pandas to return given data (or default one)"""
    if ids is None:
        ids = ["a", "b", "c", "d"]
    if x is None:
        x = [0, 1, 2, 4]
    if y is None:
        y = [3, 2, 0, 1]
    if objectives is None:
        objectives = [0.1, 0.2, 0.3, 0.5]
    if status is None:
        status = ["completed", "completed", "completed", "completed"]

    data = {
        "id": ids,
        "x": x,
        "objective": objectives,
        "status": status,
        "suggested": ids,
    }

    if not isinstance(y, str):
        data["y"] = y
    if z is not None:
        data["z"] = z

    def to_pandas(self, with_evc_tree=False):

        return pandas.DataFrame(data=data)

    monkeypatch.setattr(Experiment, "to_pandas", to_pandas)


def mock_experiment_with_random_to_pandas(monkeypatch, status=None, unbalanced=False):
    def to_pandas(self, with_evc_tree=False):
        if unbalanced:
            N = numpy.random.randint(5, 15)
        elif status is not None:
            N = len(status)
        else:
            N = 10
        ids = numpy.arange(N)
        x = numpy.random.normal(0, 0.1, size=N)
        y = numpy.random.normal(0, 0.1, size=N)
        objectives = numpy.random.normal(0, 0.1, size=N)
        if status is None:
            exp_status = ["completed"] * N
        else:
            exp_status = status

        data = pandas.DataFrame(
            data={
                "id": ids,
                "x": x,
                "y": y,
                "objective": objectives,
                "status": exp_status,
                "suggested": ids,
            }
        )

        return data

    monkeypatch.setattr(Experiment, "to_pandas", to_pandas)


def mock_model():
    """Return a mocked regressor which just predict iterated integers"""

    class Model:
        """Mocked Regressor"""

        def __init__(self):
            self.i = 0

        def predict(self, data):
            """Returns counting of predictions requested."""
            data = numpy.arange(data.shape[0]) + self.i
            self.i += data.shape[0]
            return data  #  + numpy.random.normal(0, self.i, size=data.shape[0])

    return Model()


def mock_train_regressor(monkeypatch, assert_model=None, assert_model_kwargs=None):
    """Mock the train_regressor to return the mocked regressor instead"""

    def train_regressor(model, data, **kwargs):
        """Return the mocked model, and then model argument if requested"""
        if assert_model:
            assert model == assert_model
        if assert_model_kwargs:
            assert kwargs == assert_model_kwargs
        return mock_model()

    monkeypatch.setattr(
        "orion.analysis.partial_dependency_utils.train_regressor", train_regressor
    )


@pytest.mark.usefixtures("version_XYZ")
class TestLPI:
    """Tests the ``lpi()`` method provided by the plotly backend"""

    def test_requires_argument(self):
        """Tests that the experiment data are required."""
        with pytest.raises(ValueError):
            lpi(None)

    def test_returns_plotly_object(self):
        """Tests that the plotly backend returns a plotly object"""
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = lpi(experiment, model_kwargs=dict(random_state=1))

        assert type(plot) is plotly.graph_objects.Figure

    def test_graph_layout(self, monkeypatch):
        """Tests the layout of the plot"""
        config = mock_space()
        mock_experiment(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = lpi(experiment, model_kwargs=dict(random_state=1))
            df = experiment.to_pandas()
            assert df["x"].tolist() == [0, 1, 2, 4]
            assert df["y"].tolist() == [3, 2, 0, 1]
            assert df["objective"].tolist() == [0.1, 0.2, 0.3, 0.5]

        assert_lpi_plot(plot, dims=["x", "y"])

    def test_experiment_worker_as_parameter(self, monkeypatch):
        """Tests that ``Experiment`` is a valid parameter"""
        config = mock_space()
        mock_experiment(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            experiment,
            _,
        ):
            plot = lpi(experiment, model_kwargs=dict(random_state=1))

        assert_lpi_plot(plot, dims=["x", "y"])

    def test_ignore_uncompleted_statuses(self, monkeypatch):
        """Tests that uncompleted statuses are ignored"""
        config = mock_space()
        mock_experiment(
            monkeypatch,
            ids="abcdefgh",
            x=[0, 0, 0, 1, 0, 2, 0, 3],
            y=[1, 0, 0, 2, 0, 0, 0, 3],
            objectives=[0.1, None, None, 0.2, None, 0.3, None, 0.5],
            status=[
                "completed",
                "new",
                "reserved",
                "completed",
                "broken",
                "completed",
                "interrupted",
                "completed",
            ],
        )
        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = lpi(experiment)

        assert_lpi_plot(plot, dims=["x", "y"])

    def test_multidim(self, monkeypatch):
        """Tests that dimensions with shape > 1 are flattened properly"""
        config = mock_space(y="uniform(0, 3, shape=2)")
        mock_experiment(monkeypatch, y=[[3, 3], [2, 3], [1, 2], [0, 3]])
        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = lpi(experiment, model_kwargs=dict(random_state=1))

        assert_lpi_plot(plot, dims=["x", "y[0]", "y[1]"])

    def test_fidelity(self, monkeypatch):
        """Tests that fidelity is supported"""
        config = mock_space(y="fidelity(1, 200, base=3)")
        mock_experiment(monkeypatch, y=[1, 3 ** 2, 1, 3 ** 4])
        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = lpi(experiment, model_kwargs=dict(random_state=1))

        assert_lpi_plot(plot, dims=["x", "y"])

    def test_categorical(self, monkeypatch):
        """Tests that categorical is supported"""
        config = mock_space(y='choices(["a", "b", "c"])')
        mock_experiment(monkeypatch, y=["c", "c", "a", "b"])
        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = lpi(experiment, model_kwargs=dict(random_state=1))

        assert_lpi_plot(plot, dims=["x", "y"])

    def test_categorical_multidim(self, monkeypatch):
        """Tests that multidim categorical is supported"""
        config = mock_space(y='choices(["a", "b", "c"], shape=3)')
        mock_experiment(
            monkeypatch,
            y=[["c", "b", "a"], ["c", "a", "c"], ["a", "b", "a"], ["c", "b", "b"]],
        )

        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = lpi(experiment, model_kwargs=dict(random_state=1))

        assert_lpi_plot(plot, dims=["x", "y[0]", "y[1]", "y[2]"])


@pytest.mark.usefixtures("version_XYZ")
class TestParallelCoordinates:
    """Tests the ``parallel_coordinates()`` method provided by the plotly backend"""

    def test_requires_argument(self):
        """Tests that the experiment data are required."""
        with pytest.raises(ValueError):
            parallel_coordinates(None)

    def test_returns_plotly_object(self):
        """Tests that the plotly backend returns a plotly object"""
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = parallel_coordinates(experiment)

        assert type(plot) is plotly.graph_objects.Figure

    def test_graph_layout(self):
        """Tests the layout of the plot"""
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = parallel_coordinates(experiment)

        assert_parallel_coordinates_plot(plot, order=["x", "loss"])

    def test_experiment_worker_as_parameter(self):
        """Tests that ``Experiment`` is a valid parameter"""
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            experiment,
            _,
        ):
            plot = parallel_coordinates(experiment)

        assert_parallel_coordinates_plot(plot, order=["x", "loss"])

    def test_ignore_uncompleted_statuses(self):
        """Tests that uncompleted statuses are ignored"""
        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = parallel_coordinates(experiment)

        assert_parallel_coordinates_plot(plot, order=["x", "loss"])

    def test_unsupported_order_key(self):
        """Tests that unsupported order keys are rejected"""
        with create_experiment(config, trial_config) as (_, _, experiment):
            with pytest.raises(ValueError):
                parallel_coordinates(experiment, order=["unsupported"])

    def test_order_columns(self):
        """Tests that columns are sorted according to ``order``"""
        multidim_config = copy.deepcopy(config)
        for k in "yzutv":
            multidim_config["space"][k] = "uniform(0, 200)"
        with create_experiment(multidim_config, trial_config) as (_, _, experiment):
            plot = parallel_coordinates(experiment, order="vzyx")

        assert_parallel_coordinates_plot(plot, order=["v", "z", "y", "x", "loss"])

    def test_multidim(self):
        """Tests that dimensions with shape > 1 are flattened properly"""
        multidim_config = copy.deepcopy(config)
        multidim_config["space"]["y"] = "uniform(0, 200, shape=4)"
        with create_experiment(multidim_config, trial_config) as (_, _, experiment):
            plot = parallel_coordinates(experiment)

        assert_parallel_coordinates_plot(
            plot, order=["x", "y[0]", "y[1]", "y[2]", "y[3]", "loss"]
        )

    def test_fidelity(self):
        """Tests that fidelity is set to first column by default"""
        fidelity_config = copy.deepcopy(config)
        fidelity_config["space"]["z"] = "fidelity(1, 200, base=3)"
        with create_experiment(fidelity_config, trial_config) as (_, _, experiment):
            plot = parallel_coordinates(experiment)

        assert_parallel_coordinates_plot(plot, order=["z", "x", "loss"])

    def test_categorical(self):
        """Tests that categorical is supported"""
        categorical_config = copy.deepcopy(config)
        categorical_config["space"]["z"] = 'choices(["a", "b", "c"])'
        with create_experiment(categorical_config, trial_config) as (_, _, experiment):
            plot = parallel_coordinates(experiment)

        assert_parallel_coordinates_plot(plot, order=["x", "z", "loss"])

    def test_categorical_multidim(self):
        """Tests that multidim categorical is supported"""
        categorical_config = copy.deepcopy(config)
        categorical_config["space"]["z"] = 'choices(["a", "b", "c"], shape=3)'
        with create_experiment(categorical_config, trial_config) as (_, _, experiment):
            plot = parallel_coordinates(experiment)

        assert_parallel_coordinates_plot(
            plot, order=["x", "z[0]", "z[1]", "z[2]", "loss"]
        )


@pytest.mark.usefixtures("version_XYZ")
class TestPartialDependencies:
    """Tests the ``partial_dependencies()`` method provided by the plotly backend"""

    def test_returns_plotly_object(self, monkeypatch):
        """Tests that the plotly backend returns a plotly object"""
        mock_train_regressor(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = partial_dependencies(
                experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
            )

        assert type(plot) is plotly.graph_objects.Figure

    def test_graph_layout(self, monkeypatch):
        """Tests the layout of the plot"""
        mock_train_regressor(monkeypatch)
        config = mock_space()
        mock_experiment(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = partial_dependencies(
                experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
            )
            df = experiment.to_pandas()
            assert df["x"].tolist() == [0, 1, 2, 4]
            assert df["y"].tolist() == [3, 2, 0, 1]
            assert df["objective"].tolist() == [0.1, 0.2, 0.3, 0.5]

        assert_partial_dependencies_plot(plot, dims=["x", "y"])

    def test_ignore_uncompleted_statuses(self, monkeypatch):
        """Tests that uncompleted statuses are ignored"""
        mock_train_regressor(monkeypatch)
        config = mock_space()
        mock_experiment(
            monkeypatch,
            ids="abcdefgh",
            x=[0, 0, 0, 1, 0, 2, 0, 3],
            y=[1, 0, 0, 2, 0, 0, 0, 3],
            objectives=[0.1, None, None, 0.2, None, 0.3, None, 0.5],
            status=[
                "completed",
                "new",
                "reserved",
                "completed",
                "broken",
                "completed",
                "interrupted",
                "completed",
            ],
        )
        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(experiment, n_grid_points=5)

        assert_partial_dependencies_plot(plot, dims=["x", "y"])

    def test_multidim(self, monkeypatch):
        """Tests that dimensions with shape > 1 are flattened properly"""
        mock_train_regressor(monkeypatch)
        config = mock_space(y="uniform(0, 3, shape=2)")
        mock_experiment(monkeypatch, y=[[3, 3], [2, 3], [1, 2], [0, 3]])
        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
            )

        assert_partial_dependencies_plot(plot, dims=["x", "y[0]", "y[1]"])

    def test_fidelity(self, monkeypatch):
        """Tests that fidelity is supported"""
        mock_train_regressor(monkeypatch)
        config = mock_space(y="fidelity(1, 200, base=3)")
        mock_experiment(monkeypatch, y=[1, 3 ** 2, 1, 3 ** 4])
        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
            )

        assert_partial_dependencies_plot(plot, dims=["x", "y"], log_dims=["y"])

    def test_categorical(self, monkeypatch):
        """Tests that categorical is supported"""
        mock_train_regressor(monkeypatch)
        config = mock_space(y='choices(["a", "b", "c"])')
        mock_experiment(monkeypatch, y=["c", "c", "a", "b"])
        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
            )

        # There is only 3 categories, so test must be adjusted accordingly.
        assert_partial_dependencies_plot(
            plot, dims=["x", "y"], n_grid_points={"x": 5, "y": 3}
        )

    def test_categorical_multidim(self, monkeypatch):
        """Tests that multidim categorical is supported"""
        mock_train_regressor(monkeypatch)
        config = mock_space(y='choices(["a", "b", "c"], shape=3)')
        mock_experiment(
            monkeypatch,
            y=[["c", "b", "a"], ["c", "a", "c"], ["a", "b", "a"], ["c", "b", "b"]],
        )

        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
            )

        assert_partial_dependencies_plot(
            plot,
            dims=["x", "y[0]", "y[1]", "y[2]"],
            n_grid_points={"x": 5, "y[0]": 3, "y[1]": 3, "y[2]": 3},
        )

    def test_logarithmic_scales_first(self, monkeypatch):
        """Test that log dims are turn to log scale

        Test first dim specifically because special xaxis name for first dim.
        """
        mock_train_regressor(monkeypatch)
        config = mock_space(x="loguniform(0.001, 1)", z="uniform(0, 1)")
        mock_experiment(monkeypatch, x=[0.001, 0.1, 0.01, 1], z=[0, 0.1, 0.2, 0.5])

        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
            )

        assert_partial_dependencies_plot(
            plot, dims=["x", "y", "z"], n_grid_points=5, log_dims=["x"]
        )

    def test_logarithmic_scales_any_dim(self, monkeypatch):
        """Test that log dims are turn to log scale"""
        mock_train_regressor(monkeypatch)
        config = mock_space(y="loguniform(0.001, 1)", z="uniform(0, 1)")
        mock_experiment(monkeypatch, y=[0.001, 0.1, 0.01, 1], z=[0, 0.1, 0.2, 0.5])

        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
            )

        assert_partial_dependencies_plot(
            plot, dims=["x", "y", "z"], n_grid_points=5, log_dims=["y"]
        )

    def test_int_logarithmic_scales(self, monkeypatch):
        """Test that int log dims are turn to log scale"""
        mock_train_regressor(monkeypatch)
        config = mock_space(y="loguniform(1, 1000, discrete=True)", z="uniform(0, 1)")
        mock_experiment(monkeypatch, y=[1, 10, 100, 1000], z=[0, 0.1, 0.2, 0.5])

        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
            )

        assert_partial_dependencies_plot(
            plot, dims=["x", "y", "z"], n_grid_points=5, log_dims=["y"]
        )

    def test_one_param(self, monkeypatch):
        """Test ploting a space with only 1 dim"""
        mock_train_regressor(monkeypatch)
        config = mock_space(y=None)
        mock_experiment(monkeypatch, y="drop")

        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
            )

        assert_partial_dependencies_plot(plot, dims=["x"], n_grid_points=5)

    def test_select_params(self, monkeypatch):
        """Test selecting subset"""
        mock_train_regressor(monkeypatch)
        config = mock_space(z="uniform(0, 1)")
        mock_experiment(monkeypatch, z=[0, 0.1, 0.2, 0.5])

        for params in [["x"], ["x", "y"], ["y", "z"]]:
            with create_experiment(config, trial_config) as (_, _, experiment):
                plot = partial_dependencies(
                    experiment,
                    params=params,
                    n_grid_points=5,
                    model_kwargs=dict(random_state=1),
                )

            assert_partial_dependencies_plot(plot, dims=params, n_grid_points=5)

    def test_custom_smoothing(self, monkeypatch):
        """Test changing smoothing value"""
        mock_train_regressor(monkeypatch)
        config = mock_space()
        mock_experiment(monkeypatch)

        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment,
                n_grid_points=5,
                model_kwargs=dict(random_state=1),
                smoothing=1.2,
            )

        with pytest.raises(AssertionError):
            assert_partial_dependencies_plot(plot, dims=["x", "y"], n_grid_points=5)

        assert_partial_dependencies_plot(
            plot, dims=["x", "y"], n_grid_points=5, smoothing=1.2
        )

    def test_custom_n_grid_points(self, monkeypatch):
        """Test changing n_grid_points value"""
        mock_train_regressor(monkeypatch)
        config = mock_space()
        mock_experiment(monkeypatch)

        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment,
                n_grid_points=10,
                model_kwargs=dict(random_state=1),
            )

        with pytest.raises(AssertionError):
            assert_partial_dependencies_plot(plot, dims=["x", "y"], n_grid_points=5)

        assert_partial_dependencies_plot(plot, dims=["x", "y"], n_grid_points=10)

    def test_custom_n_samples(self, monkeypatch):
        """Test changing n_samples value"""
        mock_train_regressor(monkeypatch)
        config = mock_space()
        mock_experiment(monkeypatch)

        PARAMS = ["x", "y"]
        N_SAMPLES = numpy.random.randint(20, 50)

        def mock_partial_dependency_grid(space, model, params, samples, n_points):
            print(samples)
            assert samples.shape == (N_SAMPLES, len(PARAMS))
            return partial_dependency_grid(space, model, params, samples, n_points)

        monkeypatch.setattr(
            "orion.analysis.partial_dependency_utils.partial_dependency_grid",
            mock_partial_dependency_grid,
        )

        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment,
                n_grid_points=10,
                model_kwargs=dict(random_state=1),
                n_samples=N_SAMPLES,
            )

    def test_custom_colorscale(self, monkeypatch):
        """Test changing colorscale"""
        mock_train_regressor(monkeypatch)
        config = mock_space()
        mock_experiment(monkeypatch)

        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment,
                n_grid_points=5,
                colorscale="Viridis",
                model_kwargs=dict(random_state=1),
            )

        with pytest.raises(AssertionError):
            assert_partial_dependencies_plot(
                plot, dims=["x", "y"], n_grid_points=5, custom_colorscale=False
            )

        assert_partial_dependencies_plot(
            plot, dims=["x", "y"], n_grid_points=5, custom_colorscale=True
        )

    def test_custom_model(self, monkeypatch):
        """Test changing type of regression model"""
        mock_train_regressor(monkeypatch, assert_model="BaggingRegressor")
        config = mock_space()
        mock_experiment(monkeypatch)

        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment,
                n_grid_points=5,
                model="BaggingRegressor",
                model_kwargs=dict(random_state=1),
            )

    def test_custom_model_kwargs(self, monkeypatch):
        """Test changing arguments of regression model"""
        mock_train_regressor(monkeypatch, assert_model_kwargs=dict(random_state=1))
        config = mock_space()
        mock_experiment(monkeypatch)

        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = partial_dependencies(
                experiment,
                n_grid_points=5,
                model_kwargs=dict(random_state=1),
            )


@pytest.mark.usefixtures("version_XYZ")
class TestRankings:
    """Tests the ``rankings()`` method provided by the plotly backend"""

    def test_requires_argument(self):
        """Tests that the experiment data are required."""
        with pytest.raises(ValueError):
            rankings(None)

    def test_returns_plotly_object(self, monkeypatch):
        """Tests that the plotly backend returns a plotly object"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = rankings([experiment, experiment])

        assert type(plot) is plotly.graph_objects.Figure

    def test_graph_layout(self, monkeypatch):
        """Tests the layout of the plot"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = rankings([experiment])

        assert_rankings_plot(plot, [f"{experiment.name}-v{experiment.version}"])

    def test_list_of_experiments(self, monkeypatch):
        """Tests the rankings with list of experiments"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            child = orion.client.create_experiment(
                experiment.name, branching={"branch_to": "child"}
            )

            plot = rankings([experiment, child])

        # Exps are sorted alphabetically by names.
        assert_rankings_plot(
            plot, [f"{exp.name}-v{exp.version}" for exp in [child, experiment]]
        )

    def test_list_of_experiments_name_conflict(self, monkeypatch):
        """Tests the rankings with list of experiments with the same name"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            child = orion.client.create_experiment(
                experiment.name, branching={"branch_to": experiment.name}
            )
            assert child.name == experiment.name
            assert child.version == experiment.version + 1
            plot = rankings([experiment, child])

        # Exps are sorted alphabetically by names.
        assert_rankings_plot(
            plot, [f"{exp.name}-v{exp.version}" for exp in [experiment, child]]
        )

    def test_dict_of_experiments(self, monkeypatch):
        """Tests the rankings with renamed experiments"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = rankings({"exp-1": experiment, "exp-2": experiment})

        assert_rankings_plot(plot, ["exp-1", "exp-2"])

    def test_list_of_dict_of_experiments(self, monkeypatch):
        """Tests the rankings with avg of competitions"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = rankings(
                [{"exp-1": experiment, "exp-2": experiment} for _ in range(10)]
            )

        assert_rankings_plot(plot, ["exp-1", "exp-2"], with_avg=True)

    def test_dict_of_list_of_experiments(self, monkeypatch):
        """Tests the rankings with avg of experiments separated in lists"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = rankings({"exp-1": [experiment] * 10, "exp-2": [experiment] * 10})

        assert_rankings_plot(plot, ["exp-1", "exp-2"], with_avg=True)

    def test_unbalanced_experiments(self, monkeypatch):
        """Tests the regrets with avg of unbalanced experiments"""
        mock_experiment_with_random_to_pandas(monkeypatch, unbalanced=True)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = rankings({"exp-1": [experiment] * 10, "exp-2": [experiment] * 10})

        assert_rankings_plot(plot, ["exp-1", "exp-2"], with_avg=True, balanced=0)

    def test_ignore_uncompleted_statuses(self, monkeypatch):
        """Tests that uncompleted statuses are ignored"""
        mock_experiment_with_random_to_pandas(
            monkeypatch,
            status=[
                "completed",
                "new",
                "reserved",
                "completed",
                "broken",
                "completed",
                "interrupted",
                "completed",
            ],
        )
        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = rankings([experiment])

        assert_rankings_plot(
            plot, [f"{experiment.name}-v{experiment.version}"], balanced=4
        )

    def test_unsupported_order_key(self):
        """Tests that unsupported order keys are rejected"""
        with create_experiment(config, trial_config) as (_, _, experiment):
            with pytest.raises(ValueError):
                rankings([experiment], order_by="unsupported")


@pytest.mark.usefixtures("version_XYZ")
class TestRegret:
    """Tests the ``regret()`` method provided by the plotly backend"""

    def test_requires_argument(self):
        """Tests that the experiment data are required."""
        with pytest.raises(ValueError):
            regret(None)

    def test_returns_plotly_object(self):
        """Tests that the plotly backend returns a plotly object"""
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = regret(experiment)

        assert type(plot) is plotly.graph_objects.Figure

    def test_graph_layout(self):
        """Tests the layout of the plot"""
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = regret(experiment)

        assert_regret_plot(plot)

    def test_experiment_worker_as_parameter(self):
        """Tests that ``Experiment`` is a valid parameter"""
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            experiment,
            _,
        ):
            plot = regret(experiment)

        assert_regret_plot(plot)

    def test_ignore_uncompleted_statuses(self):
        """Tests that uncompleted statuses are ignored"""
        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = regret(experiment)

        assert_regret_plot(plot)

    def test_unsupported_order_key(self):
        """Tests that unsupported order keys are rejected"""
        with create_experiment(config, trial_config) as (_, _, experiment):
            with pytest.raises(ValueError):
                regret(experiment, order_by="unsupported")


@pytest.mark.usefixtures("version_XYZ")
class TestRegrets:
    """Tests the ``regrets()`` method provided by the plotly backend"""

    def test_requires_argument(self):
        """Tests that the experiment data are required."""
        with pytest.raises(ValueError):
            regrets(None)

    def test_returns_plotly_object(self, monkeypatch):
        """Tests that the plotly backend returns a plotly object"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = regrets([experiment])

        assert type(plot) is plotly.graph_objects.Figure

    def test_graph_layout(self, monkeypatch):
        """Tests the layout of the plot"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = regrets([experiment])

        assert_regrets_plot(plot, [f"{experiment.name}-v{experiment.version}"])

    def test_list_of_experiments(self, monkeypatch):
        """Tests the regrets with list of experiments"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            child = orion.client.create_experiment(
                experiment.name, branching={"branch_to": "child"}
            )

            plot = regrets([experiment, child])

        # Exps are sorted alphabetically by names.
        assert_regrets_plot(
            plot, [f"{exp.name}-v{exp.version}" for exp in [child, experiment]]
        )

    def test_list_of_experiments_name_conflict(self, monkeypatch):
        """Tests the regrets with list of experiments with the same name"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            child = orion.client.create_experiment(
                experiment.name, branching={"branch_to": experiment.name}
            )
            assert child.name == experiment.name
            assert child.version == experiment.version + 1
            plot = regrets([experiment, child])

        # Exps are sorted alphabetically by names.
        assert_regrets_plot(
            plot, [f"{exp.name}-v{exp.version}" for exp in [experiment, child]]
        )

    def test_dict_of_experiments(self, monkeypatch):
        """Tests the regrets with renamed experiments"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = regrets({"exp-1": experiment, "exp-2": experiment})

        assert_regrets_plot(plot, ["exp-1", "exp-2"])

    def test_dict_of_list_of_experiments(self, monkeypatch):
        """Tests the regrets with avg of experiments"""
        mock_experiment_with_random_to_pandas(monkeypatch)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = regrets({"exp-1": [experiment] * 10, "exp-2": [experiment] * 10})

        assert_regrets_plot(plot, ["exp-1", "exp-2"], with_avg=True)

    def test_unbalanced_experiments(self, monkeypatch):
        """Tests the regrets with avg of unbalanced experiments"""
        mock_experiment_with_random_to_pandas(monkeypatch, unbalanced=True)
        with create_experiment(config, trial_config, ["completed"]) as (
            _,
            _,
            experiment,
        ):
            plot = regrets({"exp-1": [experiment] * 10, "exp-2": [experiment] * 10})

        assert_regrets_plot(plot, ["exp-1", "exp-2"], with_avg=True, balanced=0)

    def test_ignore_uncompleted_statuses(self, monkeypatch):
        """Tests that uncompleted statuses are ignored"""
        mock_experiment_with_random_to_pandas(
            monkeypatch,
            status=[
                "completed",
                "new",
                "reserved",
                "completed",
                "broken",
                "completed",
                "interrupted",
                "completed",
            ],
        )
        with create_experiment(config, trial_config) as (_, _, experiment):
            plot = regrets([experiment])

        assert_regrets_plot(
            plot, [f"{experiment.name}-v{experiment.version}"], balanced=4
        )

    def test_unsupported_order_key(self):
        """Tests that unsupported order keys are rejected"""
        with create_experiment(config, trial_config) as (_, _, experiment):
            with pytest.raises(ValueError):
                regrets([experiment], order_by="unsupported")

[(176, 'pytest.mark.usefixtures', 'pytest.mark.usefixtures', False, 'import pytest\n'), (292, 'pytest.mark.usefixtures', 'pytest.mark.usefixtures', False, 'import pytest\n'), (398, 'pytest.mark.usefixtures', 'pytest.mark.usefixtures', False, 'import pytest\n'), (716, 'pytest.mark.usefixtures', 'pytest.mark.usefixtures', False, 'import pytest\n'), (867, 'pytest.mark.usefixtures', 'pytest.mark.usefixtures', False, 'import pytest\n'), (923, 'pytest.mark.usefixtures', 'pytest.mark.usefixtures', False, 'import pytest\n'), (66, 'copy.deepcopy', 'copy.deepcopy', False, 'import copy\n'), (104, 'pandas.DataFrame', 'pandas.DataFrame', False, 'import pandas\n'), (117, 'numpy.arange', 'numpy.arange', False, 'import numpy\n'), (118, 'numpy.random.normal', 'numpy.random.normal', False, 'import numpy\n'), (119, 'numpy.random.normal', 'numpy.random.normal', False, 'import numpy\n'), (120, 'numpy.random.normal', 'numpy.random.normal', False, 'import numpy\n'), (126, 'pandas.DataFrame', 'pandas.DataFrame', False, 'import pandas\n'), (211, 'orion.testing.plotting.assert_lpi_plot', 'assert_lpi_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (224, 'orion.testing.plotting.assert_lpi_plot', 'assert_lpi_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (249, 'orion.testing.plotting.assert_lpi_plot', 'assert_lpi_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (258, 'orion.testing.plotting.assert_lpi_plot', 'assert_lpi_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (267, 'orion.testing.plotting.assert_lpi_plot', 'assert_lpi_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (276, 'orion.testing.plotting.assert_lpi_plot', 'assert_lpi_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (289, 'orion.testing.plotting.assert_lpi_plot', 'assert_lpi_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (321, 'orion.testing.plotting.assert_parallel_coordinates_plot', 'assert_parallel_coordinates_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (332, 'orion.testing.plotting.assert_parallel_coordinates_plot', 'assert_parallel_coordinates_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (339, 'orion.testing.plotting.assert_parallel_coordinates_plot', 'assert_parallel_coordinates_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (349, 'copy.deepcopy', 'copy.deepcopy', False, 'import copy\n'), (355, 'orion.testing.plotting.assert_parallel_coordinates_plot', 'assert_parallel_coordinates_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (359, 'copy.deepcopy', 'copy.deepcopy', False, 'import copy\n'), (364, 'orion.testing.plotting.assert_parallel_coordinates_plot', 'assert_parallel_coordinates_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (370, 'copy.deepcopy', 'copy.deepcopy', False, 'import copy\n'), (375, 'orion.testing.plotting.assert_parallel_coordinates_plot', 'assert_parallel_coordinates_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (379, 'copy.deepcopy', 'copy.deepcopy', False, 'import copy\n'), (384, 'orion.testing.plotting.assert_parallel_coordinates_plot', 'assert_parallel_coordinates_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (388, 'copy.deepcopy', 'copy.deepcopy', False, 'import copy\n'), (393, 'orion.testing.plotting.assert_parallel_coordinates_plot', 'assert_parallel_coordinates_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (434, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (460, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (472, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (484, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (497, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (515, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (535, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (550, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (565, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (580, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (616, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (636, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (645, 'numpy.random.randint', 'numpy.random.randint', False, 'import numpy\n'), (684, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (747, 'orion.testing.plotting.assert_rankings_plot', 'assert_rankings_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (764, 'orion.testing.plotting.assert_rankings_plot', 'assert_rankings_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (784, 'orion.testing.plotting.assert_rankings_plot', 'assert_rankings_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (798, 'orion.testing.plotting.assert_rankings_plot', 'assert_rankings_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (812, 'orion.testing.plotting.assert_rankings_plot', 'assert_rankings_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (824, 'orion.testing.plotting.assert_rankings_plot', 'assert_rankings_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (836, 'orion.testing.plotting.assert_rankings_plot', 'assert_rankings_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (856, 'orion.testing.plotting.assert_rankings_plot', 'assert_rankings_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (896, 'orion.testing.plotting.assert_regret_plot', 'assert_regret_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (907, 'orion.testing.plotting.assert_regret_plot', 'assert_regret_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (914, 'orion.testing.plotting.assert_regret_plot', 'assert_regret_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (954, 'orion.testing.plotting.assert_regrets_plot', 'assert_regrets_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (971, 'orion.testing.plotting.assert_regrets_plot', 'assert_regrets_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (991, 'orion.testing.plotting.assert_regrets_plot', 'assert_regrets_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (1005, 'orion.testing.plotting.assert_regrets_plot', 'assert_regrets_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (1017, 'orion.testing.plotting.assert_regrets_plot', 'assert_regrets_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (1029, 'orion.testing.plotting.assert_regrets_plot', 'assert_regrets_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (1049, 'orion.testing.plotting.assert_regrets_plot', 'assert_regrets_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (112, 'numpy.random.randint', 'numpy.random.randint', False, 'import numpy\n'), (182, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (183, 'orion.plotting.base.lpi', 'lpi', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (187, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (200, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (217, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (246, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (247, 'orion.plotting.base.lpi', 'lpi', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (255, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (264, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (273, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (286, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (298, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (299, 'orion.plotting.base.parallel_coordinates', 'parallel_coordinates', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (303, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (308, 'orion.plotting.base.parallel_coordinates', 'parallel_coordinates', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (314, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (319, 'orion.plotting.base.parallel_coordinates', 'parallel_coordinates', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (325, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (330, 'orion.plotting.base.parallel_coordinates', 'parallel_coordinates', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (336, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (337, 'orion.plotting.base.parallel_coordinates', 'parallel_coordinates', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (343, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (352, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (353, 'orion.plotting.base.parallel_coordinates', 'parallel_coordinates', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (361, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (362, 'orion.plotting.base.parallel_coordinates', 'parallel_coordinates', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (372, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (373, 'orion.plotting.base.parallel_coordinates', 'parallel_coordinates', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (381, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (382, 'orion.plotting.base.parallel_coordinates', 'parallel_coordinates', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (390, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (391, 'orion.plotting.base.parallel_coordinates', 'parallel_coordinates', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (405, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (421, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (457, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (458, 'orion.plotting.base.partial_dependencies', 'partial_dependencies', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (467, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (479, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (491, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (510, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (530, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (545, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (560, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (575, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (597, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (605, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (613, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (614, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (626, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (633, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (634, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (650, 'orion.analysis.partial_dependency_utils.partial_dependency_grid', 'partial_dependency_grid', False, 'from orion.analysis.partial_dependency_utils import partial_dependency_grid\n'), (657, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (671, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (679, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (680, 'orion.testing.plotting.assert_partial_dependencies_plot', 'assert_partial_dependencies_plot', False, 'from orion.testing.plotting import assert_lpi_plot, assert_parallel_coordinates_plot, assert_partial_dependencies_plot, assert_rankings_plot, assert_regret_plot, assert_regrets_plot\n'), (694, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (708, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (722, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (723, 'orion.plotting.base.rankings', 'rankings', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (728, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (733, 'orion.plotting.base.rankings', 'rankings', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (740, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (745, 'orion.plotting.base.rankings', 'rankings', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (752, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (761, 'orion.plotting.base.rankings', 'rankings', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (771, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (781, 'orion.plotting.base.rankings', 'rankings', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (791, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (796, 'orion.plotting.base.rankings', 'rankings', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (803, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (817, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (822, 'orion.plotting.base.rankings', 'rankings', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (829, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (834, 'orion.plotting.base.rankings', 'rankings', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (853, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (854, 'orion.plotting.base.rankings', 'rankings', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (862, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (873, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (874, 'orion.plotting.base.regret', 'regret', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (878, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (883, 'orion.plotting.base.regret', 'regret', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (889, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (894, 'orion.plotting.base.regret', 'regret', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (900, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (905, 'orion.plotting.base.regret', 'regret', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (911, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (912, 'orion.plotting.base.regret', 'regret', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (918, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (929, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (930, 'orion.plotting.base.regrets', 'regrets', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (935, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (940, 'orion.plotting.base.regrets', 'regrets', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (947, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (952, 'orion.plotting.base.regrets', 'regrets', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (959, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (968, 'orion.plotting.base.regrets', 'regrets', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (978, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (988, 'orion.plotting.base.regrets', 'regrets', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (998, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (1003, 'orion.plotting.base.regrets', 'regrets', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (1010, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (1015, 'orion.plotting.base.regrets', 'regrets', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (1022, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (1027, 'orion.plotting.base.regrets', 'regrets', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (1046, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (1047, 'orion.plotting.base.regrets', 'regrets', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (1055, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (153, 'numpy.arange', 'numpy.arange', False, 'import numpy\n'), (344, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (345, 'orion.plotting.base.parallel_coordinates', 'parallel_coordinates', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (589, 'orion.testing.create_experiment', 'create_experiment', False, 'from orion.testing import create_experiment\n'), (863, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (864, 'orion.plotting.base.rankings', 'rankings', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (919, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (920, 'orion.plotting.base.regret', 'regret', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n'), (1056, 'pytest.raises', 'pytest.raises', False, 'import pytest\n'), (1057, 'orion.plotting.base.regrets', 'regrets', False, 'from orion.plotting.base import lpi, parallel_coordinates, partial_dependencies, rankings, regret, regrets\n')]