We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
BUG: Simple model comparison shows irrelevant labels
To reproduce:
https://www.kaggle.com/itay94/notebook05f499eb19

![image](https://user-images.githubusercontent.com/20860465/147876157-58631e60-5110-4f50-b52b-1d4a7432ecc5.png)

![image](https://user-images.githubusercontent.com/20860465/147876162-b6e747b3-5b2f-446f-9c5d-c1fcbd1dffbe.png)


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `deepchecks/checks/performance/simple_model_comparison.py`
Content:
```
1 # ----------------------------------------------------------------------------
2 # Copyright (C) 2021 Deepchecks (https://www.deepchecks.com)
3 #
4 # This file is part of Deepchecks.
5 # Deepchecks is distributed under the terms of the GNU Affero General
6 # Public License (version 3 or later).
7 # You should have received a copy of the GNU Affero General Public License
8 # along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
9 # ----------------------------------------------------------------------------
10 #
11 """Module containing simple comparison check."""
12 from typing import Callable, Dict, Hashable, List
13 import numpy as np
14 import pandas as pd
15 import plotly.express as px
16 from sklearn.dummy import DummyRegressor, DummyClassifier
17 from sklearn.pipeline import Pipeline
18 from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
19 from deepchecks.checks.distribution.preprocessing import ScaledNumerics
20 from deepchecks.utils.strings import format_number
21 
22 from deepchecks import CheckResult, Dataset
23 from deepchecks.base.check import ConditionResult, TrainTestBaseCheck
24 from deepchecks.utils.metrics import task_type_check, ModelType, initialize_multi_scorers, \
25     get_scorers_list, get_scores_ratio, get_scorer_single
26 from deepchecks.utils.validation import validate_model
27 from deepchecks.errors import DeepchecksValueError
28 
29 
30 __all__ = ['SimpleModelComparison']
31 
32 
33 class SimpleModelComparison(TrainTestBaseCheck):
34     """Compare given model score to simple model score (according to given model type).
35 
36     Args:
37         simple_model_type (str):
38             Type of the simple model ['random', 'constant', 'tree'].
39                 + random - select one of the labels by random.
40                 + constant - in regression is mean value, in classification the most common value.
41                 + tree - runs a simple decision tree.
42         alternative_scorers (Dict[str, Callable], default None):
43             An optional dictionary of scorer name to scorer functions.
44             If none given, using default scorers
45         maximum_ratio (int):
46             the ratio can be up to infinity so choose maximum value to limit to.
47         max_depth (int):
48             the max depth of the tree (used only if simple model type is tree).
49         random_state (int):
50             the random state (used only if simple model type is tree or random).
51     """
52 
53     def __init__(self, simple_model_type: str = 'constant', alternative_scorers: Dict[str, Callable] = None,
54                  maximum_ratio: int = 50, max_depth: int = 3, random_state: int = 42):
55         super().__init__()
56         self.simple_model_type = simple_model_type
57         self.alternative_scorers = initialize_multi_scorers(alternative_scorers)
58         self.maximum_ratio = maximum_ratio
59         self.max_depth = max_depth
60         self.random_state = random_state
61 
62     def run(self, train_dataset, test_dataset, model) -> CheckResult:
63         """Run check.
64 
65         Args:
66             train_dataset (Dataset): The training dataset object. Must contain a label.
67             test_dataset (Dataset): The test dataset object. Must contain a label.
68             model (BaseEstimator): A scikit-learn-compatible fitted estimator instance.
69 
70         Returns:
71             CheckResult: value is a Dict of: given_model_score, simple_model_score, ratio
72                          ratio is given model / simple model (if the scorer returns negative values we divide 1 by it)
73                          if ratio is infinite max_ratio is returned
74 
75         Raises:
76             DeepchecksValueError: If the object is not a Dataset instance.
77         """
78         Dataset.validate_dataset(train_dataset)
79         Dataset.validate_dataset(test_dataset)
80         train_dataset.validate_label()
81         test_dataset.validate_label()
82         validate_model(test_dataset, model)
83 
84         # If user defined scorers used them, else use a single scorer
85         if self.alternative_scorers:
86             scorers = get_scorers_list(model, train_dataset, self.alternative_scorers, multiclass_avg=False)
87         else:
88             scorers = [get_scorer_single(model, train_dataset, multiclass_avg=False)]
89 
90         task_type = task_type_check(model, train_dataset)
91         simple_model = self._create_simple_model(train_dataset, task_type)
92 
93         models = [
94             (f'{type(model).__name__} model', 'Origin', model),
95             (f'Simple model - {self.simple_model_type}', 'Simple', simple_model)
96         ]
97 
98         # Multiclass have different return type from the scorer, list of score per class instead of single score
99         if task_type == ModelType.MULTICLASS:
100             results = []
101             for model_name, model_type, model_instance in models:
102                 for scorer in scorers:
103                     score_result: np.ndarray = scorer(model_instance, test_dataset)
104                     # Multiclass scorers return numpy array of result per class
105                     for class_i, class_score in enumerate(score_result):
106                         # The proba returns in order of the sorted classes.
107                         class_value = train_dataset.classes[class_i]
108                         results.append([model_name, model_type, class_score, scorer.name, class_value])
109 
110             results_df = pd.DataFrame(results, columns=['Model', 'Type', 'Value', 'Metric', 'Class'])
111 
112             # Plot the metrics in a graph, grouping by the model and class
113             fig = px.bar(results_df, x=['Class', 'Model'], y='Value', color='Model', barmode='group',
114                          facet_col='Metric', facet_col_spacing=0.05)
115             fig.update_xaxes(title=None, tickprefix='Class ', tickangle=60)
116             fig.update_yaxes(title=None, matches=None)
117             fig.for_each_annotation(lambda a: a.update(text=a.text.split('=')[-1]))
118             fig.for_each_yaxis(lambda yaxis: yaxis.update(showticklabels=True))
119         # Model is binary or regression
120         else:
121             results = []
122             for model_name, model_type, model_instance in models:
123                 for scorer in scorers:
124                     score_result: float = scorer(model_instance, test_dataset)
125                     results.append([model_name, model_type, score_result, scorer.name])
126 
127             results_df = pd.DataFrame(results, columns=['Model', 'Type', 'Value', 'Metric'])
128 
129             # Plot the metrics in a graph, grouping by the model
130             fig = px.bar(results_df, x='Model', y='Value', color='Model', barmode='group',
131                          facet_col='Metric', facet_col_spacing=0.05)
132             fig.update_xaxes(title=None)
133             fig.update_yaxes(title=None, matches=None)
134             fig.for_each_annotation(lambda a: a.update(text=a.text.split('=')[-1]))
135             fig.for_each_yaxis(lambda yaxis: yaxis.update(showticklabels=True))
136 
137         return CheckResult({'scores': results_df, 'type': task_type}, display=fig)
138 
139     def _create_simple_model(self, train_ds: Dataset, task_type: ModelType):
140         """Create a simple model of given type (random/constant/tree) to the given dataset.
141 
142         Args:
143             train_ds (Dataset): The training dataset object.
144             task_type (ModelType): the model type.
145         Returns:
146             Classifier object.
147 
148         Raises:
149             NotImplementedError: If the simple_model_type is not supported
150         """
151         np.random.seed(self.random_state)
152 
153         if self.simple_model_type == 'random':
154             simple_model = RandomModel()
155 
156         elif self.simple_model_type == 'constant':
157             if task_type == ModelType.REGRESSION:
158                 simple_model = DummyRegressor(strategy='mean')
159             elif task_type in {ModelType.BINARY, ModelType.MULTICLASS}:
160                 simple_model = DummyClassifier(strategy='most_frequent')
161             else:
162                 raise DeepchecksValueError(f'Unknown task type - {task_type}')
163         elif self.simple_model_type == 'tree':
164             if task_type == ModelType.REGRESSION:
165                 clf = DecisionTreeRegressor(
166                     max_depth=self.max_depth,
167                     random_state=self.random_state
168                 )
169             elif task_type in {ModelType.BINARY, ModelType.MULTICLASS}:
170                 clf = DecisionTreeClassifier(
171                     max_depth=self.max_depth,
172                     random_state=self.random_state,
173                     class_weight='balanced'
174                 )
175             else:
176                 raise DeepchecksValueError(f'Unknown task type - {task_type}')
177 
178             simple_model = Pipeline([('scaler', ScaledNumerics(train_ds.cat_features, max_num_categories=10)),
179                                      ('tree-model', clf)])
180         else:
181             raise DeepchecksValueError(
182                 f'Unknown model type - {self.simple_model_type}, expected to be one of '
183                 f"['random', 'constant', 'tree'] "
184                 f"but instead got {self.simple_model_type}"  # pylint: disable=inconsistent-quotes
185             )
186 
187         simple_model.fit(train_ds.features_columns, train_ds.label_col)
188         return simple_model
189 
190     def add_condition_ratio_not_less_than(self, min_allowed_ratio: float = 1.1, classes: List[Hashable] = None):
191         """Add condition - require min allowed ratio between the given and the simple model.
192 
193         Args:
194             min_allowed_ratio (float): Min allowed ratio between the given and the simple model -
195             ratio is given model / simple model (if the scorer returns negative values we divide 1 by it)
196             classes (List[Hashable]): Used in multiclass models to limit condition only to given classes.
197         """
198         def condition(result: Dict, max_ratio=self.maximum_ratio, class_list=classes) -> ConditionResult:
199             scores_df = result['scores']
200             task_type = result['type']
201             metrics = scores_df['Metric'].unique()
202 
203             def get_ratio(df):
204                 simple_score = df[df['Type'] == 'Simple']['Value'].iloc[0]
205                 origin_score = df[df['Type'] == 'Origin']['Value'].iloc[0]
206                 return get_scores_ratio(simple_score, origin_score, max_ratio)
207 
208             fails = []
209             if task_type == ModelType.MULTICLASS:
210                 if class_list is None:
211                     class_list = scores_df['Class'].unique()
212                 for metric in metrics:
213                     failed_classes = []
214                     for clas in class_list:
215                         score_rows = scores_df[(scores_df['Metric'] == metric) & (scores_df['Class'] == clas)]
216                         ratio = get_ratio(score_rows)
217                         if ratio < min_allowed_ratio:
218                             failed_classes.append(str(clas))
219                     if failed_classes:
220                         fails.append(f'"{metric}" - Classes: {", ".join(failed_classes)}')
221             else:
222                 for metric in metrics:
223                     score_rows = scores_df[(scores_df['Metric'] == metric)]
224                     ratio = get_ratio(score_rows)
225                     if ratio < min_allowed_ratio:
226                         fails.append(f'"{metric}"')
227 
228             if fails:
229                 msg = f'Metrics failed: {", ".join(sorted(fails))}'
230                 return ConditionResult(False, msg)
231             else:
232                 return ConditionResult(True)
233 
234         return self.add_condition('$$\\frac{\\text{model score}}{\\text{simple model score}} >= '
235                                   f'{format_number(min_allowed_ratio)}$$', condition)
236 
237 
238 class RandomModel:
239     """Model used to randomly predict from given series of labels."""
240 
241     def __init__(self):
242         self.labels = None
243 
244     def fit(self, X, y):  # pylint: disable=unused-argument,invalid-name
245         # The X is not used, but it is needed to be matching to sklearn `fit` signature
246         self.labels = y
247 
248     def predict(self, X):  # pylint: disable=invalid-name
249         return np.random.choice(self.labels, X.shape[0])
250 
251     def predict_proba(self, X):  # pylint: disable=invalid-name
252         classes = sorted(self.labels.unique().tolist())
253         predictions = self.predict(X)
254 
255         def prediction_to_proba(y_pred):
256             proba = np.zeros(len(classes))
257             proba[classes.index(y_pred)] = 1
258             return proba
259         return np.apply_along_axis(prediction_to_proba, axis=1, arr=predictions)
260
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/deepchecks/checks/performance/simple_model_comparison.py b/deepchecks/checks/performance/simple_model_comparison.py
--- a/deepchecks/checks/performance/simple_model_comparison.py
+++ b/deepchecks/checks/performance/simple_model_comparison.py
@@ -101,10 +101,8 @@
             for model_name, model_type, model_instance in models:
                 for scorer in scorers:
                     score_result: np.ndarray = scorer(model_instance, test_dataset)
-                    # Multiclass scorers return numpy array of result per class
-                    for class_i, class_score in enumerate(score_result):
-                        # The proba returns in order of the sorted classes.
-                        class_value = train_dataset.classes[class_i]
+                    # Multiclass scorers return numpy array of result per class in order of sorted classes
+                    for class_score, class_value in zip(score_result, test_dataset.classes):
                         results.append([model_name, model_type, class_score, scorer.name, class_value])
 
             results_df = pd.DataFrame(results, columns=['Model', 'Type', 'Value', 'Metric', 'Class'])
@@ -112,7 +110,7 @@
             # Plot the metrics in a graph, grouping by the model and class
             fig = px.bar(results_df, x=['Class', 'Model'], y='Value', color='Model', barmode='group',
                          facet_col='Metric', facet_col_spacing=0.05)
-            fig.update_xaxes(title=None, tickprefix='Class ', tickangle=60)
+            fig.update_xaxes(title=None, tickprefix='Class ', tickangle=60, type='category')
             fig.update_yaxes(title=None, matches=None)
             fig.for_each_annotation(lambda a: a.update(text=a.text.split('=')[-1]))
             fig.for_each_yaxis(lambda yaxis: yaxis.update(showticklabels=True))

{"golden_diff": "diff --git a/deepchecks/checks/performance/simple_model_comparison.py b/deepchecks/checks/performance/simple_model_comparison.py\n--- a/deepchecks/checks/performance/simple_model_comparison.py\n+++ b/deepchecks/checks/performance/simple_model_comparison.py\n@@ -101,10 +101,8 @@\n             for model_name, model_type, model_instance in models:\n                 for scorer in scorers:\n                     score_result: np.ndarray = scorer(model_instance, test_dataset)\n-                    # Multiclass scorers return numpy array of result per class\n-                    for class_i, class_score in enumerate(score_result):\n-                        # The proba returns in order of the sorted classes.\n-                        class_value = train_dataset.classes[class_i]\n+                    # Multiclass scorers return numpy array of result per class in order of sorted classes\n+                    for class_score, class_value in zip(score_result, test_dataset.classes):\n                         results.append([model_name, model_type, class_score, scorer.name, class_value])\n \n             results_df = pd.DataFrame(results, columns=['Model', 'Type', 'Value', 'Metric', 'Class'])\n@@ -112,7 +110,7 @@\n             # Plot the metrics in a graph, grouping by the model and class\n             fig = px.bar(results_df, x=['Class', 'Model'], y='Value', color='Model', barmode='group',\n                          facet_col='Metric', facet_col_spacing=0.05)\n-            fig.update_xaxes(title=None, tickprefix='Class ', tickangle=60)\n+            fig.update_xaxes(title=None, tickprefix='Class ', tickangle=60, type='category')\n             fig.update_yaxes(title=None, matches=None)\n             fig.for_each_annotation(lambda a: a.update(text=a.text.split('=')[-1]))\n             fig.for_each_yaxis(lambda yaxis: yaxis.update(showticklabels=True))\n", "issue": "BUG: Simple model comparison shows irrelevant labels\nTo reproduce:\r\nhttps://www.kaggle.com/itay94/notebook05f499eb19\r\n\r\n![image](https://user-images.githubusercontent.com/20860465/147876157-58631e60-5110-4f50-b52b-1d4a7432ecc5.png)\r\n\r\n![image](https://user-images.githubusercontent.com/20860465/147876162-b6e747b3-5b2f-446f-9c5d-c1fcbd1dffbe.png)\r\n\n", "before_files": [{"content": "# ----------------------------------------------------------------------------\n# Copyright (C) 2021 Deepchecks (https://www.deepchecks.com)\n#\n# This file is part of Deepchecks.\n# Deepchecks is distributed under the terms of the GNU Affero General\n# Public License (version 3 or later).\n# You should have received a copy of the GNU Affero General Public License\n# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.\n# ----------------------------------------------------------------------------\n#\n\"\"\"Module containing simple comparison check.\"\"\"\nfrom typing import Callable, Dict, Hashable, List\nimport numpy as np\nimport pandas as pd\nimport plotly.express as px\nfrom sklearn.dummy import DummyRegressor, DummyClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor\nfrom deepchecks.checks.distribution.preprocessing import ScaledNumerics\nfrom deepchecks.utils.strings import format_number\n\nfrom deepchecks import CheckResult, Dataset\nfrom deepchecks.base.check import ConditionResult, TrainTestBaseCheck\nfrom deepchecks.utils.metrics import task_type_check, ModelType, initialize_multi_scorers, \\\n    get_scorers_list, get_scores_ratio, get_scorer_single\nfrom deepchecks.utils.validation import validate_model\nfrom deepchecks.errors import DeepchecksValueError\n\n\n__all__ = ['SimpleModelComparison']\n\n\nclass SimpleModelComparison(TrainTestBaseCheck):\n    \"\"\"Compare given model score to simple model score (according to given model type).\n\n    Args:\n        simple_model_type (str):\n            Type of the simple model ['random', 'constant', 'tree'].\n                + random - select one of the labels by random.\n                + constant - in regression is mean value, in classification the most common value.\n                + tree - runs a simple decision tree.\n        alternative_scorers (Dict[str, Callable], default None):\n            An optional dictionary of scorer name to scorer functions.\n            If none given, using default scorers\n        maximum_ratio (int):\n            the ratio can be up to infinity so choose maximum value to limit to.\n        max_depth (int):\n            the max depth of the tree (used only if simple model type is tree).\n        random_state (int):\n            the random state (used only if simple model type is tree or random).\n    \"\"\"\n\n    def __init__(self, simple_model_type: str = 'constant', alternative_scorers: Dict[str, Callable] = None,\n                 maximum_ratio: int = 50, max_depth: int = 3, random_state: int = 42):\n        super().__init__()\n        self.simple_model_type = simple_model_type\n        self.alternative_scorers = initialize_multi_scorers(alternative_scorers)\n        self.maximum_ratio = maximum_ratio\n        self.max_depth = max_depth\n        self.random_state = random_state\n\n    def run(self, train_dataset, test_dataset, model) -> CheckResult:\n        \"\"\"Run check.\n\n        Args:\n            train_dataset (Dataset): The training dataset object. Must contain a label.\n            test_dataset (Dataset): The test dataset object. Must contain a label.\n            model (BaseEstimator): A scikit-learn-compatible fitted estimator instance.\n\n        Returns:\n            CheckResult: value is a Dict of: given_model_score, simple_model_score, ratio\n                         ratio is given model / simple model (if the scorer returns negative values we divide 1 by it)\n                         if ratio is infinite max_ratio is returned\n\n        Raises:\n            DeepchecksValueError: If the object is not a Dataset instance.\n        \"\"\"\n        Dataset.validate_dataset(train_dataset)\n        Dataset.validate_dataset(test_dataset)\n        train_dataset.validate_label()\n        test_dataset.validate_label()\n        validate_model(test_dataset, model)\n\n        # If user defined scorers used them, else use a single scorer\n        if self.alternative_scorers:\n            scorers = get_scorers_list(model, train_dataset, self.alternative_scorers, multiclass_avg=False)\n        else:\n            scorers = [get_scorer_single(model, train_dataset, multiclass_avg=False)]\n\n        task_type = task_type_check(model, train_dataset)\n        simple_model = self._create_simple_model(train_dataset, task_type)\n\n        models = [\n            (f'{type(model).__name__} model', 'Origin', model),\n            (f'Simple model - {self.simple_model_type}', 'Simple', simple_model)\n        ]\n\n        # Multiclass have different return type from the scorer, list of score per class instead of single score\n        if task_type == ModelType.MULTICLASS:\n            results = []\n            for model_name, model_type, model_instance in models:\n                for scorer in scorers:\n                    score_result: np.ndarray = scorer(model_instance, test_dataset)\n                    # Multiclass scorers return numpy array of result per class\n                    for class_i, class_score in enumerate(score_result):\n                        # The proba returns in order of the sorted classes.\n                        class_value = train_dataset.classes[class_i]\n                        results.append([model_name, model_type, class_score, scorer.name, class_value])\n\n            results_df = pd.DataFrame(results, columns=['Model', 'Type', 'Value', 'Metric', 'Class'])\n\n            # Plot the metrics in a graph, grouping by the model and class\n            fig = px.bar(results_df, x=['Class', 'Model'], y='Value', color='Model', barmode='group',\n                         facet_col='Metric', facet_col_spacing=0.05)\n            fig.update_xaxes(title=None, tickprefix='Class ', tickangle=60)\n            fig.update_yaxes(title=None, matches=None)\n            fig.for_each_annotation(lambda a: a.update(text=a.text.split('=')[-1]))\n            fig.for_each_yaxis(lambda yaxis: yaxis.update(showticklabels=True))\n        # Model is binary or regression\n        else:\n            results = []\n            for model_name, model_type, model_instance in models:\n                for scorer in scorers:\n                    score_result: float = scorer(model_instance, test_dataset)\n                    results.append([model_name, model_type, score_result, scorer.name])\n\n            results_df = pd.DataFrame(results, columns=['Model', 'Type', 'Value', 'Metric'])\n\n            # Plot the metrics in a graph, grouping by the model\n            fig = px.bar(results_df, x='Model', y='Value', color='Model', barmode='group',\n                         facet_col='Metric', facet_col_spacing=0.05)\n            fig.update_xaxes(title=None)\n            fig.update_yaxes(title=None, matches=None)\n            fig.for_each_annotation(lambda a: a.update(text=a.text.split('=')[-1]))\n            fig.for_each_yaxis(lambda yaxis: yaxis.update(showticklabels=True))\n\n        return CheckResult({'scores': results_df, 'type': task_type}, display=fig)\n\n    def _create_simple_model(self, train_ds: Dataset, task_type: ModelType):\n        \"\"\"Create a simple model of given type (random/constant/tree) to the given dataset.\n\n        Args:\n            train_ds (Dataset): The training dataset object.\n            task_type (ModelType): the model type.\n        Returns:\n            Classifier object.\n\n        Raises:\n            NotImplementedError: If the simple_model_type is not supported\n        \"\"\"\n        np.random.seed(self.random_state)\n\n        if self.simple_model_type == 'random':\n            simple_model = RandomModel()\n\n        elif self.simple_model_type == 'constant':\n            if task_type == ModelType.REGRESSION:\n                simple_model = DummyRegressor(strategy='mean')\n            elif task_type in {ModelType.BINARY, ModelType.MULTICLASS}:\n                simple_model = DummyClassifier(strategy='most_frequent')\n            else:\n                raise DeepchecksValueError(f'Unknown task type - {task_type}')\n        elif self.simple_model_type == 'tree':\n            if task_type == ModelType.REGRESSION:\n                clf = DecisionTreeRegressor(\n                    max_depth=self.max_depth,\n                    random_state=self.random_state\n                )\n            elif task_type in {ModelType.BINARY, ModelType.MULTICLASS}:\n                clf = DecisionTreeClassifier(\n                    max_depth=self.max_depth,\n                    random_state=self.random_state,\n                    class_weight='balanced'\n                )\n            else:\n                raise DeepchecksValueError(f'Unknown task type - {task_type}')\n\n            simple_model = Pipeline([('scaler', ScaledNumerics(train_ds.cat_features, max_num_categories=10)),\n                                     ('tree-model', clf)])\n        else:\n            raise DeepchecksValueError(\n                f'Unknown model type - {self.simple_model_type}, expected to be one of '\n                f\"['random', 'constant', 'tree'] \"\n                f\"but instead got {self.simple_model_type}\"  # pylint: disable=inconsistent-quotes\n            )\n\n        simple_model.fit(train_ds.features_columns, train_ds.label_col)\n        return simple_model\n\n    def add_condition_ratio_not_less_than(self, min_allowed_ratio: float = 1.1, classes: List[Hashable] = None):\n        \"\"\"Add condition - require min allowed ratio between the given and the simple model.\n\n        Args:\n            min_allowed_ratio (float): Min allowed ratio between the given and the simple model -\n            ratio is given model / simple model (if the scorer returns negative values we divide 1 by it)\n            classes (List[Hashable]): Used in multiclass models to limit condition only to given classes.\n        \"\"\"\n        def condition(result: Dict, max_ratio=self.maximum_ratio, class_list=classes) -> ConditionResult:\n            scores_df = result['scores']\n            task_type = result['type']\n            metrics = scores_df['Metric'].unique()\n\n            def get_ratio(df):\n                simple_score = df[df['Type'] == 'Simple']['Value'].iloc[0]\n                origin_score = df[df['Type'] == 'Origin']['Value'].iloc[0]\n                return get_scores_ratio(simple_score, origin_score, max_ratio)\n\n            fails = []\n            if task_type == ModelType.MULTICLASS:\n                if class_list is None:\n                    class_list = scores_df['Class'].unique()\n                for metric in metrics:\n                    failed_classes = []\n                    for clas in class_list:\n                        score_rows = scores_df[(scores_df['Metric'] == metric) & (scores_df['Class'] == clas)]\n                        ratio = get_ratio(score_rows)\n                        if ratio < min_allowed_ratio:\n                            failed_classes.append(str(clas))\n                    if failed_classes:\n                        fails.append(f'\"{metric}\" - Classes: {\", \".join(failed_classes)}')\n            else:\n                for metric in metrics:\n                    score_rows = scores_df[(scores_df['Metric'] == metric)]\n                    ratio = get_ratio(score_rows)\n                    if ratio < min_allowed_ratio:\n                        fails.append(f'\"{metric}\"')\n\n            if fails:\n                msg = f'Metrics failed: {\", \".join(sorted(fails))}'\n                return ConditionResult(False, msg)\n            else:\n                return ConditionResult(True)\n\n        return self.add_condition('$$\\\\frac{\\\\text{model score}}{\\\\text{simple model score}} >= '\n                                  f'{format_number(min_allowed_ratio)}$$', condition)\n\n\nclass RandomModel:\n    \"\"\"Model used to randomly predict from given series of labels.\"\"\"\n\n    def __init__(self):\n        self.labels = None\n\n    def fit(self, X, y):  # pylint: disable=unused-argument,invalid-name\n        # The X is not used, but it is needed to be matching to sklearn `fit` signature\n        self.labels = y\n\n    def predict(self, X):  # pylint: disable=invalid-name\n        return np.random.choice(self.labels, X.shape[0])\n\n    def predict_proba(self, X):  # pylint: disable=invalid-name\n        classes = sorted(self.labels.unique().tolist())\n        predictions = self.predict(X)\n\n        def prediction_to_proba(y_pred):\n            proba = np.zeros(len(classes))\n            proba[classes.index(y_pred)] = 1\n            return proba\n        return np.apply_along_axis(prediction_to_proba, axis=1, arr=predictions)\n", "path": "deepchecks/checks/performance/simple_model_comparison.py"}], "after_files": [{"content": "# ----------------------------------------------------------------------------\n# Copyright (C) 2021 Deepchecks (https://www.deepchecks.com)\n#\n# This file is part of Deepchecks.\n# Deepchecks is distributed under the terms of the GNU Affero General\n# Public License (version 3 or later).\n# You should have received a copy of the GNU Affero General Public License\n# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.\n# ----------------------------------------------------------------------------\n#\n\"\"\"Module containing simple comparison check.\"\"\"\nfrom typing import Callable, Dict, Hashable, List\nimport numpy as np\nimport pandas as pd\nimport plotly.express as px\nfrom sklearn.dummy import DummyRegressor, DummyClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor\nfrom deepchecks.checks.distribution.preprocessing import ScaledNumerics\nfrom deepchecks.utils.strings import format_number\n\nfrom deepchecks import CheckResult, Dataset\nfrom deepchecks.base.check import ConditionResult, TrainTestBaseCheck\nfrom deepchecks.utils.metrics import task_type_check, ModelType, initialize_multi_scorers, \\\n    get_scorers_list, get_scores_ratio, get_scorer_single\nfrom deepchecks.utils.validation import validate_model\nfrom deepchecks.errors import DeepchecksValueError\n\n\n__all__ = ['SimpleModelComparison']\n\n\nclass SimpleModelComparison(TrainTestBaseCheck):\n    \"\"\"Compare given model score to simple model score (according to given model type).\n\n    Args:\n        simple_model_type (str):\n            Type of the simple model ['random', 'constant', 'tree'].\n                + random - select one of the labels by random.\n                + constant - in regression is mean value, in classification the most common value.\n                + tree - runs a simple decision tree.\n        alternative_scorers (Dict[str, Callable], default None):\n            An optional dictionary of scorer name to scorer functions.\n            If none given, using default scorers\n        maximum_ratio (int):\n            the ratio can be up to infinity so choose maximum value to limit to.\n        max_depth (int):\n            the max depth of the tree (used only if simple model type is tree).\n        random_state (int):\n            the random state (used only if simple model type is tree or random).\n    \"\"\"\n\n    def __init__(self, simple_model_type: str = 'constant', alternative_scorers: Dict[str, Callable] = None,\n                 maximum_ratio: int = 50, max_depth: int = 3, random_state: int = 42):\n        super().__init__()\n        self.simple_model_type = simple_model_type\n        self.alternative_scorers = initialize_multi_scorers(alternative_scorers)\n        self.maximum_ratio = maximum_ratio\n        self.max_depth = max_depth\n        self.random_state = random_state\n\n    def run(self, train_dataset, test_dataset, model) -> CheckResult:\n        \"\"\"Run check.\n\n        Args:\n            train_dataset (Dataset): The training dataset object. Must contain a label.\n            test_dataset (Dataset): The test dataset object. Must contain a label.\n            model (BaseEstimator): A scikit-learn-compatible fitted estimator instance.\n\n        Returns:\n            CheckResult: value is a Dict of: given_model_score, simple_model_score, ratio\n                         ratio is given model / simple model (if the scorer returns negative values we divide 1 by it)\n                         if ratio is infinite max_ratio is returned\n\n        Raises:\n            DeepchecksValueError: If the object is not a Dataset instance.\n        \"\"\"\n        Dataset.validate_dataset(train_dataset)\n        Dataset.validate_dataset(test_dataset)\n        train_dataset.validate_label()\n        test_dataset.validate_label()\n        validate_model(test_dataset, model)\n\n        # If user defined scorers used them, else use a single scorer\n        if self.alternative_scorers:\n            scorers = get_scorers_list(model, train_dataset, self.alternative_scorers, multiclass_avg=False)\n        else:\n            scorers = [get_scorer_single(model, train_dataset, multiclass_avg=False)]\n\n        task_type = task_type_check(model, train_dataset)\n        simple_model = self._create_simple_model(train_dataset, task_type)\n\n        models = [\n            (f'{type(model).__name__} model', 'Origin', model),\n            (f'Simple model - {self.simple_model_type}', 'Simple', simple_model)\n        ]\n\n        # Multiclass have different return type from the scorer, list of score per class instead of single score\n        if task_type == ModelType.MULTICLASS:\n            results = []\n            for model_name, model_type, model_instance in models:\n                for scorer in scorers:\n                    score_result: np.ndarray = scorer(model_instance, test_dataset)\n                    # Multiclass scorers return numpy array of result per class in order of sorted classes\n                    for class_score, class_value in zip(score_result, test_dataset.classes):\n                        results.append([model_name, model_type, class_score, scorer.name, class_value])\n\n            results_df = pd.DataFrame(results, columns=['Model', 'Type', 'Value', 'Metric', 'Class'])\n\n            # Plot the metrics in a graph, grouping by the model and class\n            fig = px.bar(results_df, x=['Class', 'Model'], y='Value', color='Model', barmode='group',\n                         facet_col='Metric', facet_col_spacing=0.05)\n            fig.update_xaxes(title=None, tickprefix='Class ', tickangle=60, type='category')\n            fig.update_yaxes(title=None, matches=None)\n            fig.for_each_annotation(lambda a: a.update(text=a.text.split('=')[-1]))\n            fig.for_each_yaxis(lambda yaxis: yaxis.update(showticklabels=True))\n        # Model is binary or regression\n        else:\n            results = []\n            for model_name, model_type, model_instance in models:\n                for scorer in scorers:\n                    score_result: float = scorer(model_instance, test_dataset)\n                    results.append([model_name, model_type, score_result, scorer.name])\n\n            results_df = pd.DataFrame(results, columns=['Model', 'Type', 'Value', 'Metric'])\n\n            # Plot the metrics in a graph, grouping by the model\n            fig = px.bar(results_df, x='Model', y='Value', color='Model', barmode='group',\n                         facet_col='Metric', facet_col_spacing=0.05)\n            fig.update_xaxes(title=None)\n            fig.update_yaxes(title=None, matches=None)\n            fig.for_each_annotation(lambda a: a.update(text=a.text.split('=')[-1]))\n            fig.for_each_yaxis(lambda yaxis: yaxis.update(showticklabels=True))\n\n        return CheckResult({'scores': results_df, 'type': task_type}, display=fig)\n\n    def _create_simple_model(self, train_ds: Dataset, task_type: ModelType):\n        \"\"\"Create a simple model of given type (random/constant/tree) to the given dataset.\n\n        Args:\n            train_ds (Dataset): The training dataset object.\n            task_type (ModelType): the model type.\n        Returns:\n            Classifier object.\n\n        Raises:\n            NotImplementedError: If the simple_model_type is not supported\n        \"\"\"\n        np.random.seed(self.random_state)\n\n        if self.simple_model_type == 'random':\n            simple_model = RandomModel()\n\n        elif self.simple_model_type == 'constant':\n            if task_type == ModelType.REGRESSION:\n                simple_model = DummyRegressor(strategy='mean')\n            elif task_type in {ModelType.BINARY, ModelType.MULTICLASS}:\n                simple_model = DummyClassifier(strategy='most_frequent')\n            else:\n                raise DeepchecksValueError(f'Unknown task type - {task_type}')\n        elif self.simple_model_type == 'tree':\n            if task_type == ModelType.REGRESSION:\n                clf = DecisionTreeRegressor(\n                    max_depth=self.max_depth,\n                    random_state=self.random_state\n                )\n            elif task_type in {ModelType.BINARY, ModelType.MULTICLASS}:\n                clf = DecisionTreeClassifier(\n                    max_depth=self.max_depth,\n                    random_state=self.random_state,\n                    class_weight='balanced'\n                )\n            else:\n                raise DeepchecksValueError(f'Unknown task type - {task_type}')\n\n            simple_model = Pipeline([('scaler', ScaledNumerics(train_ds.cat_features, max_num_categories=10)),\n                                     ('tree-model', clf)])\n        else:\n            raise DeepchecksValueError(\n                f'Unknown model type - {self.simple_model_type}, expected to be one of '\n                f\"['random', 'constant', 'tree'] \"\n                f\"but instead got {self.simple_model_type}\"  # pylint: disable=inconsistent-quotes\n            )\n\n        simple_model.fit(train_ds.features_columns, train_ds.label_col)\n        return simple_model\n\n    def add_condition_ratio_not_less_than(self, min_allowed_ratio: float = 1.1, classes: List[Hashable] = None):\n        \"\"\"Add condition - require min allowed ratio between the given and the simple model.\n\n        Args:\n            min_allowed_ratio (float): Min allowed ratio between the given and the simple model -\n            ratio is given model / simple model (if the scorer returns negative values we divide 1 by it)\n            classes (List[Hashable]): Used in multiclass models to limit condition only to given classes.\n        \"\"\"\n        def condition(result: Dict, max_ratio=self.maximum_ratio, class_list=classes) -> ConditionResult:\n            scores_df = result['scores']\n            task_type = result['type']\n            metrics = scores_df['Metric'].unique()\n\n            def get_ratio(df):\n                simple_score = df[df['Type'] == 'Simple']['Value'].iloc[0]\n                origin_score = df[df['Type'] == 'Origin']['Value'].iloc[0]\n                return get_scores_ratio(simple_score, origin_score, max_ratio)\n\n            fails = []\n            if task_type == ModelType.MULTICLASS:\n                if class_list is None:\n                    class_list = scores_df['Class'].unique()\n                for metric in metrics:\n                    failed_classes = []\n                    for clas in class_list:\n                        score_rows = scores_df[(scores_df['Metric'] == metric) & (scores_df['Class'] == clas)]\n                        ratio = get_ratio(score_rows)\n                        if ratio < min_allowed_ratio:\n                            failed_classes.append(str(clas))\n                    if failed_classes:\n                        fails.append(f'\"{metric}\" - Classes: {\", \".join(failed_classes)}')\n            else:\n                for metric in metrics:\n                    score_rows = scores_df[(scores_df['Metric'] == metric)]\n                    ratio = get_ratio(score_rows)\n                    if ratio < min_allowed_ratio:\n                        fails.append(f'\"{metric}\"')\n\n            if fails:\n                msg = f'Metrics failed: {\", \".join(sorted(fails))}'\n                return ConditionResult(False, msg)\n            else:\n                return ConditionResult(True)\n\n        return self.add_condition('$$\\\\frac{\\\\text{model score}}{\\\\text{simple model score}} >= '\n                                  f'{format_number(min_allowed_ratio)}$$', condition)\n\n\nclass RandomModel:\n    \"\"\"Model used to randomly predict from given series of labels.\"\"\"\n\n    def __init__(self):\n        self.labels = None\n\n    def fit(self, X, y):  # pylint: disable=unused-argument,invalid-name\n        # The X is not used, but it is needed to be matching to sklearn `fit` signature\n        self.labels = y\n\n    def predict(self, X):  # pylint: disable=invalid-name\n        return np.random.choice(self.labels, X.shape[0])\n\n    def predict_proba(self, X):  # pylint: disable=invalid-name\n        classes = sorted(self.labels.unique().tolist())\n        predictions = self.predict(X)\n\n        def prediction_to_proba(y_pred):\n            proba = np.zeros(len(classes))\n            proba[classes.index(y_pred)] = 1\n            return proba\n        return np.apply_along_axis(prediction_to_proba, axis=1, arr=predictions)\n", "path": "deepchecks/checks/performance/simple_model_comparison.py"}]}