text: #! /usr/bin/env python
  # coding=utf-8
  # Copyright (c) 2019 Uber Technologies, Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  import argparse
  import logging
  import os
  import sys
  from functools import partial
  from typing import List, Union
  
  import numpy as np
  import pandas as pd
  import sklearn
  from scipy.stats import entropy
  from sklearn.calibration import calibration_curve
  from sklearn.metrics import brier_score_loss
  
  from ludwig.constants import *
  from ludwig.contrib import contrib_command, contrib_import
  from ludwig.utils import visualization_utils
  from ludwig.utils.data_utils import load_from_file, load_json, load_array
  from ludwig.utils.print_utils import logging_level_registry
  from ludwig.utils.data_utils import CACHEABLE_FORMATS, \
      figure_data_format_dataset, external_data_reader_registry
  from ludwig.utils.misc_utils import get_from_registry
  
  logger = logging.getLogger(__name__)
  
  
  def validate_conf_treshholds_and_probabilities_2d_3d(
          probabilities, treshhold_output_feature_names
  ):
      """Ensure probabilities and treshhold output_feature_names arrays have two members each.
  
      :param probabilities: List of probabilities per model
      :param threshhold_output_feature_names: List of threshhold output_feature_names per model
      :raise: RuntimeError
      """
      validation_mapping = {
          'probabilities': probabilities,
          'treshhold_output_feature_names': treshhold_output_feature_names
      }
      for item, value in validation_mapping.items():
          item_len = len(value)
          if item_len != 2:
              exception_message = 'Two {} should be provided - ' \
                                  '{} was given.'.format(item, item_len)
              logging.error(exception_message)
              raise RuntimeError(exception_message)
  
  
  def load_data_for_viz(load_type, model_file_statistics, **kwargs):
      """Load model file data in to list of .
  
      :param load_type: type of the data loader to be used.
      :param model_file_statistics: JSON file or list of json files containing any
             model experiment stats.
      :return List of training statistics loaded as json objects.
      """
      supported_load_types = dict(load_json=load_json,
                                  load_from_file=partial(
                                      load_from_file,
                                      dtype=kwargs.get('dtype', int),
                                      ground_truth_split=kwargs.get(
                                          'ground_truth_split', 2)
                                  )
                                  )
      loader = supported_load_types[load_type]
      try:
          stats_per_model = [loader(stats_f)
                             for stats_f in
                             model_file_statistics]
      except (TypeError, AttributeError):
          logging.exception(
              'Unable to open model statistics file {}!'.format(
                  model_file_statistics
              )
          )
          raise
      return stats_per_model
  
  
  def convert_to_list(item):
      """If item is not list class instance or None put inside a list.
  
      :param item: object to be checked and converted
      :return: original item if it is a list instance or list containing the item.
      """
      return item if item is None or isinstance(item, list) else [item]
  
  
  def _validate_output_feature_name_from_train_stats(
          output_feature_name,
          train_stats_per_model
  ):
      """Validate prediction output_feature_name from model train stats and return it as list.
  
      :param output_feature_name: output_feature_name containing ground truth
      :param train_stats_per_model: list of per model train stats
      :return output_feature_names: list of output_feature_name(s) containing ground truth
      """
      output_feature_names_set = set()
      for ls in train_stats_per_model:
          for _, values in ls.items():
              for key in values:
                  output_feature_names_set.add(key)
      try:
          if output_feature_name in output_feature_names_set:
              return [output_feature_name]
          else:
              return output_feature_names_set
      # raised if output_feature_name is emtpy iterable (e.g. [] in set())
      except TypeError:
          return output_feature_names_set
  
  
  def _validate_output_feature_name_from_test_stats(
          output_feature_name,
          test_stats_per_model
  ):
      """Validate prediction output_feature_name from model test stats and return it as list.
  
      :param output_feature_name: output_feature_name containing ground truth
      :param test_stats_per_model: list of per model test stats
      :return output_feature_names: list of output_feature_name(s) containing ground truth
      """
      output_feature_names_set = set()
      for ls in test_stats_per_model:
          for key in ls:
              output_feature_names_set.add(key)
      try:
          if output_feature_name in output_feature_names_set:
              return [output_feature_name]
          else:
              return output_feature_names_set
      # raised if output_feature_name is emtpy iterable (e.g. [] in set())
      except TypeError:
          return output_feature_names_set
  
  def _encode_categorical_feature(
          raw: np.array,
          str2idx: dict
  ) -> np.array:
      """encodes raw categorical string value to encoded numeric value
  
      Args:
      :param raw: (np.array) string categorical representation
      :param str2idx: (dict) dictionary that maps string representation to
          encoded value.
  
      Returns:
          np.array
      """
      return str2idx[raw]
  
  def _extract_ground_truth_values(
          ground_truth: str,
          output_feature_name: str,
          ground_truth_split: int,
          split_file: Union[str, None] = None) -> pd.Series:
      """Helper function to extract ground truth values
  
      Args:
      :param ground_truth: (str) path to source data containing ground truth.
      :param output_feature_name: (str) output feature name for ground
          truth values.
      :param ground_truth_split: (int) dataset split to use for ground truth,
          defaults to 2.
      :param split_file: (Union[str, None]) optional file path to split values.
  
      # Return
  
      :return pd.Series: ground truth values from source data set
  
      """
      # determine ground truth data format and get appropriate reader
      data_format = figure_data_format_dataset(ground_truth)
      if data_format not in CACHEABLE_FORMATS:
          raise ValueError(
              '{} is not supported for ground truth file, '
              'valid types are {}'.format(data_format, CACHEABLE_FORMATS)
          )
      reader = get_from_registry(
          data_format,
          external_data_reader_registry
      )
  
      # retrieve ground truth from source data set
      gt_df = reader(ground_truth)
  
      # extract ground truth for visualization
      if SPLIT in gt_df:
          # get split value from source data set
          split = gt_df[SPLIT]
          gt = gt_df[output_feature_name][
              split == ground_truth_split]
      elif split_file is not None:
          # retrieve from split file
          split = load_array(split_file)
          gt = gt_df[output_feature_name][
              split == ground_truth_split]
      else:
          # use all the data in ground_truth
          gt = gt_df[output_feature_name]
  
      return gt
  
  
  def generate_filename_template_path(output_dir, filename_template):
      """Ensure path to template file can be constructed given an output dir.
  
      Create output directory if yet does exist.
      :param output_dir: Directory that will contain the filename_template file
      :param filename_template: name of the file template to be appended to the
              filename template path
      :return: path to filename template inside the output dir or None if the
               output dir is None
      """
      if output_dir:
          os.makedirs(output_dir, exist_ok=True)
          return os.path.join(output_dir, filename_template)
      return None
  
  
  def compare_performance_cli(
          test_statistics: Union[str, List[str]],
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by compare_performance.
  
      # Inputs
  
      :param test_statistics: (Union[str, List[str]]) path to experiment test
          statistics file.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      test_stats_per_model = load_data_for_viz('load_json', test_statistics)
      compare_performance(test_stats_per_model, **kwargs)
  
  
  def learning_curves_cli(
          training_statistics: Union[str, List[str]],
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by learning_curves.
  
      # Inputs
  
      :param training_statistics: (Union[str, List[str]]) path to experiment
          training statistics file
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      train_stats_per_model = load_data_for_viz('load_json', training_statistics)
      learning_curves(train_stats_per_model, **kwargs)
  
  
  def compare_classifiers_performance_from_prob_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          output_feature_name: str,
          output_directory: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by compare_classifiers_from_prob.
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file
      :param ground_truth: (str) path to ground truth file
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param output_directory: (str) name of output directory containing training
          results.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
      :return None:
      """
  
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # translate string to encoded numeric value
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file=split_file
      )
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
  
      compare_classifiers_performance_from_prob(
          probabilities_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          output_directory=output_directory,
          **kwargs
      )
  
  
  def compare_classifiers_performance_from_pred_cli(
          predictions: List[str],
          ground_truth: str,
          ground_truth_metadata: str,
          ground_truth_split: int,
          split_file: str,
          output_feature_name: str,
          output_directory: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by compare_classifiers_from_pred
  
      # Inputs
  
      :param predictions: (List[str]) path to experiment predictions file.
      :param ground_truth: (str) path to ground truth file.
      :param ground_truth_metadata: (str) path to ground truth metadata file.
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param output_directory: (str) name of output directory containing training
          results.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
  
      predictions_per_model = load_data_for_viz(
          'load_from_file', predictions, dtype=str
      )
  
      compare_classifiers_performance_from_pred(
          predictions_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          output_directory=output_directory,
          **kwargs
      )
  
  
  def compare_classifiers_performance_subset_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          output_feature_name: str,
          output_directory: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by compare_classifiers_subset.
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file
      :param ground_truth: (str) path to ground truth file
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param output_directory: (str) name of output directory containing training
           results.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
  
      compare_classifiers_performance_subset(
          probabilities_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          output_directory=output_directory,
          **kwargs
      )
  
  
  def compare_classifiers_performance_changing_k_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          output_feature_name: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by compare_classifiers_changing_k.
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file
      :param ground_truth: (str) path to ground truth file
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
      compare_classifiers_performance_changing_k(
          probabilities_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          **kwargs
      )
  
  
  def compare_classifiers_multiclass_multimetric_cli(
          test_statistics: Union[str, List[str]],
          ground_truth_metadata: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by compare_classifiers_multiclass
  
      # Inputs
  
      :param test_statistics: (Union[str, List[str]]) path to experiment test
          statistics file.
      :param ground_truth_metadata: (str) path to ground truth metadata file.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      test_stats_per_model = load_data_for_viz('load_json', test_statistics)
      metadata = load_json(ground_truth_metadata)
      compare_classifiers_multiclass_multimetric(
          test_stats_per_model, metadata=metadata, **kwargs
      )
  
  
  def compare_classifiers_predictions_cli(
          predictions: List[str],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          output_feature_name: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by compare_classifiers_predictions
  
      # Inputs
  
      :param predictions: (List[str]) path to experiment predictions file.
      :param ground_truth: (str) path to grpound truth file.
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
  
      predictions_per_model = load_data_for_viz(
          'load_from_file', predictions, dtype=str
      )
  
      compare_classifiers_predictions(
          predictions_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          **kwargs
      )
  
  
  def compare_classifiers_predictions_distribution_cli(
          predictions: List[str],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          output_feature_name: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by
      compare_predictions_distribution
  
      # Inputs
  
      :param predictions: (List[str]) path to experiment predictions file.
      :param ground_truth: (str) path to grpound truth file.
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
  
      predictions_per_model = load_data_for_viz(
          'load_from_file', predictions, dtype=str
      )
      compare_classifiers_predictions_distribution(
          predictions_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          **kwargs
      )
  
  
  def confidence_thresholding_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          output_feature_name: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by confidence_thresholding.
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file.
      :param ground_truth: (str) path to ground truth file.
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
      confidence_thresholding(
          probabilities_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          **kwargs
      )
  
  
  def confidence_thresholding_data_vs_acc_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          output_feature_name: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by
      confidence_thresholding_data_vs_acc_cli.
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file.
      :param ground_truth: (str) path to ground truth file.
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
      confidence_thresholding_data_vs_acc(
          probabilities_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          **kwargs
      )
  
  
  def confidence_thresholding_data_vs_acc_subset_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          output_feature_name: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by
      confidence_thresholding_data_vs_acc_subset.
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file.
      :param ground_truth: (str) path to ground truth file.
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
      confidence_thresholding_data_vs_acc_subset(
          probabilities_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          **kwargs
      )
  
  
  def confidence_thresholding_data_vs_acc_subset_per_class_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_metadata: str,
          ground_truth_split: int,
          split_file: str,
          output_feature_name: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by compare_classifiers_multiclass
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file.
      :param ground_truth: (str) path to ground truth file.
      :param ground_truth_metadata: (str) path to ground truth metadata file.
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param output_feature_name: (str) name of the output feature to visualize.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
      confidence_thresholding_data_vs_acc_subset_per_class(
          probabilities_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          **kwargs
      )
  
  
  def confidence_thresholding_2thresholds_2d_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          threshold_output_feature_names: List[str],
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by
      confidence_thresholding_2thresholds_2d_cli
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file.
      :param ground_truth: (str) path to ground truth file.
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param threshold_output_feature_names: (List[str]) name of the output
          feature to visualizes.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth0 = _extract_ground_truth_values(
          ground_truth,
          threshold_output_feature_names[0],
          ground_truth_split,
          split_file
      )
  
      ground_truth1 = _extract_ground_truth_values(
          ground_truth,
          threshold_output_feature_names[1],
          ground_truth_split,
          split_file
      )
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
      confidence_thresholding_2thresholds_2d(
          probabilities_per_model,
          [ground_truth0, ground_truth1],
          metadata,
          threshold_output_feature_names,
          **kwargs
      )
  
  
  def confidence_thresholding_2thresholds_3d_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          threshold_output_feature_names: List[str],
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by
      confidence_thresholding_2thresholds_3d_cli
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file.
      :param ground_truth: (str) path to ground truth file.
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param threshold_output_feature_names: (List[str]) name of the output
          feature to visualizes.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth0 = _extract_ground_truth_values(
          ground_truth,
          threshold_output_feature_names[0],
          ground_truth_split,
          split_file
      )
  
      ground_truth1 = _extract_ground_truth_values(
          ground_truth,
          threshold_output_feature_names[1],
          ground_truth_split,
          split_file
      )
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
      confidence_thresholding_2thresholds_3d(
          probabilities_per_model,
          [ground_truth0, ground_truth1],
          metadata,
          threshold_output_feature_names,
          **kwargs
      )
  
  
  def binary_threshold_vs_metric_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          output_feature_name: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by binary_threshold_vs_metric_cli.
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file.
      :param ground_truth: (str) path to ground truth file.
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
  
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
      binary_threshold_vs_metric(
          probabilities_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          **kwargs
      )
  
  
  def roc_curves_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          output_feature_name: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by roc_curves_cli.
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file.
      :param ground_truth: (str) path to ground truth file.
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
  
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
      roc_curves(
          probabilities_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          **kwargs
      )
  
  
  def roc_curves_from_test_statistics_cli(
          test_statistics: Union[str, List[str]],
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by
      roc_curves_from_test_statistics_cli.
  
      # Inputs
      :param test_statistics: (Union[str, List[str]]) path to experiment test
          statistics file.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      test_stats_per_model = load_data_for_viz('load_json', test_statistics)
      roc_curves_from_test_statistics(
          test_stats_per_model, **kwargs
      )
  
  
  def calibration_1_vs_all_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          output_feature_name: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by calibration_1_vs_all_cli.
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file
      :param ground_truth: (str) path to ground truth file
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
  
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
      feature_metadata = metadata[output_feature_name]
      vfunc = np.vectorize(_encode_categorical_feature)
      ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
      calibration_1_vs_all(
          probabilities_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          **kwargs
      )
  
  
  def calibration_multiclass_cli(
          probabilities: Union[str, List[str]],
          ground_truth: str,
          ground_truth_split: int,
          split_file: str,
          ground_truth_metadata: str,
          output_feature_name: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by calibration_multiclass_cli.
  
      # Inputs
  
      :param probabilities: (Union[str, List[str]]) path to experiment
          probabilities file
      :param ground_truth: (str) path to ground truth file
      :param ground_truth_split: (str) type of ground truth split -
          `0` for training split, `1` for validation split or
          2 for `'test'` split.
      :param split_file: (str, None) file path to csv file containing split values
      :param ground_truth_metadata: (str) file path to feature metadata json file
          created during training.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
  
      # retrieve feature metadata to convert raw predictions to encoded value
      metadata = load_json(ground_truth_metadata)
  
      # retrieve ground truth from source data set
      ground_truth = _extract_ground_truth_values(
          ground_truth,
          output_feature_name,
          ground_truth_split,
          split_file
      )
  
      probabilities_per_model = load_data_for_viz(
          'load_from_file', probabilities, dtype=float
      )
      calibration_multiclass(
          probabilities_per_model,
          ground_truth,
          metadata,
          output_feature_name,
          **kwargs
      )
  
  
  def confusion_matrix_cli(
          test_statistics: Union[str, List[str]],
          ground_truth_metadata: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by confusion_matrix.
  
      # Inputs
  
      :param test_statistics: (Union[str, List[str]]) path to experiment test
          statistics file.
      :param ground_truth_metadata: (str) path to ground truth metadata file.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      test_stats_per_model = load_data_for_viz('load_json', test_statistics)
      metadata = load_json(ground_truth_metadata)
      confusion_matrix(test_stats_per_model, metadata, **kwargs)
  
  
  def frequency_vs_f1_cli(
          test_statistics: Union[str, List[str]],
          ground_truth_metadata: str,
          **kwargs: dict
  ) -> None:
      """Load model data from files to be shown by frequency_vs_f1.
  
      # Inputs
  
      :param test_statistics: (Union[str, List[str]]) path to experiment test
          statistics file.
      :param ground_truth_metadata: (str) path to ground truth metadata file.
      :param kwargs: (dict) parameters for the requested visualizations.
  
      # Return
  
      :return None:
      """
      test_stats_per_model = load_data_for_viz('load_json', test_statistics)
      metadata = load_json(ground_truth_metadata)
      frequency_vs_f1(test_stats_per_model, metadata, **kwargs)
  
  
  def learning_curves(
          train_stats_per_model: List[dict],
          output_feature_name: Union[str, None] = None,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show how model metrics change over training and validation data epochs.
  
      For each model and for each output feature and metric of the model,
      it produces a line plot showing how that metric changed over the course
      of the epochs of training on the training and validation sets.
  
      # Inputs
  
      :param train_stats_per_model: (List[dict]) list containing dictionary of
          training statistics per model.
      :param output_feature_name: (Union[str, `None`], default: `None`) name of the output feature
          to use for the visualization.  If `None`, use all output features.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
      :return: (None)
      """
      filename_template = 'learning_curves_{}_{}.' + file_format
      filename_template_path = generate_filename_template_path(
          output_directory,
          filename_template
      )
      train_stats_per_model_list = convert_to_list(train_stats_per_model)
      model_names_list = convert_to_list(model_names)
      output_feature_names = _validate_output_feature_name_from_train_stats(
          output_feature_name,
          train_stats_per_model_list
      )
  
      metrics = [LOSS, ACCURACY, HITS_AT_K, EDIT_DISTANCE]
      for output_feature_name in output_feature_names:
          for metric in metrics:
              if metric in train_stats_per_model_list[0][TRAINING][
                  output_feature_name]:
                  filename = None
                  if filename_template_path:
                      filename = filename_template_path.format(
                          output_feature_name, metric)
  
                  training_stats = [
                      learning_stats[TRAINING][output_feature_name][metric]
                      for learning_stats in
                      train_stats_per_model_list]
  
                  validation_stats = []
                  for learning_stats in train_stats_per_model_list:
                      if VALIDATION in learning_stats:
                          validation_stats.append(
                              learning_stats[VALIDATION][output_feature_name][
                                  metric]
                          )
                      else:
                          validation_stats.append(None)
  
                  visualization_utils.learning_curves_plot(
                      training_stats,
                      validation_stats,
                      metric,
                      model_names_list,
                      title='Learning Curves {}'.format(output_feature_name),
                      filename=filename
                  )
  
  
  def compare_performance(
          test_stats_per_model: List[dict],
          output_feature_name: Union[str, None] = None,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Produces model comparison barplot visualization for each overall metric
  
      For each model (in the aligned lists of test_statistics and model_names)
      it produces bars in a bar plot, one for each overall metric available
      in the test_statistics file for the specified output_feature_name.
  
      # Inputs
  
      :param test_stats_per_model: (List[dict]) dictionary containing evaluation
          performance statistics.
      :param output_feature_name: (Union[str, `None`], default: `None`) name of the output feature
          to use for the visualization.  If `None`, use all output features.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      ignore_names = ['overall_stats', 'confusion_matrix', 'per_class_stats',
                      'predictions', 'probabilities']
  
      filename_template = 'compare_performance_{}.' + file_format
      filename_template_path = generate_filename_template_path(
          output_directory,
          filename_template
      )
  
      test_stats_per_model_list = convert_to_list(test_stats_per_model)
      model_names_list = convert_to_list(model_names)
      output_feature_names = _validate_output_feature_name_from_test_stats(
          output_feature_name,
          test_stats_per_model_list
      )
  
      for output_feature_name in output_feature_names:
          metric_names_sets = list(
              set(tspr[output_feature_name].keys())
              for tspr in test_stats_per_model_list
          )
          metric_names = metric_names_sets[0]
          for metric_names_set in metric_names_sets:
              metric_names = metric_names.intersection(metric_names_set)
          metric_names.remove(LOSS)
          for name in ignore_names:
              if name in metric_names:
                  metric_names.remove(name)
          metrics_dict = {name: [] for name in metric_names}
  
          for test_stats_per_model in test_stats_per_model_list:
              for metric_name in metric_names:
                  metrics_dict[metric_name].append(
                      test_stats_per_model[output_feature_name][metric_name]
                  )
  
          # are there any metrics to compare?
          if metrics_dict:
              metrics = []
              metrics_names = []
              min_val = float("inf")
              max_val = float("-inf")
              for metric_name, metric_vals in metrics_dict.items():
                  if len(metric_vals) > 0:
                      metrics.append(metric_vals)
                      metrics_names.append(metric_name)
                      curr_min = min(metric_vals)
                      if curr_min < min_val:
                          min_val = curr_min
                      curr_max = max(metric_vals)
                      if curr_max > max_val:
                          max_val = curr_max
  
              filename = None
  
              if filename_template_path:
                  filename = filename_template_path.format(output_feature_name)
                  os.makedirs(output_directory, exist_ok=True)
  
              visualization_utils.compare_classifiers_plot(
                  metrics,
                  metrics_names,
                  model_names_list,
                  adaptive=min_val < 0 or max_val > 1,
                  title='Performance comparison on {}'.format(
                      output_feature_name),
                  filename=filename
              )
  
  
  def compare_classifiers_performance_from_prob(
          probabilities_per_model: List[np.ndarray],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          labels_limit: int = 0,
          top_n_classes: Union[List[int], int] = 3,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Produces model comparison barplot visualization from probabilities.
  
      For each model it produces bars in a bar plot, one for each overall metric
      computed on the fly from the probabilities of predictions for the specified
      `model_names`.
  
      # Inputs
  
      :param probabilities_per_model: (List[np.ndarray]) path to experiment
          probabilities file
      :param ground_truth: (pd.Series) ground truth values
      :param metadata: (dict) feature metadata dictionary
      :param output_feature_name: (str) output feature name
      :param top_n_classes: (List[int]) list containing the number of classes
          to plot.
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
  
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      top_n_classes_list = convert_to_list(top_n_classes)
      k = top_n_classes_list[0]
      model_names_list = convert_to_list(model_names)
      if labels_limit > 0:
          ground_truth[ground_truth > labels_limit] = labels_limit
  
      probs = probabilities_per_model
      accuracies = []
      hits_at_ks = []
      mrrs = []
  
      for i, prob in enumerate(probs):
  
          if labels_limit > 0 and prob.shape[1] > labels_limit + 1:
              prob_limit = prob[:, :labels_limit + 1]
              prob_limit[:, labels_limit] = prob[:, labels_limit:].sum(1)
              prob = prob_limit
  
          prob = np.argsort(prob, axis=1)
          top1 = prob[:, -1]
          topk = prob[:, -k:]
  
          accuracies.append((ground_truth == top1).sum() / len(ground_truth))
  
          hits_at_k = 0
          for j in range(len(ground_truth)):
              hits_at_k += np.in1d(ground_truth[j], topk[j])
          hits_at_ks.append(np.asscalar(hits_at_k) / len(ground_truth))
  
          mrr = 0
          for j in range(len(ground_truth)):
              ground_truth_pos_in_probs = prob[j] == ground_truth[j]
              if np.any(ground_truth_pos_in_probs):
                  mrr += (1 / -(np.asscalar(
                      np.argwhere(ground_truth_pos_in_probs)) -
                                prob.shape[1]))
          mrrs.append(mrr / len(ground_truth))
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = os.path.join(
              output_directory,
              'compare_classifiers_performance_from_prob.' + file_format
          )
  
      visualization_utils.compare_classifiers_plot(
          [accuracies, hits_at_ks, mrrs],
          [ACCURACY, HITS_AT_K, 'mrr'],
          model_names_list,
          filename=filename
      )
  
  
  def compare_classifiers_performance_from_pred(
          predictions_per_model: List[np.ndarray],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          labels_limit: int,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Produces model comparison barplot visualization from predictions.
  
      For each model it produces bars in a bar plot, one for each overall metric
      computed on the fly from the predictions for the specified
      `model_names`.
  
      # Inputs
  
      :param predictions_per_model: (List[str]) path to experiment predictions file.
      :param ground_truth: (pd.Series) ground truth values
      :param metadata: (dict) feature metadata dictionary.
      :param output_feature_name: (str) name of the output feature to visualize.
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
  
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      predictions_per_model = [
          np.ndarray.flatten(np.array(pred)) for pred in predictions_per_model
      ]
  
      if labels_limit > 0:
          ground_truth[ground_truth > labels_limit] = labels_limit
  
      preds = predictions_per_model
      model_names_list = convert_to_list(model_names)
      mapped_preds = []
      try:
          for pred in preds:
              mapped_preds.append(
                  [metadata[output_feature_name]['str2idx'][val] for val in
                   pred])
          preds = mapped_preds
      # If predictions are coming from npy file there is no need to convert to
      # numeric labels using metadata
      except (TypeError, KeyError):
          pass
      accuracies = []
      precisions = []
      recalls = []
      f1s = []
  
      for i, pred in enumerate(preds):
          accuracies.append(sklearn.metrics.accuracy_score(ground_truth, pred))
          precisions.append(
              sklearn.metrics.precision_score(ground_truth, pred,
                                              average='macro')
          )
          recalls.append(sklearn.metrics.recall_score(
              ground_truth,
              pred,
              average='macro')
          )
          f1s.append(sklearn.metrics.f1_score(
              ground_truth,
              pred,
              average='macro')
          )
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = os.path.join(
              output_directory,
              'compare_classifiers_performance_from_pred.' + file_format
          )
  
      visualization_utils.compare_classifiers_plot(
          [accuracies, precisions, recalls, f1s],
          [ACCURACY, 'precision', 'recall', 'f1'],
          model_names_list,
          filename=filename
      )
  
  
  def compare_classifiers_performance_subset(
          probabilities_per_model: List[np.array],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          top_n_classes: List[int],
          labels_limit: (int),
          subset: str,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Produces model comparison barplot visualization from train subset.
  
      For each model  it produces bars in a bar plot, one for each overall metric
       computed on the fly from the probabilities predictions for the
       specified `model_names`, considering only a subset of the full training set.
       The way the subset is obtained is using the `top_n_classes` and
       `subset` parameters.
  
       # Inputs
  
      :param probabilities_per_model: (List[numpy.array]) list of model
          probabilities.
      :param ground_truth: (Union[pd.Series, np.ndarray]) ground truth values
      :param metadata: (dict) feature metadata dictionary
      :param output_feature_name: (str) output feature name
      :param top_n_classes: (List[int]) list containing the number of classes
          to plot.
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param subset: (str) string specifying type of subset filtering.  Valid
          values are `ground_truth` or `predictions`.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      top_n_classes_list = convert_to_list(top_n_classes)
      k = top_n_classes_list[0]
      model_names_list = convert_to_list(model_names)
      if labels_limit > 0:
          ground_truth[ground_truth > labels_limit] = labels_limit
  
      subset_indices = ground_truth > 0
      gt_subset = ground_truth
      if subset == 'ground_truth':
          subset_indices = ground_truth < k
          gt_subset = ground_truth[subset_indices]
          logger.info('Subset is {:.2f}% of the data'.format(
              len(gt_subset) / len(ground_truth) * 100)
          )
  
      probs = probabilities_per_model
      accuracies = []
      hits_at_ks = []
  
      for i, prob in enumerate(probs):
  
          if labels_limit > 0 and prob.shape[1] > labels_limit + 1:
              prob_limit = prob[:, :labels_limit + 1]
              prob_limit[:, labels_limit] = prob[:, labels_limit:].sum(1)
              prob = prob_limit
  
          if subset == PREDICTIONS:
              subset_indices = np.argmax(prob, axis=1) < k
              gt_subset = ground_truth[subset_indices]
              logger.info(
                  'Subset for model_name {} is {:.2f}% of the data'.format(
                      model_names[i] if model_names and i < len(
                          model_names) else i,
                      len(gt_subset) / len(ground_truth) * 100
                  )
              )
              model_names[i] = '{} ({:.2f}%)'.format(
                  model_names[i] if model_names and i < len(model_names) else i,
                  len(gt_subset) / len(ground_truth) * 100
              )
  
          prob_subset = prob[subset_indices]
  
          prob_subset = np.argsort(prob_subset, axis=1)
          top1_subset = prob_subset[:, -1]
          top3_subset = prob_subset[:, -3:]
  
          accuracies.append(np.sum((gt_subset == top1_subset)) / len(gt_subset))
  
          hits_at_k = 0
          for j in range(len(gt_subset)):
              hits_at_k += np.in1d(gt_subset[j], top3_subset[i, :])
          hits_at_ks.append(np.asscalar(hits_at_k) / len(gt_subset))
  
      title = None
      if subset == 'ground_truth':
          title = 'Classifier performance on first {} class{} ({:.2f}%)'.format(
              k, 'es' if k > 1 else '', len(gt_subset) / len(ground_truth) * 100
          )
      elif subset == PREDICTIONS:
          title = 'Classifier performance on first {} class{}'.format(
              k,
              'es' if k > 1 else ''
          )
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = os.path.join(
              output_directory,
              'compare_classifiers_performance_subset.' + file_format
          )
  
      visualization_utils.compare_classifiers_plot(
          [accuracies, hits_at_ks],
          [ACCURACY, HITS_AT_K],
          model_names_list,
          title=title,
          filename=filename
      )
  
  
  def compare_classifiers_performance_changing_k(
          probabilities_per_model: List[np.array],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          top_k: int,
          labels_limit: int,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Produce lineplot that show Hits@K metric while k goes from 1 to `top_k`.
  
      For each model it produces a line plot that shows the Hits@K metric
      (that counts a prediction as correct if the model produces it among the
      first k) while changing k from 1 to top_k for the specified
      `output_feature_name`.
  
      # Inputs
  
      :param probabilities_per_model: (List[numpy.array]) list of model
          probabilities.
      :param ground_truth: (Union[pd.Series, np.ndarray]) ground truth values
      :param metadata: (dict) feature metadata dictionary
      :param output_feature_name: (str) output feature name
      :param top_k: (int) number of elements in the ranklist to consider.
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      k = top_k
      if labels_limit > 0:
          ground_truth[ground_truth > labels_limit] = labels_limit
      probs = probabilities_per_model
  
      hits_at_ks = []
      model_names_list = convert_to_list(model_names)
      for i, prob in enumerate(probs):
  
          if labels_limit > 0 and prob.shape[1] > labels_limit + 1:
              prob_limit = prob[:, :labels_limit + 1]
              prob_limit[:, labels_limit] = prob[:, labels_limit:].sum(1)
              prob = prob_limit
  
          prob = np.argsort(prob, axis=1)
  
          hits_at_k = [0.0] * k
          for g in range(len(ground_truth)):
              for j in range(k):
                  hits_at_k[j] += np.in1d(ground_truth[g], prob[g, -j - 1:])
          hits_at_ks.append(np.array(hits_at_k) / len(ground_truth))
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = os.path.join(
              output_directory,
              'compare_classifiers_performance_changing_k.' + file_format
          )
  
      visualization_utils.compare_classifiers_line_plot(
          np.arange(1, k + 1),
          hits_at_ks, 'hits@k',
          model_names_list,
          title='Classifier comparison (hits@k)',
          filename=filename
      )
  
  
  def compare_classifiers_multiclass_multimetric(
          test_stats_per_model: List[dict],
          metadata: dict,
          output_feature_name: str,
          top_n_classes: List[int],
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show the precision, recall and F1 of the model for the specified output_feature_name.
  
      For each model it produces four plots that show the precision,
      recall and F1 of the model on several classes for the specified output_feature_name.
  
      # Inputs
  
      :param test_stats_per_model: (List[dict]) list containing dictionary of
          evaluation performance statistics
      :param metadata: (dict) intermediate preprocess structure created during
          training containing the mappings of the input dataset.
      :param output_feature_name: (Union[str, `None`]) name of the output feature
          to use for the visualization.  If `None`, use all output features.
      :param top_n_classes: (List[int]) list containing the number of classes
          to plot.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
      :return: (None)
      """
      filename_template = 'compare_classifiers_multiclass_multimetric_{}_{}_{}.' \
                          + file_format
      filename_template_path = generate_filename_template_path(
          output_directory,
          filename_template
      )
  
      test_stats_per_model_list = convert_to_list(test_stats_per_model)
      model_names_list = convert_to_list(model_names)
      output_feature_names = _validate_output_feature_name_from_test_stats(
          output_feature_name,
          test_stats_per_model_list
      )
  
      for i, test_statistics in enumerate(
              test_stats_per_model_list):
          for output_feature_name in output_feature_names:
              model_name_name = (
                  model_names_list[i]
                  if model_names_list is not None and i < len(model_names_list)
                  else ''
              )
              if 'per_class_stats' not in test_statistics[output_feature_name]:
                  logging.warning(
                      'The output_feature_name {} in test statistics does not contain "{}", '
                      'skipping it'.format(output_feature_name, per_class_stats)
                  )
                  break
              per_class_stats = test_statistics[output_feature_name][
                  'per_class_stats']
              precisions = []
              recalls = []
              f1_scores = []
              labels = []
              for _, class_name in sorted(
                      [(metadata[output_feature_name]['str2idx'][key], key)
                       for key in per_class_stats.keys()],
                      key=lambda tup: tup[0]):
                  class_stats = per_class_stats[class_name]
                  precisions.append(class_stats['precision'])
                  recalls.append(class_stats['recall'])
                  f1_scores.append(class_stats['f1_score'])
                  labels.append(class_name)
              for k in top_n_classes:
                  k = min(k, len(precisions)) if k > 0 else len(precisions)
                  ps = precisions[0:k]
                  rs = recalls[0:k]
                  fs = f1_scores[0:k]
                  ls = labels[0:k]
  
                  filename = None
                  if filename_template_path:
                      os.makedirs(output_directory, exist_ok=True)
                      filename = filename_template_path.format(
                          model_name_name, output_feature_name, 'top{}'.format(k)
                      )
  
                  visualization_utils.compare_classifiers_multiclass_multimetric_plot(
                      [ps, rs, fs],
                      ['precision', 'recall', 'f1 score'],
                      labels=ls,
                      title='{} Multiclass Precision / Recall / '
                            'F1 Score top {} {}'.format(model_name_name, k,
                                                        output_feature_name),
                      filename=filename
                  )
  
                  p_np = np.nan_to_num(np.array(precisions, dtype=np.float32))
                  r_np = np.nan_to_num(np.array(recalls, dtype=np.float32))
                  f1_np = np.nan_to_num(np.array(f1_scores, dtype=np.float32))
                  labels_np = np.nan_to_num(np.array(labels))
  
                  sorted_indices = f1_np.argsort()
                  higher_f1s = sorted_indices[-k:][::-1]
                  filename = None
                  if filename_template_path:
                      os.makedirs(output_directory, exist_ok=True)
                      filename = filename_template_path.format(
                          model_name_name, output_feature_name,
                          'best{}'.format(k)
                      )
                  visualization_utils.compare_classifiers_multiclass_multimetric_plot(
                      [p_np[higher_f1s],
                       r_np[higher_f1s],
                       f1_np[higher_f1s]],
                      ['precision', 'recall', 'f1 score'],
                      labels=labels_np[higher_f1s].tolist(),
                      title='{} Multiclass Precision / Recall / '
                            'F1 Score best {} classes {}'.format(
                          model_name_name, k, output_feature_name),
                      filename=filename
                  )
                  lower_f1s = sorted_indices[:k]
                  filename = None
                  if filename_template_path:
                      filename = filename_template_path.format(
                          model_name_name, output_feature_name,
                          'worst{}'.format(k)
                      )
                  visualization_utils.compare_classifiers_multiclass_multimetric_plot(
                      [p_np[lower_f1s],
                       r_np[lower_f1s],
                       f1_np[lower_f1s]],
                      ['precision', 'recall', 'f1 score'],
                      labels=labels_np[lower_f1s].tolist(),
                      title='{} Multiclass Precision / Recall / F1 Score worst '
                            'k classes {}'.format(model_name_name, k,
                                                  output_feature_name),
                      filename=filename
                  )
  
                  filename = None
                  if filename_template_path:
                      filename = filename_template_path.format(
                          model_name_name, output_feature_name, 'sorted'
                      )
                  visualization_utils.compare_classifiers_multiclass_multimetric_plot(
                      [p_np[sorted_indices[::-1]],
                       r_np[sorted_indices[::-1]],
                       f1_np[sorted_indices[::-1]]],
                      ['precision', 'recall', 'f1 score'],
                      labels=labels_np[sorted_indices[::-1]].tolist(),
                      title='{} Multiclass Precision / Recall / F1 Score '
                            '{} sorted'.format(model_name_name,
                                               output_feature_name),
                      filename=filename
                  )
  
                  logging.info('\n')
                  logging.info(model_name_name)
                  tmp_str = '{0} best 5 classes: '.format(output_feature_name)
                  tmp_str += '{}'
                  logging.info(tmp_str.format(higher_f1s))
                  logging.info(f1_np[higher_f1s])
                  tmp_str = '{0} worst 5 classes: '.format(output_feature_name)
                  tmp_str += '{}'
                  logging.info(tmp_str.format(lower_f1s))
                  logging.info(f1_np[lower_f1s])
                  tmp_str = '{0} number of classes with f1 score > 0: '.format(
                      output_feature_name)
                  tmp_str += '{}'
                  logging.info(tmp_str.format(np.sum(f1_np > 0)))
                  tmp_str = '{0} number of classes with f1 score = 0: '.format(
                      output_feature_name)
                  tmp_str += '{}'
                  logging.info(tmp_str.format(np.sum(f1_np == 0)))
  
  
  def compare_classifiers_predictions(
          predictions_per_model: List[list],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          labels_limit: int,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show two models comparison of their output_feature_name predictions.
  
      # Inputs
  
      :param predictions_per_model: (List[list]) list containing the model
          predictions for the specified output_feature_name.
      :param ground_truth: (Union[pd.Series, np.ndarray]) ground truth values
      :param metadata: (dict) feature metadata dictionary
      :param output_feature_name: (str) output feature name
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      model_names_list = convert_to_list(model_names)
      name_c1 = (
          model_names_list[0] if model_names is not None and len(model_names) > 0
          else 'c1')
      name_c2 = (
          model_names_list[1] if model_names is not None and len(model_names) > 1
          else 'c2')
  
      pred_c1 = predictions_per_model[0]
      pred_c2 = predictions_per_model[1]
  
      if labels_limit > 0:
          ground_truth[ground_truth > labels_limit] = labels_limit
          pred_c1[pred_c1 > labels_limit] = labels_limit
          pred_c2[pred_c2 > labels_limit] = labels_limit
  
      # DOTO all shadows built in name - come up with a more descriptive name
      all = len(ground_truth)
      if all == 0:
          logger.error('No labels in the ground truth')
          return
  
      both_right = 0
      both_wrong_same = 0
      both_wrong_different = 0
      c1_right_c2_wrong = 0
      c1_wrong_c2_right = 0
  
      for i in range(all):
          if ground_truth[i] == pred_c1[i] and ground_truth[i] == pred_c2[i]:
              both_right += 1
          elif ground_truth[i] != pred_c1[i] and ground_truth[i] != pred_c2[i]:
              if pred_c1[i] == pred_c2[i]:
                  both_wrong_same += 1
              else:
                  both_wrong_different += 1
          elif ground_truth[i] == pred_c1[i] and ground_truth[i] != pred_c2[i]:
              c1_right_c2_wrong += 1
          elif ground_truth[i] != pred_c1[i] and ground_truth[i] == pred_c2[i]:
              c1_wrong_c2_right += 1
  
      one_right = c1_right_c2_wrong + c1_wrong_c2_right
      both_wrong = both_wrong_same + both_wrong_different
  
      logger.info('Test datapoints: {}'.format(all))
      logger.info(
          'Both right: {} {:.2f}%'.format(both_right, 100 * both_right / all))
      logger.info(
          'One right: {} {:.2f}%'.format(one_right, 100 * one_right / all))
      logger.info(
          '  {} right / {} wrong: {} {:.2f}% {:.2f}%'.format(
              name_c1,
              name_c2,
              c1_right_c2_wrong,
              100 * c1_right_c2_wrong / all,
              100 * c1_right_c2_wrong / one_right if one_right > 0 else 0
          )
      )
      logger.info(
          '  {} wrong / {} right: {} {:.2f}% {:.2f}%'.format(
              name_c1,
              name_c2,
              c1_wrong_c2_right,
              100 * c1_wrong_c2_right / all,
              100 * c1_wrong_c2_right / one_right if one_right > 0 else 0
          )
      )
      logger.info(
          'Both wrong: {} {:.2f}%'.format(both_wrong, 100 * both_wrong / all)
      )
      logger.info('  same prediction: {} {:.2f}% {:.2f}%'.format(
          both_wrong_same,
          100 * both_wrong_same / all,
          100 * both_wrong_same / both_wrong if both_wrong > 0 else 0
      )
      )
      logger.info('  different prediction: {} {:.2f}% {:.2f}%'.format(
          both_wrong_different,
          100 * both_wrong_different / all,
          100 * both_wrong_different / both_wrong if both_wrong > 0 else 0
      )
      )
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = os.path.join(
              output_directory,
              'compare_classifiers_predictions_{}_{}.{}'.format(
                  name_c1, name_c2, file_format
              )
          )
  
      visualization_utils.donut(
          [both_right, one_right, both_wrong],
          ['both right', 'one right', 'both wrong'],
          [both_right, c1_right_c2_wrong, c1_wrong_c2_right, both_wrong_same,
           both_wrong_different],
          ['both right',
           '{} right / {} wrong'.format(name_c1, name_c2),
           '{} wrong / {} right'.format(name_c1, name_c2),
           'same prediction', 'different prediction'],
          [0, 1, 1, 2, 2],
          title='{} vs {}'.format(name_c1, name_c2),
          tight_layout=kwargs.pop('tight_layout', True),
          filename=filename
      )
  
  
  def compare_classifiers_predictions_distribution(
          predictions_per_model: List[list],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          labels_limit: int,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show comparision of models predictions distribution for 10
      output_feature_name classes
  
      This visualization produces a radar plot comparing the distributions of
      predictions of the models for the first 10 classes of the specified
      output_feature_name.
  
      # Inputs
  
      :param predictions_per_model: (List[list]) list containing the model
          predictions for the specified output_feature_name.
      :param ground_truth: (Union[pd.Series, np.ndarray]) ground truth values
      :param metadata: (dict) feature metadata dictionary
      :param output_feature_name: (str) output feature name
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      model_names_list = convert_to_list(model_names)
      if labels_limit > 0:
          ground_truth[ground_truth > labels_limit] = labels_limit
          for i in range(len(predictions_per_model)):
              predictions_per_model[i][predictions_per_model[i] > labels_limit] \
                  = labels_limit
  
      max_gt = max(ground_truth)
      max_pred = max([max(alg_predictions)
                      for alg_predictions in predictions_per_model])
      max_val = max(max_gt, max_pred) + 1
  
      counts_gt = np.bincount(ground_truth, minlength=max_val)
      prob_gt = counts_gt / counts_gt.sum()
  
      counts_predictions = [np.bincount(alg_predictions, minlength=max_val)
                            for alg_predictions in predictions_per_model]
  
      prob_predictions = [alg_count_prediction / alg_count_prediction.sum()
                          for alg_count_prediction in counts_predictions]
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = os.path.join(
              output_directory,
              'compare_classifiers_predictions_distribution.' + file_format
          )
  
      visualization_utils.radar_chart(
          prob_gt,
          prob_predictions,
          model_names_list,
          filename=filename
      )
  
  
  def confidence_thresholding(
          probabilities_per_model: List[np.array],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          labels_limit: int,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show models accuracy and data coverage while increasing treshold
  
      For each model it produces a pair of lines indicating the accuracy of
      the model and the data coverage while increasing a threshold (x axis) on
      the probabilities of predictions for the specified output_feature_name.
  
      # Inputs
  
      :param probabilities_per_model: (List[numpy.array]) list of model
          probabilities.
      :param ground_truth: (Union[pd.Series, np.ndarray]) ground truth values
      :param metadata: (dict) feature metadata dictionary
      :param output_feature_name: (str) output feature name
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      if labels_limit > 0:
          ground_truth[ground_truth > labels_limit] = labels_limit
      probs = probabilities_per_model
      model_names_list = convert_to_list(model_names)
      thresholds = [t / 100 for t in range(0, 101, 5)]
  
      accuracies = []
      dataset_kept = []
  
      for i, prob in enumerate(probs):
  
          if labels_limit > 0 and prob.shape[1] > labels_limit + 1:
              prob_limit = prob[:, :labels_limit + 1]
              prob_limit[:, labels_limit] = prob[:, labels_limit:].sum(1)
              prob = prob_limit
  
          max_prob = np.max(prob, axis=1)
          predictions = np.argmax(prob, axis=1)
  
          accuracies_alg = []
          dataset_kept_alg = []
  
          for threshold in thresholds:
              threshold = threshold if threshold < 1 else 0.999
              filtered_indices = max_prob >= threshold
              filtered_gt = ground_truth[filtered_indices]
              filtered_predictions = predictions[filtered_indices]
              accuracy = (
                      (filtered_gt == filtered_predictions).sum() /
                      len(filtered_gt)
              )
  
              accuracies_alg.append(accuracy)
              dataset_kept_alg.append(len(filtered_gt) / len(ground_truth))
  
          accuracies.append(accuracies_alg)
          dataset_kept.append(dataset_kept_alg)
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = os.path.join(
              output_directory,
              'confidence_thresholding.' + file_format
          )
  
      visualization_utils.confidence_fitlering_plot(
          thresholds,
          accuracies,
          dataset_kept,
          model_names_list,
          title='Confidence_Thresholding',
          filename=filename
      )
  
  
  def confidence_thresholding_data_vs_acc(
          probabilities_per_model: List[np.array],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          labels_limit: int,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show models comparison of confidence threshold data vs accuracy.
  
      For each model it produces a line indicating the accuracy of the model
      and the data coverage while increasing a threshold on the probabilities
      of predictions for the specified output_feature_name. The difference with
      confidence_thresholding is that it uses two axes instead of three,
      not visualizing the threshold and having coverage as x axis instead of
      the threshold.
  
      # Inputs
  
      :param probabilities_per_model: (List[numpy.array]) list of model
          probabilities.
      :param ground_truth: (Union[pd.Series, np.ndarray]) ground truth values
      :param metadata: (dict) feature metadata dictionary
      :param output_feature_name: (str) output feature name
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
      :return: (None)
      """
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      if labels_limit > 0:
          ground_truth[ground_truth > labels_limit] = labels_limit
      probs = probabilities_per_model
      model_names_list = convert_to_list(model_names)
      thresholds = [t / 100 for t in range(0, 101, 5)]
  
      accuracies = []
      dataset_kept = []
  
      for i, prob in enumerate(probs):
  
          if labels_limit > 0 and prob.shape[1] > labels_limit + 1:
              prob_limit = prob[:, :labels_limit + 1]
              prob_limit[:, labels_limit] = prob[:, labels_limit:].sum(1)
              prob = prob_limit
  
          max_prob = np.max(prob, axis=1)
          predictions = np.argmax(prob, axis=1)
  
          accuracies_alg = []
          dataset_kept_alg = []
  
          for threshold in thresholds:
              threshold = threshold if threshold < 1 else 0.999
              filtered_indices = max_prob >= threshold
              filtered_gt = ground_truth[filtered_indices]
              filtered_predictions = predictions[filtered_indices]
              accuracy = ((filtered_gt == filtered_predictions).sum() /
                          len(filtered_gt))
  
              accuracies_alg.append(accuracy)
              dataset_kept_alg.append(len(filtered_gt) / len(ground_truth))
  
          accuracies.append(accuracies_alg)
          dataset_kept.append(dataset_kept_alg)
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = os.path.join(
              output_directory,
              'confidence_thresholding_data_vs_acc.' + file_format
          )
  
      visualization_utils.confidence_fitlering_data_vs_acc_plot(
          accuracies,
          dataset_kept,
          model_names_list,
          title='Confidence_Thresholding (Data vs Accuracy)',
          filename=filename
      )
  
  
  def confidence_thresholding_data_vs_acc_subset(
          probabilities_per_model: List[np.array],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          top_n_classes: List[int],
          labels_limit: int,
          subset: str,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show models comparison of confidence threshold data vs accuracy on a
      subset of data.
  
      For each model it produces a line indicating the accuracy of the model
      and the data coverage while increasing a threshold on the probabilities
      of predictions for the specified output_feature_name, considering only a subset of the
      full training set. The way the subset is obtained is using the `top_n_classes`
      and subset parameters.
       The difference with confidence_thresholding is that it uses two axes
       instead of three, not visualizing the threshold and having coverage as
       x axis instead of the threshold.
  
      If the values of subset is `ground_truth`, then only datapoints where the
      ground truth class is within the top n most frequent ones will be
      considered  as test set, and the percentage of datapoints that have been
      kept  from the original set will be displayed. If the values of subset is
       `predictions`, then only datapoints where the the model predicts a class
       that is within the top n most frequent ones will be considered as test set,
       and the percentage of datapoints that have been kept from the original set
       will be displayed for each model.
  
      # Inputs
  
      :param probabilities_per_model: (List[numpy.array]) list of model
          probabilities.
      :param ground_truth: (Union[pd.Series, np.ndarray]) ground truth values
      :param metadata: (dict) feature metadata dictionary
      :param output_feature_name: (str) output feature name
      :param top_n_classes: (List[int]) list containing the number of classes
          to plot.
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param subset: (str) string specifying type of subset filtering.  Valid
          values are `ground_truth` or `predictions`.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      top_n_classes_list = convert_to_list(top_n_classes)
      k = top_n_classes_list[0]
      if labels_limit > 0:
          ground_truth[ground_truth > labels_limit] = labels_limit
      probs = probabilities_per_model
      model_names_list = convert_to_list(model_names)
      thresholds = [t / 100 for t in range(0, 101, 5)]
  
      accuracies = []
      dataset_kept = []
  
      subset_indices = ground_truth > 0
      gt_subset = ground_truth
      if subset == 'ground_truth':
          subset_indices = ground_truth < k
          gt_subset = ground_truth[subset_indices]
          logger.info('Subset is {:.2f}% of the data'.format(
              len(gt_subset) / len(ground_truth) * 100)
          )
  
      for i, prob in enumerate(probs):
  
          if labels_limit > 0 and prob.shape[1] > labels_limit + 1:
              prob_limit = prob[:, :labels_limit + 1]
              prob_limit[:, labels_limit] = prob[:, labels_limit:].sum(1)
              prob = prob_limit
  
          if subset == PREDICTIONS:
              subset_indices = np.argmax(prob, axis=1) < k
              gt_subset = ground_truth[subset_indices]
              logger.info(
                  'Subset for model_name {} is {:.2f}% of the data'.format(
                      model_names[i] if model_names and i < len(
                          model_names) else i,
                      len(gt_subset) / len(ground_truth) * 100
                  )
              )
  
          prob_subset = prob[subset_indices]
  
          max_prob = np.max(prob_subset, axis=1)
          predictions = np.argmax(prob_subset, axis=1)
  
          accuracies_alg = []
          dataset_kept_alg = []
  
          for threshold in thresholds:
              threshold = threshold if threshold < 1 else 0.999
              filtered_indices = max_prob >= threshold
              filtered_gt = gt_subset[filtered_indices]
              filtered_predictions = predictions[filtered_indices]
              accuracy = ((filtered_gt == filtered_predictions).sum() /
                          len(filtered_gt))
  
              accuracies_alg.append(accuracy)
              dataset_kept_alg.append(len(filtered_gt) / len(ground_truth))
  
          accuracies.append(accuracies_alg)
          dataset_kept.append(dataset_kept_alg)
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = os.path.join(
              output_directory,
              'confidence_thresholding_data_vs_acc_subset.' + file_format
          )
  
      visualization_utils.confidence_fitlering_data_vs_acc_plot(
          accuracies,
          dataset_kept,
          model_names_list,
          title='Confidence_Thresholding (Data vs Accuracy)',
          filename=filename
      )
  
  
  def confidence_thresholding_data_vs_acc_subset_per_class(
          probabilities_per_model: List[np.array],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          top_n_classes: Union[int, List[int]],
          labels_limit: int,
          subset: str,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show models comparison of confidence threshold data vs accuracy on a
      subset of data per class in top n classes.
  
      For each model (in the aligned lists of probabilities and model_names)
      it produces a line indicating the accuracy of the model and the data
      coverage while increasing a threshold on the probabilities of
      predictions for the specified output_feature_name, considering only a subset of the
      full training set. The way the subset is obtained is using the
      `top_n_classes`  and `subset` parameters.  The difference with
      confidence_thresholding is that it uses two axes instead of three,
      not visualizing the threshold and having coverage as x axis instead of
      the  threshold.
  
      If the values of subset is `ground_truth`, then only datapoints where the
      ground truth class is within the top n most frequent ones will be
      considered  as test set, and the percentage of datapoints that have been
      kept from the original set will be displayed. If the values of subset is
      `predictions`, then only datapoints where the the model predicts a class that
      is within the top n most frequent ones will be considered as test set, and
      the percentage of datapoints that have been kept from the original set will
      be displayed for each model.
  
      The difference with confidence_thresholding_data_vs_acc_subset is that it
      produces one plot per class within the top_n_classes.
  
      # Inputs
  
      :param probabilities_per_model: (List[numpy.array]) list of model
          probabilities.
      :param ground_truth: (Union[pd.Series, np.ndarray]) ground truth values
      :param metadata: (dict) intermediate preprocess structure created during
          training containing the mappings of the input dataset.
      :param output_feature_name: (str) name of the output feature to use
          for the visualization.
      :param top_n_classes: (Union[int, List[int]]) number of top classes or list
          containing the number of top classes to plot.
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param subset: (str) string specifying type of subset filtering.  Valid
          values are `ground_truth` or `predictions`.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
      :return: (None)
      """
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      filename_template = \
          'confidence_thresholding_data_vs_acc_subset_per_class_{}.' + file_format
      filename_template_path = generate_filename_template_path(
          output_directory,
          filename_template
      )
      top_n_classes_list = convert_to_list(top_n_classes)
      k = top_n_classes_list[0]
      if labels_limit > 0:
          ground_truth[ground_truth > labels_limit] = labels_limit
      probs = probabilities_per_model
      model_names_list = convert_to_list(model_names)
  
      thresholds = [t / 100 for t in range(0, 101, 5)]
  
      for curr_k in range(k):
          accuracies = []
          dataset_kept = []
  
          subset_indices = ground_truth > 0
          gt_subset = ground_truth
          if subset == 'ground_truth':
              subset_indices = ground_truth == curr_k
              gt_subset = ground_truth[subset_indices]
              logger.info('Subset is {:.2f}% of the data'.format(
                  len(gt_subset) / len(ground_truth) * 100)
              )
  
          for i, prob in enumerate(probs):
  
              if labels_limit > 0 and prob.shape[1] > labels_limit + 1:
                  prob_limit = prob[:, :labels_limit + 1]
                  prob_limit[:, labels_limit] = prob[:, labels_limit:].sum(1)
                  prob = prob_limit
  
              if subset == PREDICTIONS:
                  subset_indices = np.argmax(prob, axis=1) == curr_k
                  gt_subset = ground_truth[subset_indices]
                  logger.info(
                      'Subset for model_name {} is {:.2f}% of the data'.format(
                          model_names_list[i] if model_names_list and i < len(
                              model_names_list) else i,
                          len(gt_subset) / len(ground_truth) * 100
                      )
                  )
  
              prob_subset = prob[subset_indices]
  
              max_prob = np.max(prob_subset, axis=1)
              predictions = np.argmax(prob_subset, axis=1)
  
              accuracies_alg = []
              dataset_kept_alg = []
  
              for threshold in thresholds:
                  threshold = threshold if threshold < 1 else 0.999
                  filtered_indices = max_prob >= threshold
                  filtered_gt = gt_subset[filtered_indices]
                  filtered_predictions = predictions[filtered_indices]
                  accuracy = ((filtered_gt == filtered_predictions).sum() /
                              len(filtered_gt) if len(filtered_gt) > 0 else 0)
  
                  accuracies_alg.append(accuracy)
                  dataset_kept_alg.append(len(filtered_gt) / len(ground_truth))
  
              accuracies.append(accuracies_alg)
              dataset_kept.append(dataset_kept_alg)
  
          output_feature_name_name = metadata[output_feature_name]['idx2str'][
              curr_k]
  
          filename = None
          if filename_template_path:
              os.makedirs(output_directory, exist_ok=True)
              filename = filename_template_path.format(output_feature_name_name)
  
          visualization_utils.confidence_fitlering_data_vs_acc_plot(
              accuracies, dataset_kept, model_names_list,
              decimal_digits=2,
              title='Confidence_Thresholding (Data vs Accuracy) '
                    'for class {}'.format(output_feature_name_name),
              filename=filename
          )
  
  
  def confidence_thresholding_2thresholds_2d(
          probabilities_per_model: List[np.array],
          ground_truths: Union[List[np.array], List[pd.Series]],
          metadata,
          threshold_output_feature_names: List[str],
          labels_limit: int,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show confidence threshold data vs accuracy for two output feature names.
  
      The first plot shows several semi transparent lines. They summarize the
      3d surfaces displayed by confidence_thresholding_2thresholds_3d that have
      thresholds on the confidence of the predictions of the two
      `threshold_output_feature_names`  as x and y axes and either the data
      coverage percentage or
      the accuracy as z axis. Each line represents a slice of the data
      coverage  surface projected onto the accuracy surface.
  
      # Inputs
  
      :param probabilities_per_model: (List[numpy.array]) list of model
          probabilities.
      :param ground_truth: (Union[List[np.array], List[pd.Series]]) containing
          ground truth data
      :param metadata: (dict) feature metadata dictionary
      :param threshold_output_feature_names: (List[str]) List containing two output
          feature names for visualization.
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      try:
          validate_conf_treshholds_and_probabilities_2d_3d(
              probabilities_per_model,
              threshold_output_feature_names
          )
      except RuntimeError:
          return
      probs = probabilities_per_model
      model_names_list = convert_to_list(model_names)
      filename_template = \
          'confidence_thresholding_2thresholds_2d_{}.' + file_format
      filename_template_path = generate_filename_template_path(
          output_directory,
          filename_template
      )
  
      if not isinstance(ground_truths[0], np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[threshold_output_feature_names[0]]
          vfunc = np.vectorize(_encode_categorical_feature)
          gt_1 = vfunc(ground_truths[0], feature_metadata['str2idx'])
          feature_metadata = metadata[threshold_output_feature_names[1]]
          gt_2 = vfunc(ground_truths[1], feature_metadata['str2idx'])
      else:
          gt_1 = ground_truths[0]
          gt_2 = ground_truths[1]
  
      if labels_limit > 0:
          gt_1[gt_1 > labels_limit] = labels_limit
          gt_2[gt_2 > labels_limit] = labels_limit
  
      thresholds = [t / 100 for t in range(0, 101, 5)]
      fixed_step_coverage = thresholds
      name_t1 = '{} threshold'.format(threshold_output_feature_names[0])
      name_t2 = '{} threshold'.format(threshold_output_feature_names[1])
  
      accuracies = []
      dataset_kept = []
      interps = []
      table = [[name_t1, name_t2, 'coverage', ACCURACY]]
  
      if labels_limit > 0 and probs[0].shape[1] > labels_limit + 1:
          prob_limit = probs[0][:, :labels_limit + 1]
          prob_limit[:, labels_limit] = probs[0][:, labels_limit:].sum(1)
          probs[0] = prob_limit
  
      if labels_limit > 0 and probs[1].shape[1] > labels_limit + 1:
          prob_limit = probs[1][:, :labels_limit + 1]
          prob_limit[:, labels_limit] = probs[1][:, labels_limit:].sum(1)
          probs[1] = prob_limit
  
      max_prob_1 = np.max(probs[0], axis=1)
      predictions_1 = np.argmax(probs[0], axis=1)
  
      max_prob_2 = np.max(probs[1], axis=1)
      predictions_2 = np.argmax(probs[1], axis=1)
  
      for threshold_1 in thresholds:
          threshold_1 = threshold_1 if threshold_1 < 1 else 0.999
          curr_accuracies = []
          curr_dataset_kept = []
  
          for threshold_2 in thresholds:
              threshold_2 = threshold_2 if threshold_2 < 1 else 0.999
  
              filtered_indices = np.logical_and(
                  max_prob_1 >= threshold_1,
                  max_prob_2 >= threshold_2
              )
  
              filtered_gt_1 = gt_1[filtered_indices]
              filtered_predictions_1 = predictions_1[filtered_indices]
              filtered_gt_2 = gt_2[filtered_indices]
              filtered_predictions_2 = predictions_2[filtered_indices]
  
              coverage = len(filtered_gt_1) / len(gt_1)
              accuracy = (
                             np.logical_and(
                                 filtered_gt_1 == filtered_predictions_1,
                                 filtered_gt_2 == filtered_predictions_2
                             )
                         ).sum() / len(filtered_gt_1)
  
              curr_accuracies.append(accuracy)
              curr_dataset_kept.append(coverage)
              table.append([threshold_1, threshold_2, coverage, accuracy])
  
          accuracies.append(curr_accuracies)
          dataset_kept.append(curr_dataset_kept)
          interps.append(
              np.interp(
                  fixed_step_coverage,
                  list(reversed(curr_dataset_kept)),
                  list(reversed(curr_accuracies)),
                  left=1,
                  right=0
              )
          )
  
      logger.info('CSV table')
      for row in table:
          logger.info(','.join([str(e) for e in row]))
  
      # ===========#
      # Multiline #
      # ===========#
      filename = None
      if filename_template_path:
          os.makedirs(output_directory, exist_ok=True)
          filename = filename_template_path.format('multiline')
      visualization_utils.confidence_fitlering_data_vs_acc_multiline_plot(
          accuracies,
          dataset_kept,
          model_names_list,
          title='Coverage vs Accuracy, two thresholds',
          filename=filename
      )
  
      # ==========#
      # Max line #
      # ==========#
      filename = None
      if filename_template_path:
          filename = filename_template_path.format('maxline')
      max_accuracies = np.amax(np.array(interps), 0)
      visualization_utils.confidence_fitlering_data_vs_acc_plot(
          [max_accuracies],
          [thresholds],
          model_names_list,
          title='Coverage vs Accuracy, two thresholds',
          filename=filename
      )
  
      # ==========================#
      # Max line with thresholds #
      # ==========================#
      acc_matrix = np.array(accuracies)
      cov_matrix = np.array(dataset_kept)
      t1_maxes = [1]
      t2_maxes = [1]
      for i in range(len(fixed_step_coverage) - 1):
          lower = fixed_step_coverage[i]
          upper = fixed_step_coverage[i + 1]
          indices = np.logical_and(cov_matrix >= lower, cov_matrix < upper)
          selected_acc = acc_matrix.copy()
          selected_acc[np.logical_not(indices)] = -1
          threshold_indices = np.unravel_index(
              np.argmax(selected_acc, axis=None),
              selected_acc.shape)
          t1_maxes.append(thresholds[threshold_indices[0]])
          t2_maxes.append(thresholds[threshold_indices[1]])
      model_name = model_names_list[0] if model_names_list is not None and len(
          model_names_list) > 0 else ''
  
      filename = None
      if filename_template_path:
          os.makedirs(output_directory, exist_ok=True)
          filename = filename_template_path.format('maxline_with_thresholds')
  
      visualization_utils.confidence_fitlering_data_vs_acc_plot(
          [max_accuracies, t1_maxes, t2_maxes],
          [fixed_step_coverage, fixed_step_coverage, fixed_step_coverage],
          model_names=[model_name + ' accuracy', name_t1, name_t2],
          dotted=[False, True, True],
          y_label='',
          title='Coverage vs Accuracy & Threshold',
          filename=filename
      )
  
  
  def confidence_thresholding_2thresholds_3d(
          probabilities_per_model: List[np.array],
          ground_truths: Union[List[np.array], List[pd.Series]],
          metadata,
          threshold_output_feature_names: List[str],
          labels_limit: int,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show 3d confidence threshold data vs accuracy for two output feature names.
  
      The plot shows the 3d surfaces displayed by
      confidence_thresholding_2thresholds_3d that have thresholds on the
      confidence of the predictions of the two `threshold_output_feature_names`
      as x and y axes and either the data coverage percentage or the accuracy
      as z axis.
  
      # Inputs
  
      :param probabilities_per_model: (List[numpy.array]) list of model
          probabilities.
      :param ground_truth: (Union[List[np.array], List[pd.Series]]) containing
          ground truth data
      :param metadata: (dict) feature metadata dictionary
      :param threshold_output_feature_names: (List[str]) List containing two output
          feature names for visualization.
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      try:
          validate_conf_treshholds_and_probabilities_2d_3d(
              probabilities_per_model,
              threshold_output_feature_names
          )
      except RuntimeError:
          return
      probs = probabilities_per_model
  
      if not isinstance(ground_truths[0], np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[threshold_output_feature_names[0]]
          vfunc = np.vectorize(_encode_categorical_feature)
          gt_1 = vfunc(ground_truths[0], feature_metadata['str2idx'])
          feature_metadata = metadata[threshold_output_feature_names[1]]
          gt_2 = vfunc(ground_truths[1], feature_metadata['str2idx'])
      else:
          gt_1 = ground_truths[0]
          gt_2 = ground_truths[1]
  
      if labels_limit > 0:
          gt_1[gt_1 > labels_limit] = labels_limit
          gt_2[gt_2 > labels_limit] = labels_limit
  
      thresholds = [t / 100 for t in range(0, 101, 5)]
  
      accuracies = []
      dataset_kept = []
  
      if labels_limit > 0 and probs[0].shape[1] > labels_limit + 1:
          prob_limit = probs[0][:, :labels_limit + 1]
          prob_limit[:, labels_limit] = probs[0][:, labels_limit:].sum(1)
          probs[0] = prob_limit
  
      if labels_limit > 0 and probs[1].shape[1] > labels_limit + 1:
          prob_limit = probs[1][:, :labels_limit + 1]
          prob_limit[:, labels_limit] = probs[1][:, labels_limit:].sum(1)
          probs[1] = prob_limit
  
      max_prob_1 = np.max(probs[0], axis=1)
      predictions_1 = np.argmax(probs[0], axis=1)
  
      max_prob_2 = np.max(probs[1], axis=1)
      predictions_2 = np.argmax(probs[1], axis=1)
  
      for threshold_1 in thresholds:
          threshold_1 = threshold_1 if threshold_1 < 1 else 0.999
          curr_accuracies = []
          curr_dataset_kept = []
  
          for threshold_2 in thresholds:
              threshold_2 = threshold_2 if threshold_2 < 1 else 0.999
  
              filtered_indices = np.logical_and(
                  max_prob_1 >= threshold_1,
                  max_prob_2 >= threshold_2
              )
  
              filtered_gt_1 = gt_1[filtered_indices]
              filtered_predictions_1 = predictions_1[filtered_indices]
              filtered_gt_2 = gt_2[filtered_indices]
              filtered_predictions_2 = predictions_2[filtered_indices]
  
              accuracy = (
                             np.logical_and(
                                 filtered_gt_1 == filtered_predictions_1,
                                 filtered_gt_2 == filtered_predictions_2
                             )
                         ).sum() / len(filtered_gt_1)
  
              curr_accuracies.append(accuracy)
              curr_dataset_kept.append(len(filtered_gt_1) / len(gt_1))
  
          accuracies.append(curr_accuracies)
          dataset_kept.append(curr_dataset_kept)
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = os.path.join(
              output_directory,
              'confidence_thresholding_2thresholds_3d.' + file_format
          )
  
      visualization_utils.confidence_fitlering_3d_plot(
          np.array(thresholds),
          np.array(thresholds),
          np.array(accuracies),
          np.array(dataset_kept),
          threshold_output_feature_names,
          title='Confidence_Thresholding, two thresholds',
          filename=filename
      )
  
  
  def binary_threshold_vs_metric(
          probabilities_per_model: List[np.array],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          metrics: List[str],
          positive_label: int = 1,
          model_names: List[str] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show confidence of the model against metric for the specified output_feature_name.
  
      For each metric specified in metrics (options are `f1`, `precision`, `recall`,
      `accuracy`), this visualization produces a line chart plotting a threshold
      on  the confidence of the model against the metric for the specified
      output_feature_name.  If output_feature_name is a category feature,
      positive_label, which is specified as the numeric encoded value, indicates
      the class to be considered positive class and all others will be
      considered negative. To figure out the
      association between classes and numeric encoded values check the
      ground_truth_metadata JSON file.
  
      # Inputs
  
      :param probabilities_per_model: (List[numpy.array]) list of model
          probabilities.
      :param ground_truth: (Union[pd.Series, np.ndarray]) ground truth values
      :param metadata: (dict) feature metadata dictionary
      :param output_feature_name: (str) output feature name
      :param metrics: (List[str]) metrics to display (`'f1'`, `'precision'`,
          `'recall'`, `'accuracy'`).
      :param positive_label: (int, default: `1`) numeric encoded value for the
          positive class.
      :param model_names: (List[str], default: `None`) list of the names of the
          models to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (`None`)
      """
  
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      probs = probabilities_per_model
      model_names_list = convert_to_list(model_names)
      metrics_list = convert_to_list(metrics)
      filename_template = 'binary_threshold_vs_metric_{}.' + file_format
      filename_template_path = generate_filename_template_path(
          output_directory,
          filename_template
      )
  
      thresholds = [t / 100 for t in range(0, 101, 5)]
  
      supported_metrics = {'f1', 'precision', 'recall', 'accuracy'}
  
      for metric in metrics_list:
  
          if metric not in supported_metrics:
              logger.error("Metric {} not supported".format(metric))
              continue
  
          scores = []
  
          for i, prob in enumerate(probs):
  
              scores_alg = []
  
              if len(prob.shape) == 2:
                  if prob.shape[1] > positive_label:
                      prob = prob[:, positive_label]
                  else:
                      raise Exception(
                          'the specified positive label {} is not '
                          'present in the probabilities'.format(
                              positive_label
                          )
                      )
  
              for threshold in thresholds:
                  threshold = threshold if threshold < 1 else 0.99
  
                  t_gt = ground_truth[prob >= threshold]
                  predictions = prob >= threshold
                  t_predictions = predictions[prob >= threshold]
  
                  if metric == 'f1':
                      metric_score = sklearn.metrics.f1_score(
                          t_gt,
                          t_predictions
                      )
                  elif metric == 'precision':
                      metric_score = sklearn.metrics.precision_score(
                          t_gt,
                          t_predictions
                      )
                  elif metric == 'recall':
                      metric_score = sklearn.metrics.recall_score(
                          t_gt,
                          t_predictions
                      )
                  elif metric == ACCURACY:
                      metric_score = sklearn.metrics.accuracy_score(
                          t_gt,
                          t_predictions
                      )
  
                  scores_alg.append(metric_score)
  
              scores.append(scores_alg)
  
          filename = None
          if output_directory:
              os.makedirs(output_directory, exist_ok=True)
              filename = filename_template_path.format(metric)
  
          visualization_utils.threshold_vs_metric_plot(
              thresholds,
              scores,
              model_names_list,
              title='Binary threshold vs {}'.format(metric),
              filename=filename
          )
  
  
  def roc_curves(
          probabilities_per_model: List[np.array],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          positive_label: int = 1,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show the roc curves for output features in the specified models.
  
      This visualization produces a line chart plotting the roc curves for the
      specified output feature name. If output feature name is a category feature,
      `positive_label` indicates which is the class to be considered positive
      class and all the others will be considered negative. `positive_label` is
      the encoded numeric value for category classes. The numeric value can be
      determined by association between classes and integers captured in the
      training metadata JSON file.
  
      # Inputs
  
      :param probabilities_per_model: (List[numpy.array]) list of model
          probabilities.
      :param ground_truth: (Union[pd.Series, np.ndarray]) ground truth values
      :param metadata: (dict) feature metadata dictionary
      :param output_feature_name: (str) output feature name
      :param positive_label: (int, default: `1`) numeric encoded value for the
          positive class.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      probs = probabilities_per_model
      model_names_list = convert_to_list(model_names)
      fpr_tprs = []
  
      for i, prob in enumerate(probs):
          if len(prob.shape) > 1:
              prob = prob[:, positive_label]
          fpr, tpr, _ = sklearn.metrics.roc_curve(
              ground_truth, prob,
              pos_label=positive_label
          )
          fpr_tprs.append((fpr, tpr))
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = os.path.join(
              output_directory,
              'roc_curves.' + file_format
          )
  
      visualization_utils.roc_curves(
          fpr_tprs,
          model_names_list,
          title='ROC curves',
          filename=filename
      )
  
  
  def roc_curves_from_test_statistics(
          test_stats_per_model: List[dict],
          output_feature_name: str,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show the roc curves for the specified models output binary
      `output_feature_name`.
  
      This visualization uses `output_feature_name`, `test_stats_per_model` and
      `model_names` parameters. `output_feature_name` needs to be binary feature.
      This visualization produces a line chart plotting the roc curves for the
      specified `output_feature_name`.
  
      # Inputs
  
      :param test_stats_per_model: (List[dict]) dictionary containing evaluation
          performance statistics.
      :param output_feature_name: (str) name of the output feature to use
          for the visualization.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      model_names_list = convert_to_list(model_names)
      filename_template = 'roc_curves_from_prediction_statistics.' + file_format
      filename_template_path = generate_filename_template_path(
          output_directory,
          filename_template
      )
      fpr_tprs = []
      for curr_test_statistics in test_stats_per_model:
          fpr = curr_test_statistics[output_feature_name]['roc_curve'][
              'false_positive_rate']
          tpr = curr_test_statistics[output_feature_name]['roc_curve'][
              'true_positive_rate']
          fpr_tprs.append((fpr, tpr))
  
      visualization_utils.roc_curves(
          fpr_tprs,
          model_names_list,
          title='ROC curves',
          filename=filename_template_path
      )
  
  
  def calibration_1_vs_all(
          probabilities_per_model: List[np.array],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          top_n_classes: List[int],
          labels_limit: int,
          model_names: List[str] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show models probability of predictions for the specified output_feature_name.
  
      For each class or each of the k most frequent classes if top_k is
      specified,  it produces two plots computed on the fly from the
      probabilities  of predictions for the specified output_feature_name.
  
      The first plot is a calibration curve that shows the calibration of the
      predictions considering the current class to be the true one and all
      others  to be a false one, drawing one line for each model (in the
      aligned  lists of probabilities and model_names).
  
      The second plot shows the distributions of the predictions considering
      the  current class to be the true one and all others to be a false one,
      drawing the distribution for each model (in the aligned lists of
      probabilities and model_names).
  
      # Inputs
  
      :param probabilities_per_model: (List[numpy.array]) list of model
          probabilities.
      :param ground_truth: (Union[pd.Series, np.ndarray]) ground truth values
      :param metadata: (dict) feature metadata dictionary
      :param output_feature_name: (str) output feature name
      :param top_n_classes: (list) List containing the number of classes to plot.
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param model_names: (List[str], default: `None`) list of the names of the
          models to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # String
  
      :return: (None)
      """
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      probs = probabilities_per_model
      model_names_list = convert_to_list(model_names)
      filename_template = 'calibration_1_vs_all_{}.' + file_format
      filename_template_path = generate_filename_template_path(
          output_directory,
          filename_template
      )
      if labels_limit > 0:
          ground_truth[ground_truth > labels_limit] = labels_limit
      for i, prob in enumerate(probs):
          if labels_limit > 0 and prob.shape[1] > labels_limit + 1:
              prob_limit = prob[:, :labels_limit + 1]
              prob_limit[:, labels_limit] = prob[:, labels_limit:].sum(1)
              probs[i] = prob_limit
  
      num_classes = max(ground_truth) + 1
  
      brier_scores = []
  
      classes = (min(num_classes, top_n_classes[0]) if top_n_classes[0] > 0
                 else num_classes)
  
      for class_idx in range(classes):
          fraction_positives_class = []
          mean_predicted_vals_class = []
          probs_class = []
          brier_scores_class = []
          for prob in probs:
              # ground_truth is an vector of integers, each integer is a class
              # index to have a [0,1] vector we have to check if the value equals
              # the input class index and convert the resulting boolean vector
              # into an integer vector probabilities is a n x c matrix, n is the
              # number of datapoints and c number of classes; its values are the
              # probabilities of the ith datapoint to be classified as belonging
              # to the jth class according to the learned model. For this reason
              # we need to take only the column of predictions that is about the
              # class we are interested in, the input class index
  
              gt_class = (ground_truth == class_idx).astype(int)
              prob_class = prob[:, class_idx]
  
              (
                  curr_fraction_positives,
                  curr_mean_predicted_vals
              ) = calibration_curve(gt_class, prob_class, n_bins=21)
  
              if len(curr_fraction_positives) < 2:
                  curr_fraction_positives = np.concatenate(
                      (np.array([0.]), curr_fraction_positives)
                  )
              if len(curr_mean_predicted_vals) < 2:
                  curr_mean_predicted_vals = np.concatenate(
                      (np.array([0.]), curr_mean_predicted_vals)
                  )
  
              fraction_positives_class.append(curr_fraction_positives)
              mean_predicted_vals_class.append(curr_mean_predicted_vals)
              probs_class.append(prob[:, class_idx])
              brier_scores_class.append(
                  brier_score_loss(
                      gt_class,
                      prob_class, pos_label=1
                  )
              )
  
          brier_scores.append(brier_scores_class)
  
          filename = None
          if output_directory:
              os.makedirs(output_directory, exist_ok=True)
              filename = filename_template_path.format(class_idx)
  
          visualization_utils.calibration_plot(
              fraction_positives_class,
              mean_predicted_vals_class,
              model_names_list,
              filename=filename
          )
  
          filename = None
          if output_directory:
              os.makedirs(output_directory, exist_ok=True)
              filename = filename_template_path.format(
                  'prediction_distribution_' + str(class_idx)
              )
  
          visualization_utils.predictions_distribution_plot(
              probs_class,
              model_names_list,
              filename=filename
          )
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = filename_template_path.format('brier')
  
      visualization_utils.brier_plot(
          np.array(brier_scores),
          model_names_list,
          filename=filename
      )
  
  
  def calibration_multiclass(
          probabilities_per_model: List[np.array],
          ground_truth: Union[pd.Series, np.ndarray],
          metadata: dict,
          output_feature_name: str,
          labels_limit: int,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show models probability of predictions for each class of the
      specified output_feature_name.
  
      # Inputs
  
      :param probabilities_per_model: (List[numpy.array]) list of model
          probabilities.
      :param ground_truth: (Union[pd.Series, np.ndarray]) ground truth values
      :param metadata: (dict) feature metadata dictionary
      :param output_feature_name: (str) output feature name
      :param labels_limit: (int) upper limit on the numeric encoded label value.
          Encoded numeric label values in dataset that are higher than
          `label_limit` are considered to be "rare" labels.
      :param model_names: (List[str], default: `None`) list of the names of the
          models to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      if not isinstance(ground_truth, np.ndarray):
          # not np array, assume we need to translate raw value to encoded value
          feature_metadata = metadata[output_feature_name]
          vfunc = np.vectorize(_encode_categorical_feature)
          ground_truth = vfunc(ground_truth, feature_metadata['str2idx'])
  
      probs = probabilities_per_model
      model_names_list = convert_to_list(model_names)
      filename_template = 'calibration_multiclass{}.' + file_format
      filename_template_path = generate_filename_template_path(
          output_directory,
          filename_template
      )
      if labels_limit > 0:
          ground_truth[ground_truth > labels_limit] = labels_limit
  
      prob_classes = 0
      for i, prob in enumerate(probs):
          if labels_limit > 0 and prob.shape[1] > labels_limit + 1:
              prob_limit = prob[:, :labels_limit + 1]
              prob_limit[:, labels_limit] = prob[:, labels_limit:].sum(1)
              probs[i] = prob_limit
          if probs[i].shape[1] > prob_classes:
              prob_classes = probs[i].shape[1]
  
      gt_one_hot_dim_2 = max(prob_classes, max(ground_truth) + 1)
      gt_one_hot = np.zeros((len(ground_truth), gt_one_hot_dim_2))
      gt_one_hot[np.arange(len(ground_truth)), ground_truth] = 1
      gt_one_hot_flat = gt_one_hot.flatten()
  
      fraction_positives = []
      mean_predicted_vals = []
      brier_scores = []
      for prob in probs:
          # flatten probabilities to be compared to flatten ground truth
          prob_flat = prob.flatten()
          curr_fraction_positives, curr_mean_predicted_vals = calibration_curve(
              gt_one_hot_flat,
              prob_flat,
              n_bins=21
          )
          fraction_positives.append(curr_fraction_positives)
          mean_predicted_vals.append(curr_mean_predicted_vals)
          brier_scores.append(
              brier_score_loss(
                  gt_one_hot_flat,
                  prob_flat,
                  pos_label=1
              )
          )
  
      filename = None
      if output_directory:
          os.makedirs(output_directory, exist_ok=True)
          filename = filename_template_path.format('')
  
      visualization_utils.calibration_plot(
          fraction_positives,
          mean_predicted_vals,
          model_names_list,
          filename=filename
      )
  
      filename = None
      if output_directory:
          filename = filename_template_path.format('_brier')
  
      visualization_utils.compare_classifiers_plot(
          [brier_scores],
          ['brier'],
          model_names,
          adaptive=True,
          decimals=8,
          filename=filename
      )
  
      for i, brier_score in enumerate(brier_scores):
          if i < len(model_names):
              tokenizer_name = '{}: '.format(model_names[i])
              tokenizer_name += '{}'
          else:
              tokenizer_name = '{}'
          logger.info(tokenizer_name.format(brier_score))
  
  
  def confusion_matrix(
          test_stats_per_model: List[dict],
          metadata: dict,
          output_feature_name: Union[str, None],
          top_n_classes: List[int],
          normalize: bool,
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """Show confision matrix in the models predictions for each
      `output_feature_name`.
  
      For each model (in the aligned lists of test_statistics and model_names)
      it  produces a heatmap of the confusion matrix in the predictions for
      each  output_feature_name that has a confusion matrix in test_statistics.
      The value of `top_n_classes` limits the heatmap to the n most frequent
      classes.
  
      # Inputs
  
      :param test_stats_per_model: (List[dict]) dictionary containing evaluation
        performance statistics.
      :param metadata: (dict) intermediate preprocess structure created during
          training containing the mappings of the input dataset.
      :param output_feature_name: (Union[str, `None`]) name of the output feature
          to use for the visualization.  If `None`, use all output features.
      :param top_n_classes: (List[int]) number of top classes or list
          containing the number of top classes to plot.
      :param normalize: (bool) flag to normalize rows in confusion matrix.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      test_stats_per_model_list = test_stats_per_model
      model_names_list = convert_to_list(model_names)
      filename_template = 'confusion_matrix_{}_{}_{}.' + file_format
      filename_template_path = generate_filename_template_path(
          output_directory,
          filename_template
      )
      output_feature_names = _validate_output_feature_name_from_test_stats(
          output_feature_name,
          test_stats_per_model_list
      )
  
      for i, test_statistics in enumerate(
              test_stats_per_model_list):
          for output_feature_name in output_feature_names:
              if 'confusion_matrix' in test_statistics[output_feature_name]:
                  _confusion_matrix = np.array(
                      test_statistics[output_feature_name]['confusion_matrix']
                  )
                  model_name_name = model_names_list[i] if (
                          model_names_list is not None and i < len(
                      model_names_list)
                  ) else ''
  
                  if metadata is not None and output_feature_name in metadata and 'idx2str' in \
                          metadata[output_feature_name]:
                      labels = metadata[output_feature_name]['idx2str']
                  else:
                      labels = list(range(len(_confusion_matrix)))
  
                  for k in top_n_classes:
                      k = (min(k, _confusion_matrix.shape[0])
                           if k > 0 else _confusion_matrix.shape[0])
                      cm = _confusion_matrix[:k, :k]
                      if normalize:
                          with np.errstate(divide='ignore', invalid='ignore'):
                              cm_norm = np.true_divide(cm,
                                                       cm.sum(1)[:, np.newaxis])
                              cm_norm[cm_norm == np.inf] = 0
                              cm_norm = np.nan_to_num(cm_norm)
                          cm = cm_norm
  
                      filename = None
                      if output_directory:
                          os.makedirs(output_directory, exist_ok=True)
                          filename = filename_template_path.format(
                              model_name_name,
                              output_feature_name,
                              'top' + str(k)
                          )
  
                      visualization_utils.confusion_matrix_plot(
                          cm,
                          labels[:k],
                          output_feature_name=output_feature_name,
                          filename=filename
                      )
  
                      entropies = []
                      for row in cm:
                          if np.count_nonzero(row) > 0:
                              entropies.append(entropy(row))
                          else:
                              entropies.append(0)
                      class_entropy = np.array(entropies)
                      class_desc_entropy = np.argsort(class_entropy)[::-1]
                      desc_entropy = class_entropy[class_desc_entropy]
  
                      filename = None
                      if output_directory:
                          filename = filename_template_path.format(
                              'entropy_' + model_name_name,
                              output_feature_name,
                              'top' + str(k)
                          )
  
                      visualization_utils.bar_plot(
                          class_desc_entropy,
                          desc_entropy,
                          labels=[labels[i] for i in class_desc_entropy],
                          title='Classes ranked by entropy of '
                                'Confusion Matrix row',
                          filename=filename
                      )
  
  
  def frequency_vs_f1(
          test_stats_per_model: List[dict],
          metadata: dict,
          output_feature_name: Union[str, None],
          top_n_classes: List[int],
          model_names: Union[str, List[str]] = None,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ):
      """Show prediction statistics for the specified `output_feature_name` for
      each model.
  
      For each model (in the aligned lists of `test_stats_per_model` and
      `model_names`), produces two plots statistics of predictions for the
      specified `output_feature_name`.
  
      The first plot is a line plot with one x axis representing the different
      classes and two vertical axes colored in orange and blue respectively.
      The orange one is the frequency of the class and an orange line is plotted
      to show the trend. The blue one is the F1 score for that class and a blue
      line is plotted to show the trend. The classes on the x axis are sorted by
      f1 score.
  
      The second plot has the same structure of the first one,
      but the axes are flipped and the classes on the x axis are sorted by
      frequency.
  
      # Inputs
  
      :param test_stats_per_model: (List[dict]) dictionary containing evaluation
          performance statistics.
      :param metadata: (dict) intermediate preprocess structure created during
          training containing the mappings of the input dataset.
      :param output_feature_name: (Union[str, `None`]) name of the output feature
          to use for the visualization.  If `None`, use all output features.
      :param top_n_classes: (List[int]) number of top classes or list
          containing the number of top classes to plot.
      :param model_names: (Union[str, List[str]], default: `None`) model name or
          list of the model names to use as labels.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      test_stats_per_model_list = test_stats_per_model
      model_names_list = convert_to_list(model_names)
      filename_template = 'frequency_vs_f1_{}_{}.' + file_format
      filename_template_path = generate_filename_template_path(
          output_directory,
          filename_template
      )
      output_feature_names = _validate_output_feature_name_from_test_stats(
          output_feature_name,
          test_stats_per_model_list
      )
      k = top_n_classes[0]
  
      for i, test_stats in enumerate(test_stats_per_model_list):
          for of_name in output_feature_names:
  
              # Figure out model name
              model_name = (
                  model_names_list[i]
                  if model_names_list is not None and i < len(model_names_list)
                  else ''
              )
  
              # setup directory and filename
              filename = None
              if output_directory:
                  os.makedirs(output_directory, exist_ok=True)
                  filename = filename_template_path.format(model_name, of_name)
  
              # setup local variables
              per_class_stats = test_stats[of_name]['per_class_stats']
              class_names = metadata[of_name]['idx2str']
  
              if k > 0:
                  class_names = class_names[:k]
  
              f1_scores = []
              labels = []
  
              for class_name in class_names:
                  class_stats = per_class_stats[class_name]
                  f1_scores.append(class_stats['f1_score'])
                  labels.append(class_name)
  
              # get np arrays of frequencies, f1s and labels
              idx2freq = {
                  metadata[of_name]['str2idx'][key]: val
                  for key, val in
                  metadata[of_name]['str2freq'].items()
              }
              freq_np = np.array(
                  [idx2freq[class_id]
                   for class_id in sorted(idx2freq)],
                  dtype=np.int32
              )
              f1_np = np.nan_to_num(np.array(f1_scores, dtype=np.float32))
              labels_np = np.array(labels)
  
              # sort by f1
              f1_sort_idcs = f1_np.argsort()[::-1]
              len_f1_sort_idcs = len(f1_sort_idcs)
  
              freq_sorted_by_f1 = freq_np[f1_sort_idcs]
              freq_sorted_by_f1 = freq_sorted_by_f1[:len_f1_sort_idcs]
              f1_sorted_by_f1 = f1_np[f1_sort_idcs]
              f1_sorted_by_f1 = f1_sorted_by_f1[:len_f1_sort_idcs]
              labels_sorted_by_f1 = labels_np[f1_sort_idcs]
              labels_sorted_by_f1 = labels_sorted_by_f1[:len_f1_sort_idcs]
  
              # create viz sorted by f1
              visualization_utils.double_axis_line_plot(
                  f1_sorted_by_f1,
                  freq_sorted_by_f1,
                  'F1 score',
                  'frequency',
                  labels=labels_sorted_by_f1,
                  title='{} F1 Score vs Frequency {}'.format(
                      model_name,
                      of_name
                  ),
                  filename=filename
              )
  
              # sort by freq
              freq_sort_idcs = freq_np.argsort()[::-1]
              len_freq_sort_idcs = len(freq_sort_idcs)
  
              freq_sorted_by_freq = freq_np[freq_sort_idcs]
              freq_sorted_by_freq = freq_sorted_by_freq[:len_freq_sort_idcs]
              f1_sorted_by_freq = f1_np[freq_sort_idcs]
              f1_sorted_by_freq = f1_sorted_by_freq[:len_freq_sort_idcs]
              labels_sorted_by_freq = labels_np[freq_sort_idcs]
              labels_sorted_by_freq = labels_sorted_by_freq[:len_freq_sort_idcs]
  
              # create viz sorted by freq
              visualization_utils.double_axis_line_plot(
                  freq_sorted_by_freq,
                  f1_sorted_by_freq,
                  'frequency',
                  'F1 score',
                  labels=labels_sorted_by_freq,
                  title='{} F1 Score vs Frequency {}'.format(
                      model_name,
                      of_name
                  ),
                  filename=filename
              )
  
  def hyperopt_report_cli(
      hyperopt_stats_path,
      output_directory=None,
      file_format='pdf',
      **kwargs
  ) -> None:
      """
      Produces a report about hyperparameter optimization
      creating one graph per hyperparameter to show the distribution of results
      and one additional graph of pairwise hyperparameters interactions.
  
      :param hyperopt_stats_path: path to the hyperopt results JSON file
      :param output_directory: path where to save the output plots
      :param file_format: format of the output plot, pdf or png
      :return:
      """
  
      hyperopt_report(
          hyperopt_stats_path,
          output_directory=output_directory,
          file_format=file_format
      )
  
  
  def hyperopt_report(
          hyperopt_stats_path: str,
          output_directory: str = None,
          file_format: str = 'pdf',
          **kwargs
  ) -> None:
      """
      Produces a report about hyperparameter optimization
      creating one graph per hyperparameter to show the distribution of results
      and one additional graph of pairwise hyperparameters interactions.
  
      # Inputs
  
      :param hyperopt_stats_path: (str) path to the hyperopt results JSON file.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window.
      :param file_format: (str, default: `'pdf'`) file format of output plots -
          `'pdf'` or `'png'`.
  
      # Return
  
      :return: (None)
      """
      filename_template = 'hyperopt_{}.' + file_format
      filename_template_path = generate_filename_template_path(
          output_directory,
          filename_template
      )
  
      hyperopt_stats = load_json(hyperopt_stats_path)
  
      visualization_utils.hyperopt_report(
          hyperopt_stats['hyperopt_config']['parameters'],
          hyperopt_results_to_dataframe(
              hyperopt_stats['hyperopt_results'],
              hyperopt_stats['hyperopt_config']['parameters'],
              hyperopt_stats['hyperopt_config']['metric']
          ),
          metric=hyperopt_stats['hyperopt_config']['metric'],
          filename_template=filename_template_path
      )
  
  
  def hyperopt_hiplot_cli(
          hyperopt_stats_path,
          output_directory=None,
          **kwargs
  
  ):
      """
      Produces a parallel coordinate plot about hyperparameter optimization
      creating one HTML file and optionally a CSV file to be read by hiplot
  
      :param hyperopt_stats_path: path to the hyperopt results JSON file
      :param output_directory: path where to save the output plots
      :return:
      """
  
      hyperopt_hiplot(
          hyperopt_stats_path,
          output_directory=output_directory
      )
  
  def hyperopt_hiplot(
          hyperopt_stats_path,
          output_directory=None,
          **kwargs
  ):
      """
      Produces a parallel coordinate plot about hyperparameter optimization
      creating one HTML file and optionally a CSV file to be read by hiplot
  
      # Inputs
  
      :param hyperopt_stats_path: (str) path to the hyperopt results JSON file.
      :param output_directory: (str, default: `None`) directory where to save
          plots. If not specified, plots will be displayed in a window.
  
      # Return
  
      :return: (None)
      """
      filename = 'hyperopt_hiplot.html'
      filename_path = generate_filename_template_path(
          output_directory,
          filename
      )
  
      hyperopt_stats = load_json(hyperopt_stats_path)
      hyperopt_df = hyperopt_results_to_dataframe(
          hyperopt_stats['hyperopt_results'],
          hyperopt_stats['hyperopt_config']['parameters'],
          hyperopt_stats['hyperopt_config']['metric']
      )
      visualization_utils.hyperopt_hiplot(
          hyperopt_df,
          filename=filename_path,
      )
  
  
  def hyperopt_results_to_dataframe(
          hyperopt_results,
          hyperopt_parameters,
          metric
  ):
      df = pd.DataFrame(
          [{metric: res['metric_score'], **res['parameters']}
           for res in hyperopt_results]
      )
      df = df.astype(
          {hp_name: hp_params[TYPE]
           for hp_name, hp_params in hyperopt_parameters.items()}
      )
      return df
  
  
  visualizations_registry = {
      'compare_performance':
          compare_performance_cli,
      'compare_classifiers_performance_from_prob':
          compare_classifiers_performance_from_prob_cli,
      'compare_classifiers_performance_from_pred':
          compare_classifiers_performance_from_pred_cli,
      'compare_classifiers_performance_subset':
          compare_classifiers_performance_subset_cli,
      'compare_classifiers_performance_changing_k':
          compare_classifiers_performance_changing_k_cli,
      'compare_classifiers_multiclass_multimetric':
          compare_classifiers_multiclass_multimetric_cli,
      'compare_classifiers_predictions':
          compare_classifiers_predictions_cli,
      'compare_classifiers_predictions_distribution':
          compare_classifiers_predictions_distribution_cli,
      'confidence_thresholding':
          confidence_thresholding_cli,
      'confidence_thresholding_data_vs_acc':
          confidence_thresholding_data_vs_acc_cli,
      'confidence_thresholding_data_vs_acc_subset':
          confidence_thresholding_data_vs_acc_subset_cli,
      'confidence_thresholding_data_vs_acc_subset_per_class':
          confidence_thresholding_data_vs_acc_subset_per_class_cli,
      'confidence_thresholding_2thresholds_2d':
          confidence_thresholding_2thresholds_2d_cli,
      'confidence_thresholding_2thresholds_3d':
          confidence_thresholding_2thresholds_3d_cli,
      'binary_threshold_vs_metric':
          binary_threshold_vs_metric_cli,
      'roc_curves':
          roc_curves_cli,
      'roc_curves_from_test_statistics':
          roc_curves_from_test_statistics_cli,
      'calibration_1_vs_all':
          calibration_1_vs_all_cli,
      'calibration_multiclass':
          calibration_multiclass_cli,
      'confusion_matrix':
          confusion_matrix_cli,
      'frequency_vs_f1':
          frequency_vs_f1_cli,
      'learning_curves':
          learning_curves_cli,
      'hyperopt_report':
          hyperopt_report_cli,
      'hyperopt_hiplot':
          hyperopt_hiplot_cli
  }
  
  
  def cli(sys_argv):
      parser = argparse.ArgumentParser(
          description='This script analyzes results and shows some nice plots.',
          prog='ludwig visualize',
          usage='%(prog)s [options]')
  
      parser.add_argument('-g', '--ground_truth', help='ground truth file')
      parser.add_argument(
          '-gm',
          '--ground_truth_metadata',
          help='input metadata JSON file'
      )
      parser.add_argument(
          '-sf',
          '--split_file',
          default=None,
          help='file containing split values used in conjunction with '
               'ground truth file.'
      )
  
      parser.add_argument(
          '-od',
          '--output_directory',
          help='directory where to save plots.'
               'If not specified, plots will be displayed in a window'
      )
      parser.add_argument(
          '-ff',
          '--file_format',
          help='file format of output plots',
          default='pdf',
          choices=['pdf', 'png']
      )
  
      parser.add_argument(
          '-v',
          '--visualization',
          choices=sorted(list(visualizations_registry.keys())),
          help='type of visualization',
          required=True
      )
  
      parser.add_argument(
          '-ofn',
          '--output_feature_name',
          default=[],
          help='name of the output feature to visualize'
      )
      parser.add_argument(
          '-gts',
          '--ground_truth_split',
          default=2,
          help='ground truth split - 0:train, 1:validation, 2:test split'
      )
      parser.add_argument(
          '-tf',
          '--threshold_output_feature_names',
          default=[],
          nargs='+',
          help='names of output features for 2d threshold'
      )
      parser.add_argument(
          '-pred',
          '--predictions',
          default=[],
          nargs='+',
          type=str,
          help='predictions files'
      )
      parser.add_argument(
          '-prob',
          '--probabilities',
          default=[],
          nargs='+',
          type=str,
          help='probabilities files'
      )
      parser.add_argument(
          '-trs',
          '--training_statistics',
          default=[],
          nargs='+',
          type=str,
          help='training stats files'
      )
      parser.add_argument(
          '-tes',
          '--test_statistics',
          default=[],
          nargs='+',
          type=str,
          help='test stats files'
      )
      parser.add_argument(
          '-hs',
          '--hyperopt_stats_path',
          default=None,
          type=str,
          help='hyperopt stats file'
      )
      parser.add_argument(
          '-mn',
          '--model_names',
          default=[],
          nargs='+',
          type=str,
          help='names of the models to use as labels'
      )
      parser.add_argument(
          '-tn',
          '--top_n_classes',
          default=[0],
          nargs='+',
          type=int,
          help='number of classes to plot'
      )
      parser.add_argument(
          '-k',
          '--top_k',
          default=3,
          type=int,
          help='number of elements in the ranklist to consider'
      )
      parser.add_argument(
          '-ll',
          '--labels_limit',
          default=0,
          type=int,
          help='maximum numbers of labels. '
               'If labels in dataset are higher than this number, "rare" label'
      )
      parser.add_argument(
          '-ss',
          '--subset',
          default='ground_truth',
          choices=['ground_truth', PREDICTIONS],
          help='type of subset filtering'
      )
      parser.add_argument(
          '-n',
          '--normalize',
          action='store_true',
          default=False,
          help='normalize rows in confusion matrix'
      )
      parser.add_argument(
          '-m',
          '--metrics',
          default=['f1'],
          nargs='+',
          type=str,
          help='metrics to dispay in threshold_vs_metric'
      )
      parser.add_argument(
          '-pl',
          '--positive_label',
          type=int,
          default=1,
          help='label of the positive class for the roc curve'
      )
      parser.add_argument(
          '-l',
          '--logging_level',
          default='info',
          help='the level of logging to use',
          choices=['critical', 'error', 'warning', 'info', 'debug', 'notset']
      )
  
      args = parser.parse_args(sys_argv)
  
      args.logging_level = logging_level_registry[args.logging_level]
      logging.getLogger('ludwig').setLevel(
          args.logging_level
      )
      global logger
      logger = logging.getLogger('ludwig.visualize')
  
      try:
          vis_func = visualizations_registry[args.visualization]
      except KeyError:
          logging.info('Visualization argument not recognized')
          raise
      vis_func(**vars(args))
  
  
  if __name__ == '__main__':
      contrib_import()
      contrib_command("visualize", *sys.argv)
      cli(sys.argv[1:])
  