class_docstr: Test class for second set of voting learner API tests.
class_docstr_tok: ["Test","class","for","second","set","of","voting","learner","API","tests","."]
docstr: Run checks when evaluating voting learners.
docstr_tok: ["Run","checks","when","evaluating","voting","learners","."]
code: def check_evaluate(
      self, learner_type, with_grid_search, with_soft_voting
  ):
      """Run checks when evaluating voting learners."""
      # to test the evaluate() method, we instantiate the SKLL voting learner,
      # train it on either the digits (classification) or housing (regression)
      # data set, and evaluate on the corresponding test set; then we do the
      # same in scikit-learn space and compare the objective and value for
      # on additional output metric
  
      # set various parameters based on whether we are using
      # a classifier or a regressor
      if learner_type == "classifier":
          learner_names = ["LogisticRegression", "SVC", "MultinomialNB"]
          voting_type = "soft" if with_soft_voting else "hard"
          train_fs, test_fs = TRAIN_FS_DIGITS, TEST_FS_DIGITS
          objective = "accuracy"
          extra_metric = "f1_score_macro"
          expected_voting_type = voting_type
      else:
          learner_names = ["LinearRegression", "SVR", "Ridge"]
          voting_type = "hard"
          train_fs, test_fs = TRAIN_FS_HOUSING, TEST_FS_HOUSING
          objective = "pearson"
          extra_metric = "neg_mean_squared_error"
          expected_voting_type = None
  
      # instantiate and train a SKLL voting learner
      skll_vl = VotingLearner(
          learner_names,
          voting=voting_type,
          feature_scaling="none",
          min_feature_count=0,
      )
      skll_vl.train(
          train_fs,
          grid_objective=objective,
          grid_search=with_grid_search,
          grid_search_folds=3,
      )
  
      # evaluate on the test set
      res = skll_vl.evaluate(
          test_fs,
          grid_objective=objective,
          output_metrics=[extra_metric],
      )
  
      # make sure all the parts of the results tuple
      # have the expected types
      self.assertTrue(len(res), 6)
      if learner_type == "classifier":
          self.assertTrue(isinstance(res[0], list))  # confusion matrix
          self.assertTrue(isinstance(res[1], float))  # accuracy
      else:
          self.assertEqual(res[0], None)  # no confusion matrix
          self.assertEqual(res[1], None)  # no accuracy
      self.assertTrue(isinstance(res[2], dict))  # result dict
      self.assertTrue(isinstance(res[3], dict))  # model params
      self.assertTrue(isinstance(res[4], float))  # objective
      self.assertTrue(isinstance(res[5], dict))  # metric scores
  
      # make sure the model params in the results match what we passed in
      estimators_from_params = res[3]["estimators"]
      for idx, (name, estimator) in enumerate(estimators_from_params):
          self.assertEqual(name, learner_names[idx])
          self.assertTrue(isinstance(estimator, Pipeline))
      if learner_type == "classifier":
          self.assertEqual(res[3]["voting"], expected_voting_type)
  
      # get the values for the objective and the additional metric
      skll_objective = res[4]
      skll_extra_metric = res[5][extra_metric]
  
      # now get the estimators that underlie the SKLL voting classifier
      # and use them to train a voting learner directly in scikit-learn
      named_estimators = skll_vl.model.named_estimators_
      clf1 = named_estimators[learner_names[0]]["estimator"]
      clf2 = named_estimators[learner_names[1]]["estimator"]
      clf3 = named_estimators[learner_names[2]]["estimator"]
      sklearn_model_type = (
          VotingClassifier
          if learner_type == "classifier"
          else VotingRegressor
      )
      sklearn_model_kwargs = {
          "estimators": [("clf1", clf1), ("clf2", clf2), ("clf3", clf3)]
      }
      if learner_type == "classifier":
          sklearn_model_kwargs["voting"] = voting_type
      sklearn_vl = sklearn_model_type(**sklearn_model_kwargs)
      sklearn_vl.fit(train_fs.features, train_fs.labels)
  
      # get the predictions from this voting classifier on the test set
      sklearn_predictions = sklearn_vl.predict(test_fs.features)
  
      # compute the values of the objective and the extra metric
      # on the scikit-learn side
      if learner_type == "classifier":
          sklearn_objective = accuracy_score(
              test_fs.labels, sklearn_predictions
          )
          sklearn_extra_metric = f1_score(
              test_fs.labels, sklearn_predictions, average="macro"
          )
      else:
          sklearn_objective = pearsonr(
              test_fs.labels, sklearn_predictions
          )[0]
          sklearn_extra_metric = -1 * mean_squared_error(
              test_fs.labels, sklearn_predictions
          )
  
      # check that the values match between SKLL and scikit-learn
      self.assertAlmostEqual(skll_objective, sklearn_objective)
      self.assertAlmostEqual(skll_extra_metric, sklearn_extra_metric)
  
imports: import re
  import unittest
  from itertools import product
  import numpy
  from numpy.testing import assert_raises_regex
  from scipy.stats import pearsonr
  from sklearn.ensemble import VotingClassifier, VotingRegressor
  from sklearn.metrics import accuracy_score, f1_score, mean_squared_error
  from sklearn.model_selection import ShuffleSplit, learning_curve
  from sklearn.pipeline import Pipeline
  from skll.learner import Learner
  from skll.learner.voting import VotingLearner
  from skll.utils.logging import close_and_remove_logger_handlers, get_skll_logger
  from skll.utils.testing import make_california_housing_data, make_digits_data, other_dir, output_dir
label_desc: Use image node_id 2 for calling the TestVotingLearnersAPITwo obj's underlying member method code with example usage: obj.check_evaluate(learner_type, with_grid_search, with_soft_voting) without return types