title: Problems setting up conditional search space in hyperopt
body: <p>I'll fully admit that I may be setting up the conditional space wrong here but for some reason, I just can't get this to function at all. I am attempting to use hyperopt to tune a logistic regression model and depending on the solver there are some other parameters that need to be explored. If you choose the liblinear solver you can choose penalties, and depending on the penalty you can also choose dual. When I try and run hyperopt on this search space though, it keeps giving me an error because its passing the entire dictionary as show below. Any ideas?</p>
  <p>The error I'm getting is</p>
  <pre><code>ValueError: Logistic Regression supports only liblinear, newton-cg, lbfgs and sag solvers, got {'solver': 'sag'}'  
  </code></pre>
  <p>This format worked when setting up a random forest search space so I'm at a loss.</p>
  <pre><code>import numpy as np
  import scipy as sp
  import pandas as pd
  pd.options.display.max_columns = None
  pd.options.display.max_rows = None
  import matplotlib.pyplot as plt
  %matplotlib inline
  import seaborn as sns
  sns.set(style=&quot;white&quot;)
  import pyodbc
  import statsmodels as sm
  from pandasql import sqldf
  import math
  from tqdm import tqdm
  import pickle
  
  
  from sklearn.preprocessing import RobustScaler, OneHotEncoder, MinMaxScaler
  from sklearn.utils import shuffle
  from sklearn.cross_validation import KFold, StratifiedKFold, cross_val_score, cross_val_predict, train_test_split
  from sklearn.model_selection import GridSearchCV
  from sklearn.model_selection import StratifiedKFold as StratifiedKFoldIt
  from sklearn.feature_selection import RFECV, VarianceThreshold, SelectFromModel, SelectKBest
  from sklearn.decomposition import PCA, IncrementalPCA, FactorAnalysis
  from sklearn.calibration import CalibratedClassifierCV
  from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier, AdaBoostClassifier, BaggingClassifier
  from sklearn.svm import SVC
  from sklearn.neighbors import KNeighborsClassifier
  from sklearn.naive_bayes import GaussianNB, MultinomialNB
  from sklearn.linear_model import LogisticRegression, LogisticRegressionCV, SGDClassifier
  from sklearn.metrics import precision_recall_curve, precision_score, recall_score, accuracy_score, classification_report, confusion_matrix, f1_score, log_loss
  from imblearn.over_sampling import RandomOverSampler, SMOTE, ADASYN 
  from imblearn.under_sampling import RandomUnderSampler, ClusterCentroids, NearMiss, NeighbourhoodCleaningRule, OneSidedSelection
  from xgboost.sklearn import XGBClassifier
  from hyperopt import fmin, tpe, hp, Trials, STATUS_OK
  
  
  space4lr = {
      'C': hp.uniform('C', .0001, 100.0),
      'solver' : hp.choice('solver', [
          {'solver' : 'newton-cg',},
          {'solver' : 'lbfgs',},
          {'solver' : 'sag'},
          {'solver' : 'liblinear', 'penalty' : hp.choice('penalty', [
               {'penalty' : 'l1'},
               {'penalty' : 'l2', 'dual' : hp.choice('dual', [True, False])}]
                                                        )},
      ]),
      'fit_intercept': hp.choice('fit_intercept', ['True', 'False']),
      'class_weight': hp.choice('class_weight', ['balanced', None]),
      'max_iter': 50000,
      'random_state': 84,
      'n_jobs': 8
  }
  lab = 0
  results = pd.DataFrame()
  for i in feature_elims:
  target = 'Binary_over_3'
  
  alt_targets = ['year2_PER', 'year2_GP' ,'year2_Min', 'year2_EFF' ,'year2_WS/40' ,'year2_Pts/Poss' ,'Round' ,'GRZ_Pick' 
                 ,'GRZ_Player_Rating' ,'Binary_over_2', 'Binary_over_3' ,'Binary_over_4' ,'Binary_5' ,'Draft_Strength']
  #alt_targets.remove(target)
  nondata_columns = ['display_name' ,'player_global_id', 'season' ,'season_' ,'team_global_id', 'birth_date', 'Draft_Day']
  nondata_columns.extend(alt_targets)
  
  AGG_SET_CART_PERC = sqldf(&quot;&quot;&quot;SELECT * FROM AGG_SET_PLAYED_ADJ_SOS_Jan1 t1 
                                   LEFT JOIN RANKINGS t2 ON t1.[player_global_id] = t2.[player_global_id]
                                   LEFT JOIN Phys_Training t3 ON t1.[player_global_id] = t3.[player_global_id]&quot;&quot;&quot;)
  AGG_SET_CART_PERC['HS_RSCI'] = AGG_SET_CART_PERC['HS_RSCI'].fillna(110)
  AGG_SET_CART_PERC['HS_Avg_Rank'] = AGG_SET_CART_PERC['HS_Avg_Rank'].fillna(1)
  AGG_SET_CART_PERC['HS_years_ranked'] = AGG_SET_CART_PERC['HS_years_ranked'].fillna(0)
  AGG_SET_CART_PERC = shuffle(AGG_SET_CART_PERC, random_state=8675309)
  
  rus = RandomUnderSampler(random_state=8675309)
  ros = RandomOverSampler(random_state=8675309)
  rs = RobustScaler()
  
  X = AGG_SET_CART_PERC
  y = X[target]
  X = pd.DataFrame(X.drop(nondata_columns, axis=1))
  position = pd.get_dummies(X['position'])
  for idx, row in position.iterrows():
      if row['F/C'] == 1:
          row['F'] = 1
          row['C'] = 1
      if row['G/F'] == 1:
          row['G'] = 1
          row['F'] = 1
  position = position.drop(['F/C', 'G/F'], axis=1)
  X = pd.concat([X, position], axis=1).drop(['position'], axis=1)
  X = rs.fit_transform(X, y=None)
  X = i.transform(X)
  
  def hyperopt_train_test(params):    
      clf = LogisticRegression(**params)
      #cvs = cross_val_score(xgbc, X, y, scoring='recall', cv=skf).mean()
      skf = StratifiedKFold(y, n_folds=6, shuffle=False, random_state=1)
      metrics = []
      tuning_met = []
      accuracy = []
      precision = []
      recall = []
      f1 = []
      log = []
      for i, (train, test) in enumerate(skf):
          X_train = X[train]
          y_train = y[train]
          X_test = X[test]
          y_test = y[test]
          X_train, y_train = ros.fit_sample(X_train, y_train)
          X_train, y_train = rus.fit_sample(X_train, y_train)
          clf.fit(X_train, y_train)
          y_pred = clf.predict(X_test)
          tuning_met.append((((precision_score(y_test, y_pred))*4) + recall_score(y_test, y_pred))/5)
          accuracy.append(accuracy_score(y_test, y_pred))
          precision.append(precision_score(y_test, y_pred))
          recall.append(recall_score(y_test, y_pred))
          f1.append(f1_score(y_test, y_pred))
          log.append(log_loss(y_test, y_pred))
      metrics.append(sum(tuning_met) / len(tuning_met))
      metrics.append(sum(accuracy) / len(accuracy))
      metrics.append(sum(precision) / len(precision))
      metrics.append(sum(recall) / len(recall))
      metrics.append(sum(f1) / len(f1))
      metrics.append(sum(log) / len(log))
      return(metrics)
  
  best = 0
  count = 0
  
  def f(params):
      global best, count, results, lab, met
      met = hyperopt_train_test(params.copy())
      met.append(params)
      met.append(featureset_labels[lab])
      acc = met[0]
      results = results.append([met])
      if acc &gt; best:
          print(featureset_labels[lab],'new best:', acc, 'Accuracy:', met[1], 'Precision:', met[2], 'Recall:', met[3], 'using', params, &quot;&quot;&quot;
          &quot;&quot;&quot;)
          best = acc
      else:
          print(acc, featureset_labels[lab], count)
      
      count = count + 1
      return {'loss': -acc, 'status': STATUS_OK}
   
  trials = Trials()
  best = fmin(f, space4lr, algo=tpe.suggest, max_evals=1000, trials=trials)
  print(featureset_labels[lab], ' best:')
  print(best, &quot;&quot;&quot;
  &quot;&quot;&quot;)
  lab = lab + 1
  </code></pre>