text: import pandas as pd, numpy as np
  import pickle, time
  from sklearn.model_selection import StratifiedKFold, KFold, train_test_split
  from hyperopt import fmin, tpe, Trials, STATUS_OK, STATUS_FAIL
  from datetime import datetime
  from cat_counter import CatCounter
  #from pandas.io.common import EmptyDataError
  import os
  
  
  class Experiment(object):
  
      def __init__(self, learning_task='classification', bst_name=None, n_estimators=5000, hyperopt_evals=50, 
                   compute_counters=True, counters_sort_col=None, holdout_size=0, 
                   train_path=None, test_path=None, cd_path=None, output_folder_path='./'):
          self.learning_task, self.bst_name = learning_task, bst_name
          self.compute_counters = compute_counters
          self.holdout_size = holdout_size
          self.counters_sort_col = counters_sort_col
          self.n_estimators, self.best_loss = n_estimators, np.inf
          self.best_n_estimators = None
          self.hyperopt_evals, self.hyperopt_eval_num = hyperopt_evals, 0
          self.train_path, self.test_path, self.cd_path = train_path, test_path, cd_path
          self.output_folder_path = os.path.join(output_folder_path, '')
          self.default_params, self.best_params = None, None
          self.title = None
          if self.learning_task == 'classification':
              self.metric = 'logloss'
          elif self.learning_task == 'regression':
              self.metric = 'rmse'
          else:
              raise ValueError('Task type must be "classification" or "regression"')
  
  
      def read_file(self, file_name, target_col):
          X = pd.read_csv(file_name, sep='\t', header=None)
          if self.learning_task == 'classification':
              y = np.maximum(X[target_col].values, 0)
          else:
              y = X[target_col].values
          X.drop(target_col, axis=1, inplace=True)
          return X, y
  
  
      def read_data(self):
          cols = pd.read_csv(self.cd_path, sep='\t', header=None)
          target_col = np.where(cols[1] == 'Target')[0][0]
          cat_cols = cols[cols[1] == "Categ"][0].values
  
          X_train, y_train = self.read_file(self.train_path, target_col)
          X_test, y_test = self.read_file(self.test_path, target_col)
          data = pd.concat([X_train, X_test])
          data[cat_cols] = data[cat_cols].apply(lambda x: x.astype('category').cat.codes)
          data = np.array(data).astype('float')
          X_train, X_test = data[:X_train.shape[0]], data[X_train.shape[0]:]
  
          cat_cols[cat_cols > target_col] = cat_cols[cat_cols > target_col] - 1
          return X_train, y_train, X_test, y_test, cat_cols
  
  
      def convert_to_dataset(self, data, label, cat_cols=None):
          raise NotImplementedError('Method convert_to_dataset is not implemented.')
  
  
      def preprocess_cat_cols(self, X_train, y_train, cat_cols, X_test=None, cc=None):
          if self.compute_counters == False:
              return None
          if cc is None:
              sort_values = None if self.counters_sort_col is None else X_train[:, self.counters_sort_col]
              cc = CatCounter(self.learning_task, sort_values)
              X_train[:,cat_cols] = cc.fit(X_train[:,cat_cols], y_train)
          else:
              X_train[:,cat_cols] = cc.transform(X_train[:,cat_cols])
          if not X_test is None:
              X_test[:,cat_cols] = cc.transform(X_test[:,cat_cols])
          return cc
  
  
      def split_and_preprocess(self, X_train, y_train, X_test, y_test, cat_cols, n_splits=5, random_state=0):
          if self.holdout_size > 0:
              print('Holdout is used for counters.')
              X_train, X_hout, y_train, y_hout = train_test_split(X_train, y_train, 
                                                                  test_size=self.holdout_size,
                                                                  random_state=random_state)
              cc = self.preprocess_cat_cols(X_hout, y_hout, cat_cols)
          else:
              cc = None
  
          CVSplit = KFold if self.learning_task == 'regression' else StratifiedKFold
          cv = CVSplit(n_splits=n_splits, shuffle=True, random_state=random_state)
          cv_pairs = []
  
          for train_index, test_index in cv.split(X_train, y_train):
              train, test = X_train[train_index], X_train[test_index]
              _ = self.preprocess_cat_cols(train, y_train[train_index], cat_cols, test, cc)
              dtrain = self.convert_to_dataset(train.astype(float), y_train[train_index], cat_cols)
              dtest = self.convert_to_dataset(test.astype(float), y_train[test_index], cat_cols)
              cv_pairs.append((dtrain, dtest))
  
          _ = self.preprocess_cat_cols(X_train, y_train, cat_cols, X_test, cc)
          dtrain = self.convert_to_dataset(X_train.astype(float), y_train, cat_cols)
          dtest = self.convert_to_dataset(X_test.astype(float), y_test, cat_cols)
  
          return cv_pairs, (dtrain, dtest)
  
  
      def fit(self, params, dtrain, dtest, n_estimators):
          raise NotImplementedError('Method train is not implemented.')
  
  
      def predict(self, bst, dtest, X_test):
          raise NotImplementedError('Method predict is not implemented.')
  
  
      def preprocess_params(self, params):
          raise NotImplementedError('Method preprocess_params is not implemented.')
  
  
      def run_cv(self, cv_pairs, params=None, n_estimators=None, verbose=False):
          params = params or self.default_params
          n_estimators = n_estimators or self.n_estimators
          params = self.preprocess_params(params)
          evals_results, start_time = [], time.time()
          for dtrain, dtest in cv_pairs:
              _, evals_result = self.fit(params, dtrain, dtest, n_estimators)
              evals_results.append(evals_result)
          mean_evals_results = np.mean(evals_results, axis=0)
          best_n_estimators = np.argmin(mean_evals_results) + 1
          eval_time = time.time() - start_time
  
          cv_result = {'loss': mean_evals_results[best_n_estimators - 1],
                       'best_n_estimators': best_n_estimators, 
                       'eval_time': eval_time,
                       'status': STATUS_FAIL if np.isnan(mean_evals_results[best_n_estimators - 1]) else STATUS_OK,
                       'params': params.copy()}
          self.best_loss = min(self.best_loss, cv_result['loss'])
          self.hyperopt_eval_num += 1
          cv_result.update({'hyperopt_eval_num': self.hyperopt_eval_num, 'best_loss': self.best_loss})
              
          if verbose:
              print '[{0}/{1}]\teval_time={2:.2f} sec\tcurrent_{3}={4:.6f}\tmin_{3}={5:.6f}'.format(
                          self.hyperopt_eval_num, self.hyperopt_evals, eval_time,
                          self.metric, cv_result['loss'], self.best_loss)
          return cv_result
  
  
      def run_test(self, dtrain, dtest, X_test=None, params=None, n_estimators=None, custom_metric=None, seed=0):
          params = params or self.best_params or self.default_params
          n_estimators = n_estimators or self.best_n_estimators or self.n_estimators
          params = self.preprocess_params(params)
          start_time = time.time()
          bst, evals_result = self.fit(params, dtrain, dtest, n_estimators, seed=seed)
          eval_time = time.time() - start_time
          preds = self.predict(bst, dtest, X_test)
  
          result = {'loss': evals_result[-1], 'bst': bst, 'n_estimators': n_estimators,
                    'eval_time': eval_time, 'status': STATUS_OK,  'params': params.copy(),
                    'preds': preds}
          
          if custom_metric is not None:
              if type(custom_metric) is not dict:
                  raise TypeError("custom_metric argument should be dict")
              pred = self.predict(bst, dtest, X_test)
              for title, func in custom_metric.iteritems():
                  score = func(dtest.get_label(), pred, sample_weight=None) # TODO weights
                  result[title] = score
  
          return result
  
  
      def optimize_params(self, cv_pairs, max_evals=None, verbose=True):
          max_evals = max_evals or self.hyperopt_evals
          self.trials = Trials()
          self.hyperopt_eval_num, self.best_loss = 0, np.inf
  
          _ = fmin(fn=lambda params: self.run_cv(cv_pairs, params, verbose=verbose), 
                   space=self.space, algo=tpe.suggest, max_evals=max_evals, trials=self.trials, rseed=1)
  
          self.best_params = self.trials.best_trial['result']['params']
          self.best_n_estimators = self.trials.best_trial['result']['best_n_estimators']
          return self.trials.best_trial['result']
  
  
      def dump(self, preds, elementwise_losses, test_losses, file_name):
          results = {'trials': self.trials, 'best_params': self.best_params,
                     'best_n_estimators': self.best_n_estimators,
                     'preds': preds, 'elementwise_losses': elementwise_losses, 'test_losses': test_losses}
          with open(file_name, 'wb') as f:
              pickle.dump(results, f)
  
  
      def load(self, file_name):
          with open(file_name, 'r') as f:
              results = pickle.load(f)
          self.trials = results['trials']
          self.best_params = results['best_params']
          self.best_n_estimators = results['best_n_estimators']
          preds = results['preds']
          losses = results['losses']
          test_loss = results['test_loss']
          return preds, losses, test_loss
  
  
      def print_result(self, result, name='', extra_keys=None):
          print '%s:\n' % name
          print '%s = %s' % (self.metric, result['loss'])
          if 'best_n_estimators' in result.keys():
              print 'best_n_estimators = %s' % result['best_n_estimators']
          elif 'n_estimators' in result.keys():
              print 'n_estimators = %s' % result['n_estimators']
          print 'params = %s' % result['params']
          if extra_keys is not None:
              for k in extra_keys:
                  if k in result:
                      print "%s = %f" % (k, result[k])
  
  
      def elementwise_loss(self, y, p):
          if self.learning_task == 'classification':
              p_ = np.clip(p, 1e-16, 1-1e-16)
              return - y * np.log(p_) - (1 - y) * np.log(1 - p_)
          return (y - p) ** 2
  
  
      def run(self):
          print 'Loading and preprocessing dataset...'
          X_train, y_train, X_test, y_test, cat_cols = self.read_data()
          cv_pairs, (dtrain, dtest) = self.split_and_preprocess(X_train, y_train, X_test, y_test, cat_cols)
  
          print 'Optimizing params...'
          cv_result = self.optimize_params(cv_pairs)
          self.print_result(cv_result, '\nBest result on cv')
          
          print '\nTraining algorithm with the tuned parameters for different seed...'
          preds, test_losses, elementwise_losses = [], [], []
          for seed in range(5):
              test_result = self.run_test(dtrain, dtest, X_test, seed=seed)
              preds.append(test_result['preds'])
              test_losses.append(test_result['loss'])
              elementwise_losses.append(self.elementwise_loss(y_test, preds[-1]))
              print 'For seed=%d Test\'s %s : %.5f' % (seed, self.metric, test_losses[-1])
  
          print '\nTest\'s %s mean: %.5f, Test\'s %s std: %.5f' % (self.metric, np.mean(test_losses), self.metric, np.std(test_losses))
  
          if not self.output_folder_path is None:
              date = datetime.now().strftime('%Y%m%d-%H%M%S')
              dataset_name = self.train_path.replace('/', ' ').strip().split()[-2]
              file_name = '{}{}_results_{}_{}.pkl'.format(self.output_folder_path, self.bst_name, dataset_name, date)
              self.dump(preds, elementwise_losses, test_losses, file_name)
              print 'Results are saved to %s' % file_name
  
  