code: #!/usr/bin/env python
  
  from pandas  import *
  from numpy  import *
  from djeval import *
  import csv, code
  import pickle as pickle
  from sklearn.externals import joblib
  
  NUM_GAMES=50000
  
  
  def shell():
      vars = globals()
      vars.update(locals())
      shell = code.InteractiveConsole(vars)
      shell.interact()
  
  msg("Hi! Reading eheaders")
  eheaders_filename = '/data/eheaders.p'
  eheaders_file = open(eheaders_filename, 'r')
  eheaders = pickle.load(eheaders_file)
  elos = eheaders['elos']
  result = eheaders['result']
  checkmate = eheaders['checkmate']
  openings = eheaders['openings']
  ocount = eheaders['opening_count']
  
  msg("Hi! Reading crunched movescores from %s" % sys.argv[1])
  crunched_path = sys.argv[1]
  crunched_df = read_csv(crunched_path, sep=',', engine='c', index_col=['gamenum', 'side'])
  
  do_gb = False
  if do_gb:
      msg("Hi! Reading GB scores from %s" % sys.argv[2])
      gb_path = sys.argv[2]
      gb_df = read_csv(gb_path, sep=',', engine='c', index_col=['gamenum'])
  
  msg("Hi! Reading depthstats")
  depthstats_path = '/data/depthstats.csv'
  columns = [
  'gamenum',
  'side',
  'mean_depth',
  'mean_seldepth',
  'mean_depths_agreeing_ratio',
  'mean_deepest_agree_ratio',
  'pct_sanemoves',
  'gamelength',
  'mean_num_bestmoves',
  'mean_num_bestmove_changes',
  'mean_bestmove_depths_agreeing',
  'mean_deepest_change',
  'mean_deepest_change_ratio',
  ]
  depthstats_df = read_csv(depthstats_path, sep=' ', engine='c', header=None, names=columns, index_col=False)
  depthstats_df = depthstats_df.set_index(['gamenum', 'side'])
  # we have the gamelength column in another df, drop it here to avoid conflicts
  depthstats_df.drop('gamelength', axis=1, inplace=True)
  
  do_material = True
  if do_material:
      msg("Hi! Reading material")
      material_path = '/data/material.csv'
      columns = [
      'gamenum',
      'material_break_0',
      'material_break_1',
      'material_break_2',
      'material_break_3',
      'material_break_4',
      'opening_length',
      'midgame_length',
      'endgame_length',
      'mean_acwsa',
      'mean_acwsa_0',
      'mean_acwsa_1',
      'mean_acwsa_2',
      'mean_acwsa_3',
      'mean_acwsa_4',
      'mean_acwsa_5',
      'mean_acwsa_6',
      'mean_acwsa_7',
      'mean_acwsa_8',
      'mean_acwsa_9',
      ]
      material_df = read_csv(material_path, sep=' ', engine='c', header=None, names=columns, index_col=False)
      material_df = material_df.set_index(['gamenum'])
      material_df = material_df.reindex(list(range(1, NUM_GAMES+1)))
      material_df = material_df.fillna(material_df.mean())
  
  msg("Reading ELOscored data")
  eloscored_cols = [
      'gamenum',
      'final_elo',
      'final_ply',
      'final_num_games',
      'final_elo_stdev',
      'elopath_min',
      'elopath_max',
  ]
  eloscored_df = read_csv('/data/data.pgn.eloscored21', sep=',', engine='c', header=None, names=eloscored_cols, index_col=False)
  eloscored_df = eloscored_df.set_index(['gamenum'])
  
  msg("Reading ELOscored data 4")
  eloscored4_cols = [
      'gamenum',
      'final_elo',
      'final_ply',
      'final_num_games',
      'final_elo_stdev',
  ]
  eloscored4_cols[1:] = [x + '_elo4' for x in eloscored4_cols[1:]]
  eloscored4_df = read_csv('/data/data.pgn.eloscored4', sep=',', engine='c', header=None, names=eloscored4_cols, index_col=False)
  eloscored4_df = eloscored4_df.set_index(['gamenum'])
  
  msg("Reading ELOscored data 10")
  eloscored10_cols = [
      'gamenum',
      'final_elo',
      'final_ply',
      'final_num_games',
      'final_elo_stdev',
  ]
  eloscored10_cols[1:] = [x + '_elo10' for x in eloscored10_cols[1:]]
  eloscored10_df = read_csv('/data/data.pgn.eloscored10', sep=',', engine='c', header=None, names=eloscored10_cols, index_col=False)
  eloscored10_df = eloscored10_df.set_index(['gamenum'])
  
  do_movemodel=True
  if do_movemodel:
      msg("Hi! Reading moveaggs")
      move_aggs = joblib.load('/data/move_aggs.p')
      move_aggs.fillna(move_aggs.mean(), inplace=True)
      move_aggs = move_aggs[['mean', 'median', '25', '10', 'min', 'max', 'stdev']]
      msg("Hi! Reading wmoveaggs")
      wmove_aggs = joblib.load('/data/wmove_aggs.p')
      wmove_aggs.fillna(wmove_aggs.mean(), inplace=True)
      wmove_aggs.rename(columns={'elo_pred': 'moveelo_weighted'}, inplace=True)
      wmove_aggs = wmove_aggs['moveelo_weighted']
  
  do_elochunk = False
  if do_elochunk:
      ch_agg_df = joblib.load('/data/chunk_aggs.p')
      ch_agg_df.index = ch_agg_df.index.droplevel('elo')
      ch_agg_df.columns = ['elochunk_' + x for x in ch_agg_df.columns]
  
  
  msg("Hi! Setting up playergame rows")
  
  if do_elochunk:
      elorange_cols = list(ch_agg_df.columns.values)
      msg("elorange cols are %s" % elorange_cols)
  
  msg('Preparing ELO df')
  elo_rows = [[x[0][0], x[0][1], x[1]] for x in list(elos.items())]
  elo_df = DataFrame(elo_rows, columns=['gamenum','side','elo'])
  elo_df.set_index(['gamenum','side'], inplace=True)
  
  msg('Joining DFs')
  supplemental_dfs = [depthstats_df, elo_df, crunched_df]
  if do_movemodel:
      supplemental_dfs.extend([move_aggs, wmove_aggs])
  if do_elochunk:
      supplemental_dfs.append(ch_agg_df)
  mega_df = concat(supplemental_dfs, axis=1)
  if do_material:
      mega_df = mega_df.join(material_df, how='outer')
  mega_df = mega_df.join(eloscored_df, how='outer')
  mega_df = mega_df.join(eloscored4_df, how='outer')
  mega_df = mega_df.join(eloscored10_df, how='outer')
  if do_gb:
      mega_df = mega_df.join(gb_df, how='outer')
  
  yy_df = mega_df
  msg("hi, columns are %s" % yy_df.columns)
  
  # TODO confirm that all columns are there
  
  
  def opening_feature(opening):
    if ocount[opening] < 20:
      return 'rare'
    if ocount[opening] < 200:
      return 'uncommon'
    return opening
  
  msg("Hi! Computing additional features")
  yy_df['opening_feature'] = [opening_feature(openings[x]) for x in yy_df.index.get_level_values('gamenum')]
  yy_df['opening_count'] = [ocount[openings[x]] for x in yy_df.index.get_level_values('gamenum')]
  yy_df['any_grit'] = (yy_df['grit'] > 0)
  yy_df['major_grit'] = (yy_df['grit'] > 5)
  yy_df['nmerror'] = log((-1 * yy_df['meanerror']).clip(1,60)).clip(1,4) - 2.53
  yy_df['premature_quit'] = (yy_df['gameoutcome'] == -1) & (yy_df['my_final_equity'] > -100)
  yy_df['drawn_game'] = (yy_df['gameoutcome'] == 0)
  yy_df['ended_by_checkmate'] = yy_df['won_by_checkmate'] | yy_df['lost_by_checkmate']
  yy_df['noblunders'] = (yy_df['blunderrate'] == 0)
  yy_df['final_equity'] = yy_df['my_final_equity'].abs().clip(0,300)
  yy_df['early_lead'] = yy_df['early_lead'].clip(0,100)
  yy_df['mean_depth_clipped'] = yy_df['mean_depth'].clip(0,25)
  yy_df['gamelength_clipped'] = yy_df['gamelength'].clip(20,200)
  
  
  # prepare opponent_df with selected info about opponent
  opponent_columns = ['meanerror', 'blunderrate', 'perfectrate', 'grit', 'meanecho', 'mate_created', 'mate_destroyed', 'q_error_one', 'q_error_two', 'stdeverror', 'elo', 'any_grit', 'noblunders', 'nmerror', 'mean_depths_agreeing_ratio', 'mean_deepest_agree_ratio', 'pct_sanemoves']
  if do_elochunk:
      opponent_columns.extend(elorange_cols)
  opponent_df = yy_df[opponent_columns]
  opponent_df = opponent_df.reset_index()
  opponent_df['side'] = opponent_df['side'] * -1
  opponent_df.set_index(['gamenum', 'side'], inplace=True)
  opponent_df.columns = ['opponent_' + x for x in opponent_df.columns]
  yy_df = concat([yy_df, opponent_df], axis=1)
  
  # more derived columns that use opponent comparisons
  yy_df['elo_advantage'] = (yy_df['elo'] - yy_df['opponent_elo']).clip(-500, 500)
  yy_df['max_nmerror'] = yy_df[['nmerror', 'opponent_nmerror']].max(axis=1)
  yy_df['min_nmerror'] = yy_df[['nmerror', 'opponent_nmerror']].min(axis=1)
  yy_df['max_meanecho'] = yy_df[['meanecho', 'opponent_meanecho']].max(axis=1)
  yy_df['elo_avg'] = (yy_df['elo'] + yy_df['opponent_elo'])/2.0
  yy_df['elo_advantage'] = (yy_df['elo'] - yy_df['opponent_elo'])
  yy_df['winner_elo_advantage'] = yy_df['elo_advantage'] * yy_df['gameoutcome']
  
  
  msg("Hi! Computing dummy variables")
  categorical_features = ['opening_feature']
  dummies = get_dummies(yy_df[categorical_features]).astype(np.int8)
  yy_df = yy_df.join(dummies)
  
  # fill in missing values
  msg("Hi! Filling in missing values")
  full_index = pandas.MultiIndex.from_product([list(range(1,NUM_GAMES + 1)), [1,-1]], names=['gamenum', 'side'])
  yy_df = yy_df.reindex(full_index)
  yy_elo = yy_df['elo'].copy(True)
  yy_df.fillna(yy_df.mean(numeric_only=True), inplace=True)
  yy_df.fillna(False, inplace=True)
  yy_df['elo'] = yy_elo
  
  # stupid patch for some stupid opening feature that got assigned to False by fillna ?!!?!?!?
  yy_df.loc[yy_df['opening_feature'] == False,'opening_feature'] = 'rare'
  
  msg("Hi! Writing yy_df to disk")
  yy_df.to_pickle(sys.argv[3])
  
  msg("Column counts are:")
  counts = yy_df.count(axis=0)
  print(counts)
  
   import numpy as np
  import sys
  
  if len(sys.argv) < 3:
      sys.exit("Usage: python average_predictions.py <predictions_file1> [predictions_file_2] [...] <output_file>")
  
  predictions_paths = sys.argv[1:-1]
  target_path = sys.argv[-1]
  predictions = [np.load(path) for path in predictions_paths]
  avg_predictions = np.mean(predictions, axis=0)
  np.save(target_path, avg_predictions)
   # -*- coding: utf-8 -*-
  """
  Created on Mon Aug 31 16:01:25 2015.
  
  @author: rc,alex
  """
  import os
  import sys
  if __name__ == '__main__' and __package__ is None:
      filePath = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
      sys.path.append(filePath)
  
  from glob import glob
  import numpy as np
  import pandas as pd
  from sklearn.metrics import roc_auc_score
  
  from preprocessing.aux import getEventNames
  
  cols = getEventNames()
  ids = np.load('../infos_test.npy')
  subject_test = ids[:, 1]
  series_test = ids[:, 2]
  ids = ids[:, 0]
  
  labels = np.load('../infos_val.npy')
  subjects = labels[:, -2]
  series = labels[:, -1]
  labels = labels[:, :-2]
  
  subs_val = []
  subs_test = []
  
  # get all submissions
  subs = glob('models/*.yml')
  # remove folder and extension
  subs = [sub[7:-4] for sub in subs]
  
  # load subs
  print('Computing an average of %d submissions:' % len(subs))
  for sub in subs:
      print(sub)
      subs_val.append(np.load('val/val_%s.npy' % sub)[0])
      subs_test.append(pd.read_csv('submissions/%s.csv' % sub,
                                   index_col=0).values)
  
  # average all models
  ens_val = np.mean(subs_val, axis=0)
  ens_test = np.mean(subs_test, axis=0)
  
  # stats of the average
  aucs_val = [np.mean(roc_auc_score(labels[series == s], ens_val[series == s]))
              for s in [7, 8]]
  print('AUC: %.5f (SD: %.5f), s7: %.5f ; s8: %.5f' % (np.mean(aucs_val),
                                                       np.std(aucs_val),
                                                       aucs_val[0], aucs_val[1]))
  
  # save
  np.save('val/val_YOLO.npy', [ens_val])
  sub = pd.DataFrame(data=ens_test, index=ids, columns=cols)
  sub.to_csv('submissions/YOLO.csv', index_label='id', float_format='%.8f')
   """
  Created on Thu Dec 10 10:44:27 2015
  
  @author: konrad
  """
  from keras.regularizers import l2, activity_l2
  import numpy as np
  import pandas as pd
  from keras.models import Sequential
  from keras.layers.core import Dense, Dropout, Activation
  from keras.layers.normalization import BatchNormalization
  from keras.optimizers import Adadelta
  from itertools import product
  import datetime
  
  if __name__ == '__main__':
  
      ## settings
      projPath = './'
      dataset_version = "kb9"
      model_type = "keras"
      seed_value = 123
      todate = datetime.datetime.now().strftime("%Y%m%d")
      np.random.seed(seed_value)
  
      # data
      # read the training and test sets
      xtrain = pd.read_csv(projPath + 'input/xtrain_'+ dataset_version + '.csv')
      id_train = xtrain.QuoteNumber
      ytrain = xtrain.QuoteConversion_Flag
      xtrain.drop('QuoteNumber', axis = 1, inplace = True)
      xtrain.drop('QuoteConversion_Flag', axis = 1, inplace = True)
  
      xtest = pd.read_csv(projPath + 'input/xtest_'+ dataset_version + '.csv')
      id_test = xtest.QuoteNumber
      xtest.drop('QuoteNumber', axis = 1, inplace = True)
  
      # folds
      xfolds = pd.read_csv(projPath + 'input/xfolds.csv')
      # work with 5-fold split
      fold_index = xfolds.fold5
      fold_index = np.array(fold_index) - 1
      n_folds = len(np.unique(fold_index))
  
      ## model
      nb_classes = 2
      dims = xtrain.shape[1]
      epochs = 30
  
      # parameter grids
      drop_vals = [0.01, 0.2]
      dec_vals = [0.995, 0.8]
      lr_vals = [1, 0.5, 0.1]
      reg_vals = [1e-5,1e-3]
      lay_vals = [1,2]
      param_grid = tuple([drop_vals, dec_vals, lr_vals, reg_vals, lay_vals])
      param_grid = list(product(*param_grid))
  
      # storage structure for forecasts
      mvalid = np.zeros((xtrain.shape[0],len(param_grid)))
      mfull = np.zeros((xtest.shape[0],len(param_grid)))
  
      ## build 2nd level forecasts
      for i in range(len(param_grid)):
              print("processing parameter combo:", param_grid[i])
              print("Combo:", i, "of", len(param_grid))
              # loop over folds
              # Recompile model on each fold
              for j in range(0,n_folds):
                  # configure model with j-th combo of parameters
                  x = param_grid[i]
                  model = Sequential()
                  model.add(Dense(dims * x[4], input_shape=(dims,),W_regularizer=l2(x[3])))
                  model.add(BatchNormalization())
                  model.add(Dropout(x[0]))
                  model.add(Dense(nb_classes))
                  model.add(Activation('softmax'))
                  opt=Adadelta(lr=x[2],decay=x[1],epsilon=1e-5)
                  model.compile(loss='binary_crossentropy', optimizer=opt)
  
                  idx0 = np.where(fold_index != j)
                  idx1 = np.where(fold_index == j)
                  x0 = np.array(xtrain)[idx0,:][0]
                  x1 = np.array(xtrain)[idx1,:][0]
                  y0 = np.array(ytrain)[idx0]
                  y1 = np.array(ytrain)[idx1]
                  y00 = np.zeros((x0.shape[0],2))
                  y00[:,0] = y0; y00[:,1] = 1-y0
                  # fit the model on observations associated with subject whichSubject in this fold
                  model.fit(x0, y00, nb_epoch=epochs, batch_size=1000)
                  mvalid[idx1,i] = model.predict_proba(x1)[:,0]
                  del model
                  print("finished fold:", j)
  
              print("Building full prediction model for test set.")
              # configure model with j-th combo of parameters
              x = param_grid[i]
              model = Sequential()
              model.add(Dense(dims * x[4], input_shape=(dims,),W_regularizer=l2(x[3])))
              #model.add(PReLU())
              model.add(BatchNormalization())
              model.add(Dropout(x[0]))
              model.add(Dense(nb_classes))
              model.add(Activation('softmax'))
              opt=Adadelta(lr=x[2],decay=x[1],epsilon=1e-5)
              model.compile(loss='binary_crossentropy', optimizer=opt)
  
              # fit on complete dataset
              ytrain0 = np.zeros((xtrain.shape[0],2))
              ytrain0[:,0] = ytrain
              ytrain0[:,1] = 1- ytrain
              model.fit(np.array(xtrain), ytrain0,nb_epoch=epochs, batch_size=1000)
              mfull[:,i] = model.predict_proba(np.array(xtest))[:,0]
              del model
              print("finished full prediction")
  
      ## store the results
      # add indices etc
      mvalid = pd.DataFrame(mvalid)
      mvalid.columns = [model_type + str(i) for i in range(0, mvalid.shape[1])]
      mvalid['QuoteNumber'] = id_train
      mvalid['QuoteConversion_Flag'] = ytrain
  
      mfull = pd.DataFrame(mfull)
      mfull.columns = [model_type + str(i) for i in range(0, mfull.shape[1])]
      mfull['QuoteNumber'] = id_test
  
  
      # save the files
      mvalid.to_csv(projPath + 'metafeatures/prval_' + model_type + todate + '_data' + dataset_version + '_seed' + str(seed_value) + '.csv', index = False, header = True)
      mfull.to_csv(projPath + 'metafeatures/prfull_' + model_type + todate + '_data' + dataset_version + '_seed' + str(seed_value) + '.csv', index = False, header = True)
  
apis: sklearn.externals.joblib.load
apis: numpy.save numpy.load numpy.mean
apis: numpy.load numpy.save pandas.read_csv pandas.DataFrame sklearn.metrics.roc_auc_score numpy.std numpy.mean
apis: numpy.zeros pandas.read_csv pandas.DataFrame numpy.random.seed numpy.array numpy.where numpy.unique