content: import os
  os.environ['CUDA_LAUNCH_BLOCKING'] = '1'
  import pandas as pd
  import matplotlib
  import numpy as np
  import math
  import matplotlib.pyplot as plt
  from sklearn.preprocessing import Imputer
  from sklearn.cross_validation import train_test_split
  from sklearn import preprocessing
  import plotly.tools as tls
  import pandas as pd
  from sqlalchemy import create_engine # database connection
  import datetime as dt
  import io
  import logging
  import plotly.plotly as py # interactive graphing
  from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
  from plotly.graph_objs import Bar, Scatter, Marker, Layout 
  from heraspy.model import HeraModel
  np.random.seed(1337)
  import theano
  import keras
  from keras.preprocessing.sequence import pad_sequences
  from keras.models import Model,model_from_yaml
  from keras.layers import Input, Dense, GRU, LSTM, TimeDistributed, Masking,merge
  from model import *
  import argparse
  import sys
  
  
  
  if __name__ == "__main__":
  
  
      t_start = dt.datetime.now()
      parser = argparse.ArgumentParser(prog='Weighted Model')
      parser.add_argument('-t','--table',required=True)
      args = parser.parse_args()
  
      ####################################DATA SOURCE################################
      table = vars(args)['table']
      # table = 'data_trim'
      # rsl_file = './data/gs_results_trim.csv'
      # rsl_file = './data/psql_data_trim.csv'
  
      # table = 'data_little_enc'
      # rsl_file = './data/gs_results_little.csv'
      
      # table = 'data_more'
      # rsl_file = './data/gs_results_more.csv'
      # table = 'auth'
      # rsl_file = './data/auth.csv'
  
      events_tbl = 'event'
      events_tbl = None
      rsl_file = './data/psql_{table}.csv'.format(table=table)
      ################################################################################
  
  
  
  
      print "Commencing..."
      data_dir = './data/'
      evt_name = 'Featurespace_events_output.csv'
      auth_name = 'Featurespace_auths_output.csv'
      db_name = 'c1_agg.db'
      
  
      address = "postgresql+pg8000://script@localhost:5432/ccfd"
      # disk_engine = create_engine('sqlite:///'+data_dir+db_name,convert_unicode=True)
      # disk_engine.raw_connection().connection.text_factory = str
      disk_engine = create_engine(address)
  
  
  
      #######################Settings#############################################
      samples_per_epoch = trans_num_table(table,disk_engine,mode='train',trans_mode='train')
      # epoch_limit = 10000
      # samples_per_epoch = epoch_limit
      # user_sample_size = 8000
  
  
      epoch_limit = samples_per_epoch
      user_sample_size = None
  
      nb_epoch = 300
      fraud_w_list = [1000.]
      
      ##########ENCODERS CONF
      tbl_src = 'auth'
      # tbl_src = table
      tbl_evnt = 'event'
      ##################################
     
      batch_size = 300
      batch_size_val = 1000
      print "SAMPLES per epoch:",samples_per_epoch
      print "User sample size:",user_sample_size
      print 'sequence length size',batch_size
      # samples_per_epoch = 1959
      # table = 'data_trim'
      # samples_per_epoch = 485
      
  
  
  
      lbl_pad_val = 2
      pad_val = 0
  
      # dropout_W_list = [0.3]
      dropout_W_list = [0.4,0.5,0.6,0.7]
      # dropout_W_list = [0.15,0.3,0.4,0.8]
      
  
  
  
  
  
      input_dim = 44
      hid_dims = [320]
      num_l = [7]
      lr_s = [2.5e-4]
      # lr_s = [1.25e-4,6e-5]
      # lr_s = [1e-2,1e-3,1e-4]
      # lr_s = [1e-1,1e-2,1e-3]
      num_opt = 1
      opts = lambda x,lr:[keras.optimizers.RMSprop(lr=lr, rho=0.9, epsilon=1e-08),
                      # keras.optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08),
                      # keras.optimizers.Nadam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004)
                      ][x]
      # add_info = str(int(seq_len_param))+'_class_w_'+str(fraud_w)                
                    
      
  
      print 'Populating encoders'
  
      path_encoders ='./data/encoders/{tbl_src}+{tbl_evnt}'.format(tbl_src=tbl_src,tbl_evnt=tbl_evnt) 
      if os.path.exists(path_encoders):
          encoders = load_encoders(path_encoders)
      else:
          encoders = populate_encoders_scale(tbl_src,disk_engine,tbl_evnt)
          with open(path_encoders, 'wb') as output:
              pickle.dump(encoders, output, pickle.HIGHEST_PROTOCOL)
              print 'ENCODERS SAVED to {path}!'.format(path=path_encoders)
  
      # sys.exit()
      gru_dict = {}
      lstm_dict = {}
      for fraud_w in fraud_w_list:
          add_info = 'Mask=pad_class_w_'+str(fraud_w)+'ES-OFF'  
          class_weight = {0 : 1.,
              1: fraud_w,
              2: 0.}
          for dropout_W in dropout_W_list:
              for hidden_dim in hid_dims:
              # gru
                  for opt_id in range(num_opt):
                      for lr in lr_s:
                          optimizer = opts(opt_id,lr)
                          for num_layers in num_l:
                              for rnn in ['gru']:
  
                                  short_title = 'bi_'+rnn.upper()+'_'+str(hidden_dim)+'_'+str(num_layers)+'_DO-'+str(dropout_W)+'_w'+str(class_weight[1])
                                  title = 'Bidirectional_Class'+str(class_weight[1])+'_'+rnn.upper()+'_'+str(hidden_dim)+'_'+str(num_layers)+'_'+str(type(optimizer).__name__)+'_'+str(lr)+'_epochs_'+str(nb_epoch)+'_DO-'+str(dropout_W)
                                  print title
                                  input_layer = Input(shape=(int(seq_len_param), input_dim),name='main_input')
                                  mask = Masking(mask_value=pad_val)(input_layer)
                                  x = mask
                                  for i in range(num_layers):
                                      if rnn == 'gru':
                                          prev_frw = GRU(hidden_dim,#input_length=50,
                                                              return_sequences=True,go_backwards=False,stateful=False,
                                                              unroll=False,consume_less='gpu',
                                                              init='glorot_uniform', inner_init='orthogonal', activation='tanh',
                                                      inner_activation='hard_sigmoid', W_regularizer=None, U_regularizer=None,
                                                      b_regularizer=None, dropout_W=dropout_W, dropout_U=0.0)(x)
                                          prev_bck = GRU(hidden_dim,#input_length=50,
                                                              return_sequences=True,go_backwards=True,stateful=False,
                                                              unroll=False,consume_less='gpu',
                                                              init='glorot_uniform', inner_init='orthogonal', activation='tanh',
                                                      inner_activation='hard_sigmoid', W_regularizer=None, U_regularizer=None,
                                                      b_regularizer=None, dropout_W=dropout_W, dropout_U=0.0)(x)
                                      else:
                                          prev_frw = LSTM(hidden_dim, return_sequences=True,go_backwards=False,stateful=False,
                                              init='glorot_uniform', inner_init='orthogonal', 
                                              forget_bias_init='one', activation='tanh', inner_activation='hard_sigmoid',
                                              W_regularizer=None, U_regularizer=None, b_regularizer=None, dropout_W=dropout_W, dropout_U=0.0)(x)
                                          prev_bck = LSTM(hidden_dim, return_sequences=True,go_backwards=True,stateful=False,
                                              init='glorot_uniform', inner_init='orthogonal', 
                                              forget_bias_init='one', activation='tanh', inner_activation='hard_sigmoid',
                                              W_regularizer=None, U_regularizer=None, b_regularizer=None, dropout_W=dropout_W, dropout_U=0.0)(x)
                                      x = merge([prev_frw, prev_bck], mode='concat')
                                  output_layer = TimeDistributed(Dense(3,activation='softmax'))(x)
                                  model = Model(input=[input_layer],output=[output_layer])
                                  model.compile(optimizer=optimizer,
                                                  loss='sparse_categorical_crossentropy',
                                                  metrics=['accuracy'],
                                                  sample_weight_mode="temporal")
  
                                  ########save architecture ######
                                  arch_dir = './data/models/archs/'+short_title+'.yml'
                                  yaml_string = model.to_yaml()
                                  with open(arch_dir, 'wb') as output:
                                      pickle.dump(yaml_string, output, pickle.HIGHEST_PROTOCOL)
                                      print 'model saved!'
                                  ##############        
  
                                  user_mode = 'train'
                                  trans_mode = 'train'
                                  data_gen = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                                                   batch_size=batch_size,usr_ratio=80,class_weight=class_weight,lbl_pad_val = lbl_pad_val, pad_val = pad_val,
                                                   sub_sample=user_sample_size,epoch_size=epoch_limit,events_tbl=events_tbl)
                                                   # sub_sample=user_sample_size,epoch_size=samples_per_epoch)
                                  ########validation data
                                  print 'Generating Validation set!'
                                  user_mode = 'test'
                                  trans_mode = 'test'
                                  val_gen = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                                                   batch_size=batch_size_val,usr_ratio=80,class_weight=class_weight,lbl_pad_val = lbl_pad_val, pad_val = pad_val,
                                                   sub_sample=None,epoch_size=None,events_tbl=events_tbl)
                                  validation_data = next(val_gen)
                                  print '################GENERATED#######################'
                                  ###############CALLBACKS
                                  patience = 30
                                  early_Stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=patience, verbose=0, mode='auto')
  
                                  save_path = './data/models/'+table+'/'
                                  var_name = '.{epoch:02d}-{val_loss:.5f}.hdf5'
                                  checkpoint = keras.callbacks.ModelCheckpoint(save_path+short_title+var_name, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')
  
                                  root_url = 'http://localhost:9000'
                                  remote_log = keras.callbacks.RemoteMonitor(root=root_url)
  
                                  # callbacks = [early_Stop,checkpoint]
                                  callbacks = [early_Stop,checkpoint,remote_log]
                                  callbacks = []
                                  history = model.fit_generator(data_gen, samples_per_epoch, nb_epoch, verbose=1, callbacks=callbacks,validation_data=validation_data, nb_val_samples=None, class_weight=None, max_q_size=10000)
  
                                  py.sign_in('bottydim', 'o1kuyms9zv') 
  
                                  auc_list = []
                                  print '#########################TRAIN STATS################'
                                  user_mode = 'train'
                                  trans_mode = 'train'
                                  val_samples = trans_num_table(table,disk_engine,mode=user_mode,trans_mode=trans_mode)
                                  print '# samples',val_samples
                                  plt_filename = './figures/GS/'+table+'/'+'ROC_'+user_mode+'_'+trans_mode+'_'+title+'_'+add_info+".png"
  
                                  data_gen = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                                                   batch_size=batch_size,usr_ratio=80,class_weight=None,lbl_pad_val = lbl_pad_val, pad_val = pad_val,events_tbl=events_tbl) 
  
                                  eval_list  = eval_auc_generator(model, data_gen, val_samples, max_q_size=10000,plt_filename=plt_filename)
                                  auc_val = eval_list[0]
                                  clc_report = eval_list[1]
                                  acc = eval_list[2]
                                  print "AUC:",auc_val 
                                  print 'CLassification report'
                                  print clc_report
                                  print 'Accuracy'
                                  print acc
                                  auc_list.append(str(auc_val))
                                  print '##################EVALUATION USERS#########################'
  
                                  user_mode = 'test'
                                  trans_mode = 'train'
                                  val_samples = trans_num_table(table,disk_engine,mode=user_mode,trans_mode=trans_mode)
                                  print '# samples',val_samples
                                  plt_filename = './figures/GS/'+table+'/'+'ROC_'+user_mode+'_'+trans_mode+'_'+title+'_'+add_info+".png"
  
                                  eval_gen = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                                                   batch_size=batch_size,usr_ratio=80,class_weight=None,lbl_pad_val = lbl_pad_val, pad_val = pad_val,events_tbl=events_tbl) 
  
                                  eval_list  = eval_auc_generator(model, eval_gen, val_samples, max_q_size=10000,plt_filename=plt_filename)
                                  auc_val = eval_list[0]
                                  clc_report = eval_list[1]
                                  acc = eval_list[2]
                                  print "AUC:",auc_val 
                                  print 'CLassification report'
                                  print clc_report
                                  print 'Accuracy'
                                  print acc
                                  auc_list.append(str(auc_val))
                                  print '#####################################################'
                                  print '##################EVALUATION Transactions#########################'
  
                                  user_mode = 'train'
                                  trans_mode = 'test'
                                  val_samples = trans_num_table(table,disk_engine,mode=user_mode,trans_mode=trans_mode)
                                  print '# samples',val_samples
                                  plt_filename = './figures/GS/'+table+'/'+'ROC_'+user_mode+'_'+trans_mode+'_'+title+'_'+add_info+".png"
  
                                  eval_gen = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                                                   batch_size=batch_size,usr_ratio=80,class_weight=None,lbl_pad_val = lbl_pad_val, pad_val = pad_val,events_tbl=events_tbl) 
  
                                  eval_list  = eval_auc_generator(model, eval_gen, val_samples, max_q_size=10000,plt_filename=plt_filename)
                                  auc_val = eval_list[0]
                                  clc_report = eval_list[1]
                                  acc = eval_list[2]
                                  print "AUC:",auc_val 
                                  print 'CLassification report'
                                  print clc_report
                                  print 'Accuracy'
                                  print acc
                                  auc_list.append(str(auc_val))
                                  print '#####################################################'
                                  print '##################EVALUATION Pure#########################'
  
                                  user_mode = 'test'
                                  trans_mode = 'test'
                                  val_samples = trans_num_table(table,disk_engine,mode=user_mode,trans_mode=trans_mode)
                                  print '# samples',val_samples
                                  plt_filename = './figures/GS/'+table+'/'+'ROC_'+user_mode+'_'+trans_mode+'_'+title+'_'+add_info+".png"
                                  
                                  eval_gen = data_generator(user_mode,trans_mode,disk_engine,encoders,table=table,
                                                   batch_size=batch_size,usr_ratio=80,class_weight=None,lbl_pad_val = lbl_pad_val, pad_val = pad_val,events_tbl=events_tbl)
                                  
                                  eval_list  = eval_auc_generator(model, eval_gen, val_samples, max_q_size=10000,plt_filename=plt_filename)
                                  auc_val = eval_list[0]
                                  clc_report = eval_list[1]
                                  acc = eval_list[2]
                                  print "AUC:",auc_val 
                                  print 'CLassification report'
                                  print clc_report
                                  print 'Accuracy'
                                  print acc
                                  auc_list.append(str(auc_val))
                                  print '#####################################################'
                                  with io.open(rsl_file, 'a', encoding='utf-8') as file:
                                      auc_string = ','.join(auc_list)
                                      title_csv = title.replace('_',',')+','+str(history.history['acc'][-1])+','+str(history.history['loss'][-1])+','+str(auc_val)+','+str(acc)+','+auc_string+'\n'
                                      file.write(unicode(title_csv))
                                      print 'logged @ {file}'.format(file=rsl_file)
                                  trim_point = -15
                                  fig = {
                                      'data': [Scatter(
                                          x=history.epoch[trim_point:],
                                          y=history.history['loss'][trim_point:])],
                                      'layout': {'title': title}
                                      }
                                  py.image.save_as(fig,filename='./results/figures/'+table+'/'+short_title+'_'+'LOSS'+'_'+add_info+".png")
                                  trim_point = 0
                                  fig = {
                                      'data': [Scatter(
                                          x=history.epoch[trim_point:],
                                          y=history.history['loss'][trim_point:])],
                                      'layout': {'title': title}
                                      }
                                  py.image.save_as(fig,filename='./results/figures/'+table+'/'+short_title+'_'+'LOSS'+'_'+'FULL'+".png")                            
  
                                  # iplot(fig,filename='figures/'+title,image='png')
                                  # title = title.replace('Loss','Acc')
                                  fig = {
                                      'data': [Scatter(
                                          x=history.epoch[trim_point:],
                                          y=history.history['acc'][trim_point:])],
                                      'layout': {'title': title}
                                      }
                                  filename_val='./results/figures/'+table+'/'+short_title+'_'+'ACC'+'_'+add_info+".png"
                                  py.image.save_as(fig,filename=filename_val)    
                                  print 'exported @',filename_val
                                  fig = {
                                      'data': [Scatter(
                                          x=history.epoch[trim_point:],
                                          y=history.history['val_loss'][trim_point:])],
                                      'layout': {'title': title}
                                      }
                                  py.image.save_as(fig,filename='./results/figures/'+table+'/'+short_title+'_'+'VAL LOSS'+'_'+add_info+".png")   
                                  print 'time taken: {time}'.format(time=days_hours_minutes_seconds(dt.datetime.now()-t_start))