content: """
  A non-blending lightGBM model that incorporates portions and ideas from various public kernels
  This kernel gives LB: 0.977 when the parameter 'debug' below is set to 0 but this implementation requires a machine with ~32 GB of memory
  """
  
  import pandas as pd
  import time
  import numpy as np
  from sklearn.cross_validation import train_test_split
  import lightgbm as lgb
  import gc
  import matplotlib.pyplot as plt
  import os
  
  debug=1 
  if debug:
      print('*** debug parameter set: this is a test run for debugging purposes ***')
  
  def lgb_modelfit_nocv(params, dtrain, dvalid, predictors, target='target', objective='binary', metrics='auc',
                   feval=None, early_stopping_rounds=20, num_boost_round=3000, verbose_eval=10, categorical_features=None):
      lgb_params = {
          'boosting_type': 'gbdt',
          'objective': objective,
          'metric':metrics,
          'learning_rate': 0.2,
          #'is_unbalance': 'true',  #because training data is unbalance (replaced with scale_pos_weight)
          'num_leaves': 31,  # we should let it be smaller than 2^(max_depth)
          'max_depth': -1,  # -1 means no limit
          'min_child_samples': 20,  # Minimum number of data need in a child(min_data_in_leaf)
          'max_bin': 255,  # Number of bucketed bin for feature values
          'subsample': 0.6,  # Subsample ratio of the training instance.
          'subsample_freq': 0,  # frequence of subsample, <=0 means no enable
          'colsample_bytree': 0.3,  # Subsample ratio of columns when constructing each tree.
          'min_child_weight': 5,  # Minimum sum of instance weight(hessian) needed in a child(leaf)
          'subsample_for_bin': 200000,  # Number of samples for constructing bin
          'min_split_gain': 0,  # lambda_l1, lambda_l2 and min_gain_to_split to regularization
          'reg_alpha': 0,  # L1 regularization term on weights
          'reg_lambda': 0,  # L2 regularization term on weights
          'nthread': 4,
          'verbose': 0,
          'metric':metrics
      }
  
      lgb_params.update(params)
  
      print("preparing validation datasets")
  
      xgtrain = lgb.Dataset(dtrain[predictors].values, label=dtrain[target].values,
                            feature_name=predictors,
                            categorical_feature=categorical_features
                            )
      xgvalid = lgb.Dataset(dvalid[predictors].values, label=dvalid[target].values,
                            feature_name=predictors,
                            categorical_feature=categorical_features
                            )
  
      evals_results = {}
  
      bst1 = lgb.train(lgb_params, 
                       xgtrain, 
                       valid_sets=[xgtrain, xgvalid], 
                       valid_names=['train','valid'], 
                       evals_result=evals_results, 
                       num_boost_round=num_boost_round,
                       early_stopping_rounds=early_stopping_rounds,
                       verbose_eval=10, 
                       feval=feval)
  
      print("\nModel Report")
      print("bst1.best_iteration: ", bst1.best_iteration)
      print(metrics+":", evals_results['valid'][metrics][bst1.best_iteration-1])
  
      return (bst1,bst1.best_iteration)
  
  def DO(frm,to,fileno):
      dtypes = {
              'ip'            : 'uint32',
              'app'           : 'uint16',
              'device'        : 'uint16',
              'os'            : 'uint16',
              'channel'       : 'uint16',
              'is_attributed' : 'uint8',
              'click_id'      : 'uint32',
              }
  
      print('loading train data...',frm,to)
      train_df = pd.read_csv("../input/train.csv", parse_dates=['click_time'], skiprows=range(1,frm), nrows=to-frm, dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'])
  
      print('loading test data...')
      if debug:
          test_df = pd.read_csv("../input/test.csv", nrows=100000, parse_dates=['click_time'], dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])
      else:
          test_df = pd.read_csv("../input/test.csv", parse_dates=['click_time'], dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])
  
      len_train = len(train_df)
      train_df=train_df.append(test_df)
  
      del test_df
      gc.collect()
      
      print('Extracting new features...')
      train_df['hour'] = pd.to_datetime(train_df.click_time).dt.hour.astype('uint8')
      train_df['day'] = pd.to_datetime(train_df.click_time).dt.day.astype('uint8')
      
      gc.collect()
      
      naddfeat=9
      for i in range(0,naddfeat):
          if i==0: selcols=['ip', 'channel']; QQ=4;
          if i==1: selcols=['ip', 'device', 'os', 'app']; QQ=5;
          if i==2: selcols=['ip', 'day', 'hour']; QQ=4;
          if i==3: selcols=['ip', 'app']; QQ=4;
          if i==4: selcols=['ip', 'app', 'os']; QQ=4;
          if i==5: selcols=['ip', 'device']; QQ=4;
          if i==6: selcols=['app', 'channel']; QQ=4;
          if i==7: selcols=['ip', 'os']; QQ=5;
          if i==8: selcols=['ip', 'device', 'os', 'app']; QQ=4;
          print('selcols',selcols,'QQ',QQ)
          
          filename='X%d_%d_%d.csv'%(i,frm,to)
          
          if os.path.exists(filename):
              if QQ==5: 
                  gp=pd.read_csv(filename,header=None)
                  train_df['X'+str(i)]=gp
              else: 
                  gp=pd.read_csv(filename)
                  train_df = train_df.merge(gp, on=selcols[0:len(selcols)-1], how='left')
          else:
              if QQ==0:
                  gp = train_df[selcols].groupby(by=selcols[0:len(selcols)-1])[selcols[len(selcols)-1]].count().reset_index().\
                      rename(index=str, columns={selcols[len(selcols)-1]: 'X'+str(i)})
                  train_df = train_df.merge(gp, on=selcols[0:len(selcols)-1], how='left')
              if QQ==1:
                  gp = train_df[selcols].groupby(by=selcols[0:len(selcols)-1])[selcols[len(selcols)-1]].mean().reset_index().\
                      rename(index=str, columns={selcols[len(selcols)-1]: 'X'+str(i)})
                  train_df = train_df.merge(gp, on=selcols[0:len(selcols)-1], how='left')
              if QQ==2:
                  gp = train_df[selcols].groupby(by=selcols[0:len(selcols)-1])[selcols[len(selcols)-1]].var().reset_index().\
                      rename(index=str, columns={selcols[len(selcols)-1]: 'X'+str(i)})
                  train_df = train_df.merge(gp, on=selcols[0:len(selcols)-1], how='left')
              if QQ==3:
                  gp = train_df[selcols].groupby(by=selcols[0:len(selcols)-1])[selcols[len(selcols)-1]].skew().reset_index().\
                      rename(index=str, columns={selcols[len(selcols)-1]: 'X'+str(i)})
                  train_df = train_df.merge(gp, on=selcols[0:len(selcols)-1], how='left')
              if QQ==4:
                  gp = train_df[selcols].groupby(by=selcols[0:len(selcols)-1])[selcols[len(selcols)-1]].nunique().reset_index().\
                      rename(index=str, columns={selcols[len(selcols)-1]: 'X'+str(i)})
                  train_df = train_df.merge(gp, on=selcols[0:len(selcols)-1], how='left')
              if QQ==5:
                  gp = train_df[selcols].groupby(by=selcols[0:len(selcols)-1])[selcols[len(selcols)-1]].cumcount()
                  train_df['X'+str(i)]=gp.values
              
              if not debug:
                   gp.to_csv(filename,index=False)
              
          del gp
          gc.collect()    
  
      print('doing nextClick')
      predictors=[]
      
      new_feature = 'nextClick'
      filename='nextClick_%d_%d.csv'%(frm,to)
  
      if os.path.exists(filename):
          print('loading from save file')
          QQ=pd.read_csv(filename).values
      else:
          D=2**26
          train_df['category'] = (train_df['ip'].astype(str) + "_" + train_df['app'].astype(str) + "_" + train_df['device'].astype(str) \
              + "_" + train_df['os'].astype(str)).apply(hash) % D
          click_buffer= np.full(D, 3000000000, dtype=np.uint32)
  
          train_df['epochtime']= train_df['click_time'].astype(np.int64) // 10 ** 9
          next_clicks= []
          for category, t in zip(reversed(train_df['category'].values), reversed(train_df['epochtime'].values)):
              next_clicks.append(click_buffer[category]-t)
              click_buffer[category]= t
          del(click_buffer)
          QQ= list(reversed(next_clicks))
  
          if not debug:
              print('saving')
              pd.DataFrame(QQ).to_csv(filename,index=False)
  
      train_df[new_feature] = QQ
      predictors.append(new_feature)
  
      train_df[new_feature+'_shift'] = pd.DataFrame(QQ).shift(+1).values
      predictors.append(new_feature+'_shift')
      
      del QQ
      gc.collect()
  
      print('grouping by ip-day-hour combination...')
      gp = train_df[['ip','day','hour','channel']].groupby(by=['ip','day','hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_tcount'})
      train_df = train_df.merge(gp, on=['ip','day','hour'], how='left')
      del gp
      gc.collect()
  
      print('grouping by ip-app combination...')
      gp = train_df[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})
      train_df = train_df.merge(gp, on=['ip','app'], how='left')
      del gp
      gc.collect()
  
      print('grouping by ip-app-os combination...')
      gp = train_df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})
      train_df = train_df.merge(gp, on=['ip','app', 'os'], how='left')
      del gp
      gc.collect()
  
      # Adding features with var and mean hour (inspired from nuhsikander's script)
      print('grouping by : ip_day_chl_var_hour')
      gp = train_df[['ip','day','hour','channel']].groupby(by=['ip','day','channel'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_tchan_count'})
      train_df = train_df.merge(gp, on=['ip','day','channel'], how='left')
      del gp
      gc.collect()
  
      print('grouping by : ip_app_os_var_hour')
      gp = train_df[['ip','app', 'os', 'hour']].groupby(by=['ip', 'app', 'os'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_app_os_var'})
      train_df = train_df.merge(gp, on=['ip','app', 'os'], how='left')
      del gp
      gc.collect()
  
      print('grouping by : ip_app_channel_var_day')
      gp = train_df[['ip','app', 'channel', 'day']].groupby(by=['ip', 'app', 'channel'])[['day']].var().reset_index().rename(index=str, columns={'day': 'ip_app_channel_var_day'})
      train_df = train_df.merge(gp, on=['ip','app', 'channel'], how='left')
      del gp
      gc.collect()
  
      print('grouping by : ip_app_chl_mean_hour')
      gp = train_df[['ip','app', 'channel','hour']].groupby(by=['ip', 'app', 'channel'])[['hour']].mean().reset_index().rename(index=str, columns={'hour': 'ip_app_channel_mean_hour'})
      print("merging...")
      train_df = train_df.merge(gp, on=['ip','app', 'channel'], how='left')
      del gp
      gc.collect()
  
      print("vars and data type: ")
      train_df.info()
      train_df['ip_tcount'] = train_df['ip_tcount'].astype('uint16')
      train_df['ip_app_count'] = train_df['ip_app_count'].astype('uint16')
      train_df['ip_app_os_count'] = train_df['ip_app_os_count'].astype('uint16')
  
      target = 'is_attributed'
      predictors.extend(['app','device','os', 'channel', 'hour', 'day', 
                    'ip_tcount', 'ip_tchan_count', 'ip_app_count',
                    'ip_app_os_count', 'ip_app_os_var',
                    'ip_app_channel_var_day','ip_app_channel_mean_hour'])
      categorical = ['app', 'device', 'os', 'channel', 'hour', 'day']
      for i in range(0,naddfeat):
          predictors.append('X'+str(i))
          
      print('predictors',predictors)
  
      test_df = train_df[len_train:]
      val_df = train_df[(len_train-val_size):len_train]
      train_df = train_df[:(len_train-val_size)]
  
      print("train size: ", len(train_df))
      print("valid size: ", len(val_df))
      print("test size : ", len(test_df))
  
      sub = pd.DataFrame()
      sub['click_id'] = test_df['click_id'].astype('int')
  
      gc.collect()
  
      print("Training...")
      start_time = time.time()
  
      params = {
          'learning_rate': 0.20,
          #'is_unbalance': 'true', # replaced with scale_pos_weight argument
          'num_leaves': 7,  # 2^max_depth - 1
          'max_depth': 3,  # -1 means no limit
          'min_child_samples': 100,  # Minimum number of data need in a child(min_data_in_leaf)
          'max_bin': 100,  # Number of bucketed bin for feature values
          'subsample': 0.7,  # Subsample ratio of the training instance.
          'subsample_freq': 1,  # frequence of subsample, <=0 means no enable
          'colsample_bytree': 0.9,  # Subsample ratio of columns when constructing each tree.
          'min_child_weight': 0,  # Minimum sum of instance weight(hessian) needed in a child(leaf)
          'scale_pos_weight':200 # because training data is extremely unbalanced 
      }
      (bst,best_iteration) = lgb_modelfit_nocv(params, 
                              train_df, 
                              val_df, 
                              predictors, 
                              target, 
                              objective='binary', 
                              metrics='auc',
                              early_stopping_rounds=30, 
                              verbose_eval=True, 
                              num_boost_round=1000, 
                              categorical_features=categorical)
  
      print('[{}]: model training time'.format(time.time() - start_time))
      del train_df
      del val_df
      gc.collect()
      
      print('Plot feature importances...')
      ax = lgb.plot_importance(bst, max_num_features=100)
      plt.show()
  
      print("Predicting...")
      sub['is_attributed'] = bst.predict(test_df[predictors],num_iteration=best_iteration)
      if not debug:
          print("writing...")
          sub.to_csv('sub_it%d.csv.gz'%(fileno),index=False,compression='gzip')
      print("done...")
      return sub
  
  nrows=184903891-1
  nchunk=40000000
  val_size=2500000
  
  frm=nrows-75000000
  if debug:
      frm=0
      nchunk=100000
      val_size=10000
  
  to=frm+nchunk
  
  sub=DO(frm,to,0)
  