text: #!/usr/bin/env python3
  # -*- coding: utf-8 -*-
  """
  Created on Sun Mar  3 11:32:16 2019
  
  @author: Kazuki
  """
  
  import numpy as np
  import pandas as pd
  import os, gc
  from glob import glob
  from tqdm import tqdm
  
  import sys
  sys.path.append(f'/home/{os.environ.get("USER")}/PythonLibrary')
  import lgbextension as ex
  import lightgbm as lgb
  
  from sklearn.externals import joblib
  from sklearn.metrics import roc_auc_score
  
  import utils , utils_cat
  utils.start(__file__)
  #==============================================================================
  
  SUBMIT_FILE_PATH = '../output/0303-1.csv.gz'
  
  COMMENT = 'nejumi + f014 f020'
  
  EXE_SUBMIT = True
  
  SEED = np.random.randint(9999)
  print('SEED:', SEED)
  np.random.seed(SEED)
  
  param = {
           'boosting_type': 'gbdt',
           'class_weight': None,
           'colsample_bytree': 0.71,
           'learning_rate': 0.05,
           'max_depth': -1,
           'min_child_samples': 10,
           'min_child_weight': 5,
           'min_split_gain': 0,
           'n_jobs': -1,
           'num_leaves': 64,
           'objective': 'binary',
           'reg_alpha': 0,
           'reg_lambda': 0,
           'subsample': 0.71,
           'subsample_for_bin': 50000,
           'subsample_freq': 1,
           'max_bin': 255,
           'metric': 'auc',
           'nthread': -1,
           'verbose': -1,
           }
  
  NROUND = 19999
  NFOLD = 5
  VERBOSE_EVAL = 100
  ESR = 100
  
  col_drop = [
  #        'f003_date_min', 
  #        'f003_date_max',
  #        'f003_date_max-m-min',
  #        'f003_max-m-AvSigVersion',
  #        'f003_max-m-Census_OSVersion',
  #        'f003_max-m-OsBuildLab'
          ]
  
  USE_PREF = ['f014', 'f020']
  
  RESULT_DICT = {}
  RESULT_DICT['file'] = SUBMIT_FILE_PATH
  
  
  # =============================================================================
  # load
  # =============================================================================
  
  files_tr = sorted(glob('../data/train_f*.f'))
  
  # USE_PREF
  li = []
  for i in files_tr:
      for j in USE_PREF:
          if j in i:
              li.append(i)
              break
  files_tr = li
  
  [print(i,f) for i,f in enumerate(files_tr)]
  
  X_train = pd.concat([
                  pd.read_feather(f) for f in tqdm(files_tr, mininterval=30)
                 ]+[joblib.load('../external/X_train_nejumi.pkl.gz')], axis=1)
  
  y_train = utils.load_target()['HasDetections']
  
  
  # drop
  if len(col_drop) > 0:
      X_train.drop(col_drop, axis=1, inplace=True)
  
  
  if X_train.columns.duplicated().sum()>0:
      raise Exception(f'duplicated!: { X_train.columns[X_train.columns.duplicated()] }')
  print('no dup :) ')
  print(f'X_train.shape {X_train.shape}')
  
  gc.collect()
  
  CAT = list( set(X_train.columns)&set(utils_cat.ALL))
  print(f'CAT: {CAT}')
  
  
  COL = X_train.columns.tolist()
  
  RESULT_DICT['feature size'] = len(COL)
  RESULT_DICT['category feature size'] = len(CAT)
  
  
  # =============================================================================
  # all sample
  # =============================================================================
  
  
  dtrain = lgb.Dataset(X_train, y_train.values, 
                       categorical_feature=CAT, 
                       free_raw_data=False)
  
  gc.collect()
  
  #models = []
  #for i in range(LOOP):
  #    param['seed'] = np.random.randint(9999)
  #    model = lgb.train(params=param, train_set=dtrain, 
  #                      num_boost_round=NROUND,
  #                      )
  #    model.save_model(f'../data/lgb{i}.model')
  #    models.append(model)
  
  # CV
  param['seed'] = np.random.randint(9999)
  ret, models = lgb.cv(param, dtrain, NROUND,
                       nfold=NFOLD,
                       stratified=True, shuffle=True,
                       feval=ex.eval_auc,
                       early_stopping_rounds=ESR, 
                       verbose_eval=VERBOSE_EVAL,
                       categorical_feature=CAT,
                       seed=SEED)
  
  for i, model in enumerate(models):
      model.save_model(f'../data/lgb{i}.model')
  
  
  #models = []
  #for i in range(LOOP):
  #    model = lgb.Booster(model_file=f'../data/lgb{i}.model')
  #    models.append(model)
  
  
  
  
  
  
  
  imp = ex.getImp(models)
  imp['split'] /= imp['split'].max()
  imp['gain'] /= imp['gain'].max()
  imp['total'] = imp['split'] + imp['gain']
  imp.sort_values('total', ascending=False, inplace=True)
  imp.reset_index(drop=True, inplace=True)
  
  
  imp.to_csv(f'LOG/imp_{__file__}.csv', index=False)
  utils.savefig_imp(imp, f'LOG/imp_{__file__}.png', x='total')
  
  RESULT_DICT['nfold'] = NFOLD
  RESULT_DICT['seed'] = SEED
  RESULT_DICT['eta'] = param['learning_rate']
  RESULT_DICT['best NROUND'] = len(ret['auc-mean'])
  RESULT_DICT['train AUC'] = ret['auc-mean'][-1]
  
  
  del dtrain, X_train, y_train; gc.collect()
  
  
  # =============================================================================
  # test
  # =============================================================================
  
  
  files_te = sorted(glob('../data/test_f*.f'))
  
  # USE_PREF
  li = []
  for i in files_te:
      for j in USE_PREF:
          if j in i:
              li.append(i)
              break
  files_te = li
  
  [print(i,f) for i,f in enumerate(files_te)]
  
  X_test = pd.concat([
                  pd.read_feather(f) for f in tqdm(files_te, mininterval=30)
                 ]+[joblib.load('../external/X_test_nejumi.pkl.gz')], axis=1)[COL]
  
  
  if X_test.columns.duplicated().sum()>0:
      raise Exception(f'duplicated!: { X_test.columns[X_test.columns.duplicated()] }')
  print('no dup :) ')
  print(f'X_test.shape {X_test.shape}')
  
  
  y_pred = pd.Series(0, index=X_test.index)
  
  for model in tqdm(models):
      y_pred += pd.Series(model.predict(X_test)).rank()
  y_pred /= y_pred.max()
  
  sub = pd.read_csv('../input/sample_submission.csv.zip')
  sub['HasDetections'] = y_pred.values
  
  
  print(f'corr with {utils.SUB_BEST}')
  sub_best = pd.read_csv(utils.SUB_BEST)
  print('with mybest:', sub['HasDetections'].corr( sub_best['HasDetections'], method='spearman') )
  sub_best['HasDetections'] = np.load(utils.SUB_nejumi)
  print('with nejumi:', sub['HasDetections'].corr( sub_best['HasDetections'], method='spearman') )
  
  
  
  print("""
  # =============================================================================
  # write down these info to benchmark.xlsx
  # =============================================================================
  """)
  
  [print(f'{k:<25}: {RESULT_DICT[k]}') for k in RESULT_DICT]
  
  print("""
  # =============================================================================
  """)
  
  
  
  # =============================================================================
  # submission
  # =============================================================================
  if EXE_SUBMIT:
      sub.to_csv(SUBMIT_FILE_PATH, index=False, compression='gzip')
      print('submit')
      utils.submit(SUBMIT_FILE_PATH, COMMENT)
  else:
      utils.to_pkl_gzip(sub[['HasDetections']], SUBMIT_FILE_PATH.replace('.csv.gz', f'_{SEED}.pkl'))
  
  
  
  
  
  #==============================================================================
  utils.end(__file__)
  #utils.stop_instance()
  
  