code: #!/usr/bin/env python3
  # -*- coding: utf-8 -*-
  """
  Created on Tue Feb  5 22:33:48 2019
  
  @author: Kazuki
  """
  
  import numpy as np
  import pandas as pd
  import os, gc
  from glob import glob
  from tqdm import tqdm
  
  import sys
  sys.path.append(f'/home/{os.environ.get("USER")}/PythonLibrary')
  import lgbextension as ex
  import lightgbm as lgb
  from multiprocessing import cpu_count
  
  from sklearn.metrics import roc_auc_score
  
  import utils , utils_cat
  utils.start(__file__)
  #==============================================================================
  
  SEED = np.random.randint(9999)
  print('SEED:', SEED)
  
  
  DROP = [
  #        'f002_EngineVersion', 'f002_AvSigVersion', 'f002_AppVersion',
  #        
  #        'f003_AvSigVersion', 'f003_OsBuildLab', 'f003_Census_OSVersion',
  #        'f003_date_min', 'f003_date_max'
          ]
  
  NFOLD = 5
  
  LOOP = 1
  
  param = {
           'objective': 'binary',
           'metric': 'auc',
           
           'learning_rate': 0.05,
           'max_depth': -1,
           'num_leaves': 2**6 -1,
           'max_bin': 127,
           
           'min_child_weight': 10,
           'min_data_in_leaf': 150,
           'reg_lambda': 0.5,  # L2 regularization term on weights.
           'reg_alpha': 0.5,  # L1 regularization term on weights.
           
           'colsample_bytree': 0.9,
           'subsample': 0.7,
  #         'nthread': 32,
           'nthread': cpu_count(),
           'bagging_freq': 1,
           'verbose':-1,
           }
  
  NROUND = 500
  ESR = 50
  VERBOSE_EVAL = 25
  
  TRAIN_TH = 0.6
  VALID_TH = 0.8
  
  outpath_tr = '../data/train_f005_1.f'
  outpath_te = '../data/test_f005_1.f'
  
  # =============================================================================
  # load
  # =============================================================================
  
  files_tr = sorted(glob('../data/f005/train_f005*.f'))[20:40]
  
  
  [print(i,f) for i,f in enumerate(files_tr)]
  
  X_train = pd.concat([
                  pd.read_feather(f).sample(frac=0.5, random_state=SEED) for f in tqdm(files_tr, mininterval=60)
                 ], axis=1)
  y_train = utils.load_target().sample(frac=0.5, random_state=SEED)['HasDetections']
  
  if len(DROP)>0:
      X_train.drop(DROP, axis=1, inplace=True)
  
  
  #adv = pd.read_csv('../data/oof_802_adv.py.csv').iloc[:8921483].oof
  #adv_th = adv.quantile(VALID_TH)
  #
  #X_valid = X_train[adv>adv.quantile(VALID_TH)]
  #y_valid = y_train[adv>adv.quantile(VALID_TH)]
  #
  #X_train = X_train[adv<=adv.quantile(TRAIN_TH)]
  #y_train = y_train[adv<=adv.quantile(TRAIN_TH)]
  
  if X_train.columns.duplicated().sum()>0:
      raise Exception(f'duplicated!: { X_train.columns[X_train.columns.duplicated()] }')
  print('no dup :) ')
  print(f'X_train.shape {X_train.shape}')
  #print(f'X_valid.shape {X_valid.shape}')
  
  gc.collect()
  
  CAT = list( set(X_train.columns)&set(utils_cat.ALL))
  print(f'CAT: {CAT}')
  
  
  # =============================================================================
  # hold out
  # =============================================================================
  
  
  dtrain = lgb.Dataset(X_train, y_train.values, 
                       categorical_feature=CAT, 
                       free_raw_data=False)
  
  #dvalid = lgb.Dataset(X_valid, y_valid.values, 
  #                     categorical_feature=CAT, 
  #                     free_raw_data=False)
  gc.collect()
  
  
  
  
  model = lgb.train(params=param, train_set=dtrain, num_boost_round=NROUND, 
  #                  valid_sets=[dtrain, dvalid], 
  #                  valid_names=['train','valid'], 
  #                  feval=ex.eval_auc,
                    categorical_feature=CAT, 
  #                  early_stopping_rounds=ESR,
                    verbose_eval=VERBOSE_EVAL)
  
  
  
  imp = ex.getImp(model)
  imp['split'] /= imp['split'].max()
  imp['gain'] /= imp['gain'].max()
  imp['total'] = imp['split'] + imp['gain']
  imp.sort_values('total', ascending=False, inplace=True)
  imp.reset_index(drop=True, inplace=True)
  
  
  imp.to_csv(f'LOG/imp_{__file__}.csv', index=False)
  
  
  # =============================================================================
  # 
  # =============================================================================
  
  imp = pd.read_csv('LOG/imp_005-2_agg_each_lgb_1.py.csv')
  COL = imp.head(30).feature.tolist()
  
  X_train = pd.concat([
                  pd.read_feather(f) for f in tqdm(files_tr, mininterval=60)
                 ], axis=1)[COL]
  
  X_train.to_feather(outpath_tr)
  
  
  
  files_te = sorted(glob('../data/f005/test_f005*.f'))[20:40]
  
  X_test = pd.concat([
                  pd.read_feather(f) for f in tqdm(files_te, mininterval=60)
                 ], axis=1)[COL]
  
  X_test.to_feather(outpath_te)
  
  
  #==============================================================================
  utils.end(__file__)
  #utils.stop_instance()
  
  
apis: pandas.read_csv pandas.read_feather numpy.random.randint