code: # https://github.com/pytorch/ignite/blob/master/examples/notebooks/FashionMNIST.ipynb
  # https://www.kaggle.com/stuarthallows/using-xgboost-with-scikit-learn
  import time
  import sys, os
  
  from sklearn.metrics import f1_score
  
  idx = os.getcwd().index("trade")
  PROJECT_HOME = os.getcwd()[:idx] + "trade"
  sys.path.append(PROJECT_HOME)
  
  import xgboost as xgb
  from pytz import timezone
  from sklearn.model_selection import StratifiedKFold, GridSearchCV, train_test_split
  from sklearn.ensemble import GradientBoostingClassifier
  from common.global_variables import *
  import matplotlib.pyplot as plt
  from common.utils import *
  from web.db.database import Model, trade_db_session
  
  import numpy as np
  import os
  from common.logger import get_logger
  from codes.upbit.upbit_api import Upbit
  from codes.upbit.upbit_order_book_based_data import UpbitOrderBookBasedData
  from common.utils import save_model
  
  import torch.nn.modules.loss
  import warnings
  import gc
  warnings.filterwarnings("ignore")
  
  logger = get_logger("make_models")
  
  def mkdir_models():
      if not os.path.exists(os.path.join(PROJECT_HOME, LOCAL_MODEL_SOURCE)):
          os.makedirs(os.path.join(PROJECT_HOME, LOCAL_MODEL_SOURCE))
  
      if not os.path.exists(os.path.join(PROJECT_HOME, LOCAL_MODEL_SOURCE, 'graphs')):
          os.makedirs(os.path.join(PROJECT_HOME, LOCAL_MODEL_SOURCE, 'graphs'))
  
      if not os.path.exists(os.path.join(PROJECT_HOME, LOCAL_MODEL_SOURCE, 'SCALERS')):
          os.makedirs(os.path.join(PROJECT_HOME, LOCAL_MODEL_SOURCE, 'SCALERS'))
  
  
  def save_graph(coin_name, model_type, valid_loss_min, last_valid_accuracy, last_save_epoch, valid_size, one_count_rate,
                 avg_train_losses, train_accuracy_list, avg_valid_losses, valid_accuracy_list):
      files = glob.glob(os.path.join(PROJECT_HOME, LOCAL_MODEL_SOURCE, model_type, coin_name + '*'))
      for f in files:
          os.remove(f)
  
      plt.clf()
  
      fig, ax_lst = plt.subplots(2, 2, figsize=(30, 10), gridspec_kw={'hspace': 0.35})
  
      ax_lst[0][0].plot(range(len(avg_train_losses)), avg_train_losses)
      ax_lst[0][0].set_title('AVG. TRAIN LOSSES', fontweight="bold", size=10)
  
      ax_lst[0][1].plot(range(len(train_accuracy_list)), train_accuracy_list)
      ax_lst[0][1].set_title('TRAIN ACCURACY CHANGE', fontweight="bold", size=10)
      ax_lst[0][1].set_xlabel('EPISODES', size=10)
  
      ax_lst[1][0].plot(range(len(avg_valid_losses)), avg_valid_losses)
      ax_lst[1][0].set_title('AVG. VALIDATION LOSSES', fontweight="bold", size=10)
  
      ax_lst[1][1].plot(range(len(valid_accuracy_list)), valid_accuracy_list)
      ax_lst[1][1].set_title('VALIDATION ACCURACY CHANGE', fontweight="bold", size=10)
      ax_lst[1][1].set_xlabel('EPISODES', size=10)
  
      filename = os.path.join(PROJECT_HOME, LOCAL_MODEL_SOURCE, model_type, 'graphs', "{0}_{1}_{2:.2f}_{3:.2f}_{4}_{5:.2f}.png".format(
          coin_name,
          last_save_epoch,
          valid_loss_min,
          last_valid_accuracy,
          valid_size,
          one_count_rate
      ))
  
      plt.savefig(filename)
      plt.close('all')
  
  
  def train(optimizer, model, criterion, train_losses, x_train_normalized, y_up_train):
      model.train()
      optimizer.zero_grad()
      out = model.forward(x_train_normalized)
  
      loss = criterion(out, y_up_train)
      loss.backward()
      optimizer.step()
      train_losses.append(loss.item())
  
      out = torch.sigmoid(out)
      t = torch.tensor(0.5).to(DEVICE)
      output_index = (out > t).float() * 1
  
      return y_up_train.size(0), (output_index == y_up_train).sum().float()
  
  
  def post_train_processing(train_losses, avg_train_losses, train_accuracy_list, correct, total_size):
      train_loss = np.average(train_losses)
      avg_train_losses.append(train_loss)
  
      train_accuracy = 100 * correct / total_size
      train_accuracy_list.append(train_accuracy)
  
      return train_loss, train_accuracy
  
  
  def validate(epoch, model, criterion, valid_losses, x_valid_normalized, y_up_valid):
      model.eval()
      out = model.forward(x_valid_normalized)
      loss = criterion(out, y_up_valid)
      valid_losses.append(loss.item())
  
      out = torch.sigmoid(out)
      t = torch.tensor(0.5).to(DEVICE)
      output_index = (out > t).float() * 1
  
      if VERBOSE: logger.info("Epoch {0} - Y_pred: {1}, Y_true: {2}".format(epoch, output_index, y_up_valid))
  
      return y_up_valid.size(0), (output_index == y_up_valid).sum().float()
  
  
  def post_validation_processing(valid_losses, avg_valid_losses, valid_accuracy_list, correct, total_size):
      valid_loss = np.average(valid_losses)
      avg_valid_losses.append(valid_loss)
  
      valid_accuracy = 100 * correct / total_size
      valid_accuracy_list.append(valid_accuracy)
  
      return valid_loss, valid_accuracy
  
  
  def make_model(model_type, x_normalized, y_up, global_total_size, X_train, y_train, X_test, y_test, one_rate):
      if VERBOSE:
          logger.info("[[[{0}]]]".format(model_type))
  
      model = load_model(model_type=model_type)
      if model:
          model.fit(X_train, y_train)
          y_pred = model.predict(X_test)
          best_model = model
          best_f1_score = f1_score(y_test, y_pred)
      else:
          gc.collect()
          coin_model_start_time = time.time()
  
          X = x_normalized.reshape(global_total_size, -1)
          y = y_up
  
          if model_type == "GB":
              param_grid = {
                  'learning_rate': [0.01, 0.05, 0.1],
                  'max_depth': np.linspace(1, 8, 4, endpoint=True),
                  'n_estimators': [32, 64, 128],
                  'max_features': list(range(int(x_normalized.shape[1] / 2), x_normalized.shape[1], 2)),
              }
  
              model_classifier = GradientBoostingClassifier()
          elif model_type == "XGBOOST":
              param_grid = {
                  'objective': ['binary:logistic'],
                  'max_depth': [4, 6, 8],
                  'min_child_weight': [9, 11],
                  'subsample': [0.8, 0.9],
                  'colsample_bytree': [0.7],
                  'n_estimators': [50, 100, 200]
              }
              model_classifier = xgb.XGBClassifier()
          else:
              param_grid = {}
              model_classifier = None
  
          clf = GridSearchCV(
              model_classifier,
              param_grid,
              cv=StratifiedKFold(n_splits=4, shuffle=True),
              scoring='f1',
              refit=True,
              verbose=True
          )
  
          clf.fit(X, y)
  
          coin_model_elapsed_time = time.time() - coin_model_start_time
          coin_model_elapsed_time_str = time.strftime("%H:%M:%S", time.gmtime(coin_model_elapsed_time))
  
          msg_str = "{0} - make_model - Elapsed Time: {1}, Best Score: {2}\n".format(
              model_type, coin_model_elapsed_time_str, clf.best_score_
          )
          logger.info(msg_str)
          logger.info(clf.best_params_)
          if PUSH_SLACK_MESSAGE: SLACK.send_message("me", msg_str)
  
          best_model = clf.best_estimator_
          best_f1_score = clf.best_score_
  
      model_filename = save_model(best_model, model_type=model_type)
  
      now = dt.datetime.now(timezone('Asia/Seoul'))
      now_str = now.strftime(fmt)
      current_time_str = now_str.replace("T", " ")
  
      model = trade_db_session.query(Model).filter(Model.model_type == model_type).first()
  
      if model:
          model.model_filename = model_filename
          model.one_rate = float(one_rate)
          model.train_size = model.train_size + global_total_size
          model.datetime = dt.datetime.strptime(current_time_str, fmt.replace("T", " "))
          model.window_size = WINDOW_SIZE
          model.future_target_size = FUTURE_TARGET_SIZE
          model.up_rate = UP_RATE
          model.feature_size = INPUT_SIZE
          model.best_score = float(best_f1_score)
  
          trade_db_session.commit()
      else:
          model = Model(
              model_type=model_type,
              model_filename=model_filename,
              one_rate=float(one_rate),
              train_size=global_total_size,
              datetime=dt.datetime.strptime(current_time_str, fmt.replace("T", " ")),
              window_size=WINDOW_SIZE,
              future_target_size=FUTURE_TARGET_SIZE,
              up_rate=UP_RATE,
              feature_size=INPUT_SIZE,
              best_score=float(best_f1_score)
          )
          trade_db_session.add(model)
          trade_db_session.commit()
  
      if VERBOSE:
          msg_str = "TRAINED_SIZE: {0}, BEST_SCORE: {1}\n".format(model.train_size, model.best_score)
          logger.info(msg_str)
          SLACK.send_message("me", msg_str)
  
  
  def main(coin_names):
      start_time = time.time()
  
      heading_msg = "\n**************************\n"
      heading_msg += "WINDOW SIZE:{0}, FUTURE_TARGET_SIZE:{1}, UP_RATE:{2}, INPUT_SIZE:{3}, DEVICE:{4}, MODEL_SOURCE:{5}".format(
          WINDOW_SIZE,
          FUTURE_TARGET_SIZE,
          UP_RATE,
          INPUT_SIZE,
          DEVICE,
          LOCAL_MODEL_SOURCE
      )
      logger.info(heading_msg)
  
      x_normalized_list = []
      y_up_list = []
      one_rate_list = []
      global_total_size = 0
      for idx, coin_name in enumerate(coin_names):
          gc.collect()
  
          upbit_order_book_data = UpbitOrderBookBasedData(coin_name)
          x_normalized_original, y_up_original, one_rate, total_size = upbit_order_book_data.get_dataset(split=False)
  
          if VERBOSE:
              logger.info("{0}, {1}: x_normalized_original: {2}, y_up_original: {3}, one_rate: {4}, total_size: {5}".format(
                  idx,
                  coin_name,
                  x_normalized_original.shape,
                  y_up_original.shape,
                  one_rate,
                  total_size
              ))
  
          x_normalized_list.append(x_normalized_original)
          y_up_list.append(y_up_original)
          one_rate_list.append(one_rate)
          global_total_size += total_size
  
      gc.collect()
      x_normalized = np.concatenate(x_normalized_list)
  
      gc.collect()
      y_up = np.concatenate(y_up_list)
  
      one_rate = float(np.mean(one_rate_list))
  
      if VERBOSE:
          logger.info("TOTAL -- x_normalized: {0}, y_up: {1}, one_rate: {2}, total_size: {3}".format(
              x_normalized.shape, y_up.shape, one_rate, global_total_size
          ))
  
      ##### MAKE MODELS
      X_train, X_test, y_train, y_test = train_test_split(x_normalized, y_up, test_size=0.2, random_state=0)
  
      X_train = X_train.reshape(X_train.shape[0], -1)
      X_test = X_test.reshape(X_test.shape[0], -1)
  
      ## GB
      make_model("GB", x_normalized, y_up, global_total_size, X_train, y_train, X_test, y_test, one_rate)
  
      ## XGBOOST
      make_model("XGBOOST", x_normalized, y_up, global_total_size, X_train, y_train, X_test, y_test, one_rate)
  
      elapsed_time = time.time() - start_time
      elapsed_time_str = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
  
      logger.info("####################################################################")
      logger.info("Elapsed Time: {0}".format(elapsed_time_str))
      logger.info("####################################################################\n")
  
      slack_msg = "MODEL CONSTRUCTION COMES TO END: - ELAPSED_TIME:{0} @ {1}".format(
          elapsed_time_str, SOURCE
      )
      if PUSH_SLACK_MESSAGE: SLACK.send_message("me", slack_msg)
  
  
  if __name__ == "__main__":
      mkdir_models()
  
      if PUSH_SLACK_MESSAGE: SLACK.send_message("me", "MAKE MODELS STARTED @ {0}".format(SOURCE))
  
      upbit = Upbit(CLIENT_ID_UPBIT, CLIENT_SECRET_UPBIT, fmt)
  
      while True:
          main(coin_names=upbit.get_all_coin_names(parts=1))
  
apis: sklearn.model_selection.StratifiedKFold sklearn.model_selection.train_test_split sklearn.metrics.f1_score sklearn.ensemble.GradientBoostingClassifier