text: import os.path
  import json
  import pandas as pd
  import xgboost as xgb
  import joblib
  from IPython import get_ipython
  from sklearn.preprocessing import scale
  from sklearn.model_selection import KFold
  from time import time
  from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix, \
      classification_report, accuracy_score, auc, roc_auc_score, roc_curve
  from sklearn.externals import joblib
  from sklearn.linear_model import LogisticRegression
  from sklearn.ensemble import RandomForestClassifier
  from sklearn.svm import SVC
  from sklearn.svm import LinearSVC
  from sklearn.neighbors import KNeighborsClassifier
  from sklearn.naive_bayes import GaussianNB
  from sklearn.tree import DecisionTreeClassifier
  from sklearn.ensemble import GradientBoostingClassifier
  from sklearn.linear_model import SGDClassifier
  
  from sklearn.model_selection import GridSearchCV
  from sklearn.model_selection import RandomizedSearchCV
  from sklearn.model_selection import cross_val_score
  
  from sklearn.datasets import make_classification
  # from matplotlib import pyplot
  from sklearn.model_selection import train_test_split
  
  from sklearn import metrics
  from sklearn.metrics import make_scorer
  from scipy.stats import expon
  from clean_data import make_directory
  
  from sklearn.datasets import make_classification
  import warnings
  
  warnings.filterwarnings("ignore", category=DeprecationWarning)
  
  ipy = get_ipython()
  if ipy is not None:
      ipy.run_line_magic('matplotlib', 'inline')
  
  
  def prepare_data(data, drop_na=True):
      ''' Drops unnecessary columns, Fill or Drop rows containing N/A, and pre-processes the columns.'''
      data = data.drop(columns=['Date', 'HomeTeam', 'AwayTeam'])
      data = data.drop(columns=['FTHG', 'FTAG'])
      data = data.drop(columns=['HT_goal_for', 'AT_goal_for', 'HT_goal_against', 'AT_goal_against'])
      # data = data.drop(columns=['HT_3_win_streak', 'HT_5_win_streak', 'HT_3_lose_Streak', 'HT_5_lose_Streak',
      #                         'AT_3_win_streak', 'AT_5_win_streak', 'AT_3_lose_Streak', 'AT_5_lose_Streak'])
  
      data = data.loc[data['HT_match_played'] == data['HT_match_played']]
  
      if drop_na:
          data = data.dropna()
      else:
          data.fillna(value=-99999, inplace=True)
  
      # Columns that are not normalized: (Ordinal, Categorical)
      # [FTR, HT_match_played, AT_match_played, HT_3_win_streak, HT_5_win_streak,
      # HT_3_lose_Streak, HT_5_lose_Streak, AT_3_win_streak, AT_5_win_streak, AT_3_lose_Streak, AT_5_lose_Streak]
  
      # Columns that are normalised: (Continuous variables)
      normalized_columns = ['HomeOVA', 'AwayOVA', 'OVA_diff']
      normalized_columns += ['HT_current_standing', 'AT_current_standing']
      normalized_columns += ['HT_goal_diff', 'HT_win_rate_season', 'AT_goal_diff', 'AT_win_rate_season']
      normalized_columns += ['HT_past_standing', 'HT_past_goal_diff', 'HT_past_win_rate',
                             'AT_past_standing', 'AT_past_goal_diff', 'AT_past_win_rate']
      normalized_columns += ['HT_5_win_rate', 'AT_5_win_rate', 'HT_win_rate_against', 'AT_win_rate_against']
      normalized_columns += ['current_standing_diff', 'win_rate_season_diff', 'goal_diff_diff']
      normalized_columns += ['past_standing_diff', 'past_goal_diff_diff', 'past_win_rate_diff']
      # normalized_columns += ['HT_goal_for', 'AT_goal_for', 'HT_goal_against', 'AT_goal_against']
  
      for column in normalized_columns:
          data[column] = scale(list(data[column]))
  
      return data
  
  
  def train_classifier(clf, x_train, y_train):
      # This function is fitting a classifier to the training data
  
      # A clock is started - train the classifier - stop the clock.
      start = time()
      clf.fit(x_train, y_train)
      end = time()
  
      print("Trained model in {:.4f} seconds".format(end - start))
  
  
  def predict_labels(clf, features, target):
      # This function makes a prediction using a fit classifier based on F! score.
  
      # A clocked is started - it then makes a prediction - stops the clock.
      start = time()
      y_pred = clf.predict(features)
      end = time()
      print("Made predictions in {:.4f} seconds.".format(end - start))
  
      return f1_score(target, y_pred, labels=['H', 'D', 'A'], average=None), sum(target == y_pred) / float(len(y_pred)), \
             clf.score(features, target), y_pred
  
  
  def train_and_predict(clf, x_train, y_train, x_test, y_test):
      # Train and predict using a classifier based on F1 score.
  
      # This indicates the classifier and the training set size.
      print("Training a {} using a training set size of {}. . .".format(clf.__class__.__name__, len(x_train)))
  
      # Train the classifier
      train_classifier(clf, x_train, y_train)
  
      # Print the results of prediction for both training and testing
      f1, acc, confidence, _ = predict_labels(clf, x_train, y_train)
      # print("F1 score and accuracy score for training set: {} , {}.".format(f1, acc))
      # print("Confidence score for training set: {}.".format(confidence))
      print()
  
      f1, acc, confidence, predictions = predict_labels(clf, x_test, y_test)
      print("F1 score and accuracy score for test set: {} , {}.".format(f1, acc))
      # print("Confidence score for test set: {}.".format(confidence))
      print()
  
      return confidence, predictions
  
  
  def get_grid_clf(clf, scoring, param, x_all, y_all):
      grid_search = GridSearchCV(clf,
                                 scoring=scoring,
                                 param_grid=param,
                                 verbose=100)
      grid_obj = grid_search.fit(x_all, y_all)
  
      clf = grid_obj.best_estimator_
      params = grid_obj.best_params_
      print(clf)
      print(params)
  
      return clf
  
  
  def get_random_clf(clf, scoring, param, x_all, y_all):
      random_search = RandomizedSearchCV(clf, param,
                                         n_iter=10,
                                         scoring=scoring,
                                         verbose=100)
      random_obj = random_search.fit(x_all, y_all)
  
      clf = random_obj.best_estimator_
      params = random_obj.best_params_
      print(clf)
      print(params)
  
      return clf
  
  
  def process_print_result(clfs, res):
      def average(lst):
          return sum(lst) / len(lst)
  
      avg_dict = {}
      best_clf_so_far = 0
      best_avg_so_far = -1
      for i in range(len(clfs)):
          clf_name = clfs[i].__class__.__name__
          if clf_name in avg_dict:
              clf_name += json.dumps(clfs[i].get_params())
          avg = average(res[i])
          avg_dict[clf_name] = avg
          if avg > best_avg_so_far:
              best_avg_so_far = avg
              best_clf_so_far = i
  
      for clf_name in sorted(avg_dict, key=avg_dict.get, reverse=True):
          print("{}: {}".format(clf_name, avg_dict[clf_name]))
  
      return avg_dict, clfs[best_clf_so_far]
  
  
  def getCLF(finalFilePath, model_confidence_csv_path, clf_file, recalculate=True):
      if not recalculate:
          # prediction result (y_result) not available
          return joblib.load(clf_file), None
  
      # First load the data from csv file
      data = pd.read_csv(finalFilePath)
  
      # Drop columns that are not needed and normalized each columns
      data = prepare_data(data, drop_na=True)
      data = data.loc[(data['FTR'] == 'H') | (data['FTR'] == 'D') | (data['FTR'] == 'A')]
  
      #   Divide data into features and label
      x_all = data.drop(columns=['FTR'])
      y_all = data['FTR']
  
      # List of Classifiers that we are going to run
      classifiers = [
          # Logistic Regressions
          LogisticRegression(),
          # Best param in this grid search
          LogisticRegression(penalty='l2', solver='newton-cg', multi_class='ovr',
                             C=0.1, warm_start=True),
          LogisticRegression(penalty='l2', solver='lbfgs', multi_class='multinomial',
                             C=0.4, warm_start=False),
          # SVC
          SVC(probability=True),
          SVC(C=0.3, class_weight=None, decision_function_shape='ovo', degree=1,
              kernel='rbf', probability=True, shrinking=True, tol=0.0005),
          SVC(C=0.28, class_weight=None, decision_function_shape='ovo', degree=1,
              kernel='rbf', probability=True, shrinking=True, tol=0.0002),
          # XGBoost
          xgb.XGBClassifier(),
          xgb.XGBClassifier(learning_rate=0.01, n_estimators=1000, max_depth=2,
                            min_child_weight=5, gamma=0, subsample=0.8, colsample_bytree=0.7,
                            scale_pos_weight=0.8, reg_alpha=1e-5, booster='gbtree', objective='multi:softprob'),
          KNeighborsClassifier(),
          RandomForestClassifier(),
          GaussianNB(),
          DecisionTreeClassifier(),
          GradientBoostingClassifier(),
          LinearSVC(),
          SGDClassifier()
      ]
  
      # clf_L = LogisticRegression()
      # parameters_L = {'penalty': ['l2'],
      #                'solver': ['lbfgs', 'newton-cg', 'sag'],
      #                'multi_class': ['ovr', 'multinomial'],
      #                'C': [x * 0.1 + 0.1 for x in range(10)],
      #                'warm_start': [True, False],
      #                'fit_intercept': [True, False],
      #                'class_weight': ['balanced', None]}
      # f1_scorer_l = make_scorer(f1_score, labels=['H', 'D', 'A'], average='micro')
      # clf_L = get_grid_clf(clf_L, f1_scorer_l, parameters_L, x_all, y_all)
      # classifiers.append(clf_L)
  
      ##    SVC
      # clf_L = SVC()
      # parameters_L = {
      #    'C': [x * 0.01 + 0.27 for x in range(5)],
      #    'kernel': ['linear', 'poly', 'rbf', 'sigmoid'],
      #    'degree': [x + 1 for x in range(3)],
      #    'shrinking': [True, False],
      #    'tol': [x * 0.0005 + 0.0005 for x in range(3)],
      #    'class_weight': ['balanced', None],
      #    'decision_function_shape': ['ovo', 'ovr']
      # }
      # f1_scorer_l = make_scorer(f1_score, labels=['H', 'D', 'A'], average='micro')
      # clf_L = get_grid_clf(clf_L, f1_scorer_l, parameters_L, x_all, y_all)
      # classifiers.append(clf_L)
  
      ##    XGBoost
      clf_L = xgb.XGBClassifier()
      parameters_L = {
          'learning_rate': [0.01],
          'n_estimators': [1000],
          'max_depth': [2],
          'min_child_weight': [5],
          'gamma': [0],
          'subsample': [0.8],
          'colsample_bytree': [0.7],
          'scale_pos_weight': [0.8],
          'reg_alpha': [1e-5],
          'booster': ['gbtree'],
          'objective': ['multi:softprob']
      }
      f1_scorer_l = make_scorer(f1_score, labels=['H', 'D', 'A'], average='micro')
      clf_L = get_grid_clf(clf_L, f1_scorer_l, parameters_L, x_all, y_all)
      classifiers.append(clf_L)
  
      # We are going to record accuracies of each classifier prediction iteration
      len_classifiers = len(classifiers)
      result = [[] for _ in range(len_classifiers)]
      y_results = [[] for _ in range(len_classifiers + 1)]
  
      # Using 10-fold cross validation (Dividing the data into sub groups (90% to fit, 10% to test), and run
      # prediction with each classifiers using the sub groups as a dataset)
      # This will test the skill of the classifiers.
      split = 10
      kf = KFold(n_splits=split, shuffle=True)
      for split_index, (train_index, test_index) in enumerate(kf.split(x_all)):
          print("Processing {}/{} of KFold Cross Validation...".format(split_index + 1, split))
          x_train, x_test = x_all.iloc[train_index], x_all.iloc[test_index]
          y_train, y_test = y_all.iloc[train_index], y_all.iloc[test_index]
          y_results[len_classifiers] += y_test.tolist()
  
          for index, clf in enumerate(classifiers):
              print("KFold: {}/{}. clf_index: {}/{}.".format(split_index + 1, split, index + 1, len(classifiers)))
              confidence, predicted_result = train_and_predict(clf, x_train, y_train, x_test, y_test)
              result[index].append(confidence)
              y_results[index] += predicted_result.tolist()
  
      #   Make a dictionary of average accuracies for each classifiers
      avg_dict, best_clf = process_print_result(classifiers, result)
  
      #   Put the result into csv file
      if os.path.isfile(model_confidence_csv_path):
          df = pd.read_csv(model_confidence_csv_path)
          newdf = pd.DataFrame(avg_dict, index=[df.shape[1]])
          df = pd.concat([df, newdf], ignore_index=True, sort=False)
      else:
          make_directory(model_confidence_csv_path)
          df = pd.DataFrame(avg_dict, index=[0])
      df.to_csv(model_confidence_csv_path, index=False)
  
      #    Saves the classifier using joblib module
      if recalculate:
          joblib.dump(best_clf, clf_file)
      #   Return the best classifier
      return best_clf, y_results
  
  
  def predict_next_round(clf, final_path, current_raw_cleaned_path, statistics=False, stat_path=None, first=True):
      # First indicates whether the one being predicted is the upcoming round
  
      # Load final data csv
      global date_so_far
      df = pd.read_csv(final_path)
  
      # Get the row count of the dataframe
      len_df = df.shape[0]
  
      # Normalize each columns and remove rows that should not be predicted yet
      df = prepare_data(df, drop_na=False)
      df = df.loc[(df['FTR'] != 'H') & (df['FTR'] != 'D') & (df['FTR'] != 'A')]
      df = df.drop(columns=['FTR'])
  
      if statistics:
          if stat_path is not None:
              make_directory(stat_path)
          else:
              raise ValueError("specify 'stat_path' to save prediction result. Exiting...")
  
      if len(df) > 0:
          df_indices = [x - len_df for x in df.index]
  
          prediction = clf.predict(df).tolist()
          prediction_probability = clf.predict_proba(df).tolist()
          clf_classes = clf.classes_
  
          df_to_predict = pd.read_csv(current_raw_cleaned_path)
          len_df = df_to_predict.shape[0]
  
          print("{:20} {:20} {:20} {}".format("Home", "Away", "Predict", "Probability"))
          for (index, result, pred_prob) in zip(df_indices, prediction, prediction_probability):
              home_team = df_to_predict.at[index + len_df, 'HomeTeam']
              away_team = df_to_predict.at[index + len_df, 'AwayTeam']
              date_so_far = df_to_predict.at[index + len_df, 'Date']
  
              df_to_predict.at[index + len_df, 'FTR'] = result
              df_to_predict.at[index + len_df, 'FTHG'] = 0
              df_to_predict.at[index + len_df, 'FTAG'] = 0
  
              for (outcome, prob) in zip(clf_classes, pred_prob):
                  df_to_predict.at[index + len_df, 'prob_' + outcome] = prob
  
              print("{:20} {:20} {:20} {}".format(home_team, away_team, home_team
              if result == "H" else away_team, max(pred_prob)))
  
          if statistics:
              if first:
                  if os.path.exists(stat_path):
                      os.remove(stat_path)
                  df_to_predict.to_csv(stat_path, index=False)
              else:
                  if os.path.isfile(stat_path):
                      stat_df = pd.read_csv(stat_path)
                      stat_df.update(df_to_predict)
                      stat_df.to_csv(stat_path, index=False)
                  else:
                      raise ValueError('FATAL ERROR: either set first=True, or feed stat_path.')
  
          df_to_predict = df_to_predict.drop(columns=['prob_' + outcome for outcome in clf_classes])
  
          df_to_predict.to_csv(current_raw_cleaned_path, index=False)
          return True, date_so_far
      else:
          print("There are no more games to make prediction")
          return False, None