code: def FE_remove_variables_using_SULOV_method(
      df,
      preds_in,
      modeltype,
      target,
      corr_limit=0.70,
      verbose=0,
      dask_xgboost_flag=False,
  ):
      """
      FE stands for Feature Engineering - it means this function performs feature engineering
      ###########################################################################################
      #####              SULOV stands for Searching Uncorrelated List Of Variables  #############
      This highly efficient method removes variables that are highly correlated using a series of
      pair-wise correlation knockout rounds. It is extremely fast and hence can work on thousands
      of variables in less than a minute, even on a laptop. You need to send in a list of numeric
      variables and that's all! The method defines high Correlation as anything over 0.70 (absolute)
      but this can be changed. If two variables have absolute correlation higher than this, they
      will be marked, and using a process of elimination, one of them will get knocked out:
      To decide order of variables to keep, we use mutuail information score to select. MIS returns
      a ranked list of these correlated variables: when we select one, we knock out others
      that it is correlated to. Then we select next var. This way we knock out correlated variables.
      Finally we are left with uncorrelated variables that are also highly important in mutual score.
      ########  YOU MUST INCLUDE THE ABOVE MESSAGE IF YOU COPY THIS CODE IN YOUR LIBRARY ##########
      """
  
      df = copy.deepcopy(df)
      df_target = df[target]
      numvars = (
          df[preds_in].select_dtypes(include="float").columns.tolist()
      )
      remvars = left_subtract(preds_in, numvars)
      df = df[numvars]
      ### for some reason, doing a mass fillna of vars doesn't work! Hence doing it individually!
      null_vars = np.array(numvars)[df.isnull().sum() > 0]
      for each_num in null_vars:
          ### Remember that fillna only works at the dataframe level!
          df[[each_num]] = df[[each_num]].fillna(0)
      target = copy.deepcopy(target)
  
      print(
          "#######################################################################################"
      )
      print(
          "#####  SULOV:  Searching for Uncorrelated List Of Variables in %s features ############"
          % len(numvars)
      )
      print(
          "#######################################################################################"
      )
      ### This is a shorter version of getting unduplicated and highly correlated vars ##
      correlation_dataframe = (
          df.corr().abs().unstack().sort_values().drop_duplicates()
      )
      corrdf = pd.DataFrame(correlation_dataframe[:].reset_index())
      corrdf.columns = ["var1", "var2", "coeff"]
      corrdf1 = corrdf[corrdf["coeff"] >= corr_limit]
      correlated_pair = list(
          zip(
              corrdf1["var1"].values.tolist(),
              corrdf1["var2"].values.tolist(),
          )
      )
      corr_pair_dict = dict(return_dictionary_list(correlated_pair))
      corr_list = find_remove_duplicates(
          corrdf1["var1"].values.tolist()
          + corrdf1["var2"].values.tolist()
      )
      keys_in_dict = list(corr_pair_dict.keys())
      reverse_correlated_pair = [(y, x) for (x, y) in correlated_pair]
      reverse_corr_pair_dict = dict(
          return_dictionary_list(reverse_correlated_pair)
      )
      #### corr_pair_dict is used later to make the network diagram to see which vars are correlated to which
      for key, val in reverse_corr_pair_dict.items():
          if key in keys_in_dict:
              if len(key) > 1:
                  corr_pair_dict[key] += val
          else:
              corr_pair_dict[key] = val
  
      ###### This is for ordering the variables in the highest to lowest importance to target ###
      if len(corr_list) == 0:
          final_list = list(correlation_dataframe)
          print(
              "Selecting all (%d) variables since none of numeric vars are highly correlated..."
              % len(numvars)
          )
          return numvars
      else:
          if isinstance(target, list):
              target = target[0]
          max_feats = len(corr_list)
          if modeltype == "Regression":
              sel_function = mutual_info_regression
              fs = SelectKBest(score_func=sel_function, k=max_feats)
          else:
              sel_function = mutual_info_classif
              fs = SelectKBest(score_func=sel_function, k=max_feats)
          ##### you must ensure there are no infinite nor null values in corr_list df ##
          df_fit = df[corr_list]
          ### Now check if there are any NaN values in the dataset #####
  
          if df_fit.isnull().sum().sum() > 0:
              df_fit = df_fit.dropna()
          else:
              print("    there are no null values in dataset...")
          ##### Reduce memory usage and find mutual information score ####
          # try:
          #    df_fit = reduce_mem_usage(df_fit)
          # except:
          #    print('Reduce memory erroring. Continuing...')
          ##### Ready to perform fit and find mutual information score ####
          try:
              fs.fit(df_fit, df_target)
          except:
              print(
                  "    SelectKBest() function is erroring. Returning with all %s variables..."
                  % len(numvars)
              )
              return numvars
          try:
              #################################################################################
              #######   This is the main section where we use mutual info score to select vars
              #################################################################################
              mutual_info = dict(zip(corr_list, fs.scores_))
              #### The first variable in list has the highest correlation to the target variable ###
              sorted_by_mutual_info = [
                  key
                  for (key, val) in sorted(
                      mutual_info.items(),
                      key=lambda kv: kv[1],
                      reverse=True,
                  )
              ]
              #####   Now we select the final list of correlated variables ###########
              selected_corr_list = []
              #### You have to make multiple copies of this sorted list since it is iterated many times ####
              orig_sorted = copy.deepcopy(sorted_by_mutual_info)
              copy_sorted = copy.deepcopy(sorted_by_mutual_info)
              copy_pair = copy.deepcopy(corr_pair_dict)
              #### select each variable by the highest mutual info and see what vars are correlated to it
              for each_corr_name in copy_sorted:
                  ### add the selected var to the selected_corr_list
                  selected_corr_list.append(each_corr_name)
                  for each_remove in copy_pair[each_corr_name]:
                      #### Now remove each variable that is highly correlated to the selected variable
                      if each_remove in copy_sorted:
                          copy_sorted.remove(each_remove)
              ##### Now we combine the uncorrelated list to the selected correlated list above
              rem_col_list = left_subtract(numvars, corr_list)
              final_list = rem_col_list + selected_corr_list
              removed_cols = left_subtract(numvars, final_list)
          except Exception as e:
              print("    SULOV Method crashing due to %s" % e)
              #### Dropping highly correlated Features fast using simple linear correlation ###
              removed_cols = remove_highly_correlated_vars_fast(
                  df, corr_limit
              )
              final_list = left_subtract(numvars, removed_cols)
          if len(removed_cols) > 0:
              print(
                  "    Removing (%d) highly correlated variables:"
                  % (len(removed_cols))
              )
              if len(removed_cols) <= 30:
                  print("    %s" % removed_cols)
              if len(final_list) <= 30:
                  print(
                      "    Following (%d) vars selected: %s"
                      % (len(final_list), final_list)
                  )
          ##############    D R A W   C O R R E L A T I O N   N E T W O R K ##################
          selected = copy.deepcopy(final_list)
          try:
              import networkx as nx
  
              #### Now start building the graph ###################
              gf = nx.Graph()
              ### the mutual info score gives the size of the bubble ###
              multiplier = 2100
              for each in orig_sorted:
                  gf.add_node(
                      each,
                      size=int(max(1, mutual_info[each] * multiplier)),
                  )
              ######### This is where you calculate the size of each node to draw
              sizes = [
                  mutual_info[x] * multiplier for x in list(gf.nodes())
              ]
              ####  The sizes of the bubbles for each node is determined by its mutual information score value
              corr = df_fit.corr()
              high_corr = corr[abs(corr) > corr_limit]
              ## high_corr is the dataframe of a few variables that are highly correlated to each other
              combos = combinations(corr_list, 2)
              ### this gives the strength of correlation between 2 nodes ##
              multiplier = 20
              for var1, var2 in combos:
                  if np.isnan(high_corr.loc[var1, var2]):
                      pass
                  else:
                      gf.add_edge(
                          var1,
                          var2,
                          weight=multiplier * high_corr.loc[var1, var2],
                      )
              ######## Now start building the networkx graph ##########################
              widths = nx.get_edge_attributes(gf, "weight")
              nodelist = gf.nodes()
              cols = 5
              height_size = 5
              width_size = 15
              rows = int(len(corr_list) / cols)
              if rows < 1:
                  rows = 1
              plt.figure(
                  figsize=(width_size, min(20, height_size * rows))
              )
              pos = nx.shell_layout(gf)
              nx.draw_networkx_nodes(
                  gf,
                  pos,
                  nodelist=nodelist,
                  node_size=sizes,
                  node_color="blue",
                  alpha=0.5,
              )
              nx.draw_networkx_edges(
                  gf,
                  pos,
                  edgelist=widths.keys(),
                  width=list(widths.values()),
                  edge_color="lightblue",
                  alpha=0.6,
              )
              pos_higher = {}
              x_off = 0.04  # offset on the x axis
              y_off = 0.04  # offset on the y axis
              for k, v in pos.items():
                  pos_higher[k] = (v[0] + x_off, v[1] + y_off)
              if len(selected) == 0:
                  nx.draw_networkx_labels(
                      gf,
                      pos=pos_higher,
                      labels=dict(zip(nodelist, nodelist)),
                      font_color="black",
                  )
              else:
                  nx.draw_networkx_labels(
                      gf,
                      pos=pos_higher,
                      labels=dict(
                          zip(
                              nodelist,
                              [
                                  x + " (selected)"
                                  if x in selected
                                  else x + " (removed)"
                                  for x in nodelist
                              ],
                          )
                      ),
                      font_color="black",
                  )
              plt.box(True)
              plt.title(
                  """In SULOV, we repeatedly remove features with lower mutual info scores among highly correlated pairs (see figure),
                          SULOV selects the feature with higher mutual info score related to target when choosing between a pair. """,
                  fontsize=10,
              )
              plt.suptitle(
                  "How SULOV Method Works by Removing Highly Correlated Features",
                  fontsize=20,
                  y=1.03,
              )
              red_patch = mpatches.Patch(
                  color="blue",
                  label="Bigger circle denotes higher mutual info score with target",
              )
              blue_patch = mpatches.Patch(
                  color="lightblue",
                  label="Thicker line denotes higher correlation between two variables",
              )
              plt.legend(handles=[red_patch, blue_patch], loc="best")
              plt.show()
              #####    N E T W O R K     D I A G R A M    C O M P L E T E   #################
              return final_list + remvars
          except Exception as e:
              print(
                  "    Networkx library visualization crashing due to %s"
                  % e
              )
              print(
                  "Continuing with SULOV. %d features selected"
                  % len(final_list)
              )
              return final_list + remvars
  
imports: import numpy
  import pandas
  import random
  import warnings
  from sklearn.exceptions import DataConversionWarning
  import logging
  import pdb
  import copy
  import time
  from sklearn.feature_selection import chi2, mutual_info_regression, mutual_info_classif
  from sklearn.feature_selection import SelectKBest
  from itertools import combinations
  import matplotlib.patches
  import matplotlib.pyplot
  from collections import defaultdict
  from collections import OrderedDict
  import time
label_desc: Use image node_id 5 for calling a global function with example usage: FE_remove_variables_using_SULOV_method(df, preds_in, modeltype, target, corr_limit, verbose, dask_xgboost_flag) and returns: numvars, unknown, numvars, unknown