text: """Hide-and-Seek Privacy Challenge Codebase.
  
  Reference: James Jordon, Daniel Jarrett, Jinsung Yoon, Ari Ercole, Cheng Zhang, Danielle Belgrave, Mihaela van der Schaar, 
  "Hide-and-Seek Privacy Challenge: Synthetic Data Generation vs. Patient Re-identification with Clinical Time-series Data," 
  Neural Information Processing Systems (NeurIPS) Competition, 2020.
  
  Link: https://www.vanderschaar-lab.com/announcing-the-neurips-2020-hide-and-seek-privacy-challenge/
  
  Last updated Date: Oct 17th 2020
  Code author: Jinsung Yoon, Evgeny Saveliev
  Contact: jsyoon0823@gmail.com, e.s.saveliev@gmail.com
  
  
  -----------------------------
  
  (1) data_preprocess: Load the data and preprocess into a 3d numpy array
  (2) imputater: Impute missing data 
  """
  # Local packages
  import os
  from typing import Union, Tuple, List
  import warnings
  warnings.filterwarnings("ignore")
  
  # 3rd party modules
  import numpy as np
  import pandas as pd
  from tqdm import tqdm
  from scipy import stats
  from sklearn.preprocessing import StandardScaler, MinMaxScaler
  
  
  def data_preprocess(
      file_name: str,
      ts_column_name: str,
      id_column_name: str,
      max_seq_len: int,
      padding_value: float=-1.0,
      impute_method: str="mode", 
      scaling_method: str="minmax", 
  ) -> Tuple[np.ndarray, np.ndarray, List]:
      """Load the data and preprocess into 3d numpy array.
      Preprocessing includes:
      1. Remove outliers
      2. Extract sequence length for each patient id
      3. Impute missing data 
      4. Normalize data
      6. Sort dataset according to sequence length
  
      Args:
      - file_name (str): CSV file name
      - ts_column_name (str): name of Time Series column in the training data
      - id_column_name (str): name of ID column in the training data
      - max_seq_len (int): maximum sequence length
      - impute_method (str): The imputation method ("median" or "mode") 
      - scaling_method (str): The scaler method ("standard" or "minmax")
  
      Returns:
      - processed_data: preprocessed data
      - time: ndarray of ints indicating the length for each data
      - params: the parameters to rescale the data 
      """
  
      #########################
      # Load data
      #########################
  
      # Load csv
      print("Loading data as DataFrame...\n")
      original_data = pd.read_csv(file_name)
  
      #########################
      # Remove outliers from dataset
      #########################
      
      total_records = original_data.shape[0]
      z_scores = stats.zscore(original_data, axis=0, nan_policy='omit')
      z_filter = np.nanmax(np.abs(z_scores), axis=1) < 3
      original_data = original_data[z_filter]
      print(f"Dropped {total_records - original_data.shape[0]} records (outliers)\n")
  
      # Parameters
      unique_ids = np.unique(original_data[id_column_name])
      ids_count = len(unique_ids)
      dim = len(original_data.columns) - 1
  
      #########################
      # Impute, scale and pad data
      #########################
      
      # Initialize scaler
      if scaling_method == "minmax":
          scaler = MinMaxScaler()
          scaler.fit(original_data)
          params = [scaler.data_min_, scaler.data_max_]
      
      elif scaling_method == "standard":
          scaler = StandardScaler()
          scaler.fit(original_data)
          params = [scaler.mean_, scaler.var_]
  
      # Imputation values
      if impute_method == "median":
          impute_vals = original_data.median()
      elif impute_method == "mode":
          impute_vals = stats.mode(original_data).mode[0]
      else:
          raise ValueError("Imputation method should be `median` or `mode`")    
  
      # TODO: Sanity check for padding value
      # if np.any(ori_data == padding_value):
      #     print(f"Padding value `{padding_value}` found in data")
      #     padding_value = np.nanmin(ori_data.to_numpy()) - 1
      #     print(f"Changed padding value to: {padding_value}\n")
      
      # Output initialization
      output = np.empty([ids_count, max_seq_len, dim])  # Shape:[no, max_seq_len, dim]
      output.fill(padding_value)
      time = []
  
      # For each uniq id
      for i in tqdm(range(ids_count)):
          # Extract the time-series data with a certain id_column_name
          id_data = original_data[original_data[id_column_name] == unique_ids[i]].to_numpy()
  
          # Impute missing data
          id_data = imputer(id_data, impute_vals)
  
          # Normalize data
          id_data = scaler.transform(id_data)
          
          # Extract time and assign to the preprocessed data (Excluding ID)
          ts_len = len(id_data)
  
          # Pad data to `max_seq_len`
          if ts_len >= max_seq_len:
              output[i, :, :] = id_data[:max_seq_len, 1:]  # Shape: [1, max_seq_len, dim]
              time.append(max_seq_len)
          else:
              output[i, :ts_len, :] = id_data[:, 1:]  # Shape: [1, max_seq_len, dim]
              time.append(ts_len)
  
      return output, time, params, max_seq_len, padding_value
  
  
  def imputer(
      curr_data: np.ndarray, 
      impute_vals: List, 
      zero_fill: bool = True
  ) -> np.ndarray:
      """Impute missing data given values for each columns.
  
      Args:
          curr_data (np.ndarray): Data before imputation.
          impute_vals (list): Values to be filled for each column.
          zero_fill (bool, optional): Whather to Fill with zeros the cases where 
              impute_val is nan. Defaults to True.
  
      Returns:
          np.ndarray: Imputed data.
      """
  
      curr_data = pd.DataFrame(data=curr_data)
      impute_vals = pd.Series(impute_vals)
      
      # Impute data
      imputed_data = curr_data.fillna(impute_vals)
  
      # Zero-fill, in case the `impute_vals` for a particular feature is `nan`.
      imputed_data = imputed_data.fillna(0.0)
  
      # Check for any N/A values
      if imputed_data.isnull().any().any():
          raise ValueError("NaN values remain after imputation")
  
      return imputed_data.to_numpy()
  