class_docstr: Class that implements the optimization for optimized pre-processing.
  
  Based on:
  http://papers.nips.cc/paper/6988-optimized-pre-processing-for-discrimination-prevention
  
  and
  
  https://github.com/fair-preprocessing/nips2017
  
  The particular formulation implemented here is:
  1. l1 distance between input and transformed distributions
  2. "Excess distortion constraint" - eqn 5 in paper.
  3. Discrimination constraints for all combinations of groups specified
     (there is no distinction between protected and unprotected groups). The
     constraints are given in eqn 2, 3 in the paper. We use a single epsilon
     value for all combinations of y and d values
  
  See section 4.3 in supplementary material of the paper for an example
  
  Attributes:
      features (list): All features
      df (DataFrame): Input data
      dfJoint (DataFrame): Empirical joint distribution
      D_features (list): protected attribute names
      X_features (list): feature names for input data
      Y_features (list): feature names for binary label
      X_values (list): Values that features can take
      Y_values (list): Values that the label can take
      D_values (list): Values that protected attributes can take
      XY_features (list): Combination of X, and Y features
      DXY_features (list): Combination of D, X, and Y features
      XY_values (list): Combination of X, and Y values
      DXY_values (list): Combination of D, X, and Y values
      y_index (Int64Index): Indices for the Y values
      XY_index (MultiIndex): Indices for the combination of X, and Y values
      DXY_index (MultiIndex): Indices for the combination of D, X, and Y
          values
      YD_features_index (MultiIndex): Indices for the combination of Y, and D
          values
  
      clist (list): Distance thresholds for individual distortion
      CMlist (list): List of constraint matrices corresponding to each
          threshold in clist
      dfD (DataFrame): distortion matrix with indices and columns
      dlist (list): Probability bounds given in eq. 5 of the paper for
          each threshold in clist
      epsilon (float): epsilon value used in discrimination constraint
  
      dfD_to_Y_address (Dataframe): matrix for p_yd, with y varying in the
          columns
      dfMask_Pxyd_to_Pyd (DataFrame): Mask to transform P_XYD to P_YD
      dfMask_Pxyd_to_Pxy (DataFrame): Mask to convert from P_XYD to P_XY
      dfPxyd (DataFrame): Representation of only frequencies from dfJoint
      dfMask_Pxyd_to_Py (DataFrame): Mask to convert from P_XYD to P_Y
      dfMask_Pxy_to_Py (DataFrame): Mask to convert from P_XY to P_Y
      dfMask_Pxyd_to_Pd (DataFrame): Mask to convert from P_XYD to P_D
      dfP (DataFrame): Mapping transformation learned from the data
class_docstr_tok: ["Class","that","implements","the","optimization","for","optimized","pre-processing",".","Based","on",":","http",":","\/\/papers.nips.cc\/paper\/6988-optimized-pre-processing-for-discrimination-prevention","and","https",":","\/\/github.com\/fair-preprocessing\/nips2017","The","particular","formulation","implemented","here","is",":","1.","l1","distance","between","input","and","transformed","distributions","2",".","``","Excess","distortion","constraint","''","-","eqn","5","in","paper",".","3",".","Discrimination","constraints","for","all","combinations","of","groups","specified","(","there","is","no","distinction","between","protected","and","unprotected","groups",")",".","The","constraints","are","given","in","eqn","2",",","3","in","the","paper",".","We","use","a","single","epsilon","value","for","all","combinations","of","y","and","d","values","See","section","4.3","in","supplementary","material","of","the","paper","for","an","example","Attributes",":","features","(","list",")",":","All","features","df","(","DataFrame",")",":","Input","data","dfJoint","(","DataFrame",")",":","Empirical","joint","distribution","D_features","(","list",")",":","protected","attribute","names","X_features","(","list",")",":","feature","names","for","input","data","Y_features","(","list",")",":","feature","names","for","binary","label","X_values","(","list",")",":","Values","that","features","can","take","Y_values","(","list",")",":","Values","that","the","label","can","take","D_values","(","list",")",":","Values","that","protected","attributes","can","take","XY_features","(","list",")",":","Combination","of","X",",","and","Y","features","DXY_features","(","list",")",":","Combination","of","D",",","X",",","and","Y","features","XY_values","(","list",")",":","Combination","of","X",",","and","Y","values","DXY_values","(","list",")",":","Combination","of","D",",","X",",","and","Y","values","y_index","(","Int64Index",")",":","Indices","for","the","Y","values","XY_index","(","MultiIndex",")",":","Indices","for","the","combination","of","X",",","and","Y","values","DXY_index","(","MultiIndex",")",":","Indices","for","the","combination","of","D",",","X",",","and","Y","values","YD_features_index","(","MultiIndex",")",":","Indices","for","the","combination","of","Y",",","and","D","values","clist","(","list",")",":","Distance","thresholds","for","individual","distortion","CMlist","(","list",")",":","List","of","constraint","matrices","corresponding","to","each","threshold","in","clist","dfD","(","DataFrame",")",":","distortion","matrix","with","indices","and","columns","dlist","(","list",")",":","Probability","bounds","given","in","eq",".","5","of","the","paper","for","each","threshold","in","clist","epsilon","(","float",")",":","epsilon","value","used","in","discrimination","constraint","dfD_to_Y_address","(","Dataframe",")",":","matrix","for","p_yd",",","with","y","varying","in","the","columns","dfMask_Pxyd_to_Pyd","(","DataFrame",")",":","Mask","to","transform","P_XYD","to","P_YD","dfMask_Pxyd_to_Pxy","(","DataFrame",")",":","Mask","to","convert","from","P_XYD","to","P_XY","dfPxyd","(","DataFrame",")",":","Representation","of","only","frequencies","from","dfJoint","dfMask_Pxyd_to_Py","(","DataFrame",")",":","Mask","to","convert","from","P_XYD","to","P_Y","dfMask_Pxy_to_Py","(","DataFrame",")",":","Mask","to","convert","from","P_XY","to","P_Y","dfMask_Pxyd_to_Pd","(","DataFrame",")",":","Mask","to","convert","from","P_XYD","to","P_D","dfP","(","DataFrame",")",":","Mapping","transformation","learned","from","the","data"]
docstr: Main optimization routine to estimate the pre-processing
  transformation.
  
  The particular formulation implemented here is:
  1. l1 distance between input and transformed distributions
  2. "Excess distortion constraint" - eqn 5 in paper.
  3. Discrimination constraints for all combinations of groups specified
     (there is no distinction between protected and unprotected groups).
     The constraints are given in eqn 2, 3 in the paper. We use a single
     /\epsilon value for all combinations of y and d values
  
  See section 4.3 in supplementary material of the paper for an example
  
  Args:
      epsilon (float): Distance thresholds for individual distortion
      dlist (list): Probability bounds given in eq. 5 of the paper for
          each threshold in clist
      verbose (bool): Verbosity flag
docstr_tok: ["Main","optimization","routine","to","estimate","the","pre-processing","transformation",".","The","particular","formulation","implemented","here","is",":","1.","l1","distance","between","input","and","transformed","distributions","2",".","``","Excess","distortion","constraint","''","-","eqn","5","in","paper",".","3",".","Discrimination","constraints","for","all","combinations","of","groups","specified","(","there","is","no","distinction","between","protected","and","unprotected","groups",")",".","The","constraints","are","given","in","eqn","2",",","3","in","the","paper",".","We","use","a","single","\/\\epsilon","value","for","all","combinations","of","y","and","d","values","See","section","4.3","in","supplementary","material","of","the","paper","for","an","example","Args",":","epsilon","(","float",")",":","Distance","thresholds","for","individual","distortion","dlist","(","list",")",":","Probability","bounds","given","in","eq",".","5","of","the","paper","for","each","threshold","in","clist","verbose","(","bool",")",":","Verbosity","flag"]
code: def optimize(self, epsilon=1.0, dlist=[], verbose=True):
      """Main optimization routine to estimate the pre-processing
      transformation.
  
      The particular formulation implemented here is:
      1. l1 distance between input and transformed distributions
      2. "Excess distortion constraint" - eqn 5 in paper.
      3. Discrimination constraints for all combinations of groups specified
         (there is no distinction between protected and unprotected groups).
         The constraints are given in eqn 2, 3 in the paper. We use a single
         /\epsilon value for all combinations of y and d values
  
      See section 4.3 in supplementary material of the paper for an example
  
      Args:
          epsilon (float): Distance thresholds for individual distortion
          dlist (list): Probability bounds given in eq. 5 of the paper for
              each threshold in clist
          verbose (bool): Verbosity flag
      """
      self.epsilon = epsilon
      self.dlist = dlist
  
      # main conditional map
      Pmap = Variable((self.dfP.shape[0], self.dfP.shape[1]))
      # marginal distribution of (Xh Yh)
      PXhYh = Variable((self.dfMask_Pxyd_to_Pxy.shape[1],))
      # rows represent p_(y|D)
      PYhgD = Variable(
          (
              self.dfD_to_Y_address.shape[1],
              self.dfD_to_Y_address.shape[0],
          )
      )
  
      # marginal distribution
      dfMarginal = self.dfJoint.groupby(self.DXY_features)[
          "Frequency"
      ].sum()
      PxydMarginal = (
          pd.concat([self.dfP, dfMarginal], axis=1)
          .fillna(0)["Frequency"]
          .values
      )
      self.PxydMarginal = PxydMarginal
      PdMarginal = PxydMarginal.dot(self.dfMask_Pxyd_to_Pd).T
      PxyMarginal = PxydMarginal.dot(self.dfMask_Pxyd_to_Pxy).T
  
      # add constraints
      # 1. valid distribution
      constraints = [cp.sum(Pmap, axis=1) == 1]
      constraints.append(Pmap >= 0)
  
      # 2. definition of marginal PxhYh
      constraints.append(
          PXhYh == cp.sum(np.diag(PxydMarginal) * Pmap, axis=0).T
      )
  
      # add the conditional mapping
      constraints.append(
          PYhgD
          == np.diag(np.ravel(PdMarginal) ** (-1))
          .dot(self.dfMask_Pxyd_to_Pd.values.T)
          .dot(np.diag(PxydMarginal))
          * Pmap
          * self.dfMask_Pxy_to_Py.values
      )
  
      # 3. add excess distorion
      # print(PxyMarginal)
      # Pxy_xhyh = np.nan_to_num(np.diag(PxyMarginal**(-1))).dot(self.dfMask_Pxyd_to_Pxy.values.T).dot(np.diag(PxydMarginal))*Pmap
      Pxy_xhyh = (
          np.nan_to_num(np.diag((PxyMarginal + 1e-10) ** (-1)))
          .dot(self.dfMask_Pxyd_to_Pxy.values.T)
          .dot(np.diag(PxydMarginal + 1e-10))
          * Pmap
      )
  
      for i in range(len(self.CMlist)):
          constraints.append(
              cp.sum(cp.multiply(self.CMlist[i], Pxy_xhyh), axis=1)
              <= self.dlist[i]
          )
  
      # 4. Discrimination control
      for d in range(self.dfMask_Pxyd_to_Pd.shape[1]):
          for d2 in range(self.dfMask_Pxyd_to_Pd.shape[1]):
              if d > d2:
                  continue
              # constraints.append(PYhgD[d,:].T<=PYhgD[d2,:].T*(1+self.epsilon))
              # constraints.append(PYhgD[d2,:].T<=PYhgD[d,:].T*(1+self.epsilon))
              constraints.append(
                  PYhgD[d, :].T - PYhgD[d2, :].T <= self.epsilon
              )
              constraints.append(
                  PYhgD[d2, :].T - PYhgD[d, :].T <= self.epsilon
              )
  
      # 5. Objective is l1 distance between the original
      # and perturbed distributions
      obj = Minimize(cp.norm(PXhYh - PxyMarginal, 1) / 2)
  
      prob = Problem(obj, constraints)
      prob.solve(verbose=verbose)
  
      if prob.status in ["optimal", "optimal_inaccurate"]:
          print(
              "Optimized Preprocessing: Objective converged to %f"
              % (prob.value)
          )
      else:
          raise RuntimeError(
              "Optimized Preprocessing: Optimization did not "
              "converge"
          )
  
      self.dfP.loc[:, :] = Pmap.value
      self.optimum = prob.value
      self.const = []
  
      for i in range(len(self.CMlist)):
          self.const.append(
              cp.sum(
                  cp.multiply(self.CMlist[i], Pxy_xhyh), axis=1
              ).value.max()
          )
  
file_path: AIF360/aif360/algorithms/preprocessing/optim_preproc_helpers/opt_tools.py
imports: import numpy
  import pandas
  import cvxpy
  from cvxpy import Problem, Minimize, Variable
label_desc: Use image node_id 5 for calling the OptTools obj's underlying member method code with example usage: obj.optimize(epsilon, dlist, verbose) without return types