import tensorflow as tf

        reg += self.L1_in * tf.reduce_mean(tf.abs(self.W_in) * self.input_Connectivity)
        reg += self.L1_rec * tf.reduce_mean(tf.abs(self.W_rec) * self.rec_Connectivity)
        if self.dale_ratio:
            reg += self.L1_out * tf.reduce_mean(tf.matmul(tf.abs(self.W_out) * self.output_Connectivity, self.Dale_out))
        else:
            reg += self.L1_out * tf.reduce_mean(tf.abs(self.W_out) * self.output_Connectivity)

        # L2 weight regularization
        reg += self.L2_in * tf.reduce_mean(tf.square(tf.abs(self.W_in) * self.input_Connectivity))
        reg += self.L2_rec * tf.reduce_mean(tf.square(tf.abs(self.W_rec) * self.rec_Connectivity))
