import tensorflow as tf

        if self.dale_ratio:
            reg += self.L1_out * tf.reduce_mean(tf.matmul(tf.abs(self.W_out) * self.output_Connectivity, self.Dale_out))
        else:
            reg += self.L1_out * tf.reduce_mean(tf.abs(self.W_out) * self.output_Connectivity)

        # L2 weight regularization
        reg += self.L2_in * tf.reduce_mean(tf.square(tf.abs(self.W_in) * self.input_Connectivity))
        reg += self.L2_rec * tf.reduce_mean(tf.square(tf.abs(self.W_rec) * self.rec_Connectivity))
        if self.dale_ratio:
            reg += self.L2_out * tf.reduce_mean(tf.square(
                tf.matmul(tf.abs(self.W_out) * self.output_Connectivity, self.Dale_out)))
        else:
            reg += self.L2_out * tf.reduce_mean(tf.square(tf.abs(self.W_out) * self.output_Connectivity))

        # L2 firing rate regularization
        reg += self.L2_firing_rate * tf.reduce_mean(tf.square(tf.nn.relu(self.states)))

        # susillo regularization
        reg += self.sussillo_constant * self.sussillo_reg()

        return reg

    # implement one step of the RNN
    def rnn_step(self, rnn_in, state):
