title: Wasserstein GAN problem with last discriminator layer and clipping
body: <p>When I use linear or No activation in the last Discriminator layer using weight clipping Discriminator accuracy goes to 1 and Generator goes to 0. In case when I remove weight clipping, Generator accuracy goes to 1 and discriminator goes to 0 around 300 iterations. But when I use sigmoid activation as the last layer in the discriminator with clipping Generator accuracy goes to 1 and without clipping the generator loss get stuck while accuracies going as they should around 0.5.
  NOTE - in all cases, results are produced  and all of the show WARNING:tensorflow:Discrepancy between trainable weights and collected trainable weights, did you set <code>model.trainable</code> without calling <code>model.compile</code> after ?</p>
  
  <p>Code is given here, please do not mind the indentation on copying and pasting it's everywhere - </p>
  
  <pre><code>class WGAN():
  
    def  __init__(self, 
                input_dim,
                disc_filter,
                disc_kernel,
                disc_strides,
                disc_dropout,
                disc_lr,
                gen_filter,
                gen_kernel,
                gen_strides,
                gen_upsample,
                gen_lr,
                z_dim,
                batch_size):
  
  
        self.input_dim = input_dim
        self.disc_filter = disc_filter
        self.disc_kernel = disc_kernel
        self.disc_strides = disc_strides
        self.disc_dropout = disc_dropout
        self.disc_lr = disc_lr
        self.gen_filter = gen_filter
        self.gen_kernel = gen_kernel
        self.gen_strides = gen_strides
        self.gen_upsample = gen_upsample
        self.gen_lr = gen_lr
        self.z_dim = z_dim
        self.batch_size = batch_size
        self.weight_init = RandomNormal(mean=0., stddev=0.02)
  
        self.d_losses = []
        self.g_losses = []
        self.epoch = 0
  
        self.Discriminator()
        self.Generator()
  
        self.full_model()
  
    def wasserstein(self, y_true, y_pred):
      return -K.mean(y_true * y_pred)
  
    def Discriminator(self):
  
      disc_input = Input(shape=self.input_dim, name='discriminator_input')
      x = disc_input
  
      for i in range(len(self.disc_filter)):
          x = Conv2D(filters=self.disc_filter[i], kernel_size=self.disc_kernel[i], strides=self.disc_strides[i], padding='same', name='disc_'+str(i))(x)
          x = LeakyReLU()(x)
          x = Dropout(self.disc_dropout)(x)
          x = BatchNormalization()(x)
  
      x = Flatten()(x)
      disc_output = Dense(1, activation='sigmoid', kernel_initializer = self.weight_init)(x)
      self.discriminator = Model(disc_input, disc_output)
  
    def Generator(self):
  
      gen_input = Input(shape=(self.z_dim,), name='generator_input')
      x = gen_input
  
      x = Dense(7*7*self.batch_size, kernel_initializer = self.weight_init)(x)
      x = LeakyReLU()(x)
      x = BatchNormalization()(x)
      x = Reshape(target_shape=(7,7,self.batch_size))(x)
  
      for i in range(len(self.gen_filter)):
          if self.gen_upsample[i]==2:
              x = UpSampling2D(size=self.gen_upsample[i], name='upsample_'+str(i/2))(x)
              x = Conv2D(filters=self.gen_filter[i], kernel_size=self.gen_kernel[i], strides=self.gen_strides[i], padding='same', name='gen_'+str(i))(x)
  
          else:
              x = Conv2DTranspose(filters=self.gen_filter[i], kernel_size=self.gen_kernel[i], strides=self.gen_strides[i], padding='same', name='gen_'+str(i))(x)
  
          if i&lt;len(self.gen_filter)-1:
              x = BatchNormalization()(x)
              x = LeakyReLU()(x)
  
          else:
              x = Activation("tanh")(x)
  
      gen_output = x
      self.generator = Model(gen_input, gen_output)
  
  
    def set_trainable(self, model, val):
      model.trainable=val
      for l in model.layers:
          l.trainable=val
  
    def full_model(self):
  
      ### COMPILE DISCRIMINATOR
      self.discriminator.compile(optimizer= Adam(self.disc_lr), loss = self.wasserstein, metrics=['accuracy'])
  
      ### COMPILE THE FULL GAN
  
      self.set_trainable(self.discriminator, False)
      self.discriminator.compile(optimizer= Adam(self.disc_lr), loss = self.wasserstein, metrics=['accuracy'])
  
      model_input = Input(shape=(self.z_dim,), name='model_input')
      model_output = self.discriminator(self.generator(model_input))
      self.model = Model(model_input, model_output)
  
      self.model.compile(optimizer= Adam(self.disc_lr), loss = self.wasserstein, metrics=['accuracy'])
  
      self.set_trainable(self.discriminator, True)
  
    def train_generator(self, batch_size):
      valid = np.ones((batch_size,1))
      noise = np.random.normal(0, 1, (batch_size, self.z_dim))
      return self.model.train_on_batch(noise, valid)
  
    def train_discriminator(self, x_train, batch_size, using_generator):
      valid = np.ones((batch_size,1))
      fake = np.zeros((batch_size,1))
  
      if using_generator:
          true_imgs = next(x_train)[0]
          if true_imgs.shape[0] != batch_size:
              true_imgs = next(x_train)[0]
      else:
          idx = np.random.randint(0, x_train.shape[0], batch_size)
          true_imgs = x_train[idx]
  
      noise = np.random.normal(0, 1, (batch_size, self.z_dim))
      gen_imgs = self.generator.predict(noise)
  
      d_loss_real, d_acc_real =  self.discriminator.train_on_batch(true_imgs, valid)
      d_loss_fake, d_acc_fake =  self.discriminator.train_on_batch(gen_imgs, fake)
      d_loss = 0.5 * (d_loss_real + d_loss_fake)
      d_acc = 0.5 * (d_acc_real + d_acc_fake)
  
      for l in self.discriminator.layers:
          weights = l.get_weights()
          weights = [np.clip(w, -0.01, 0.01) for w in weights]
          l.set_weights(weights)
  
      return [d_loss, d_loss_real, d_loss_fake, d_acc, d_acc_real, d_acc_fake]
  
    def train(self, x_train, batch_size, epochs, print_every_n_batches = 50, using_generator = False):
  
      for epoch in range(self.epoch, self.epoch + epochs):
  
          d = self.train_discriminator(x_train, batch_size, using_generator)
          g = self.train_generator(batch_size)
          if self.epoch % print_every_n_batches == 0:
              print ("%d [D loss: (%.3f)(R %.3f, F %.3f)] [D acc: (%.3f)(%.3f, %.3f)] [G loss: %.3f] [G acc: %.3f]" % (epoch, d[0], d[1], d[2], d[3], d[4], d[5], g[0], g[1]))
  
          self.d_losses.append(d)
          self.g_losses.append(g)
  
          self.epoch+=1
  </code></pre>
  
  <hr>