text: # Modified StyleGAN2 Projector with CLIP, addl. losses, kmeans, etc.
  # by Peter Baylies, 2021 -- @pbaylies on Twitter
  
  # Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
  #
  # NVIDIA CORPORATION and its licensors retain all intellectual property
  # and proprietary rights in and to this software, related documentation
  # and any modifications thereto.  Any use, reproduction, disclosure or
  # distribution of this software and related documentation without an express
  # license agreement from NVIDIA CORPORATION is strictly prohibited.
  
  """Project given image to the latent space of pretrained network pickle."""
  
  import copy
  import os
  from time import perf_counter
  
  import click
  import imageio
  import numpy as np
  import PIL.Image
  from PIL import ImageFilter
  import torch
  import torch.nn.functional as F
  
  import dnnlib
  import legacy
  
  image_mean = torch.tensor([0.48145466, 0.4578275, 0.40821073]).cuda()
  image_std = torch.tensor([0.26862954, 0.26130258, 0.27577711]).cuda()
  
  def score_images(G, model, text, latents, device, label_class = 0, batch_size = 8):
    scores = []
    all_images = []
    for i in range(latents.shape[0]//batch_size):
      images = G.synthesis(torch.tensor(latents[i*batch_size:(i+1)*batch_size,:,:], dtype=torch.float32, device=device), noise_mode='const')
      with torch.no_grad():
          image_input = (torch.clamp(images, -1, 1) + 1) * 0.5
          image_input = F.interpolate(image_input, size=(256, 256), mode='area')
          image_input = image_input[:, :, 16:240, 16:240] # 256 -> 224, center crop
          image_input -= image_mean[None, :, None, None]
          image_input /= image_std[None, :, None, None]
          score = model(image_input, text)[0]
          scores.append(score.cpu().numpy())
          all_images.append(images.cpu().numpy())
  
    scores = np.array(scores)
    scores = scores.reshape(-1, *scores.shape[2:]).squeeze()
    scores = 1 - scores / np.linalg.norm(scores)
    all_images = np.array(all_images)
    all_images = all_images.reshape(-1, *all_images.shape[2:])
    return scores, all_images
  
  def project(
      G,
      target_image: torch.Tensor, # [C,H,W] and dynamic range [0,255], W & H must match G output resolution
      target_text,
      *,
      num_steps                  = 300,
      w_avg_samples              = 8192,
      initial_learning_rate      = 0.02,
      initial_latent             = None,
      initial_noise_factor       = 0.01,
      lr_rampdown_length         = 0.10,
      lr_rampup_length           = 0.5,
      noise_ramp_length          = 0.75,
      latent_range               = 2.0,
      max_noise                  = 0.5,
      min_threshold              = 0.6,
      use_vgg                    = True,
      use_clip                   = True,
      use_pixel                  = True,
      use_penalty                = True,
      use_center                 = True,
      regularize_noise_weight    = 1e5,
      kmeans                     = True,
      kmeans_clusters            = 64,
      verbose                    = False,
      device: torch.device
  ):
      if target_image is not None:
          assert target_image.shape == (G.img_channels, G.img_resolution, G.img_resolution)
      else:
          use_vgg = False
          use_pixel = False
  
      # reduce errors unless using clip
      if use_clip:
          import clip
  
      def logprint(*args):
          if verbose:
              print(*args)
  
      G = copy.deepcopy(G).eval().requires_grad_(False).to(device) # type: ignore
  
      # Compute w stats.
      logprint(f'Computing W midpoint and stddev using {w_avg_samples} samples...')
      z_samples = np.random.RandomState(123).randn(w_avg_samples, G.z_dim)
      labels = None
      if (G.mapping.c_dim):
          labels = torch.from_numpy(0.5*np.random.RandomState(123).randn(w_avg_samples, G.mapping.c_dim)).to(device)
      w_samples = G.mapping(torch.from_numpy(z_samples).to(device), labels)  # [N, L, C]
      w_samples = w_samples.cpu().numpy().astype(np.float32)                 # [N, L, C]
      w_samples_1d = w_samples[:, :1, :].astype(np.float32)
  
      w_avg = np.mean(w_samples, axis=0, keepdims=True)      # [1, L, C]
      w_std = (np.sum((w_samples - w_avg) ** 2) / w_avg_samples) ** 0.5
  
      kmeans_latents = None
      if initial_latent is not None:
          w_avg = initial_latent
      else:
          if kmeans and use_clip and target_text is not None:
              from kmeans_pytorch import kmeans
              # data
              data_size, dims, num_clusters = w_avg_samples, G.z_dim, kmeans_clusters
              x = w_samples_1d
              x = torch.from_numpy(x)
  
              # kmeans
              logprint(f'Performing kmeans clustering using {w_avg_samples} latents into {kmeans_clusters} clusters...')
              cluster_ids_x, cluster_centers = kmeans(
                  X=x, num_clusters=num_clusters, distance='euclidean', device=device
              )
              #logprint(f'\nGenerating images from kmeans latents...')
              kmeans_latents = torch.tensor(cluster_centers, dtype=torch.float32, device=device, requires_grad=True)
  
      # Setup noise inputs.
      noise_bufs = { name: buf for (name, buf) in G.synthesis.named_buffers() if 'noise_const' in name }
  
      # Load VGG16 feature detector.
      if use_vgg:
          url = 'https://nvlabs-fi-cdn.nvidia.com/stylegan2-ada-pytorch/pretrained/metrics/vgg16.pt'
          with dnnlib.util.open_url(url) as f:
              vgg16 = torch.jit.load(f).eval().to(device)
  
      # Load CLIP
      if use_clip:
          model, transform = clip.load("ViT-B/16", device=device)
  
      # Features for target image.
      if target_image is not None:
          target_images = target_image.unsqueeze(0).to(device).to(torch.float32)
          small_target = F.interpolate(target_images, size=(64, 64), mode='area')
          if use_center:
              center_target = F.interpolate(target_images, size=(448, 448), mode='area')[:, :, 112:336, 112:336]
          target_images = F.interpolate(target_images, size=(256, 256), mode='area')
          target_images = target_images[:, :, 16:240, 16:240] # 256 -> 224, center crop
  
      if use_vgg:
          vgg_target_features = vgg16(target_images, resize_images=False, return_lpips=True)
          if use_center:
              vgg_target_center = vgg16(center_target, resize_images=False, return_lpips=True)
  
      if use_clip:
          if target_image is not None:
              with torch.no_grad():
                  clip_target_features = model.encode_image(((target_images / 255.0) - image_mean[None, :, None, None]) / image_std[None, :, None, None]).float()
                  if use_center:
                      clip_target_center = model.encode_image(((center_target / 255.0) - image_mean[None, :, None, None]) / image_std[None, :, None, None]).float()
  
      if kmeans_latents is not None and use_clip and target_text is not None:
          scores, kmeans_images = score_images(G, model, target_text, kmeans_latents.repeat([1, G.mapping.num_ws, 1]), device=device)
          ind = np.argpartition(scores, 4)[:4]
          w_avg = torch.median(kmeans_latents[ind],dim=0,keepdim=True)[0].repeat([1, G.mapping.num_ws, 1])
  
      w_opt = torch.tensor(w_avg, dtype=torch.float32, device=device, requires_grad=True) # pylint: disable=not-callable
      w_avg_tensor = w_opt.clone()
      w_out = torch.zeros([num_steps] + list(w_opt.shape[1:]), dtype=torch.float32, device=device)
      optimizer = torch.optim.AdamW([w_opt] + list(noise_bufs.values()), betas=(0.9, 0.999), lr=initial_learning_rate)
  
      # Init noise.
      for buf in noise_bufs.values():
          buf[:] = torch.randn_like(buf)
          buf.requires_grad = True
  
      for step in range(num_steps):
          # Learning rate schedule.
          t = step / num_steps
          w_noise_scale = max_noise * w_std * initial_noise_factor * max(0.0, 1.0 - t / noise_ramp_length) ** 2
          lr_ramp = min(1.0, (1.0 - t) / lr_rampdown_length)
          lr_ramp = 0.5 - 0.5 * np.cos(lr_ramp * np.pi)
          lr_ramp = lr_ramp * min(1.0, t / lr_rampup_length)
          lr = initial_learning_rate * lr_ramp
          for param_group in optimizer.param_groups:
              param_group['lr'] = lr
  
          # Synth images from opt_w.
          w_noise = torch.randn_like(w_opt) * w_noise_scale
          ws = w_opt + w_noise
          synth_images = G.synthesis(torch.clamp(ws,-latent_range,latent_range), noise_mode='const')
  
          # Downsample image to 256x256 if it's larger than that. CLIP was built for 224x224 images.
          synth_images = (torch.clamp(synth_images, -1, 1) + 1) * (255/2)
          small_synth = F.interpolate(synth_images, size=(64, 64), mode='area')
          if use_center:
              center_synth = F.interpolate(synth_images, size=(448, 448), mode='area')[:, :, 112:336, 112:336]
          synth_images = F.interpolate(synth_images, size=(256, 256), mode='area')
  
          # Features for synth images.
          synth_images = synth_images[:, :, 16:240, 16:240] # 256 -> 224, center crop
  
          dist = 0
  
          if use_vgg:
              vgg_synth_features = vgg16(synth_images, resize_images=False, return_lpips=True)
              vgg_dist =  (vgg_target_features - vgg_synth_features).square().sum()
              if use_center:
                  vgg_synth_center = vgg16(center_synth, resize_images=False, return_lpips=True)
                  vgg_dist += (vgg_target_center - vgg_synth_center).square().sum()
              vgg_dist *= 6
              dist += F.relu(vgg_dist*vgg_dist - min_threshold)
  
          if use_clip:
              clip_synth_image = ((synth_images / 255.0) - image_mean[None, :, None, None]) / image_std[None, :, None, None]
              clip_synth_features = model.encode_image(clip_synth_image).float()
              adj_center = 2.0
  
              if use_center:
                  clip_cynth_center_image = ((center_synth / 255.0) - image_mean[None, :, None, None]) / image_std[None, :, None, None]
                  adj_center = 1.0
                  clip_synth_center = model.encode_image(clip_cynth_center_image).float()
  
              if target_image is not None:
                  clip_dist =  (clip_target_features - clip_synth_features).square().sum()
                  if use_center:
                      clip_dist += (clip_target_center - clip_synth_center).square().sum()
                  dist += F.relu(0.5 + adj_center*clip_dist - min_threshold)
  
              if target_text is not None:
                  clip_text = 1 - model(clip_synth_image, target_text)[0].sum() / 100
                  if use_center:
                      clip_text += 1 - model(clip_cynth_center_image, target_text)[0].sum() / 100
                  dist += 2*F.relu(adj_center*clip_text*clip_text - min_threshold / adj_center)
  
          if use_pixel:
              pixel_dist =  (target_images - synth_images).abs().sum() / 2000000.0
              if use_center:
                  pixel_dist += (center_target - center_synth).abs().sum() / 2000000.0
              pixel_dist += (small_target - small_synth).square().sum() / 800000.0
              pixel_dist /= 4
              dist += F.relu(lr_ramp * pixel_dist - min_threshold)
  
          if use_penalty:
              l1_penalty = (w_opt - w_avg_tensor).abs().sum() / 5000.0
              dist += F.relu(lr_ramp * l1_penalty - min_threshold)
  
          # Noise regularization.
          reg_loss = 0.0
          for v in noise_bufs.values():
              noise = v[None,None,:,:] # must be [1,1,H,W] for F.avg_pool2d()
              while True:
                  reg_loss += (noise*torch.roll(noise, shifts=1, dims=3)).mean()**2
                  reg_loss += (noise*torch.roll(noise, shifts=1, dims=2)).mean()**2
                  if noise.shape[2] <= 8:
                      break
                  noise = F.avg_pool2d(noise, kernel_size=2)
          #print(vgg_dist, clip_dist, pixel_dist, l1_penalty, reg_loss * regularize_noise_weight)
          loss = dist + reg_loss * regularize_noise_weight
  
          # Step
          optimizer.zero_grad(set_to_none=True)
          loss.backward()
          optimizer.step()
          logprint(f'step {step+1:>4d}/{num_steps}: dist {dist:<4.2f} loss {float(loss):<5.2f}')
          with torch.no_grad():
              torch.clamp(w_opt,-latent_range,latent_range,out=w_opt)
          # Save projected W for each optimization step.
          w_out[step] = w_opt.detach()[0]
          # Normalize noise.
          with torch.no_grad():
              for buf in noise_bufs.values():
                  buf -= buf.mean()
                  buf *= buf.square().mean().rsqrt()
  
      return w_out
  
  #----------------------------------------------------------------------------
  
  @click.command()
  @click.option('--network', 'network_pkl', help='Network pickle filename', required=True)
  @click.option('--target-image', 'target_fname', help='Target image file to project to', required=False, metavar='FILE', default=None)
  @click.option('--target-text',            help='Target text to project to', required=False, default=None)
  @click.option('--initial-latent',         help='Initial latent', default=None)
  @click.option('--lr',                     help='Learning rate', type=float, default=0.1, show_default=True)
  @click.option('--num-steps',              help='Number of optimization steps', type=int, default=1000, show_default=True)
  @click.option('--seed',                   help='Random seed', type=int, default=303, show_default=True)
  @click.option('--save-video',             help='Save an mp4 video of optimization progress', type=bool, default=True, show_default=True)
  @click.option('--outdir',                 help='Where to save the output images', required=True, metavar='DIR')
  @click.option('--use-vgg',                help='Use VGG16 in the loss', type=bool, default=True, show_default=True)
  @click.option('--use-clip',               help='Use CLIP in the loss', type=bool, default=True, show_default=True)
  @click.option('--use-pixel',              help='Use L1/L2 distance on pixels in the loss', type=bool, default=True, show_default=True)
  @click.option('--use-penalty',            help='Use a penalty on latent values distance from the mean in the loss', type=bool, default=True, show_default=True)
  @click.option('--use-center',            help='Optimize against an additional center image crop', type=bool, default=True, show_default=True)
  @click.option('--use-kmeans',            help='Perform kmeans clustering for selecting initial latents', type=bool, default=True, show_default=True)
  def run_projection(
      network_pkl: str,
      target_fname: str,
      target_text: str,
      initial_latent: str,
      outdir: str,
      save_video: bool,
      seed: int,
      lr: float,
      num_steps: int,
      use_vgg: bool,
      use_clip: bool,
      use_pixel: bool,
      use_penalty: bool,
      use_center: bool,
      use_kmeans: bool,
  ):
      """Project given image to the latent space of pretrained network pickle.
  
      Examples:
  
      \b
      python projector.py --outdir=out --target=~/mytargetimg.png \\
          --network=https://nvlabs-fi-cdn.nvidia.com/stylegan2-ada-pytorch/pretrained/ffhq.pkl
      """
      np.random.seed(seed)
      torch.manual_seed(seed)
  
      # Load networks.
      print('Loading networks from "%s"...' % network_pkl)
      device = torch.device('cuda')
      with dnnlib.util.open_url(network_pkl) as fp:
          G = legacy.load_network_pkl(fp)['G_ema'].requires_grad_(False).to(device) # type: ignore
  
      # Load target image.
      target_image = None
      if target_fname:
          target_pil = PIL.Image.open(target_fname).convert('RGB').filter(ImageFilter.SHARPEN)
  
          w, h = target_pil.size
          s = min(w, h)
          target_pil = target_pil.crop(((w - s) // 2, (h - s) // 2, (w + s) // 2, (h + s) // 2))
          target_pil = target_pil.resize((G.img_resolution, G.img_resolution), PIL.Image.LANCZOS)
          target_uint8 = np.array(target_pil, dtype=np.uint8)
          target_image = torch.tensor(target_uint8.transpose([2, 0, 1]), device=device)
  
      if target_text:
          import clip
          target_text = torch.cat([clip.tokenize(target_text)]).to(device)
  
      if initial_latent is not None:
          initial_latent = np.load(initial_latent)
          initial_latent = initial_latent[initial_latent.files[0]]
  
      # Optimize projection.
      start_time = perf_counter()
      projected_w_steps = project(
          G,
          target_image=target_image,
          target_text=target_text,
          initial_latent=initial_latent,
          initial_learning_rate=lr,
          num_steps=num_steps,
          use_vgg=use_vgg,
          use_clip=use_clip,
          use_pixel=use_pixel,
          use_penalty=use_penalty,
          use_center=use_center,
          kmeans=use_kmeans,
          device=device,
          verbose=True
      )
      print (f'Elapsed: {(perf_counter()-start_time):.1f} s')
  
      os.makedirs(outdir, exist_ok=True)
      # Save final projected frame and W vector.
      if target_fname:
          target_pil.save(f'{outdir}/target.png')
      projected_w = projected_w_steps[-1]
      synth_image = G.synthesis(projected_w.unsqueeze(0), noise_mode='const')
      synth_image = (synth_image + 1) * (255/2)
      synth_image = synth_image.permute(0, 2, 3, 1).clamp(0, 255).to(torch.uint8)[0].cpu().numpy()
      PIL.Image.fromarray(synth_image, 'RGB').save(f'{outdir}/proj.png')
      np.savez(f'{outdir}/projected_w.npz', w=projected_w.unsqueeze(0).cpu().numpy())
  
      # Render debug output: optional video and projected image and W vector.
      if save_video:
          video = imageio.get_writer(f'{outdir}/proj.mp4', mode='I', fps=10, codec='libx264', bitrate='16M')
          print (f'Saving optimization progress video "{outdir}/proj.mp4"')
          for projected_w in projected_w_steps:
              synth_image = G.synthesis(projected_w.unsqueeze(0), noise_mode='const')
              synth_image = (synth_image + 1) * (255/2)
              synth_image = synth_image.permute(0, 2, 3, 1).clamp(0, 255).to(torch.uint8)[0].cpu().numpy()
              if target_fname:
                  video.append_data(np.concatenate([target_uint8, synth_image], axis=1))
              else:
                  video.append_data(synth_image)
          video.close()
  
  #----------------------------------------------------------------------------
  
  if __name__ == "__main__":
      run_projection() # pylint: disable=no-value-for-parameter
  
  #----------------------------------------------------------------------------
  