text: import torch
  import torch.nn as nn
  import torch.nn.functional as F
  from torch.autograd import Variable
  from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
  
  from utils import to_gpu
  import json
  import os
  import numpy as np
  
  
  class MLP_D(nn.Module):
      def __init__(self, ninput, noutput, layers,
                   activation=nn.LeakyReLU(0.2), gpu=True):
          super(MLP_D, self).__init__()
          self.ninput = ninput
          self.noutput = noutput
  
          layer_sizes = [ninput] + [int(x) for x in layers.split('-')]
          self.layers = []
  
          for i in range(len(layer_sizes)-1):
              layer = nn.Linear(layer_sizes[i], layer_sizes[i+1])
              self.layers.append(layer)
              self.add_module("layer"+str(i+1), layer)
  
              # No batch normalization after first layer
              if i != 0:
                  bn = nn.BatchNorm1d(layer_sizes[i+1], eps=1e-05, momentum=0.1)
                  self.layers.append(bn)
                  self.add_module("bn"+str(i+1), bn)
  
              self.layers.append(activation)
              self.add_module("activation"+str(i+1), activation)
  
          layer = nn.Linear(layer_sizes[-1], noutput)
          self.layers.append(layer)
          self.add_module("layer"+str(len(self.layers)), layer)
  
          self.init_weights()
  
      def forward(self, x):
          for i, layer in enumerate(self.layers):
              x = layer(x)
          x = torch.mean(x)
          return x
  
      def init_weights(self):
          init_std = 0.02
          for layer in self.layers:
              try:
                  layer.weight.data.normal_(0, init_std)
                  layer.bias.data.fill_(0)
              except:
                  pass
  
  
  class MLP_G(nn.Module):
      def __init__(self, ninput, noutput, layers,
                   activation=nn.ReLU(), gpu=True):
          super(MLP_G, self).__init__()
          self.ninput = ninput
          self.noutput = noutput
  
          layer_sizes = [ninput] + [int(x) for x in layers.split('-')]
          self.layers = []
  
          for i in range(len(layer_sizes)-1):
              layer = nn.Linear(layer_sizes[i], layer_sizes[i+1])
              self.layers.append(layer)
              self.add_module("layer"+str(i+1), layer)
  
              bn = nn.BatchNorm1d(layer_sizes[i+1], eps=1e-05, momentum=0.1)
              self.layers.append(bn)
              self.add_module("bn"+str(i+1), bn)
  
              self.layers.append(activation)
              self.add_module("activation"+str(i+1), activation)
  
          layer = nn.Linear(layer_sizes[-1], noutput)
          self.layers.append(layer)
          self.add_module("layer"+str(len(self.layers)), layer)
  
          self.init_weights()
  
      def forward(self, x):
          for i, layer in enumerate(self.layers):
              x = layer(x)
          return x
  
      def init_weights(self):
          init_std = 0.02
          for layer in self.layers:
              try:
                  layer.weight.data.normal_(0, init_std)
                  layer.bias.data.fill_(0)
              except:
                  pass
  
  
  class Seq2Seq(nn.Module):
      def __init__(self, emsize, nhidden, ntokens, nlayers, noise_r=0.2,
                   hidden_init=False, dropout=0, gpu=True):
          super(Seq2Seq, self).__init__()
          self.nhidden = nhidden
          self.emsize = emsize
          self.ntokens = ntokens
          self.nlayers = nlayers
          self.noise_r = noise_r
          self.hidden_init = hidden_init
          self.dropout = dropout
          self.gpu = gpu
  
          self.start_symbols = to_gpu(gpu, Variable(torch.ones(10, 1).long()))
  
          # Vocabulary embedding
          self.embedding = nn.Embedding(ntokens, emsize)
          self.embedding_decoder = nn.Embedding(ntokens, emsize)
  
          # RNN Encoder and Decoder
          self.encoder = nn.LSTM(input_size=emsize,
                                 hidden_size=nhidden,
                                 num_layers=nlayers,
                                 dropout=dropout,
                                 batch_first=True)
  
          decoder_input_size = emsize+nhidden
          self.decoder = nn.LSTM(input_size=decoder_input_size,
                                 hidden_size=nhidden,
                                 num_layers=1,
                                 dropout=dropout,
                                 batch_first=True)
  
          # Initialize Linear Transformation
          self.linear = nn.Linear(nhidden, ntokens)
  
          self.init_weights()
  
      def init_weights(self):
          initrange = 0.1
  
          # Initialize Vocabulary Matrix Weight
          self.embedding.weight.data.uniform_(-initrange, initrange)
          self.embedding_decoder.weight.data.uniform_(-initrange, initrange)
  
          # Initialize Encoder and Decoder Weights
          for p in self.encoder.parameters():
              p.data.uniform_(-initrange, initrange)
          for p in self.decoder.parameters():
              p.data.uniform_(-initrange, initrange)
  
          # Initialize Linear Weight
          self.linear.weight.data.uniform_(-initrange, initrange)
          self.linear.bias.data.fill_(0)
  
      def init_hidden(self, bsz):
          zeros1 = Variable(torch.zeros(1, bsz, self.nhidden))
          zeros2 = Variable(torch.zeros(1, bsz, self.nhidden))
          return (to_gpu(self.gpu, zeros1), to_gpu(self.gpu, zeros2))
  
      def init_state(self, bsz):
          zeros = Variable(torch.zeros(1, bsz, self.nhidden))
          return to_gpu(self.gpu, zeros)
  
      def store_grad_norm(self, grad):
          norm = torch.norm(grad, 2, 1)
          self.grad_norm = norm.detach().data.mean()
          return grad
  
      def forward(self, indices, lengths, noise, encode_only=False):
          batch_size, maxlen = indices.size()
  
          hidden = self.encode(indices, lengths, noise)
  
          if encode_only:
              return hidden
  
          if hidden.requires_grad:
              hidden.register_hook(self.store_grad_norm)
  
          decoded = self.decode(hidden, batch_size, maxlen,
                                indices=indices, lengths=lengths)
  
          return decoded
  
      def encode(self, indices, lengths, noise): 
          embeddings = self.embedding(indices)
          packed_embeddings = pack_padded_sequence(input=embeddings,
                                                   lengths=lengths,
                                                   batch_first=True)
  
          packed_output, state = self.encoder(packed_embeddings)
          hidden = state[0][-1]
          hidden = hidden / torch.norm(hidden, p=2, dim=1, keepdim=True)
          
          if noise and self.noise_r > 0:
              gauss_noise = torch.normal(means=torch.zeros(hidden.size()),
                                         std=self.noise_r)
              hidden = hidden + Variable(gauss_noise.cuda())
  
          return hidden
  
      def decode(self, hidden, batch_size, maxlen, indices=None, lengths=None):
          # batch x hidden
          all_hidden = hidden.unsqueeze(1).repeat(1, maxlen, 1)
  
          if self.hidden_init:
              # initialize decoder hidden state to encoder output
              state = (hidden.unsqueeze(0), self.init_state(batch_size))
          else:
              state = self.init_hidden(batch_size)
  
          embeddings = self.embedding_decoder(indices)
          augmented_embeddings = torch.cat([embeddings, all_hidden], 2)
          packed_embeddings = pack_padded_sequence(input=augmented_embeddings,
                                                   lengths=lengths,
                                                   batch_first=True)
  
          packed_output, state = self.decoder(packed_embeddings, state)
          output, lengths = pad_packed_sequence(packed_output, batch_first=True)
  
          # reshape to batch_size*maxlen x nhidden before linear over vocab
          decoded = self.linear(output.contiguous().view(-1, self.nhidden))
          decoded = decoded.view(batch_size, maxlen, self.ntokens)
  
          return decoded
  
      def generate(self, hidden, maxlen, sample=True, temp=1.0):
          """Generate through decoder; no backprop"""
  
          batch_size = hidden.size(0)
  
          if self.hidden_init:
              # initialize decoder hidden state to encoder output
              state = (hidden.unsqueeze(0), self.init_state(batch_size))
          else:
              state = self.init_hidden(batch_size)
  
          # <sos>
          self.start_symbols.data.resize_(batch_size, 1)
          self.start_symbols.data.fill_(1)
  
          embedding = self.embedding_decoder(self.start_symbols)
          inputs = torch.cat([embedding, hidden.unsqueeze(1)], 2)
  
          # unroll
          all_indices = []
          for i in range(maxlen):
              output, state = self.decoder(inputs, state)
              overvocab = self.linear(output.squeeze(1))
              if not sample:
                  vals, indices = torch.max(overvocab, 1)
              else:
                  probs = F.softmax(overvocab / temp, dim=-1)
                  indices = torch.multinomial(probs, 1)
              indices = indices.unsqueeze(1)
              all_indices.append(indices)
  
              embedding = self.embedding_decoder(indices)
              inputs = torch.cat([embedding, hidden.unsqueeze(1)], 2)
  
          max_indices = torch.cat(all_indices, 1)
          return max_indices
      
      def noise_anneal(self, fac):
          self.noise_r *= fac
  
  
  def generate(autoencoder, gan_gen, z, vocab, sample, maxlen):
      """
      Assume noise is batch_size x z_size
      """
      if type(z) == Variable:
          noise = z
      elif type(z) == torch.FloatTensor or type(z) == torch.cuda.FloatTensor:
          noise = Variable(z, volatile=True)
      elif type(z) == np.ndarray:
          noise = Variable(torch.from_numpy(z).float(), volatile=True)
      else:
          raise ValueError("Unsupported input type (noise): {}".format(type(z)))
  
      gan_gen.eval()
      autoencoder.eval()
  
      # generate from random noise
      fake_hidden = gan_gen(noise)
      max_indices = autoencoder.generate(hidden=fake_hidden,
                                         maxlen=maxlen,
                                         sample=sample)
  
      max_indices = max_indices.data.cpu().numpy()
      sentences = []
      for idx in max_indices:
          # generated sentence
          words = [vocab[x] for x in idx]
          # truncate sentences to first occurrence of <eos>
          truncated_sent = []
          for w in words:
              if w != '<eos>':
                  truncated_sent.append(w)
              else:
                  break
          sent = " ".join(truncated_sent)
          sentences.append(sent)
  
      return sentences
  