text: import argparse
  import hashlib
  import math
  import os
  import time
  
  import numpy as np
  import torch
  
  import data
  from utils import batchify, get_batch, repackage_hidden
  
  parser = argparse.ArgumentParser(description='PyTorch PennTreeBank RNN/LSTM Language Model')
  parser.add_argument('--data', type=str, default='data/penn/',
                      help='location of the data corpus')
  parser.add_argument('--load', type=str, default='PTB_att.pt',
                      help='trained model for inference')
  parser.add_argument('--model', type=str, default='LSTM',
                      help='type of recurrent net (LSTM, QRNN, GRU)')
  parser.add_argument('--emsize', type=int, default=400,
                      help='size of word embeddings')
  parser.add_argument('--nhid', type=int, default=1150,
                      help='number of hidden units per layer')
  parser.add_argument('--nlayers', type=int, default=3,
                      help='number of layers')
  parser.add_argument('--lr', type=float, default=30,
                      help='initial learning rate')
  parser.add_argument('--clip', type=float, default=0.25,
                      help='gradient clipping')
  parser.add_argument('--epochs', type=int, default=1000,
                      help='upper epoch limit')
  parser.add_argument('--batch_size', type=int, default=80, metavar='N',
                      help='batch size')
  parser.add_argument('--bptt', type=int, default=70,
                      help='sequence length')
  parser.add_argument('--dropout', type=float, default=0.4,
                      help='dropout applied to layers (0 = no dropout)')
  parser.add_argument('--dropouth', type=float, default=0.3,
                      help='dropout for rnn layers (0 = no dropout)')
  parser.add_argument('--dropouti', type=float, default=0.65,
                      help='dropout for input embedding layers (0 = no dropout)')
  parser.add_argument('--dropoute', type=float, default=0.1,
                      help='dropout to remove words from embedding layer (0 = no dropout)')
  parser.add_argument('--wdrop', type=float, default=0.5,
                      help='amount of weight dropout to apply to the RNN hidden to hidden matrix')
  parser.add_argument('--seed', type=int, default=1111,
                      help='random seed')
  parser.add_argument('--nonmono', type=int, default=5,
                      help='random seed')
  parser.add_argument('--cuda', action='store_false',
                      help='use CUDA')
  parser.add_argument('--log-interval', type=int, default=200, metavar='N',
                      help='report interval')
  randomhash = ''.join(str(time.time()).split('.'))
  parser.add_argument('--save', type=str, default=randomhash + '.pt',
                      help='path to save the final model')
  parser.add_argument('--alpha', type=float, default=2,
                      help='alpha L2 regularization on RNN activation (alpha = 0 means no regularization)')
  parser.add_argument('--beta', type=float, default=1,
                      help='beta slowness regularization applied on RNN activiation (beta = 0 means no regularization)')
  parser.add_argument('--wdecay', type=float, default=1.2e-6,
                      help='weight decay applied to all weights')
  parser.add_argument('--resume', type=str, default='',
                      help='path of model to resume')
  parser.add_argument('--optimizer', type=str, default='sgd',
                      help='optimizer to use (sgd, adam)')
  parser.add_argument('--when', nargs="+", type=int, default=[-1],
                      help='When (which epochs) to divide the learning rate by 10 - accepts multiple')
  args = parser.parse_args()
  args.tied = True
  
  
  def model_load(fn):
      global model, criterion, optimizer
      with open(fn, 'rb') as f:
          model, criterion, optimizer = torch.load(f)
  
  
  def evaluate(data_source, batch_size=20):
      # Turn on evaluation mode which disables dropout.
  
      state_dict = model.state_dict()
      for k, i in state_dict.items():
  
          '''print(k)
          print(i.shape)'''
          sz = i.shape
          t = i.cpu().numpy()
          max_coeff_abs = np.amax(np.absolute(t))
  
          # print(i.data.numpy())
  
          b = t
  
          if (len(sz) > 1):
              pass
              'divide to four submatrices'
              print(i.size())
              print(np.count_nonzero(b) / b.size)
  
      model.cuda()
      model.eval()
      if args.model == 'QRNN': model.reset()
      total_loss = 0
      ntokens = len(corpus.dictionary)
      hidden = model.init_hidden(batch_size)
  
      for i in range(0, data_source.size(0) - 1, args.bptt):
          data, targets = get_batch(data_source, i, args, evaluation=True)
          data = data.cuda()
          output, hidden = model(data, hidden)
          total_loss += len(data) * criterion(model.decoder.weight.cpu(), model.decoder.bias.cpu(), output.cpu(),
                                              targets.cpu()).data
          hidden = repackage_hidden(hidden)
      return total_loss.item() / len(data_source)
  
  
  fn = 'corpus.{}.data'.format(hashlib.md5(args.data.encode()).hexdigest())
  if os.path.exists(fn):
      print('Loading cached dataset...')
      corpus = torch.load(fn)
  else:
      print('Producing dataset...')
      corpus = data.Corpus(args.data)
      torch.save(corpus, fn)
  
  eval_batch_size = 20
  test_batch_size = 20
  train_data = batchify(corpus.train, args.batch_size, args)
  val_data = batchify(corpus.valid, eval_batch_size, args)
  test_data = batchify(corpus.test, test_batch_size, args)
  
  # Load the best saved model.
  model_load(args.load)
  
  
  def count_zeros(model_state_dict):
      total_params = 0
      total_zeros = 0
      for layer, params in model_state_dict.items():
          total = params.numel()
          zeros = (params == 0).sum()
          print(f'{layer}: {total} params, {zeros} zeros ({100 * zeros / total}%)')
          total_params += total
          total_zeros += zeros
      print(f'total: {total_params} params, {total_zeros} zeros ({100 * total_zeros / total_params}%)')
  
  
  state_dict = model.state_dict()
  count_zeros(model.state_dict())
  
  print(model)
  
  # Run on test data.
  test_loss = evaluate(test_data, test_batch_size)
  print('=' * 89)
  print('| End of training | test loss {:5.2f} | test ppl {:8.2f} | test bpc {:8.3f}'.format(
      test_loss, math.exp(test_loss), test_loss / math.log(2)))
  print('=' * 89)
  # Run on test data.
  val_loss = evaluate(val_data, test_batch_size)
  print('=' * 89)
  print('| End of training | val loss {:5.2f} | val ppl {:8.2f} | val bpc {:8.3f}'.format(
      val_loss, math.exp(val_loss), val_loss / math.log(2)))
  print('=' * 89)
  