code: from __future__ import print_function
  
  import errno
  import os
  from PIL import Image
  import torch
  import torch.nn as nn
  import re
  
  import json
  import pickle as  cPickle
  import numpy as np
  import utils
  import h5py
  import operator
  import functools
  from torch._six import string_classes
  import torch.nn.functional as F
  import collections
  
  #from pycocotools.coco import COCO
  # from scipy.sparse import coo_matrix
  # from sklearn.metrics.pairwise import cosine_similarity
  from torch.utils.data.dataloader import default_collate
  
  
  EPS = 1e-7
  
  
  def assert_eq(real, expected):
      assert real == expected, '%s (true) vs %s (expected)' % (real, expected)
  
  
  def assert_array_eq(real, expected):
      assert (np.abs(real-expected) < EPS).all(), \
          '%s (true) vs %s (expected)' % (real, expected)
  
  
  def load_folder(folder, suffix):
      imgs = []
      for f in sorted(os.listdir(folder)):
          if f.endswith(suffix):
              imgs.append(os.path.join(folder, f))
      return imgs
  
  
  def load_imageid(folder):
      images = load_folder(folder, 'jpg')
      img_ids = set()
      for img in images:
          img_id = int(img.split('/')[-1].split('.')[0].split('_')[-1])
          img_ids.add(img_id)
      return img_ids
  
  
  def pil_loader(path):
      with open(path, 'rb') as f:
          with Image.open(f) as img:
              return img.convert('RGB')
  
  
  def weights_init(m):
      """custom weights initialization."""
      cname = m.__class__
      if cname == nn.Linear or cname == nn.Conv2d or cname == nn.ConvTranspose2d:
          m.weight.data.normal_(0.0, 0.02)
      elif cname == nn.BatchNorm2d:
          m.weight.data.normal_(1.0, 0.02)
          m.bias.data.fill_(0)
      else:
          print('%s is not initialized.' % cname)
  
  
  def init_net(net, net_file):
      if net_file:
          net.load_state_dict(torch.load(net_file))
      else:
          net.apply(weights_init)
  
  
  def create_dir(path):
      if not os.path.exists(path):
          try:
              os.makedirs(path)
          except OSError as exc:
              if exc.errno != errno.EEXIST:
                  raise
  
  
  class Logger(object):
      def __init__(self, output_name):
          dirname = os.path.dirname(output_name)
          if not os.path.exists(dirname):
              os.mkdir(dirname)
  
          self.log_file = open(output_name, 'w')
          self.infos = {}
  
      def append(self, key, val):
          vals = self.infos.setdefault(key, [])
          vals.append(val)
  
      def log(self, extra_msg=''):
          msgs = [extra_msg]
          for key, vals in self.infos.iteritems():
              msgs.append('%s %.6f' % (key, np.mean(vals)))
          msg = '\n'.join(msgs)
          self.log_file.write(msg + '\n')
          self.log_file.flush()
          self.infos = {}
          return msg
  
      def write(self, msg):
          self.log_file.write(msg + '\n')
          self.log_file.flush()
          print(msg)
  
  def print_model(model, logger):
      print(model)
      nParams = 0
      for w in model.parameters():
          nParams += functools.reduce(operator.mul, w.size(), 1)
      if logger:
          logger.write('nParams=\t'+str(nParams))
  
  
  def save_model(path, model, epoch, optimizer=None):
      model_dict = {
              'epoch': epoch,
              'model_state': model.state_dict()
          }
      if optimizer is not None:
          model_dict['optimizer_state'] = optimizer.state_dict()
  
      torch.save(model_dict, path)
  
  def rho_select(pad, lengths):
      # Index of the last output for each sequence.
      idx_ = (lengths-1).view(-1,1).expand(pad.size(0), pad.size(2)).unsqueeze(1)
      extracted = pad.gather(1, idx_).squeeze(1)
      return extracted
  
  def trim_collate(batch):
      "Puts each data field into a tensor with outer dimension batch size"
      _use_shared_memory = True
      error_msg = "batch must contain tensors, numbers, dicts or lists; found {}"
      elem_type = type(batch[0])
      if torch.is_tensor(batch[0]):
          out = None
          if 1 < batch[0].dim(): # image features
              max_num_boxes = max([x.size(0) for x in batch])
              if _use_shared_memory:
                  # If we're in a background process, concatenate directly into a
                  # shared memory tensor to avoid an extra copy
                  numel = len(batch) * max_num_boxes * batch[0].size(-1)
                  storage = batch[0].storage()._new_shared(numel)
                  out = batch[0].new(storage)
              # warning: F.pad returns Variable!
              return torch.stack([F.pad(x, (0,0,0,max_num_boxes-x.size(0))).data for x in batch], 0, out=out)
          else:
              if _use_shared_memory:
                  # If we're in a background process, concatenate directly into a
                  # shared memory tensor to avoid an extra copy
                  numel = sum([x.numel() for x in batch])
                  storage = batch[0].storage()._new_shared(numel)
                  out = batch[0].new(storage)
              return torch.stack(batch, 0, out=out)
      elif elem_type.__module__ == 'numpy' and elem_type.__name__ != 'str_' \
              and elem_type.__name__ != 'string_':
          elem = batch[0]
          if elem_type.__name__ == 'ndarray':
              # array of string classes and object
              if re.search('[SaUO]', elem.dtype.str) is not None:
                  raise TypeError(error_msg.format(elem.dtype))
  
              return torch.stack([torch.from_numpy(b) for b in batch], 0)
          if elem.shape == ():  # scalars
              py_type = float if elem.dtype.name.startswith('float') else int
              return numpy_type_map[elem.dtype.name](list(map(py_type, batch)))
      elif isinstance(batch[0], int):
          return torch.LongTensor(batch)
      elif isinstance(batch[0], float):
          return torch.DoubleTensor(batch)
      elif isinstance(batch[0], string_classes):
          return batch
      elif isinstance(batch[0], collections.Mapping):
          return {key: default_collate([d[key] for d in batch]) for key in batch[0]}
      elif isinstance(batch[0], collections.Sequence):
          transposed = zip(*batch)
          return [trim_collate(samples) for samples in transposed]
  
      raise TypeError((error_msg.format(type(batch[0]))))
  
  
  def sparse_mx_to_torch_sparse_tensor(sparse_mx):
      """Convert a scipy sparse matrix to a torch sparse tensor."""
      sparse_mx = sparse_mx.tocoo().astype(np.float32)
      indices = torch.from_numpy(
          np.vstack((sparse_mx.row, sparse_mx.col)).astype(np.int64))
      values = torch.from_numpy(sparse_mx.data)
      shape = torch.Size(sparse_mx.shape)
      return torch.sparse.FloatTensor(indices, values, shape)
  
  
  def mask_softmax(x, lengths):  # , dim=1)
      mask = torch.zeros_like(x).to(device=x.device, non_blocking=True)
      t_lengths = lengths[:, :, None].expand_as(mask)
      arange_id = torch.arange(mask.size(1)).to(device=x.device, non_blocking=True)
      arange_id = arange_id[None, :, None].expand_as(mask)
  
      mask[arange_id < t_lengths] = 1
      # https://stackoverflow.com/questions/42599498/numercially-stable-softmax
      # https://stackoverflow.com/questions/34968722/how-to-implement-the-softmax-function-in-python
      # exp(x - max(x)) instead of exp(x) is a trick
      # to improve the numerical stability while giving
      # the same outputs
      x2 = torch.exp(x - torch.max(x))
      x3 = x2 * mask
      epsilon = 1e-5
      x3_sum = torch.sum(x3, dim=1, keepdim=True) + epsilon
      x4 = x3 / x3_sum.expand_as(x3)
      return x4
  
  
  class GradReverseMask(torch.autograd.Function):
      """
      This layer is used to create an adversarial loss.
  
      """
  
      @staticmethod
      def forward(ctx, x, mask, weight):
          """
          The mask should be composed of 0 or 1.
          The '1' will get their gradient reversed..
          """
          ctx.save_for_backward(mask)
          ctx.weight = weight
          return x.view_as(x)
  
      @staticmethod
      def backward(ctx, grad_output):
          mask, = ctx.saved_tensors
          mask_c = mask.clone().detach().float()
          mask_c[mask == 0] = 1.0
          mask_c[mask == 1] = - float(ctx.weight)
          return grad_output * mask_c[:, None].float(), None, None
  
  
  def grad_reverse_mask(x, mask, weight=1):
      return GradReverseMask.apply(x, mask, weight)
  
  
  class GradReverse(torch.autograd.Function):
      """
      This layer is used to create an adversarial loss.
      """
  
      @staticmethod
      def forward(ctx, x):
          return x.view_as(x)
  
      @staticmethod
      def backward(ctx, grad_output):
          return grad_output.neg()
  
  
  def grad_reverse(x):
      return GradReverse.apply(x)
  
  
  class GradMulConst(torch.autograd.Function):
      """
      This layer is used to create an adversarial loss.
      """
  
      @staticmethod
      def forward(ctx, x, const):
          ctx.const = const
          return x.view_as(x)
  
      @staticmethod
      def backward(ctx, grad_output):
          return grad_output * ctx.const, None
  
  
  def grad_mul_const(x, const):
      return GradMulConst.apply(x, const)
  
apis: torch.sum numpy.vstack torch.utils.data.dataloader.default_collate torch.stack torch.Size torch.load torch.zeros_like torch.save numpy.abs torch.DoubleTensor torch.sparse.FloatTensor torch.is_tensor torch.from_numpy torch.max torch.LongTensor numpy.mean