text: # YOLOv5 🚀 by Ultralytics, GPL-3.0 license
  """
  Experimental modules
  """
  import math
  import numpy as np
  import torch
  import torch.nn as nn
  
  from models.common import Conv
  from utils.downloads import attempt_download
  
  
  class CrossConv(nn.Module):
      # Cross Convolution Downsample
      def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False):
          # ch_in, ch_out, kernel, stride, groups, expansion, shortcut
          super().__init__()
          c_ = int(c2 * e)  # hidden channels
          self.cv1 = Conv(c1, c_, (1, k), (1, s))
          self.cv2 = Conv(c_, c2, (k, 1), (s, 1), g=g)
          self.add = shortcut and c1 == c2
  
      def forward(self, x):
          return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))
  
  
  class Sum(nn.Module):
      # Weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070
      def __init__(self, n, weight=False):  # n: number of inputs
          super().__init__()
          self.weight = weight  # apply weights boolean
          self.iter = range(n - 1)  # iter object
          if weight:
              self.w = nn.Parameter(-torch.arange(1., n) / 2, requires_grad=True)  # layer weights
  
      def forward(self, x):
          y = x[0]  # no weight
          if self.weight:
              w = torch.sigmoid(self.w) * 2
              for i in self.iter:
                  y = y + x[i + 1] * w[i]
          else:
              for i in self.iter:
                  y = y + x[i + 1]
          return y
  
  
  class MixConv2d(nn.Module):
      # Mixed Depth-wise Conv https://arxiv.org/abs/1907.09595
      def __init__(self, c1, c2, k=(1, 3), s=1, equal_ch=True):  # ch_in, ch_out, kernel, stride, ch_strategy
          super().__init__()
          n = len(k)  # number of convolutions
          if equal_ch:  # equal c_ per group
              i = torch.linspace(0, n - 1E-6, c2).floor()  # c2 indices
              c_ = [(i == g).sum() for g in range(n)]  # intermediate channels
          else:  # equal weight.numel() per group
              b = [c2] + [0] * n
              a = np.eye(n + 1, n, k=-1)
              a -= np.roll(a, 1, axis=1)
              a *= np.array(k) ** 2
              a[0] = 1
              c_ = np.linalg.lstsq(a, b, rcond=None)[0].round()  # solve for equal weight indices, ax = b
  
          self.m = nn.ModuleList(
              [nn.Conv2d(c1, int(c_), k, s, k // 2, groups=math.gcd(c1, int(c_)), bias=False) for k, c_ in zip(k, c_)])
          self.bn = nn.BatchNorm2d(c2)
          self.act = nn.SiLU()
  
      def forward(self, x):
          return self.act(self.bn(torch.cat([m(x) for m in self.m], 1)))
  
  
  class Ensemble(nn.ModuleList):
      # Ensemble of models
      def __init__(self):
          super().__init__()
  
      def forward(self, x, augment=False, profile=False, visualize=False):
          y = []
          for module in self:
              y.append(module(x, augment, profile, visualize)[0])
          # y = torch.stack(y).max(0)[0]  # max ensemble
          # y = torch.stack(y).mean(0)  # mean ensemble
          y = torch.cat(y, 1)  # nms ensemble
          return y, None  # inference, train output
  
  
  def attempt_load(weights, map_location=None, inplace=True, fuse=True):
      from models.yolo import Detect, Model
  
      # Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a
      model = Ensemble()
      for w in weights if isinstance(weights, list) else [weights]:
          ckpt = torch.load(attempt_download(w), map_location=map_location)  # load
          if fuse:
              model.append(ckpt['ema' if ckpt.get('ema') else 'model'].float().fuse().eval())  # FP32 model
          else:
              model.append(ckpt['ema' if ckpt.get('ema') else 'model'].float().eval())  # without layer fuse
  
      # Compatibility updates
      for m in model.modules():
          if type(m) in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU, Detect, Model]:
              m.inplace = inplace  # pytorch 1.7.0 compatibility
              if type(m) is Detect:
                  if not isinstance(m.anchor_grid, list):  # new Detect Layer compatibility
                      delattr(m, 'anchor_grid')
                      setattr(m, 'anchor_grid', [torch.zeros(1)] * m.nl)
          elif type(m) is Conv:
              m._non_persistent_buffers_set = set()  # pytorch 1.6.0 compatibility
  
      if len(model) == 1:
          return model[-1]  # return model
      else:
          print(f'Ensemble created with {weights}\n')
          for k in ['names']:
              setattr(model, k, getattr(model[-1], k))
          model.stride = model[torch.argmax(torch.tensor([m.stride.max() for m in model])).int()].stride  # max stride
          return model  # return ensemble
  