class_docstr: MobileNetV3Space implements the largest search space in `TuNAS <https://arxiv.org/abs/2008.06120>`__.
  
  The search dimensions include widths, expand ratios, kernel sizes, SE ratio.
  Some of them can be turned off via arguments to narrow down the search space.
  
  Different from ProxylessNAS search space, this space is implemented with :class:`~nni.nas.nn.pytorch.ValueChoice`.
  
  We use the following snipppet as reference.
  https://github.com/google-research/google-research/blob/20736344591f774f4b1570af64624ed1e18d2867/tunas/mobile_search_space_v3.py#L728
  
  We have ``num_blocks`` which equals to the length of ``self.blocks`` (the main body of the network).
  For simplicity, the following parameter specification assumes ``num_blocks`` equals 8 (body + head).
  If a shallower body is intended, arrays including ``base_widths``, ``squeeze_excite``, ``depth_range``,
  ``stride``, ``activation`` should also be shortened accordingly.
  
  Parameters
  ----------
  num_labels
      Dimensions for classification head.
  base_widths
      Widths of each stage, from stem, to body, to head.
      Length should be 9, i.e., ``num_blocks + 1`` (because there is a stem width in front).
  width_multipliers
      A range of widths multiplier to choose from. The choice is independent for each stage.
      Or it can be a fixed float. This will be applied on ``base_widths``,
      and we would also make sure that widths can be divided by 8.
  expand_ratios
      A list of expand ratios to choose from. Independent for every **block**.
  squeeze_excite
      Indicating whether the current stage can have an optional SE layer.
      Expect array of length 6 for stage 0 to 5. Each element can be one of ``force``, ``optional``, ``none``.
  depth_range
      A range (e.g., ``(1, 4)``),
      or a list of range (e.g., ``[(1, 3), (1, 4), (1, 4), (1, 3), (0, 2)]``).
      If a list, the length should be 5. The depth are specified for stage 1 to 5.
  stride
      Stride for all stages (including stem and head). Length should be same as ``base_widths``.
  activation
      Activation (class) for all stages. Length is same as ``base_widths``.
  se_from_exp
      Calculate SE channel reduction from expanded (mid) channels.
  dropout_rate
      Dropout rate at classification head.
  bn_eps
      Epsilon of batch normalization.
  bn_momentum
      Momentum of batch normalization.
class_docstr_tok: ["MobileNetV3Space","implements","the","largest","search","space","in","`","TuNAS","<","https",":","\/\/arxiv.org\/abs\/2008.06120",">","`","__",".","The","search","dimensions","include","widths",",","expand","ratios",",","kernel","sizes",",","SE","ratio",".","Some","of","them","can","be","turned","off","via","arguments","to","narrow","down","the","search","space",".","Different","from","ProxylessNAS","search","space",",","this","space","is","implemented","with",":","class",":","`","~nni.nas.nn.pytorch.ValueChoice","`",".","We","use","the","following","snipppet","as","reference",".","https",":","\/\/github.com\/google-research\/google-research\/blob\/20736344591f774f4b1570af64624ed1e18d2867\/tunas\/mobile_search_space_v3.py","#","L728","We","have","``","num_blocks","``","which","equals","to","the","length","of","``","self.blocks","``","(","the","main","body","of","the","network",")",".","For","simplicity",",","the","following","parameter","specification","assumes","``","num_blocks","``","equals","8","(","body","+","head",")",".","If","a","shallower","body","is","intended",",","arrays","including","``","base_widths","``",",","``","squeeze_excite","``",",","``","depth_range","``",",","``","stride","``",",","``","activation","``","should","also","be","shortened","accordingly",".","Parameters","--","--","--","--","--","num_labels","Dimensions","for","classification","head",".","base_widths","Widths","of","each","stage",",","from","stem",",","to","body",",","to","head",".","Length","should","be","9",",","i.e.",",","``","num_blocks","+","1","``","(","because","there","is","a","stem","width","in","front",")",".","width_multipliers","A","range","of","widths","multiplier","to","choose","from",".","The","choice","is","independent","for","each","stage",".","Or","it","can","be","a","fixed","float",".","This","will","be","applied","on","``","base_widths","``",",","and","we","would","also","make","sure","that","widths","can","be","divided","by","8.","expand_ratios","A","list","of","expand","ratios","to","choose","from",".","Independent","for","every","*","*","block","*","*",".","squeeze_excite","Indicating","whether","the","current","stage","can","have","an","optional","SE","layer",".","Expect","array","of","length","6","for","stage","0","to","5",".","Each","element","can","be","one","of","``","force","``",",","``","optional","``",",","``","none","``",".","depth_range","A","range","(","e.g.",",","``","(","1",",","4",")","``",")",",","or","a","list","of","range","(","e.g.",",","``","[","(","1",",","3",")",",","(","1",",","4",")",",","(","1",",","4",")",",","(","1",",","3",")",",","(","0",",","2",")","]","``",")",".","If","a","list",",","the","length","should","be","5",".","The","depth","are","specified","for","stage","1","to","5.","stride","Stride","for","all","stages","(","including","stem","and","head",")",".","Length","should","be","same","as","``","base_widths","``",".","activation","Activation","(","class",")","for","all","stages",".","Length","is","same","as","``","base_widths","``",".","se_from_exp","Calculate","SE","channel","reduction","from","expanded","(","mid",")","channels",".","dropout_rate","Dropout","rate","at","classification","head",".","bn_eps","Epsilon","of","batch","normalization",".","bn_momentum","Momentum","of","batch","normalization","."]
code: def __init__(
      self,
      num_labels: int = 1000,
      base_widths: Tuple[int, ...] = (
          16,
          16,
          16,
          32,
          64,
          128,
          256,
          512,
          1024,
      ),
      width_multipliers: Union[Tuple[float, ...], float] = (
          0.5,
          0.625,
          0.75,
          1.0,
          1.25,
          1.5,
          2.0,
      ),
      expand_ratios: Tuple[float, ...] = (1.0, 2.0, 3.0, 4.0, 5.0, 6.0),
      squeeze_excite: Tuple[
          Literal["force", "optional", "none"], ...
      ] = ("none", "none", "optional", "none", "optional", "optional"),
      depth_range: Union[List[Tuple[int, int]], Tuple[int, int]] = (
          1,
          4,
      ),
      stride: Tuple[int, ...] = (2, 1, 2, 2, 2, 1, 2, 1, 1),
      activation: Tuple[Literal["hswish", "swish", "relu"], ...] = (
          "hswish",
          "relu",
          "relu",
          "relu",
          "hswish",
          "hswish",
          "hswish",
          "hswish",
          "hswish",
      ),
      se_from_exp: bool = True,
      dropout_rate: float = 0.2,
      bn_eps: float = 1e-3,
      bn_momentum: float = 0.1,
  ):
      super().__init__()
  
      self.num_blocks = (
          len(base_widths) - 1
      )  # without stem, equal to len(self.blocks)
      assert self.num_blocks >= 4
  
      assert (
          len(base_widths)
          == len(stride)
          == len(activation)
          == self.num_blocks + 1
      )
  
      # The final two blocks can't have SE
      assert len(squeeze_excite) == self.num_blocks - 2 and all(
          se in ["force", "optional", "none"] for se in squeeze_excite
      )
  
      # The first and final two blocks can't have variational depth
      if isinstance(depth_range[0], int):
          depth_range = cast(Tuple[int, int], depth_range)
          assert (
              len(depth_range) == 2
              and depth_range[1] >= depth_range[0] >= 1
          )
          self.depth_range = [depth_range] * (self.num_blocks - 3)
      else:
          assert len(depth_range) == self.num_blocks - 3
          self.depth_range = cast(List[Tuple[int, int]], depth_range)
          for d in self.depth_range:
              d = cast(Tuple[int, int], d)
              # pylint: disable=unsubscriptable-object
              assert (
                  len(d) == 2 and d[1] >= d[0] >= 1
              ), f"{d} does not satisfy depth constraints"
  
      self.widths = []
      for i, base_width in enumerate(base_widths):
          if isinstance(width_multipliers, float):
              self.widths.append(
                  make_divisible(base_width * width_multipliers, 8)
              )
          else:
              self.widths.append(
                  # According to tunas, stem and stage 0 share one width multiplier
                  # https://github.com/google-research/google-research/blob/20736344/tunas/mobile_search_space_v3.py#L791
                  make_divisible(
                      nni.choice(
                          f"s{max(i - 1, 0)}_width_mult",
                          list(width_multipliers),
                      )
                      * base_width,
                      8,
                  )
              )
  
      self.expand_ratios = expand_ratios
      self.se_from_exp = se_from_exp
  
      # NOTE: The built-in hardswish produces slightly different output from 3rd-party implementation
      # But I guess it doesn't really matter.
      # https://github.com/rwightman/pytorch-image-models/blob/b7cb8d03/timm/models/layers/activations.py#L79
  
      self.stem = ConvBNReLU(
          3,
          self.widths[0],
          nni.choice(f"stem_ks", [3, 5]),
          stride=stride[0],
          activation_layer=_act_fn(activation[0]),
      )
  
      blocks: List[nn.Module] = [
          # Stage 0
          # FIXME: this should be an optional layer.
          # https://github.com/google-research/google-research/blob/20736344/tunas/mobile_search_space_v3.py#L791
          DepthwiseSeparableConv(
              self.widths[0],
              self.widths[1],
              nni.choice(f"s0_i0_ks", [3, 5, 7]),
              stride=stride[1],
              squeeze_excite=cast(
                  Callable[[MaybeIntChoice, MaybeIntChoice], nn.Module],
                  partial(
                      _se_or_skip,
                      optional=squeeze_excite[0] == "optional",
                      se_from_exp=self.se_from_exp,
                      label=f"s0_i0_se",
                  ),
              )
              if squeeze_excite[0] != "none"
              else None,
              activation_layer=_act_fn(activation[1]),
          ),
      ]
  
      blocks += [
          # Stage 1-5 (by default)
          self._make_stage(
              i,
              self.widths[i],
              self.widths[i + 1],
              squeeze_excite[i],
              stride[i + 1],
              _act_fn(activation[i + 1]),
          )
          for i in range(1, self.num_blocks - 2)
      ]
  
      # Head
      blocks += [
          ConvBNReLU(
              self.widths[self.num_blocks - 2],
              self.widths[self.num_blocks - 1],
              kernel_size=1,
              stride=stride[self.num_blocks - 1],
              activation_layer=_act_fn(activation[self.num_blocks - 1]),
          ),
          nn.AdaptiveAvgPool2d(1),
          # In some implementation, this is a linear instead.
          # Should be equivalent.
          ConvBNReLU(
              self.widths[self.num_blocks - 1],
              self.widths[self.num_blocks],
              kernel_size=1,
              stride=stride[self.num_blocks],
              norm_layer=nn.Identity,
              activation_layer=_act_fn(activation[self.num_blocks]),
          ),
      ]
  
      self.blocks = nn.Sequential(*blocks)
  
      self.classifier = nn.Sequential(
          nn.Dropout(dropout_rate),
          MutableLinear(
              cast(int, self.widths[self.num_blocks]), num_labels
          ),
      )
  
      reset_parameters(self, bn_momentum=bn_momentum, bn_eps=bn_eps)
  
imports: from functools import partial
  from typing import Tuple, Optional, Callable, Union, List, Type, cast
  from typing_extensions import Literal
  import torch
  from torch import nn
  import nni
  from nni.nas.nn.pytorch import ModelSpace, Repeat, LayerChoice, MutableLinear, MutableConv2d
  from .proxylessnas import ConvBNReLU, InvertedResidual, DepthwiseSeparableConv, MaybeIntChoice, make_divisible, reset_parameters
  from .utils.pretrained import load_pretrained_weight
label_desc: Use image node_id 1 to create a new MobileNetV3Space object from inherited base classes: ModelSpace with example: obj = MobileNetV3Space(num_labels, base_widths, width_multipliers, expand_ratios, squeeze_excite, depth_range, stride, activation, se_from_exp, dropout_rate, bn_eps, bn_momentum)