class_docstr: This class defines the `Vision Transformer architecture <https://arxiv.org/abs/2010.11929>`_. Our model implementation
  is inspired from `Early Convolutions Help Transformers See Better <https://arxiv.org/abs/2106.14881>`_
  
  .. note::
      Our implementation is different from the original implementation in two ways:
      1. Kernel size is odd.
      2. Our positional encoding implementation allows us to use ViT with any multiple input scales
      3. We do not use StochasticDepth
      4. We do not add positional encoding to class token (if enabled), as suggested in `DeiT-3 paper <https://arxiv.org/abs/2204.07118>`_
class_docstr_tok: ["This","class","defines","the","`","Vision","Transformer","architecture","<","https",":","\/\/arxiv.org\/abs\/2010.11929",">","`","_",".","Our","model","implementation","is","inspired","from","`","Early","Convolutions","Help","Transformers","See","Better","<","https",":","\/\/arxiv.org\/abs\/2106.14881",">","`","_","..","note",":",":","Our","implementation","is","different","from","the","original","implementation","in","two","ways",":","1",".","Kernel","size","is","odd",".","2",".","Our","positional","encoding","implementation","allows","us","to","use","ViT","with","any","multiple","input","scales","3",".","We","do","not","use","StochasticDepth","4",".","We","do","not","add","positional","encoding","to","class","token","(","if","enabled",")",",","as","suggested","in","`","DeiT-3","paper","<","https",":","\/\/arxiv.org\/abs\/2204.07118",">","`","_"]
docstr: This function adjusts the learning rate of each layer in transformer module.
  Layer-wise learning is a bit involved and requires a knowledge of how each layer is consumed
  during the forward pass. We adjust the learning rate of patch embedding and transformer layers
  while keeping the classifier and SimpleFPN at 1.0. This is because layer_wise_lr is typically
  applied during fine-tuning for down-stream tasks.
  
  For ViT (classification tasks), the path is like this:
  Patch Embedding --> Transformer --> PostNorm --> Classifier
  
  For ViT (detection tasks), the path is like this:
  Patch Embedding --> Transformer --> PostNorm --> SimpleFPN
docstr_tok: ["This","function","adjusts","the","learning","rate","of","each","layer","in","transformer","module",".","Layer-wise","learning","is","a","bit","involved","and","requires","a","knowledge","of","how","each","layer","is","consumed","during","the","forward","pass",".","We","adjust","the","learning","rate","of","patch","embedding","and","transformer","layers","while","keeping","the","classifier","and","SimpleFPN","at","1.0",".","This","is","because","layer_wise_lr","is","typically","applied","during","fine-tuning","for","down-stream","tasks",".","For","ViT","(","classification","tasks",")",",","the","path","is","like","this",":","Patch","Embedding","--",">","Transformer","--",">","PostNorm","--",">","Classifier","For","ViT","(","detection","tasks",")",",","the","path","is","like","this",":","Patch","Embedding","--",">","Transformer","--",">","PostNorm","--",">","SimpleFPN"]
code: def _apply_layer_wise_lr(
      self,
      weight_decay: Optional[float] = 0.0,
      no_decay_bn_filter_bias: Optional[bool] = False,
      *args,
      **kwargs,
  ) -> Tuple[List, List]:
      """
      This function adjusts the learning rate of each layer in transformer module.
      Layer-wise learning is a bit involved and requires a knowledge of how each layer is consumed
      during the forward pass. We adjust the learning rate of patch embedding and transformer layers
      while keeping the classifier and SimpleFPN at 1.0. This is because layer_wise_lr is typically
      applied during fine-tuning for down-stream tasks.
  
      For ViT (classification tasks), the path is like this:
      Patch Embedding --> Transformer --> PostNorm --> Classifier
  
      For ViT (detection tasks), the path is like this:
      Patch Embedding --> Transformer --> PostNorm --> SimpleFPN
  
      """
      n_layers = 1 + len(self.transformer)
      layer_wise_lr = [
          round(self.layer_wise_lr_decay_rate ** (n_layers - i), 5)
          for i in range(n_layers)
      ]
      module_name = kwargs.pop("module_name", "")
  
      param_list = []
      param_lr_list = []
  
      if self.neural_augmentor:
          neural_aug_params = parameter_list(
              named_parameters=self.neural_augmentor.named_parameters,
              weight_decay=weight_decay,
              no_decay_bn_filter_bias=no_decay_bn_filter_bias,
              module_name=module_name + "neural_augmentor.",
              *args,
              **kwargs,
          )
          param_list.extend(neural_aug_params)
          param_lr_list.extend(
              [layer_wise_lr[0]] * len(neural_aug_params)
          )
  
      # Patch embedding related parameters
      embedding_params = parameter_list(
          named_parameters=self.patch_emb.named_parameters,
          weight_decay=weight_decay,
          no_decay_bn_filter_bias=no_decay_bn_filter_bias,
          module_name=module_name + "patch_emb.",
          *args,
          **kwargs,
      )
      param_list.extend(embedding_params)
      param_lr_list.extend([layer_wise_lr[0]] * len(embedding_params))
  
      # positional embedding parameters
      pos_emb_params = parameter_list(
          named_parameters=self.pos_embed.named_parameters,
          weight_decay=weight_decay,
          no_decay_bn_filter_bias=no_decay_bn_filter_bias,
          module_name=module_name + "pos_embed.",
          *args,
          **kwargs,
      )
      param_list.extend(pos_emb_params)
      param_lr_list.extend([layer_wise_lr[0]] * len(pos_emb_params))
  
      if self.cls_token is not None:
          # CLS token params
          cls_token_params = parameter_list(
              named_parameters=self.cls_token.named_parameters,
              weight_decay=0.0,
              no_decay_bn_filter_bias=no_decay_bn_filter_bias,
              module_name=module_name + "cls_token.",
              *args,
              **kwargs,
          )
          param_list.extend(cls_token_params)
          param_lr_list.extend(
              [layer_wise_lr[0]] * len(cls_token_params)
          )
  
      # transformer related parameters
      for layer_id, transformer_layer in enumerate(self.transformer):
          layer_lr = layer_wise_lr[layer_id + 1]
          transformer_layer_params = parameter_list(
              named_parameters=transformer_layer.named_parameters,
              weight_decay=weight_decay,
              no_decay_bn_filter_bias=no_decay_bn_filter_bias,
              module_name=module_name + f"transformer.{layer_id}.",
              *args,
              **kwargs,
          )
          param_list.extend(transformer_layer_params)
          param_lr_list.extend(
              [layer_lr] * len(transformer_layer_params)
          )
  
      # transformer post-norm params
      post_transformer_norm_params = parameter_list(
          named_parameters=self.post_transformer_norm.named_parameters,
          weight_decay=weight_decay,
          no_decay_bn_filter_bias=no_decay_bn_filter_bias,
          module_name=module_name + "post_transformer_norm.",
          *args,
          **kwargs,
      )
      param_list.extend(post_transformer_norm_params)
      param_lr_list.extend(
          [layer_wise_lr[-1]] * len(post_transformer_norm_params)
      )
  
      if self.classifier is not None:
          # classifier parameters
          classifier_params = parameter_list(
              named_parameters=self.classifier.named_parameters,
              weight_decay=0.0,
              no_decay_bn_filter_bias=no_decay_bn_filter_bias,
              module_name=module_name + "classifier.",
              *args,
              **kwargs,
          )
          param_list.extend(classifier_params)
          param_lr_list.extend([1.0] * len(classifier_params))
  
      if self.simple_fpn is not None:
          # simple FPN parameters
          simple_fpn_params = parameter_list(
              named_parameters=self.simple_fpn.named_parameters,
              weight_decay=0.0,
              no_decay_bn_filter_bias=no_decay_bn_filter_bias,
              module_name=module_name + "simple_fpn.",
              *args,
              **kwargs,
          )
          param_list.extend(simple_fpn_params)
          param_lr_list.extend([1.0] * len(simple_fpn_params))
      return param_list, param_lr_list
  
imports: import argparse
  from typing import Dict, List, Optional, Tuple, Union
  import numpy
  import torch
  from torch import Tensor, nn
  from torch.utils.checkpoint import checkpoint_sequential
  from cvnets.layers import ConvLayer2d, Dropout, Identity, LinearLayer, MaxPool2d, PositionalEmbedding, TransposeConvLayer2d, get_normalization_layer
  from cvnets.misc.common import parameter_list
  from cvnets.misc.init_utils import initialize_conv_layer
  from cvnets.models import MODEL_REGISTRY
  from cvnets.models.classification.base_image_encoder import BaseImageEncoder
  from cvnets.models.classification.config.vit import get_configuration
  from cvnets.modules import TransformerEncoder
  from utils import logger
label_desc: Use image node_id 4 for calling the VisionTransformer obj's underlying member method code with example usage: obj._apply_layer_wise_lr(weight_decay, no_decay_bn_filter_bias) and returns: param_list,  param_lr_list