class_docstr: This class defines the `ByteFormer <https://arxiv.org/pdf/2306.00238.pdf>`_
  architecture.
class_docstr_tok: ["This","class","defines","the","`","ByteFormer","<","https",":","\/\/arxiv.org\/pdf\/2306.00238.pdf",">","`","_","architecture","."]
code: def __init__(self, opts: argparse.Namespace, *args, **kwargs) -> None:
      super().__init__(opts, *args, **kwargs)
  
      byteformer_config = get_configuration(opts)
      embed_dim = byteformer_config["embed_dim"]
      ffn_dim = byteformer_config["ffn_dim"]
      n_transformer_layers = byteformer_config["n_transformer_layers"]
      num_heads = byteformer_config["n_attn_heads"]
      attn_dropout = byteformer_config["attn_dropout"]
      dropout = byteformer_config["dropout"]
      ffn_dropout = byteformer_config["ffn_dropout"]
      norm_layer = byteformer_config["norm_layer"]
  
      # This is usually 257 in the case of byte inputs (2**8 + 1 mask token).
      vocab_size = getattr(
          opts, "model.classification.byteformer.vocab_size"
      )
      self.embeddings = embedding.Embedding(
          opts,
          num_embeddings=vocab_size,
          embedding_dim=embed_dim,
          padding_idx=-1,
      )
      # Reinitialize everything except the padding index.
      init.trunc_normal_(
          self.embeddings.weight[:-1], std=math.sqrt(1.0 / embed_dim)
      )
  
      self.dummy_input_token_length = getattr(
          opts,
          "model.classification.byteformer.dummy_input_token_length",
      )
  
      # Add token reduction convolution.
      self.conv_kernel_size = getattr(
          opts, "model.classification.byteformer.conv_kernel_size"
      )
      if self.conv_kernel_size == 0:
          # We skip the convolution.
          self.token_reduction_net = None
      if self.conv_kernel_size is not None:
          self.token_reduction_net = nn.Conv1d(
              embed_dim,
              get_configuration(opts)["embed_dim"],
              kernel_size=self.conv_kernel_size,
              stride=self.conv_kernel_size // 2,
              bias=False,
          )
  
      # Add the positional embeddings.
      self.max_num_tokens = getattr(
          opts, "model.classification.byteformer.max_num_tokens"
      )
      self.sinusoidal_pos_embed = getattr(
          opts, "model.classification.byteformer.sinusoidal_pos_emb"
      )
      self.pos_embed = positional_embedding.PositionalEmbedding(
          opts=opts,
          num_embeddings=self.max_num_tokens,
          embedding_dim=embed_dim,
          sequence_first=False,
          padding_idx=None,
          is_learnable=not self.sinusoidal_pos_embed,
          interpolation_mode="bilinear",
      )
  
      pos_emb_drop_p = getattr(
          opts, "model.classification.byteformer.dropout"
      )
      self.emb_dropout = nn.Dropout(p=pos_emb_drop_p)
  
      # Build the transformer backbone.
      window_sizes = getattr(
          opts, "model.classification.byteformer.window_sizes"
      )
      window_shifts = getattr(
          opts, "model.classification.byteformer.window_shifts"
      )
      downsample = getattr(
          opts, "model.classification.byteformer.downsample"
      )
  
      if len(window_sizes) == 1:
          window_sizes = window_sizes * n_transformer_layers
  
      for x in [window_sizes, window_shifts, downsample]:
          if len(x) != n_transformer_layers:
              raise ValueError(
                  f"Invalid argument length {len(x)} != {n_transformer_layers}"
              )
  
      stochastic_dropout = getattr(
          opts, "model.classification.byteformer.stochastic_dropout"
      )
      per_layer_stochastic_drop_rate = [
          round(x, 3)
          for x in np.linspace(
              0, stochastic_dropout, n_transformer_layers
          )
      ]
  
      blocks = []
      self.downsamplers = nn.ModuleDict()
      for layer_idx in range(n_transformer_layers):
          blocks.append(
              WindowedTransformerEncoder(
                  opts=opts,
                  embed_dim=embed_dim,
                  ffn_latent_dim=ffn_dim,
                  num_heads=num_heads,
                  attn_dropout=attn_dropout,
                  dropout=dropout,
                  ffn_dropout=ffn_dropout,
                  transformer_norm_layer=norm_layer,
                  stochastic_dropout=per_layer_stochastic_drop_rate[
                      layer_idx
                  ],
                  window_size=window_sizes[layer_idx],
                  window_shift=window_shifts[layer_idx],
              )
          )
          if downsample is not None and downsample[layer_idx]:
              self.downsamplers[
                  self.get_downsampler_name(layer_idx)
              ] = token_merging.TokenMerging(embed_dim)
      self.transformer = nn.Sequential(*blocks)
  
      self.post_transformer_norm = get_normalization_layer(
          opts=opts, num_features=embed_dim, norm_type=norm_layer
      )
  
      num_classes = getattr(opts, "model.classification.n_classes")
      self.classifier = LinearLayer(embed_dim, num_classes)
  
imports: import argparse
  import math
  from typing import Dict, Optional, Tuple, Union
  import numpy
  import torch
  from torch import Tensor, nn
  from torch.nn import init
  from cvnets.layers import LinearLayer, embedding, get_normalization_layer, normalization, positional_embedding, token_merging
  from cvnets.models import MODEL_REGISTRY, BaseAnyNNModel
  from cvnets.models.classification.config.byteformer import get_configuration
  from cvnets.modules import WindowedTransformerEncoder
label_desc: Use image node_id 1 to create a new ByteFormer object from inherited base classes: BaseAnyNNModel with example: obj = ByteFormer(opts)