code: def __init__(
      self,
      weight,
      lora_dim=0,
      lora_scaling=1,
      lora_droppout=0,
      bias=None,
  ):
      super(LinearLayer_LoRA, self).__init__()
      self.weight = weight
      self.bias = bias
  
      if lora_dim <= 0:
          raise ValueError(
              "You are training to use LoRA, whose reduced dim should be larger than 1"
          )
  
      try:
          # for zero stage 3
          rows, columns = weight.ds_shape
      except:
          rows, columns = weight.shape
      self.lora_right_weight = torch.nn.Parameter(
          torch.zeros(columns, lora_dim)
      )  # apply transpose so in forward we do not need to transpose again
      self.lora_left_weight = torch.nn.Parameter(
          torch.zeros(lora_dim, rows)
      )
      self.lora_scaling = lora_scaling / lora_dim
  
      if lora_droppout > 0:
          self.lora_dropout = torch.nn.Dropout(lora_droppout)
      else:
          self.lora_dropout = torch.nn.Identity()
  
      self.reset_parameters()
      # disable the original weight gradient
      self.weight.requires_grad = False
      # fuse LoRA to the original weight
      self.fuse_lora = False
  
imports: import os
  import math
  import torch
  import torch.nn.functional
  import pytest
  import deepspeed
  from deepspeed.runtime.zero import GatheredParameters
  from deepspeed.ops.op_builder import OpBuilder
  from deepspeed.utils import safe_get_full_grad
  import numpy.testing
  from unit.common import DistributedTest
  from deepspeed.ops.op_builder import InferenceBuilder
  from transformers import AutoConfig, AutoTokenizer, AutoModelForCausalLM
label_desc: Use image node_id 1 to create a new LinearLayer_LoRA object from inherited base classes: torch.nn with example: obj = LinearLayer_LoRA(weight, lora_dim, lora_scaling, lora_droppout, bias)