

'''simple docstring'''

import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)

def 					__UpperCamelCase				(						lowercase__    :	Any			):






     '''simple docstring'''





     if model.config.model_type == "gpt2":
          return model.transformer.h[0].mlp.c_fc
     return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
       import torch
       import torch.nn as nn


       class 	lowerCAmelCase  (   nn.Module ):







              def __init__(				self      :			List[Any]    ,   __lowercase      :			nn.Module    ,   __lowercase      :			int   ):







                   """simple docstring"""







                   super().__init__()
                   __lowercase      		=module
                   __lowercase      		=nn.Sequential(
                       nn.Linear(module.in_features    ,   __lowercase    ,   bias=__lowercase   )    ,   nn.Linear(__lowercase    ,   module.out_features    ,   bias=__lowercase   )    ,   )
                   __lowercase      		=(2.0 / (5 * min(module.in_features    ,   module.out_features   ))) ** 0.5
                   nn.init.normal_(self.adapter[0].weight    ,   std=__lowercase   )
                   nn.init.zeros_(self.adapter[1].weight   )
                   self.adapter.to(module.weight.device   )







              def       snake_case (				self      :			Tuple    ,   __lowercase      :			Tuple    ,   *__lowercase      :			str    ,   **__lowercase      :			Any   ):







                   """simple docstring"""







                   return self.module(__lowercase    ,   *__lowercase    ,   **__lowercase   ) + self.adapter(__lowercase   )


@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 	lowerCAmelCase  (   unittest.TestCase ):
       # We keep the constants inside the init function and model loading inside setUp function

       # We need to test on relatively large models (aka >1b parameters otherwise the quantiztion may not work as expected)
       # Therefore here we use only bloom-1b3 to test our module
       lowerCAmelCase_												=			"bigscience/bloom-1b7"

       # Constant values
       lowerCAmelCase_												=			2.1_0_9_6_5_9_5_5_2_6_9_2_5_7_4

       lowerCAmelCase_												=			"Hello my name is"
       lowerCAmelCase_												=			set()
       EXPECTED_OUTPUTS.add("Hello my name is John and I am a professional photographer. I" )
       EXPECTED_OUTPUTS.add("Hello my name is John.\nI am a friend of your father.\n" )
       EXPECTED_OUTPUTS.add("Hello my name is John Doe, I am a student at the University" )
       lowerCAmelCase_												=			1_0







       def       snake_case (				self      :			Union[str, Any]   ):







            """simple docstring"""







            __lowercase      		=AutoTokenizer.from_pretrained(self.model_name   )






class 	lowerCAmelCase  (   A ):







       def       snake_case (				self      :			Optional[Any]   ):







            """simple docstring"""







            super().setUp()

            # Models and tokenizer
            __lowercase      		=AutoModelForCausalLM.from_pretrained(
                self.model_name    ,   torch_dtype=torch.floataa    ,   device_map='auto'   )
            __lowercase      		=AutoModelForCausalLM.from_pretrained(self.model_name    ,   load_in_abit=__lowercase    ,   device_map='auto'   )







       def       snake_case (				self      :			Optional[int]   ):







            """simple docstring"""







            del self.model_fpaa
            del self.model_abit

            gc.collect()
            torch.cuda.empty_cache()







       def       snake_case (				self      :			Dict   ):







            """simple docstring"""







            __lowercase      		=self.model_abit.config

            self.assertTrue(hasattr(__lowercase    ,   'quantization_config'   )   )

            __lowercase      		=config.to_dict()
            __lowercase      		=config.to_diff_dict()

            __lowercase      		=config.to_json_string()







       def       snake_case (				self      :			Dict   ):







            """simple docstring"""







            from bitsandbytes.nn import Paramsabit

            __lowercase      		=self.model_fpaa.get_memory_footprint()
            __lowercase      		=self.model_abit.get_memory_footprint()

            self.assertAlmostEqual(mem_fpaa / mem_abit    ,   self.EXPECTED_RELATIVE_DIFFERENCE   )
            __lowercase      		=get_some_linear_layer(self.model_abit   )
            self.assertTrue(linear.weight.__class__ == Paramsabit   )







       def       snake_case (				self      :			Optional[int]   ):







            """simple docstring"""







            from transformers import TaPreTrainedModel

            self.model_fpaa.get_memory_footprint()
            self.model_abit.get_memory_footprint()

            for name, module in self.model_abit.named_modules():
                 if isinstance(__lowercase    ,   torch.nn.Linear   ):
                      if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
                           # 4-bit parameters are packed in uint8 variables
                           self.assertTrue(module.weight.dtype == torch.uinta   )







       def       snake_case (				self      :			str   ):







            """simple docstring"""







            __lowercase      		=self.tokenizer(self.input_text    ,   return_tensors='pt'   )
            __lowercase      		=self.model_abit.generate(input_ids=encoded_input['input_ids'].to(0   )    ,   max_new_tokens=10   )

            self.assertIn(self.tokenizer.decode(output_sequences[0]    ,   skip_special_tokens=__lowercase   )    ,   self.EXPECTED_OUTPUTS   )







       def       snake_case (				self      :			Optional[Any]   ):







            """simple docstring"""







            __lowercase      		=BitsAndBytesConfig()
            __lowercase      		=True

            __lowercase      		=AutoModelForCausalLM.from_pretrained(
                self.model_name    ,   quantization_config=__lowercase    ,   device_map='auto'   )

            __lowercase      		=self.tokenizer(self.input_text    ,   return_tensors='pt'   )
            __lowercase      		=model_abit_from_config.generate(
                input_ids=encoded_input['input_ids'].to(0   )    ,   max_new_tokens=10   )

            self.assertIn(self.tokenizer.decode(output_sequences[0]    ,   skip_special_tokens=__lowercase   )    ,   self.EXPECTED_OUTPUTS   )







       def       snake_case (				self      :			Optional[Any]   ):







            """simple docstring"""







            with self.assertRaises(__lowercase   ), tempfile.TemporaryDirectory() as tmpdirname:
                 self.model_abit.save_pretrained(__lowercase   )







       def       snake_case (				self      :			Tuple   ):







            """simple docstring"""







            __lowercase      		=BitsAndBytesConfig()

            with self.assertRaises(__lowercase   ):
                 __lowercase      		=AutoModelForCausalLM.from_pretrained(
                     self.model_name    ,   quantization_config=__lowercase    ,   load_in_abit=__lowercase    ,   device_map='auto'    ,   bnb_abit_quant_type='nf4'    ,   )







       def       snake_case (				self      :			Optional[int]   ):







            """simple docstring"""







            with self.assertRaises(__lowercase   ):
                 # Tries with `str`
                 self.model_abit.to('cpu'   )

            with self.assertRaises(__lowercase   ):
                 # Tries with a `dtype``
                 self.model_abit.to(torch.floataa   )

            with self.assertRaises(__lowercase   ):
                 # Tries with a `device`
                 self.model_abit.to(torch.device('cuda:0'   )   )

            with self.assertRaises(__lowercase   ):
                 # Tries with a `device`
                 self.model_abit.float()

            with self.assertRaises(__lowercase   ):
                 # Tries with a `device`
                 self.model_abit.half()

            # Test if we did not break anything
            __lowercase      		=self.tokenizer(self.input_text    ,   return_tensors='pt'   )

            __lowercase      		=self.model_fpaa.to(torch.floataa   )
            __lowercase      		=self.model_fpaa.generate(input_ids=encoded_input['input_ids'].to(0   )    ,   max_new_tokens=10   )

            # Check this does not throw an error
            __lowercase      		=self.model_fpaa.to('cpu'   )

            # Check this does not throw an error
            __lowercase      		=self.model_fpaa.half()

            # Check this does not throw an error
            __lowercase      		=self.model_fpaa.float()







       def       snake_case (				self      :			int   ):







            """simple docstring"""







            __lowercase      		=AutoModelForSeqaSeqLM.from_pretrained('t5-small'    ,   load_in_abit=__lowercase    ,   device_map='auto'   )
            self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa   )






@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 	lowerCAmelCase  (   unittest.TestCase ):







       @classmethod
       def       snake_case (				cls      :			Tuple   ):







            """simple docstring"""







            __lowercase      		='t5-small'
            __lowercase      		='google/flan-t5-small'  # flan-t5 uses dense-act instead of dense-relu-dense
            __lowercase      		=AutoTokenizer.from_pretrained(cls.model_name   )
            __lowercase      		='Translate in German: Hello, my dog is cute'







       def       snake_case (				self      :			str   ):







            """simple docstring"""







            gc.collect()
            torch.cuda.empty_cache()







       def       snake_case (				self      :			Optional[Any]   ):







            """simple docstring"""







            from transformers import TaForConditionalGeneration

            __lowercase      		=TaForConditionalGeneration._keep_in_fpaa_modules
            __lowercase      		=None

            # test with `t5-small`
            __lowercase      		=TaForConditionalGeneration.from_pretrained(self.model_name    ,   load_in_abit=__lowercase    ,   device_map='auto'   )
            __lowercase      		=self.tokenizer(self.input_text    ,   return_tensors='pt'   ).to(0   )
            __lowercase      		=model.generate(**__lowercase   )

            # test with `flan-t5-small`
            __lowercase      		=TaForConditionalGeneration.from_pretrained(
                self.dense_act_model_name    ,   load_in_abit=__lowercase    ,   device_map='auto'   )
            __lowercase      		=self.tokenizer(self.input_text    ,   return_tensors='pt'   ).to(0   )
            __lowercase      		=model.generate(**__lowercase   )
            __lowercase      		=modules







       def       snake_case (				self      :			List[Any]   ):







            """simple docstring"""







            import bitsandbytes as bnb

            from transformers import TaForConditionalGeneration

            # test with `t5-small`
            __lowercase      		=TaForConditionalGeneration.from_pretrained(self.model_name    ,   load_in_abit=__lowercase    ,   device_map='auto'   )

            # there was a bug with decoders - this test checks that it is fixed
            self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q    ,   bnb.nn.Linearabit   )   )

            __lowercase      		=self.tokenizer(self.input_text    ,   return_tensors='pt'   ).to(0   )
            __lowercase      		=model.generate(**__lowercase   )

            # test with `flan-t5-small`
            __lowercase      		=TaForConditionalGeneration.from_pretrained(
                self.dense_act_model_name    ,   load_in_abit=__lowercase    ,   device_map='auto'   )
            __lowercase      		=self.tokenizer(self.input_text    ,   return_tensors='pt'   ).to(0   )
            __lowercase      		=model.generate(**__lowercase   )






class 	lowerCAmelCase  (   A ):







       def       snake_case (				self      :			Any   ):







            """simple docstring"""







            super().setUp()
            # model_name
            __lowercase      		='bigscience/bloom-560m'
            __lowercase      		='t5-small'

            # Different types of model

            __lowercase      		=AutoModel.from_pretrained(self.model_name    ,   load_in_abit=__lowercase    ,   device_map='auto'   )
            # Sequence classification model
            __lowercase      		=AutoModelForSequenceClassification.from_pretrained(
                self.model_name    ,   load_in_abit=__lowercase    ,   device_map='auto'   )
            # CausalLM model
            __lowercase      		=AutoModelForCausalLM.from_pretrained(self.model_name    ,   load_in_abit=__lowercase    ,   device_map='auto'   )
            # Seq2seq model
            __lowercase      		=AutoModelForSeqaSeqLM.from_pretrained(
                self.seq_to_seq_name    ,   load_in_abit=__lowercase    ,   device_map='auto'   )







       def       snake_case (				self      :			List[Any]   ):







            """simple docstring"""







            del self.base_model
            del self.sequence_model
            del self.model_abit
            del self.seq_to_seq_model

            gc.collect()
            torch.cuda.empty_cache()







       def       snake_case (				self      :			List[Any]   ):







            """simple docstring"""







            from bitsandbytes.nn import Paramsabit

            self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit   )

            # Other heads should be nn.Parameter
            self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter   )
            self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter   )
            self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter   )






class 	lowerCAmelCase  (   A ):







       def       snake_case (				self      :			Union[str, Any]   ):







            """simple docstring"""







            super().setUp()







       def       snake_case (				self      :			List[str]   ):







            """simple docstring"""







            del self.pipe

            gc.collect()
            torch.cuda.empty_cache()







       def       snake_case (				self      :			Any   ):







            """simple docstring"""







            __lowercase      		=pipeline(
                'text-generation'    ,   model=self.model_name    ,   model_kwargs={'device_map': 'auto', 'load_in_4bit': True, 'torch_dtype': torch.floataa}    ,   max_new_tokens=self.MAX_NEW_TOKENS    ,   )

            # Real second forward pass
            __lowercase      		=self.pipe(self.input_text   )
            self.assertIn(pipeline_output[0]['generated_text']    ,   self.EXPECTED_OUTPUTS   )






@require_torch_multi_gpu
class 	lowerCAmelCase  (   A ):







       def       snake_case (				self      :			Tuple   ):







            """simple docstring"""







            super().setUp()







       def       snake_case (				self      :			List[str]   ):







            """simple docstring"""







            __lowercase      		=AutoModelForCausalLM.from_pretrained(
                self.model_name    ,   load_in_abit=__lowercase    ,   device_map='balanced'   )

            # Check correct device map
            self.assertEqual(set(model_parallel.hf_device_map.values()   )    ,   {0, 1}   )

            # Check that inference pass works on the model
            __lowercase      		=self.tokenizer(self.input_text    ,   return_tensors='pt'   )

            # Second real batch
            __lowercase      		=model_parallel.generate(input_ids=encoded_input['input_ids'].to(0   )    ,   max_new_tokens=10   )
            self.assertIn(self.tokenizer.decode(output_parallel[0]    ,   skip_special_tokens=__lowercase   )    ,   self.EXPECTED_OUTPUTS   )






class 	lowerCAmelCase  (   A ):







       def       snake_case (				self      :			Optional[int]   ):







            """simple docstring"""







            __lowercase      		='facebook/opt-350m'
            super().setUp()







       def       snake_case (				self      :			str   ):







            """simple docstring"""







            if version.parse(importlib.metadata.version('bitsandbytes'   )   ) < version.parse('0.37.0'   ):
                 return

            # Step 1: freeze all parameters
            __lowercase      		=AutoModelForCausalLM.from_pretrained(self.model_name    ,   load_in_abit=__lowercase   )

            self.assertEqual(set(model.hf_device_map.values()   )    ,   {torch.cuda.current_device()}   )

            for param in model.parameters():
                 __lowercase      		=False  # freeze the model - train adapters later
                 if param.ndim == 1:
                      # cast the small parameters (e.g. layernorm) to fp32 for stability
                      __lowercase      		=param.data.to(torch.floataa   )

        # Step 2: add adapters
            for _, module in model.named_modules():
                 if "OPTAttention" in repr(type(__lowercase   )   ):
                      __lowercase      		=LoRALayer(module.q_proj    ,   rank=16   )
                      __lowercase      		=LoRALayer(module.k_proj    ,   rank=16   )
                      __lowercase      		=LoRALayer(module.v_proj    ,   rank=16   )

        # Step 3: dummy batch
            __lowercase      		=self.tokenizer('Test batch '    ,   return_tensors='pt'   ).to(0   )

            # Step 4: Check if the gradient is not None
            with torch.cuda.amp.autocast():
                 __lowercase      		=model.forward(**__lowercase   )
                 out.logits.norm().backward()

            for module in model.modules():
                 if isinstance(__lowercase    ,   __lowercase   ):
                      self.assertTrue(module.adapter[1].weight.grad is not None   )
                      self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0   )
                 elif isinstance(__lowercase    ,   nn.Embedding   ):
                      self.assertTrue(module.weight.grad is None   )






class 	lowerCAmelCase  (   A ):
       lowerCAmelCase_												=			"gpt2-xl"
       lowerCAmelCase_												=			3.3_1_9_1_8_5_4_8_5_4_1_5_2_1_8_7





'''simple docstring'''

class 	lowerCAmelCase  :







       def __init__(				self      :			List[Any]    ,   __lowercase      :			str    ,   __lowercase      :			Any    ,   __lowercase      :			str   ):







            """simple docstring"""







            __lowercase      		=name
            __lowercase      		=value
            __lowercase      		=weight







       def __repr__(				self      :			List[str]   ):







            """simple docstring"""







            return f'''{self.__class__.__name__}({self.name}, {self.value}, {self.weight})'''







       def       snake_case (				self      :			List[Any]   ):







            """simple docstring"""







            return self.value







       def       snake_case (				self      :			str   ):







            """simple docstring"""







            return self.name







       def       snake_case (				self      :			Optional[Any]   ):







            """simple docstring"""







            return self.weight







       def       snake_case (				self      :			Tuple   ):







            """simple docstring"""







            return self.value / self.weight






def 					__UpperCamelCase				(						lowercase__    :	Dict,       lowercase__    :	List[str],       lowercase__    :	str			):






     '''simple docstring'''





     __lowercase      		=[]
     for i in range(len(lowercase__			)			):
          menu.append(Things(name[i],       value[i],       weight[i]			)			)
     return menu

def 					__UpperCamelCase				(						lowercase__    :	List[Any],       lowercase__    :	List[Any],       lowercase__    :	str			):






     '''simple docstring'''





     __lowercase      		=sorted(lowercase__,       key=lowercase__,       reverse=lowercase__			)
     __lowercase      		=[]
     __lowercase		,  __lowercase      		=0.0, 0.0
     for i in range(len(lowercase__			)			):
          if (total_cost + items_copy[i].get_weight()) <= max_cost:
               result.append(items_copy[i]			)
               total_cost += items_copy[i].get_weight()
               total_value += items_copy[i].get_value()
     return (result, total_value)

def 					__UpperCamelCase				(						):






     '''simple docstring'''







if __name__ == "__main__":
       import doctest

       doctest.testmod()


