
"""simple docstring"""



import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)


def lowerCamelCase_				(     _lowerCamelCase			:	Tuple				):
       if model.config.model_type == "gpt2":
              return model.transformer.h[0].mlp.c_fc
       return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
 import torch
 import torch.nn as nn





 class 					lowerCAmelCase			(				nn.Module					):
      """simple docstring"""







      def __init__(      self ,		UpperCamelCase__ ,		UpperCamelCase__							)					->  int:







             '''simple docstring'''





             super().__init__()
             lowerCamelCase_       							=		module
             lowerCamelCase_       							=		nn.Sequential(
                 nn.Linear(module.in_features ,		snake_case__ ,		bias=snake_case__							) ,		nn.Linear(snake_case__ ,		module.out_features ,		bias=snake_case__							) ,		)
             lowerCamelCase_       							=		(2.0 / (5 * min(module.in_features ,		module.out_features							))) ** 0.5
             nn.init.normal_(self.adapter[0].weight ,		std=snake_case__							)
             nn.init.zeros_(self.adapter[1].weight							)
             self.adapter.to(module.weight.device							)




      def    _lowerCAmelCase		(      self ,		UpperCamelCase__ ,		*UpperCamelCase__ ,		**UpperCamelCase__							)					->  Dict:







             '''simple docstring'''





             return self.module(snake_case__ ,		*snake_case__ ,		**snake_case__							) + self.adapter(snake_case__							)





@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 					lowerCAmelCase			(				unittest.TestCase					):
     """simple docstring"""







     __lowercase						:Tuple          =    "bigscience/bloom-1b7"

     # Constant values
     __lowercase						:Tuple          =    2.109_6595_5269_2574

     __lowercase						:int          =    "Hello my name is"
     __lowercase						:Dict          =    set()
     EXPECTED_OUTPUTS.add("Hello my name is John and I am a professional photographer. I"					)
     EXPECTED_OUTPUTS.add("Hello my name is John.\nI am a friend of your father.\n"					)
     EXPECTED_OUTPUTS.add("Hello my name is John Doe, I am a student at the University"					)
     __lowercase						:Optional[int]          =    10




     def    _lowerCAmelCase		(      self							)					->  Optional[int]:







            '''simple docstring'''





            lowerCamelCase_       							=		AutoTokenizer.from_pretrained(self.model_name							)





class 					lowerCAmelCase			(				SCREAMING_SNAKE_CASE_					):
     """simple docstring"""







     def    _lowerCAmelCase		(      self							)					->  Tuple:







            '''simple docstring'''





            super().setUp()

            # Models and tokenizer
            lowerCamelCase_       							=		AutoModelForCausalLM.from_pretrained(
                self.model_name ,		torch_dtype=torch.floataa ,		device_map='''auto'''							)
            lowerCamelCase_       							=		AutoModelForCausalLM.from_pretrained(self.model_name ,		load_in_abit=snake_case__ ,		device_map='''auto'''							)




     def    _lowerCAmelCase		(      self							)					->  List[str]:







            '''simple docstring'''





            del self.model_fpaa
            del self.model_abit

            gc.collect()
            torch.cuda.empty_cache()




     def    _lowerCAmelCase		(      self							)					->  int:







            '''simple docstring'''





            lowerCamelCase_       							=		self.model_abit.config

            self.assertTrue(hasattr(snake_case__ ,		'''quantization_config'''							)							)

            lowerCamelCase_       							=		config.to_dict()
            lowerCamelCase_       							=		config.to_diff_dict()

            lowerCamelCase_       							=		config.to_json_string()




     def    _lowerCAmelCase		(      self							)					->  str:







            '''simple docstring'''





            from bitsandbytes.nn import Paramsabit

            lowerCamelCase_       							=		self.model_fpaa.get_memory_footprint()
            lowerCamelCase_       							=		self.model_abit.get_memory_footprint()

            self.assertAlmostEqual(mem_fpaa / mem_abit ,		self.EXPECTED_RELATIVE_DIFFERENCE							)
            lowerCamelCase_       							=		get_some_linear_layer(self.model_abit							)
            self.assertTrue(linear.weight.__class__ == Paramsabit							)




     def    _lowerCAmelCase		(      self							)					->  Any:







            '''simple docstring'''





            from transformers import TaPreTrainedModel

            self.model_fpaa.get_memory_footprint()
            self.model_abit.get_memory_footprint()

            for name, module in self.model_abit.named_modules():
                   if isinstance(snake_case__ ,		torch.nn.Linear							):
                          if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
                                 # 4-bit parameters are packed in uint8 variables
                                 self.assertTrue(module.weight.dtype == torch.uinta							)




     def    _lowerCAmelCase		(      self							)					->  Union[str, Any]:







            '''simple docstring'''





            lowerCamelCase_       							=		self.tokenizer(self.input_text ,		return_tensors='''pt'''							)
            lowerCamelCase_       							=		self.model_abit.generate(input_ids=encoded_input['''input_ids'''].to(0							) ,		max_new_tokens=10							)

            self.assertIn(self.tokenizer.decode(output_sequences[0] ,		skip_special_tokens=snake_case__							) ,		self.EXPECTED_OUTPUTS							)




     def    _lowerCAmelCase		(      self							)					->  List[Any]:







            '''simple docstring'''





            lowerCamelCase_       							=		BitsAndBytesConfig()
            lowerCamelCase_       							=		True

            lowerCamelCase_       							=		AutoModelForCausalLM.from_pretrained(
                self.model_name ,		quantization_config=snake_case__ ,		device_map='''auto'''							)

            lowerCamelCase_       							=		self.tokenizer(self.input_text ,		return_tensors='''pt'''							)
            lowerCamelCase_       							=		model_abit_from_config.generate(
                input_ids=encoded_input['''input_ids'''].to(0							) ,		max_new_tokens=10							)

            self.assertIn(self.tokenizer.decode(output_sequences[0] ,		skip_special_tokens=snake_case__							) ,		self.EXPECTED_OUTPUTS							)




     def    _lowerCAmelCase		(      self							)					->  int:







            '''simple docstring'''





            with self.assertRaises(snake_case__							), tempfile.TemporaryDirectory() as tmpdirname:
                   self.model_abit.save_pretrained(snake_case__							)




     def    _lowerCAmelCase		(      self							)					->  Optional[Any]:







            '''simple docstring'''





            lowerCamelCase_       							=		BitsAndBytesConfig()

            with self.assertRaises(snake_case__							):
                   lowerCamelCase_       							=		AutoModelForCausalLM.from_pretrained(
                       self.model_name ,		quantization_config=snake_case__ ,		load_in_abit=snake_case__ ,		device_map='''auto''' ,		bnb_abit_quant_type='''nf4''' ,		)




     def    _lowerCAmelCase		(      self							)					->  Tuple:







            '''simple docstring'''





            with self.assertRaises(snake_case__							):
                   # Tries with `str`
                   self.model_abit.to('''cpu'''							)

            with self.assertRaises(snake_case__							):
                   # Tries with a `dtype``
                   self.model_abit.to(torch.floataa							)

            with self.assertRaises(snake_case__							):
                   # Tries with a `device`
                   self.model_abit.to(torch.device('''cuda:0'''							)							)

            with self.assertRaises(snake_case__							):
                   # Tries with a `device`
                   self.model_abit.float()

            with self.assertRaises(snake_case__							):
                   # Tries with a `device`
                   self.model_abit.half()

            # Test if we did not break anything
            lowerCamelCase_       							=		self.tokenizer(self.input_text ,		return_tensors='''pt'''							)

            lowerCamelCase_       							=		self.model_fpaa.to(torch.floataa							)
            lowerCamelCase_       							=		self.model_fpaa.generate(input_ids=encoded_input['''input_ids'''].to(0							) ,		max_new_tokens=10							)

            # Check this does not throw an error
            lowerCamelCase_       							=		self.model_fpaa.to('''cpu'''							)

            # Check this does not throw an error
            lowerCamelCase_       							=		self.model_fpaa.half()

            # Check this does not throw an error
            lowerCamelCase_       							=		self.model_fpaa.float()




     def    _lowerCAmelCase		(      self							)					->  int:







            '''simple docstring'''





            lowerCamelCase_       							=		AutoModelForSeqaSeqLM.from_pretrained('''t5-small''' ,		load_in_abit=snake_case__ ,		device_map='''auto'''							)
            self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa							)





@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 					lowerCAmelCase			(				unittest.TestCase					):
     """simple docstring"""







     @classmethod
     def    _lowerCAmelCase		(      cls							)					->  Optional[int]:







            '''simple docstring'''





            lowerCamelCase_       							=		'''t5-small'''
            lowerCamelCase_       							=		'''google/flan-t5-small'''  # flan-t5 uses dense-act instead of dense-relu-dense
            lowerCamelCase_       							=		AutoTokenizer.from_pretrained(cls.model_name							)
            lowerCamelCase_       							=		'''Translate in German: Hello, my dog is cute'''




     def    _lowerCAmelCase		(      self							)					->  int:







            '''simple docstring'''





            gc.collect()
            torch.cuda.empty_cache()




     def    _lowerCAmelCase		(      self							)					->  List[Any]:







            '''simple docstring'''





            from transformers import TaForConditionalGeneration

            lowerCamelCase_       							=		TaForConditionalGeneration._keep_in_fpaa_modules
            lowerCamelCase_       							=		None

            # test with `t5-small`
            lowerCamelCase_       							=		TaForConditionalGeneration.from_pretrained(self.model_name ,		load_in_abit=snake_case__ ,		device_map='''auto'''							)
            lowerCamelCase_       							=		self.tokenizer(self.input_text ,		return_tensors='''pt'''							).to(0							)
            lowerCamelCase_       							=		model.generate(**snake_case__							)

            # test with `flan-t5-small`
            lowerCamelCase_       							=		TaForConditionalGeneration.from_pretrained(
                self.dense_act_model_name ,		load_in_abit=snake_case__ ,		device_map='''auto'''							)
            lowerCamelCase_       							=		self.tokenizer(self.input_text ,		return_tensors='''pt'''							).to(0							)
            lowerCamelCase_       							=		model.generate(**snake_case__							)
            lowerCamelCase_       							=		modules




     def    _lowerCAmelCase		(      self							)					->  Dict:







            '''simple docstring'''





            import bitsandbytes as bnb

            from transformers import TaForConditionalGeneration

            # test with `t5-small`
            lowerCamelCase_       							=		TaForConditionalGeneration.from_pretrained(self.model_name ,		load_in_abit=snake_case__ ,		device_map='''auto'''							)

            # there was a bug with decoders - this test checks that it is fixed
            self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q ,		bnb.nn.Linearabit							)							)

            lowerCamelCase_       							=		self.tokenizer(self.input_text ,		return_tensors='''pt'''							).to(0							)
            lowerCamelCase_       							=		model.generate(**snake_case__							)

            # test with `flan-t5-small`
            lowerCamelCase_       							=		TaForConditionalGeneration.from_pretrained(
                self.dense_act_model_name ,		load_in_abit=snake_case__ ,		device_map='''auto'''							)
            lowerCamelCase_       							=		self.tokenizer(self.input_text ,		return_tensors='''pt'''							).to(0							)
            lowerCamelCase_       							=		model.generate(**snake_case__							)





class 					lowerCAmelCase			(				SCREAMING_SNAKE_CASE_					):
     """simple docstring"""







     def    _lowerCAmelCase		(      self							)					->  Optional[int]:







            '''simple docstring'''





            super().setUp()
            # model_name
            lowerCamelCase_       							=		'''bigscience/bloom-560m'''
            lowerCamelCase_       							=		'''t5-small'''

            # Different types of model

            lowerCamelCase_       							=		AutoModel.from_pretrained(self.model_name ,		load_in_abit=snake_case__ ,		device_map='''auto'''							)
            # Sequence classification model
            lowerCamelCase_       							=		AutoModelForSequenceClassification.from_pretrained(
                self.model_name ,		load_in_abit=snake_case__ ,		device_map='''auto'''							)
            # CausalLM model
            lowerCamelCase_       							=		AutoModelForCausalLM.from_pretrained(self.model_name ,		load_in_abit=snake_case__ ,		device_map='''auto'''							)
            # Seq2seq model
            lowerCamelCase_       							=		AutoModelForSeqaSeqLM.from_pretrained(
                self.seq_to_seq_name ,		load_in_abit=snake_case__ ,		device_map='''auto'''							)




     def    _lowerCAmelCase		(      self							)					->  Union[str, Any]:







            '''simple docstring'''





            del self.base_model
            del self.sequence_model
            del self.model_abit
            del self.seq_to_seq_model

            gc.collect()
            torch.cuda.empty_cache()




     def    _lowerCAmelCase		(      self							)					->  Any:







            '''simple docstring'''





            from bitsandbytes.nn import Paramsabit

            self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit							)

            # Other heads should be nn.Parameter
            self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter							)
            self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter							)
            self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter							)





class 					lowerCAmelCase			(				SCREAMING_SNAKE_CASE_					):
     """simple docstring"""







     def    _lowerCAmelCase		(      self							)					->  List[str]:







            '''simple docstring'''





            super().setUp()




     def    _lowerCAmelCase		(      self							)					->  str:







            '''simple docstring'''





            del self.pipe

            gc.collect()
            torch.cuda.empty_cache()




     def    _lowerCAmelCase		(      self							)					->  Any:







            '''simple docstring'''





            lowerCamelCase_       							=		pipeline(
                '''text-generation''' ,		model=self.model_name ,		model_kwargs={'''device_map''': '''auto''', '''load_in_4bit''': True, '''torch_dtype''': torch.floataa} ,		max_new_tokens=self.MAX_NEW_TOKENS ,		)

            # Real second forward pass
            lowerCamelCase_       							=		self.pipe(self.input_text							)
            self.assertIn(pipeline_output[0]['''generated_text'''] ,		self.EXPECTED_OUTPUTS							)





@require_torch_multi_gpu
class 					lowerCAmelCase			(				SCREAMING_SNAKE_CASE_					):
     """simple docstring"""







     def    _lowerCAmelCase		(      self							)					->  Union[str, Any]:







            '''simple docstring'''





            super().setUp()




     def    _lowerCAmelCase		(      self							)					->  Tuple:







            '''simple docstring'''





            lowerCamelCase_       							=		AutoModelForCausalLM.from_pretrained(
                self.model_name ,		load_in_abit=snake_case__ ,		device_map='''balanced'''							)

            # Check correct device map
            self.assertEqual(set(model_parallel.hf_device_map.values()							) ,		{0, 1}							)

            # Check that inference pass works on the model
            lowerCamelCase_       							=		self.tokenizer(self.input_text ,		return_tensors='''pt'''							)

            # Second real batch
            lowerCamelCase_       							=		model_parallel.generate(input_ids=encoded_input['''input_ids'''].to(0							) ,		max_new_tokens=10							)
            self.assertIn(self.tokenizer.decode(output_parallel[0] ,		skip_special_tokens=snake_case__							) ,		self.EXPECTED_OUTPUTS							)





class 					lowerCAmelCase			(				SCREAMING_SNAKE_CASE_					):
     """simple docstring"""







     def    _lowerCAmelCase		(      self							)					->  Union[str, Any]:







            '''simple docstring'''





            lowerCamelCase_       							=		'''facebook/opt-350m'''
            super().setUp()




     def    _lowerCAmelCase		(      self							)					->  Tuple:







            '''simple docstring'''





            if version.parse(importlib.metadata.version('''bitsandbytes'''							)							) < version.parse('''0.37.0'''							):
                   return

            # Step 1: freeze all parameters
            lowerCamelCase_       							=		AutoModelForCausalLM.from_pretrained(self.model_name ,		load_in_abit=snake_case__							)

            self.assertEqual(set(model.hf_device_map.values()							) ,		{torch.cuda.current_device()}							)

            for param in model.parameters():
                   lowerCamelCase_       							=		False  # freeze the model - train adapters later
                   if param.ndim == 1:
                          # cast the small parameters (e.g. layernorm) to fp32 for stability
                          lowerCamelCase_       							=		param.data.to(torch.floataa							)

        # Step 2: add adapters
            for _, module in model.named_modules():
                   if "OPTAttention" in repr(type(snake_case__							)							):
                          lowerCamelCase_       							=		LoRALayer(module.q_proj ,		rank=16							)
                          lowerCamelCase_       							=		LoRALayer(module.k_proj ,		rank=16							)
                          lowerCamelCase_       							=		LoRALayer(module.v_proj ,		rank=16							)

        # Step 3: dummy batch
            lowerCamelCase_       							=		self.tokenizer('''Test batch ''' ,		return_tensors='''pt'''							).to(0							)

            # Step 4: Check if the gradient is not None
            with torch.cuda.amp.autocast():
                   lowerCamelCase_       							=		model.forward(**snake_case__							)
                   out.logits.norm().backward()

            for module in model.modules():
                   if isinstance(snake_case__ ,		snake_case__							):
                          self.assertTrue(module.adapter[1].weight.grad is not None							)
                          self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0							)
                   elif isinstance(snake_case__ ,		nn.Embedding							):
                          self.assertTrue(module.weight.grad is None							)





class 					lowerCAmelCase			(				SCREAMING_SNAKE_CASE_					):
     """simple docstring"""







     __lowercase						:Tuple          =    "gpt2-xl"
     __lowercase						:Dict          =    3.3191_8548_5415_2187
def 	lowerCamelCase					(	UpperCamelCase     :				int							)   ->    bool:
		_lowerCamelCase				   =  (1 + 24 * n) ** 0.5
		return ((1 + root) / 6) % 1 == 0


def 	lowerCamelCase					(	UpperCamelCase     :				int = 50_00							)   ->    int:
		_lowerCamelCase				   =  [(i * (3 * i - 1)) // 2 for i in range(1			,       UpperCamelCase							)]
		for i, pentagonal_i in enumerate(UpperCamelCase							):
				for j in range(UpperCamelCase			,       len(UpperCamelCase							)							):
						_lowerCamelCase				   =  pentagonal_nums[j]
						_lowerCamelCase				   =  pentagonal_i + pentagonal_j
						_lowerCamelCase				   =  pentagonal_j - pentagonal_i
						if is_pentagonal(UpperCamelCase							) and is_pentagonal(UpperCamelCase							):
								return b

		return -1


if __name__ == "__main__":
				print(F'''{solution() = }''')