


import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)




def    a   (							A__							)	-> Optional[Any]:




      '''simple docstring'''

      if model.config.model_type == "gpt2":
            return model.transformer.h[0].mlp.c_fc
      return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
    import torch
    import torch.nn as nn

    class 							lowercase  (						nn.Module			):







         def __init__(						self :						Tuple   ,   _lowercase :						nn.Module   ,   _lowercase :						int      ):
               super().__init__()
               SCREAMING_SNAKE_CASE__       :  Any						     =		module
               SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		nn.Sequential(
                   nn.Linear(module.in_features   ,   __UpperCAmelCase   ,   bias=__UpperCAmelCase      )   ,   nn.Linear(__UpperCAmelCase   ,   module.out_features   ,   bias=__UpperCAmelCase      )   ,   )
               SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		(2.0 / (5 * min(module.in_features   ,   module.out_features      ))) ** 0.5
               nn.init.normal_(self.adapter[0].weight   ,   std=__UpperCAmelCase      )
               nn.init.zeros_(self.adapter[1].weight      )
               self.adapter.to(module.weight.device      )







         def 							lowercase__       (						self :						Tuple   ,   _lowercase :						List[Any]   ,   *_lowercase :						int   ,   **_lowercase :						Tuple      ):
               return self.module(__UpperCAmelCase   ,   *__UpperCAmelCase   ,   **__UpperCAmelCase      ) + self.adapter(__UpperCAmelCase      )

@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 							lowercase  (						unittest.TestCase			):
     lowerCamelCase			:			List[Any]      = '''bigscience/bloom-1b7'''

     # Constant values
     lowerCamelCase			:			List[str]      = 2.109_659_552_692_574

     lowerCamelCase			:			Optional[Any]      = '''Hello my name is'''
     lowerCamelCase			:			Union[str, Any]      = set()
     EXPECTED_OUTPUTS.add('''Hello my name is John and I am a professional photographer. I'''			)
     EXPECTED_OUTPUTS.add('''Hello my name is John.\nI am a friend of your father.\n'''			)
     EXPECTED_OUTPUTS.add('''Hello my name is John Doe, I am a student at the University'''			)
     lowerCamelCase			:			Optional[Any]      = 10







     def 							lowercase__       (						self :						List[str]      ):
           SCREAMING_SNAKE_CASE__       :  str						     =		AutoTokenizer.from_pretrained(self.model_name      )




class 							lowercase  (						__UpperCamelCase			):







     def 							lowercase__       (						self :						Dict      ):
           super().setUp()

           # Models and tokenizer
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		AutoModelForCausalLM.from_pretrained(
               self.model_name   ,   torch_dtype=torch.floataa   ,   device_map='''auto'''      )
           SCREAMING_SNAKE_CASE__       :  Any						     =		AutoModelForCausalLM.from_pretrained(self.model_name   ,   load_in_abit=__UpperCAmelCase   ,   device_map='''auto'''      )







     def 							lowercase__       (						self :						Dict      ):
           del self.model_fpaa
           del self.model_abit

           gc.collect()
           torch.cuda.empty_cache()







     def 							lowercase__       (						self :						Optional[Any]      ):
           SCREAMING_SNAKE_CASE__       :  Dict						     =		self.model_abit.config

           self.assertTrue(hasattr(__UpperCAmelCase   ,   '''quantization_config'''      )      )

           SCREAMING_SNAKE_CASE__       :  List[str]						     =		config.to_dict()
           SCREAMING_SNAKE_CASE__       :  str						     =		config.to_diff_dict()

           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		config.to_json_string()







     def 							lowercase__       (						self :						Optional[int]      ):
           from bitsandbytes.nn import Paramsabit

           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		self.model_fpaa.get_memory_footprint()
           SCREAMING_SNAKE_CASE__       :  List[str]						     =		self.model_abit.get_memory_footprint()

           self.assertAlmostEqual(mem_fpaa / mem_abit   ,   self.EXPECTED_RELATIVE_DIFFERENCE      )
           SCREAMING_SNAKE_CASE__       :  Tuple						     =		get_some_linear_layer(self.model_abit      )
           self.assertTrue(linear.weight.__class__ == Paramsabit      )







     def 							lowercase__       (						self :						int      ):
           from transformers import TaPreTrainedModel

           self.model_fpaa.get_memory_footprint()
           self.model_abit.get_memory_footprint()

           for name, module in self.model_abit.named_modules():
                 if isinstance(__UpperCAmelCase   ,   torch.nn.Linear      ):
                       if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
                             # 4-bit parameters are packed in uint8 variables
                             self.assertTrue(module.weight.dtype == torch.uinta      )







     def 							lowercase__       (						self :						Any      ):
           SCREAMING_SNAKE_CASE__       :  List[Any]						     =		self.tokenizer(self.input_text   ,   return_tensors='''pt'''      )
           SCREAMING_SNAKE_CASE__       :  List[Any]						     =		self.model_abit.generate(input_ids=encoded_input['''input_ids'''].to(0      )   ,   max_new_tokens=10      )

           self.assertIn(self.tokenizer.decode(output_sequences[0]   ,   skip_special_tokens=__UpperCAmelCase      )   ,   self.EXPECTED_OUTPUTS      )







     def 							lowercase__       (						self :						Optional[Any]      ):
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		BitsAndBytesConfig()
           SCREAMING_SNAKE_CASE__       :  Dict						     =		True

           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		AutoModelForCausalLM.from_pretrained(
               self.model_name   ,   quantization_config=__UpperCAmelCase   ,   device_map='''auto'''      )

           SCREAMING_SNAKE_CASE__       :  List[str]						     =		self.tokenizer(self.input_text   ,   return_tensors='''pt'''      )
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		model_abit_from_config.generate(
               input_ids=encoded_input['''input_ids'''].to(0      )   ,   max_new_tokens=10      )

           self.assertIn(self.tokenizer.decode(output_sequences[0]   ,   skip_special_tokens=__UpperCAmelCase      )   ,   self.EXPECTED_OUTPUTS      )







     def 							lowercase__       (						self :						str      ):



           with self.assertRaises(__UpperCAmelCase      ), tempfile.TemporaryDirectory() as tmpdirname:
                 self.model_abit.save_pretrained(__UpperCAmelCase      )







     def 							lowercase__       (						self :						Tuple      ):
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		BitsAndBytesConfig()

           with self.assertRaises(__UpperCAmelCase      ):
                 SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		AutoModelForCausalLM.from_pretrained(
                     self.model_name   ,   quantization_config=__UpperCAmelCase   ,   load_in_abit=__UpperCAmelCase   ,   device_map='''auto'''   ,   bnb_abit_quant_type='''nf4'''   ,   )







     def 							lowercase__       (						self :						List[Any]      ):



           with self.assertRaises(__UpperCAmelCase      ):
                 # Tries with `str`
                 self.model_abit.to('''cpu'''      )

           with self.assertRaises(__UpperCAmelCase      ):
                 # Tries with a `dtype``
                 self.model_abit.to(torch.floataa      )

           with self.assertRaises(__UpperCAmelCase      ):
                 # Tries with a `device`
                 self.model_abit.to(torch.device('''cuda:0'''      )      )

           with self.assertRaises(__UpperCAmelCase      ):
                 # Tries with a `device`
                 self.model_abit.float()

           with self.assertRaises(__UpperCAmelCase      ):
                 # Tries with a `device`
                 self.model_abit.half()

           # Test if we did not break anything
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		self.tokenizer(self.input_text   ,   return_tensors='''pt'''      )

           SCREAMING_SNAKE_CASE__       :  List[Any]						     =		self.model_fpaa.to(torch.floataa      )
           SCREAMING_SNAKE_CASE__       :  List[Any]						     =		self.model_fpaa.generate(input_ids=encoded_input['''input_ids'''].to(0      )   ,   max_new_tokens=10      )

           # Check this does not throw an error
           SCREAMING_SNAKE_CASE__       :  Dict						     =		self.model_fpaa.to('''cpu'''      )

           # Check this does not throw an error
           SCREAMING_SNAKE_CASE__       :  Any						     =		self.model_fpaa.half()

           # Check this does not throw an error
           SCREAMING_SNAKE_CASE__       :  Dict						     =		self.model_fpaa.float()







     def 							lowercase__       (						self :						str      ):
           SCREAMING_SNAKE_CASE__       :  Tuple						     =		AutoModelForSeqaSeqLM.from_pretrained('''t5-small'''   ,   load_in_abit=__UpperCAmelCase   ,   device_map='''auto'''      )
           self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa      )




@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 							lowercase  (						unittest.TestCase			):







     @classmethod
     def 							lowercase__       (						cls :						Any      ):
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		"t5-small"
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		"google/flan-t5-small"  # flan-t5 uses dense-act instead of dense-relu-dense
           SCREAMING_SNAKE_CASE__       :  int						     =		AutoTokenizer.from_pretrained(cls.model_name      )
           SCREAMING_SNAKE_CASE__       :  int						     =		"Translate in German: Hello, my dog is cute"







     def 							lowercase__       (						self :						Optional[int]      ):
           gc.collect()
           torch.cuda.empty_cache()







     def 							lowercase__       (						self :						List[str]      ):
           from transformers import TaForConditionalGeneration

           SCREAMING_SNAKE_CASE__       :  Tuple						     =		TaForConditionalGeneration._keep_in_fpaa_modules
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		None

           # test with `t5-small`
           SCREAMING_SNAKE_CASE__       :  List[str]						     =		TaForConditionalGeneration.from_pretrained(self.model_name   ,   load_in_abit=__UpperCAmelCase   ,   device_map='''auto'''      )
           SCREAMING_SNAKE_CASE__       :  str						     =		self.tokenizer(self.input_text   ,   return_tensors='''pt'''      ).to(0      )
           SCREAMING_SNAKE_CASE__       :  List[str]						     =		model.generate(**__UpperCAmelCase      )

           # test with `flan-t5-small`
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		TaForConditionalGeneration.from_pretrained(
               self.dense_act_model_name   ,   load_in_abit=__UpperCAmelCase   ,   device_map='''auto'''      )
           SCREAMING_SNAKE_CASE__       :  Tuple						     =		self.tokenizer(self.input_text   ,   return_tensors='''pt'''      ).to(0      )
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		model.generate(**__UpperCAmelCase      )
           SCREAMING_SNAKE_CASE__       :  Any						     =		modules







     def 							lowercase__       (						self :						Dict      ):
           import bitsandbytes as bnb

           from transformers import TaForConditionalGeneration

           # test with `t5-small`
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		TaForConditionalGeneration.from_pretrained(self.model_name   ,   load_in_abit=__UpperCAmelCase   ,   device_map='''auto'''      )

           # there was a bug with decoders - this test checks that it is fixed
           self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q   ,   bnb.nn.Linearabit      )      )

           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		self.tokenizer(self.input_text   ,   return_tensors='''pt'''      ).to(0      )
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		model.generate(**__UpperCAmelCase      )

           # test with `flan-t5-small`
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		TaForConditionalGeneration.from_pretrained(
               self.dense_act_model_name   ,   load_in_abit=__UpperCAmelCase   ,   device_map='''auto'''      )
           SCREAMING_SNAKE_CASE__       :  str						     =		self.tokenizer(self.input_text   ,   return_tensors='''pt'''      ).to(0      )
           SCREAMING_SNAKE_CASE__       :  List[str]						     =		model.generate(**__UpperCAmelCase      )




class 							lowercase  (						__UpperCamelCase			):







     def 							lowercase__       (						self :						Optional[Any]      ):
           super().setUp()
           # model_name
           SCREAMING_SNAKE_CASE__       :  int						     =		"bigscience/bloom-560m"
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		"t5-small"

           # Different types of model

           SCREAMING_SNAKE_CASE__       :  int						     =		AutoModel.from_pretrained(self.model_name   ,   load_in_abit=__UpperCAmelCase   ,   device_map='''auto'''      )
           # Sequence classification model
           SCREAMING_SNAKE_CASE__       :  int						     =		AutoModelForSequenceClassification.from_pretrained(
               self.model_name   ,   load_in_abit=__UpperCAmelCase   ,   device_map='''auto'''      )
           # CausalLM model
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		AutoModelForCausalLM.from_pretrained(self.model_name   ,   load_in_abit=__UpperCAmelCase   ,   device_map='''auto'''      )
           # Seq2seq model
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		AutoModelForSeqaSeqLM.from_pretrained(
               self.seq_to_seq_name   ,   load_in_abit=__UpperCAmelCase   ,   device_map='''auto'''      )







     def 							lowercase__       (						self :						str      ):
           del self.base_model
           del self.sequence_model
           del self.model_abit
           del self.seq_to_seq_model

           gc.collect()
           torch.cuda.empty_cache()







     def 							lowercase__       (						self :						Dict      ):
           from bitsandbytes.nn import Paramsabit

           self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit      )

           # Other heads should be nn.Parameter
           self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter      )
           self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter      )
           self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter      )




class 							lowercase  (						__UpperCamelCase			):







     def 							lowercase__       (						self :						int      ):
           super().setUp()







     def 							lowercase__       (						self :						Dict      ):
           del self.pipe

           gc.collect()
           torch.cuda.empty_cache()







     def 							lowercase__       (						self :						Optional[Any]      ):
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		pipeline(
               '''text-generation'''   ,   model=self.model_name   ,   model_kwargs={'''device_map''': '''auto''', '''load_in_4bit''': True, '''torch_dtype''': torch.floataa}   ,   max_new_tokens=self.MAX_NEW_TOKENS   ,   )

           # Real second forward pass
           SCREAMING_SNAKE_CASE__       :  Any						     =		self.pipe(self.input_text      )
           self.assertIn(pipeline_output[0]['''generated_text''']   ,   self.EXPECTED_OUTPUTS      )




@require_torch_multi_gpu
class 							lowercase  (						__UpperCamelCase			):







     def 							lowercase__       (						self :						Optional[int]      ):
           super().setUp()







     def 							lowercase__       (						self :						str      ):
           SCREAMING_SNAKE_CASE__       :  Any						     =		AutoModelForCausalLM.from_pretrained(
               self.model_name   ,   load_in_abit=__UpperCAmelCase   ,   device_map='''balanced'''      )

           # Check correct device map
           self.assertEqual(set(model_parallel.hf_device_map.values()      )   ,   {0, 1}      )

           # Check that inference pass works on the model
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		self.tokenizer(self.input_text   ,   return_tensors='''pt'''      )

           # Second real batch
           SCREAMING_SNAKE_CASE__       :  Dict						     =		model_parallel.generate(input_ids=encoded_input['''input_ids'''].to(0      )   ,   max_new_tokens=10      )
           self.assertIn(self.tokenizer.decode(output_parallel[0]   ,   skip_special_tokens=__UpperCAmelCase      )   ,   self.EXPECTED_OUTPUTS      )




class 							lowercase  (						__UpperCamelCase			):







     def 							lowercase__       (						self :						Union[str, Any]      ):
           SCREAMING_SNAKE_CASE__       :  str						     =		"facebook/opt-350m"
           super().setUp()







     def 							lowercase__       (						self :						Optional[Any]      ):



           if version.parse(importlib.metadata.version('''bitsandbytes'''      )      ) < version.parse('''0.37.0'''      ):
                 return

           # Step 1: freeze all parameters
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		AutoModelForCausalLM.from_pretrained(self.model_name   ,   load_in_abit=__UpperCAmelCase      )

           self.assertEqual(set(model.hf_device_map.values()      )   ,   {torch.cuda.current_device()}      )

           for param in model.parameters():
                 SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		False  # freeze the model - train adapters later
                 if param.ndim == 1:
                       # cast the small parameters (e.g. layernorm) to fp32 for stability
                       SCREAMING_SNAKE_CASE__       :  int						     =		param.data.to(torch.floataa      )

        # Step 2: add adapters
           for _, module in model.named_modules():
                 if "OPTAttention" in repr(type(__UpperCAmelCase      )      ):
                       SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		LoRALayer(module.q_proj   ,   rank=16      )
                       SCREAMING_SNAKE_CASE__       :  Dict						     =		LoRALayer(module.k_proj   ,   rank=16      )
                       SCREAMING_SNAKE_CASE__       :  Tuple						     =		LoRALayer(module.v_proj   ,   rank=16      )

        # Step 3: dummy batch
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		self.tokenizer('''Test batch '''   ,   return_tensors='''pt'''      ).to(0      )

           # Step 4: Check if the gradient is not None
           with torch.cuda.amp.autocast():
                 SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		model.forward(**__UpperCAmelCase      )
                 out.logits.norm().backward()

           for module in model.modules():
                 if isinstance(__UpperCAmelCase   ,   __UpperCAmelCase      ):
                       self.assertTrue(module.adapter[1].weight.grad is not None      )
                       self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0      )
                 elif isinstance(__UpperCAmelCase   ,   nn.Embedding      ):
                       self.assertTrue(module.weight.grad is None      )




class 							lowercase  (						__UpperCamelCase			):
     lowerCamelCase			:			int      = '''gpt2-xl'''
     lowerCamelCase			:			Union[str, Any]      = 3.3_191_854_854_152_187














import unittest

from transformers import is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, require_torch, slow


if is_flax_available():
			import optax
			from flax.training.common_utils import onehot

			from transformers import AutoTokenizer, FlaxMTaForConditionalGeneration
			from transformers.models.ta.modeling_flax_ta import shift_tokens_right




@require_torch
@require_sentencepiece
@require_tokenizers
@require_flax
class 							snake_case_       (    unittest.TestCase      ):

						"""simple docstring"""



						@slow
						def        UpperCAmelCase__					(self:					Optional[int]   )    ->     List[str]:




											'''simple docstring'''



											__a					:						Optional[Any]      =					FlaxMTaForConditionalGeneration.from_pretrained("google/mt5-small"   )
											__a					:						Dict      =					AutoTokenizer.from_pretrained("google/mt5-small"   )

											__a					:						Any      =					tokenizer("Hello there" ,					return_tensors="np"   ).input_ids
											__a					:						Union[str, Any]      =					tokenizer("Hi I am" ,					return_tensors="np"   ).input_ids

											__a					:						Union[str, Any]      =					shift_tokens_right(__UpperCAmelCase ,					model.config.pad_token_id ,					model.config.decoder_start_token_id   )

											__a					:						Tuple      =					model(__UpperCAmelCase ,					decoder_input_ids=__UpperCAmelCase   ).logits
											__a					:						str      =					optax.softmax_cross_entropy(__UpperCAmelCase ,					onehot(__UpperCAmelCase ,					logits.shape[-1]   )   ).mean()

											__a					:						List[str]      =					-(labels.shape[-1] * loss.item())

											__a					:						Optional[int]      =					-84.91_27
											self.assertTrue(abs(mtf_score - EXPECTED_SCORE   ) < 1E-4   )





