import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)


def 				lowerCAmelCase__       (			a__ )      ->Tuple:




  '''simple docstring'''

  if model.config.model_type == "gpt2":
    return model.transformer.h[0].mlp.c_fc
  return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
   import torch
   import torch.nn as nn




   class 					_UpperCAmelCase			(     nn.Module  ):



    '''simple docstring'''




    def __init__(		self				:   Optional[Any]							, lowercase_				:   nn.Module							, lowercase_				:   int)						->	Any:


      """simple docstring"""







      super().__init__()
      _UpperCamelCase           =    module
      _UpperCamelCase           =    nn.Sequential(
          nn.Linear(module.in_features							, lowercase_							, bias=lowercase_)							, nn.Linear(lowercase_							, module.out_features							, bias=lowercase_)							, )
      _UpperCamelCase           =    (2.0 / (5 * min(module.in_features							, module.out_features))) ** 0.5
      nn.init.normal_(self.adapter[0].weight							, std=lowercase_)
      nn.init.zeros_(self.adapter[1].weight)
      self.adapter.to(module.weight.device)







    def 		__UpperCAmelCase				(		self				:   str							, lowercase_				:   Union[str, Any]							, *lowercase_				:   Optional[Any]							, **lowercase_				:   List[Any])						->	Optional[int]:


      """simple docstring"""







      return self.module(lowercase_							, *lowercase_							, **lowercase_) + self.adapter(lowercase_)




@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 					_UpperCAmelCase			(     unittest.TestCase  ):



 '''simple docstring'''




 __A			       =					'''bigscience/bloom-1b7'''

 # Constant values
 __A			       =					2.109_659_552_692_574

 __A			       =					'''Hello my name is'''
 __A			       =					set()
 EXPECTED_OUTPUTS.add('''Hello my name is John and I am a professional photographer. I'''  )
 EXPECTED_OUTPUTS.add('''Hello my name is John.\nI am a friend of your father.\n'''  )
 EXPECTED_OUTPUTS.add('''Hello my name is John Doe, I am a student at the University'''  )
 __A			       =					10







 def 		__UpperCAmelCase				(		self				:   str)						->	List[Any]:


   """simple docstring"""







   _UpperCamelCase           =    AutoTokenizer.from_pretrained(self.model_name)




class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''




 def 		__UpperCAmelCase				(		self				:   Optional[int])						->	Dict:


   """simple docstring"""







   super().setUp()

   # Models and tokenizer
   _UpperCamelCase           =    AutoModelForCausalLM.from_pretrained(
       self.model_name							, torch_dtype=torch.floataa							, device_map="auto")
   _UpperCamelCase           =    AutoModelForCausalLM.from_pretrained(self.model_name							, load_in_abit=lowercase_							, device_map="auto")







 def 		__UpperCAmelCase				(		self				:   Tuple)						->	Dict:


   """simple docstring"""







   del self.model_fpaa
   del self.model_abit

   gc.collect()
   torch.cuda.empty_cache()







 def 		__UpperCAmelCase				(		self				:   List[str])						->	Tuple:


   """simple docstring"""







   _UpperCamelCase           =    self.model_abit.config

   self.assertTrue(hasattr(lowercase_							, "quantization_config"))

   _UpperCamelCase           =    config.to_dict()
   _UpperCamelCase           =    config.to_diff_dict()

   _UpperCamelCase           =    config.to_json_string()







 def 		__UpperCAmelCase				(		self				:   List[str])						->	Optional[int]:


   """simple docstring"""







   from bitsandbytes.nn import Paramsabit

   _UpperCamelCase           =    self.model_fpaa.get_memory_footprint()
   _UpperCamelCase           =    self.model_abit.get_memory_footprint()

   self.assertAlmostEqual(mem_fpaa / mem_abit							, self.EXPECTED_RELATIVE_DIFFERENCE)
   _UpperCamelCase           =    get_some_linear_layer(self.model_abit)
   self.assertTrue(linear.weight.__class__ == Paramsabit)







 def 		__UpperCAmelCase				(		self				:   Optional[int])						->	List[Any]:


   """simple docstring"""







   from transformers import TaPreTrainedModel

   self.model_fpaa.get_memory_footprint()
   self.model_abit.get_memory_footprint()

   for name, module in self.model_abit.named_modules():
     if isinstance(lowercase_							, torch.nn.Linear):
       if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
         # 4-bit parameters are packed in uint8 variables
         self.assertTrue(module.weight.dtype == torch.uinta)







 def 		__UpperCAmelCase				(		self				:   Tuple)						->	Tuple:


   """simple docstring"""







   _UpperCamelCase           =    self.tokenizer(self.input_text							, return_tensors="pt")
   _UpperCamelCase           =    self.model_abit.generate(input_ids=encoded_input["input_ids"].to(0)							, max_new_tokens=10)

   self.assertIn(self.tokenizer.decode(output_sequences[0]							, skip_special_tokens=lowercase_)							, self.EXPECTED_OUTPUTS)







 def 		__UpperCAmelCase				(		self				:   Tuple)						->	List[str]:


   """simple docstring"""







   _UpperCamelCase           =    BitsAndBytesConfig()
   _UpperCamelCase           =    True

   _UpperCamelCase           =    AutoModelForCausalLM.from_pretrained(
       self.model_name							, quantization_config=lowercase_							, device_map="auto")

   _UpperCamelCase           =    self.tokenizer(self.input_text							, return_tensors="pt")
   _UpperCamelCase           =    model_abit_from_config.generate(
       input_ids=encoded_input["input_ids"].to(0)							, max_new_tokens=10)

   self.assertIn(self.tokenizer.decode(output_sequences[0]							, skip_special_tokens=lowercase_)							, self.EXPECTED_OUTPUTS)







 def 		__UpperCAmelCase				(		self				:   Union[str, Any])						->	Tuple:


   """simple docstring"""







   with self.assertRaises(lowercase_), tempfile.TemporaryDirectory() as tmpdirname:
     self.model_abit.save_pretrained(lowercase_)







 def 		__UpperCAmelCase				(		self				:   List[Any])						->	Optional[Any]:


   """simple docstring"""







   _UpperCamelCase           =    BitsAndBytesConfig()

   with self.assertRaises(lowercase_):
     _UpperCamelCase           =    AutoModelForCausalLM.from_pretrained(
         self.model_name							, quantization_config=lowercase_							, load_in_abit=lowercase_							, device_map="auto"							, bnb_abit_quant_type="nf4"							, )







 def 		__UpperCAmelCase				(		self				:   Optional[Any])						->	List[str]:


   """simple docstring"""







   with self.assertRaises(lowercase_):
     # Tries with `str`
     self.model_abit.to("cpu")

   with self.assertRaises(lowercase_):
     # Tries with a `dtype``
     self.model_abit.to(torch.floataa)

   with self.assertRaises(lowercase_):
     # Tries with a `device`
     self.model_abit.to(torch.device("cuda:0"))

   with self.assertRaises(lowercase_):
     # Tries with a `device`
     self.model_abit.float()

   with self.assertRaises(lowercase_):
     # Tries with a `device`
     self.model_abit.half()

   # Test if we did not break anything
   _UpperCamelCase           =    self.tokenizer(self.input_text							, return_tensors="pt")

   _UpperCamelCase           =    self.model_fpaa.to(torch.floataa)
   _UpperCamelCase           =    self.model_fpaa.generate(input_ids=encoded_input["input_ids"].to(0)							, max_new_tokens=10)

   # Check this does not throw an error
   _UpperCamelCase           =    self.model_fpaa.to("cpu")

   # Check this does not throw an error
   _UpperCamelCase           =    self.model_fpaa.half()

   # Check this does not throw an error
   _UpperCamelCase           =    self.model_fpaa.float()







 def 		__UpperCAmelCase				(		self				:   Dict)						->	Tuple:


   """simple docstring"""







   _UpperCamelCase           =    AutoModelForSeqaSeqLM.from_pretrained("t5-small"							, load_in_abit=lowercase_							, device_map="auto")
   self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa)




@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 					_UpperCAmelCase			(     unittest.TestCase  ):



 '''simple docstring'''




 @classmethod
 def 		__UpperCAmelCase				(		cls				:   List[str])						->	Union[str, Any]:


   """simple docstring"""







   _UpperCamelCase           =    "t5-small"
   _UpperCamelCase           =    "google/flan-t5-small"  # flan-t5 uses dense-act instead of dense-relu-dense
   _UpperCamelCase           =    AutoTokenizer.from_pretrained(cls.model_name)
   _UpperCamelCase           =    "Translate in German: Hello, my dog is cute"







 def 		__UpperCAmelCase				(		self				:   Dict)						->	Dict:


   """simple docstring"""







   gc.collect()
   torch.cuda.empty_cache()







 def 		__UpperCAmelCase				(		self				:   Optional[Any])						->	str:


   """simple docstring"""







   from transformers import TaForConditionalGeneration

   _UpperCamelCase           =    TaForConditionalGeneration._keep_in_fpaa_modules
   _UpperCamelCase           =    None

   # test with `t5-small`
   _UpperCamelCase           =    TaForConditionalGeneration.from_pretrained(self.model_name							, load_in_abit=lowercase_							, device_map="auto")
   _UpperCamelCase           =    self.tokenizer(self.input_text							, return_tensors="pt").to(0)
   _UpperCamelCase           =    model.generate(**lowercase_)

   # test with `flan-t5-small`
   _UpperCamelCase           =    TaForConditionalGeneration.from_pretrained(
       self.dense_act_model_name							, load_in_abit=lowercase_							, device_map="auto")
   _UpperCamelCase           =    self.tokenizer(self.input_text							, return_tensors="pt").to(0)
   _UpperCamelCase           =    model.generate(**lowercase_)
   _UpperCamelCase           =    modules







 def 		__UpperCAmelCase				(		self				:   Dict)						->	Union[str, Any]:


   """simple docstring"""







   import bitsandbytes as bnb

   from transformers import TaForConditionalGeneration

   # test with `t5-small`
   _UpperCamelCase           =    TaForConditionalGeneration.from_pretrained(self.model_name							, load_in_abit=lowercase_							, device_map="auto")

   # there was a bug with decoders - this test checks that it is fixed
   self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q							, bnb.nn.Linearabit))

   _UpperCamelCase           =    self.tokenizer(self.input_text							, return_tensors="pt").to(0)
   _UpperCamelCase           =    model.generate(**lowercase_)

   # test with `flan-t5-small`
   _UpperCamelCase           =    TaForConditionalGeneration.from_pretrained(
       self.dense_act_model_name							, load_in_abit=lowercase_							, device_map="auto")
   _UpperCamelCase           =    self.tokenizer(self.input_text							, return_tensors="pt").to(0)
   _UpperCamelCase           =    model.generate(**lowercase_)




class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''




 def 		__UpperCAmelCase				(		self				:   Optional[int])						->	List[Any]:


   """simple docstring"""







   super().setUp()
   # model_name
   _UpperCamelCase           =    "bigscience/bloom-560m"
   _UpperCamelCase           =    "t5-small"

   # Different types of model

   _UpperCamelCase           =    AutoModel.from_pretrained(self.model_name							, load_in_abit=lowercase_							, device_map="auto")
   # Sequence classification model
   _UpperCamelCase           =    AutoModelForSequenceClassification.from_pretrained(
       self.model_name							, load_in_abit=lowercase_							, device_map="auto")
   # CausalLM model
   _UpperCamelCase           =    AutoModelForCausalLM.from_pretrained(self.model_name							, load_in_abit=lowercase_							, device_map="auto")
   # Seq2seq model
   _UpperCamelCase           =    AutoModelForSeqaSeqLM.from_pretrained(
       self.seq_to_seq_name							, load_in_abit=lowercase_							, device_map="auto")







 def 		__UpperCAmelCase				(		self				:   Union[str, Any])						->	List[str]:


   """simple docstring"""







   del self.base_model
   del self.sequence_model
   del self.model_abit
   del self.seq_to_seq_model

   gc.collect()
   torch.cuda.empty_cache()







 def 		__UpperCAmelCase				(		self				:   Union[str, Any])						->	Dict:


   """simple docstring"""







   from bitsandbytes.nn import Paramsabit

   self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit)

   # Other heads should be nn.Parameter
   self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter)
   self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter)
   self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter)




class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''




 def 		__UpperCAmelCase				(		self				:   Union[str, Any])						->	int:


   """simple docstring"""







   super().setUp()







 def 		__UpperCAmelCase				(		self				:   str)						->	List[str]:


   """simple docstring"""







   del self.pipe

   gc.collect()
   torch.cuda.empty_cache()







 def 		__UpperCAmelCase				(		self				:   Dict)						->	Optional[Any]:


   """simple docstring"""







   _UpperCamelCase           =    pipeline(
       "text-generation"							, model=self.model_name							, model_kwargs={"device_map": "auto", "load_in_4bit": True, "torch_dtype": torch.floataa}							, max_new_tokens=self.MAX_NEW_TOKENS							, )

   # Real second forward pass
   _UpperCamelCase           =    self.pipe(self.input_text)
   self.assertIn(pipeline_output[0]["generated_text"]							, self.EXPECTED_OUTPUTS)




@require_torch_multi_gpu
class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''




 def 		__UpperCAmelCase				(		self				:   Any)						->	Optional[Any]:


   """simple docstring"""







   super().setUp()







 def 		__UpperCAmelCase				(		self				:   str)						->	Optional[Any]:


   """simple docstring"""







   _UpperCamelCase           =    AutoModelForCausalLM.from_pretrained(
       self.model_name							, load_in_abit=lowercase_							, device_map="balanced")

   # Check correct device map
   self.assertEqual(set(model_parallel.hf_device_map.values())							, {0, 1})

   # Check that inference pass works on the model
   _UpperCamelCase           =    self.tokenizer(self.input_text							, return_tensors="pt")

   # Second real batch
   _UpperCamelCase           =    model_parallel.generate(input_ids=encoded_input["input_ids"].to(0)							, max_new_tokens=10)
   self.assertIn(self.tokenizer.decode(output_parallel[0]							, skip_special_tokens=lowercase_)							, self.EXPECTED_OUTPUTS)




class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''




 def 		__UpperCAmelCase				(		self				:   Tuple)						->	Optional[Any]:


   """simple docstring"""







   _UpperCamelCase           =    "facebook/opt-350m"
   super().setUp()







 def 		__UpperCAmelCase				(		self				:   List[str])						->	Tuple:


   """simple docstring"""







   if version.parse(importlib.metadata.version("bitsandbytes")) < version.parse("0.37.0"):
     return

   # Step 1: freeze all parameters
   _UpperCamelCase           =    AutoModelForCausalLM.from_pretrained(self.model_name							, load_in_abit=lowercase_)

   self.assertEqual(set(model.hf_device_map.values())							, {torch.cuda.current_device()})

   for param in model.parameters():
     _UpperCamelCase           =    False  # freeze the model - train adapters later
     if param.ndim == 1:
       # cast the small parameters (e.g. layernorm) to fp32 for stability
       _UpperCamelCase           =    param.data.to(torch.floataa)

        # Step 2: add adapters
   for _, module in model.named_modules():
     if "OPTAttention" in repr(type(lowercase_)):
       _UpperCamelCase           =    LoRALayer(module.q_proj							, rank=16)
       _UpperCamelCase           =    LoRALayer(module.k_proj							, rank=16)
       _UpperCamelCase           =    LoRALayer(module.v_proj							, rank=16)

        # Step 3: dummy batch
   _UpperCamelCase           =    self.tokenizer("Test batch "							, return_tensors="pt").to(0)

   # Step 4: Check if the gradient is not None
   with torch.cuda.amp.autocast():
     _UpperCamelCase           =    model.forward(**lowercase_)
     out.logits.norm().backward()

   for module in model.modules():
     if isinstance(lowercase_							, lowercase_):
       self.assertTrue(module.adapter[1].weight.grad is not None)
       self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0)
     elif isinstance(lowercase_							, nn.Embedding):
       self.assertTrue(module.weight.grad is None)




class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''




 __A			       =					'''gpt2-xl'''
 __A			       =					3.3_191_854_854_152_187





import time
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, torch_device

from ..test_modeling_common import ids_tensor


if is_torch_available():
   import torch

   from transformers.generation import (
       MaxLengthCriteria,
       MaxNewTokensCriteria,
       MaxTimeCriteria,
       StoppingCriteriaList,
       validate_stopping_criteria,
   )




@require_torch
class 					_UpperCAmelCase			(     unittest.TestCase  ):



 '''simple docstring'''




 def 		__UpperCAmelCase				(		self				:   List[Any]							, lowercase_				:   Dict)						->	Optional[Any]:


   """simple docstring"""







   _UpperCamelCase           =    3
   _UpperCamelCase           =    250

   _UpperCamelCase           =    ids_tensor((batch_size, length)							, lowercase_)
   _UpperCamelCase           =    torch.ones((batch_size, length)							, device=lowercase_							, dtype=torch.float) / length
   return input_ids, scores







 def 		__UpperCAmelCase				(		self				:   Any)						->	Optional[int]:


   """simple docstring"""







   _UpperCamelCase      ,						_UpperCamelCase           =    self._get_tensors(5)

   _UpperCamelCase           =    StoppingCriteriaList(
       [
           MaxLengthCriteria(max_length=10),
           MaxTimeCriteria(max_time=0.1),
       ])

   self.assertFalse(criteria(lowercase_							, lowercase_))

   _UpperCamelCase      ,						_UpperCamelCase           =    self._get_tensors(9)
   self.assertFalse(criteria(lowercase_							, lowercase_))

   _UpperCamelCase      ,						_UpperCamelCase           =    self._get_tensors(10)
   self.assertTrue(criteria(lowercase_							, lowercase_))







 def 		__UpperCAmelCase				(		self				:   Tuple)						->	int:


   """simple docstring"""







   _UpperCamelCase           =    MaxLengthCriteria(max_length=10)

   _UpperCamelCase      ,						_UpperCamelCase           =    self._get_tensors(5)
   self.assertFalse(criteria(lowercase_							, lowercase_))

   _UpperCamelCase      ,						_UpperCamelCase           =    self._get_tensors(9)
   self.assertFalse(criteria(lowercase_							, lowercase_))

   _UpperCamelCase      ,						_UpperCamelCase           =    self._get_tensors(10)
   self.assertTrue(criteria(lowercase_							, lowercase_))







 def 		__UpperCAmelCase				(		self				:   Optional[int])						->	int:


   """simple docstring"""







   _UpperCamelCase           =    MaxNewTokensCriteria(start_length=5							, max_new_tokens=5)

   _UpperCamelCase      ,						_UpperCamelCase           =    self._get_tensors(5)
   self.assertFalse(criteria(lowercase_							, lowercase_))

   _UpperCamelCase      ,						_UpperCamelCase           =    self._get_tensors(9)
   self.assertFalse(criteria(lowercase_							, lowercase_))

   _UpperCamelCase      ,						_UpperCamelCase           =    self._get_tensors(10)
   self.assertTrue(criteria(lowercase_							, lowercase_))

   _UpperCamelCase           =    StoppingCriteriaList([criteria])
   self.assertEqual(criteria_list.max_length							, 10)







 def 		__UpperCAmelCase				(		self				:   int)						->	str:


   """simple docstring"""







   _UpperCamelCase      ,						_UpperCamelCase           =    self._get_tensors(5)

   _UpperCamelCase           =    MaxTimeCriteria(max_time=0.1)
   self.assertFalse(criteria(lowercase_							, lowercase_))

   _UpperCamelCase           =    MaxTimeCriteria(max_time=0.1							, initial_timestamp=time.time() - 0.2)
   self.assertTrue(criteria(lowercase_							, lowercase_))







 def 		__UpperCAmelCase				(		self				:   Optional[int])						->	Tuple:


   """simple docstring"""







   validate_stopping_criteria(StoppingCriteriaList([MaxLengthCriteria(10)])							, 10)

   with self.assertWarns(lowercase_):
     validate_stopping_criteria(StoppingCriteriaList([MaxLengthCriteria(10)])							, 11)

   _UpperCamelCase           =    validate_stopping_criteria(StoppingCriteriaList()							, 11)

   self.assertEqual(len(lowercase_)							, 1)




