






import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)





def    __lowerCAmelCase   (    __snake_case							):
     if model.config.model_type == "gpt2":
          return model.transformer.h[0].mlp.c_fc
     return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
 import torch
 import torch.nn as nn



 class 				_UpperCamelCase      (nn.Module      ):


       def __init__(						self       ,    __UpperCamelCase       ,    __UpperCamelCase						)->						Tuple:
            super().__init__()
            __lowerCAmelCase             =       module
            __lowerCAmelCase             =       nn.Sequential(
                nn.Linear(module.in_features       ,    __UpperCamelCase       ,    bias=__UpperCamelCase						)       ,    nn.Linear(__UpperCamelCase       ,    module.out_features       ,    bias=__UpperCamelCase						)       ,    )
            __lowerCAmelCase             =       (2.0 / (5 * min(module.in_features       ,    module.out_features						))) ** 0.5
            nn.init.normal_(self.adapter[0].weight       ,    std=__UpperCamelCase						)
            nn.init.zeros_(self.adapter[1].weight						)
            self.adapter.to(module.weight.device						)


       def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    *__UpperCamelCase       ,    **__UpperCamelCase						)->						Any:
            return self.module(__UpperCamelCase       ,    *__UpperCamelCase       ,    **__UpperCamelCase						) + self.adapter(__UpperCamelCase						)



@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 				_UpperCamelCase      (unittest.TestCase      ):
      # We keep the constants inside the init function and model loading inside setUp function

      # We need to test on relatively large models (aka >1b parameters otherwise the quantiztion may not work as expected)
      # Therefore here we use only bloom-1b3 to test our module
      snake_case_	=				"""bigscience/bloom-1b7"""

      # Constant values
      snake_case_	=				2.1_0_9_6_5_9_5_5_2_6_9_2_5_7_4

      snake_case_	=				"""Hello my name is"""
      snake_case_	=				set()
      EXPECTED_OUTPUTS.add("""Hello my name is John and I am a professional photographer. I"""      )
      EXPECTED_OUTPUTS.add("""Hello my name is John.\nI am a friend of your father.\n"""      )
      EXPECTED_OUTPUTS.add("""Hello my name is John Doe, I am a student at the University"""      )
      snake_case_	=				10


      def 			__UpperCAmelCase		(						self						)->						Any:
           # Models and tokenizer
           __lowerCAmelCase             =       AutoTokenizer.from_pretrained(self.model_name						)



class 				_UpperCamelCase      (a_      ):


      def 			__UpperCAmelCase		(						self						)->						List[str]:
           super().setUp()

           # Models and tokenizer
           __lowerCAmelCase             =       AutoModelForCausalLM.from_pretrained(
               self.model_name       ,    torch_dtype=torch.floataa       ,    device_map="auto"						)
           __lowerCAmelCase             =       AutoModelForCausalLM.from_pretrained(self.model_name       ,    load_in_abit=__UpperCamelCase       ,    device_map="auto"						)


      def 			__UpperCAmelCase		(						self						)->						int:
           del self.model_fpaa
           del self.model_abit

           gc.collect()
           torch.cuda.empty_cache()


      def 			__UpperCAmelCase		(						self						)->						Dict:
           __lowerCAmelCase             =       self.model_abit.config

           self.assertTrue(hasattr(__UpperCamelCase       ,    "quantization_config"						)						)

           __lowerCAmelCase             =       config.to_dict()
           __lowerCAmelCase             =       config.to_diff_dict()

           __lowerCAmelCase             =       config.to_json_string()


      def 			__UpperCAmelCase		(						self						)->						Dict:
           from bitsandbytes.nn import Paramsabit

           __lowerCAmelCase             =       self.model_fpaa.get_memory_footprint()
           __lowerCAmelCase             =       self.model_abit.get_memory_footprint()

           self.assertAlmostEqual(mem_fpaa / mem_abit       ,    self.EXPECTED_RELATIVE_DIFFERENCE						)
           __lowerCAmelCase             =       get_some_linear_layer(self.model_abit						)
           self.assertTrue(linear.weight.__class__ == Paramsabit						)


      def 			__UpperCAmelCase		(						self						)->						int:
           from transformers import TaPreTrainedModel

           self.model_fpaa.get_memory_footprint()
           self.model_abit.get_memory_footprint()

           for name, module in self.model_abit.named_modules():
                if isinstance(__UpperCamelCase       ,    torch.nn.Linear						):
                     if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
                          # 4-bit parameters are packed in uint8 variables
                          self.assertTrue(module.weight.dtype == torch.uinta						)


      def 			__UpperCAmelCase		(						self						)->						int:
           __lowerCAmelCase             =       self.tokenizer(self.input_text       ,    return_tensors="pt"						)
           __lowerCAmelCase             =       self.model_abit.generate(input_ids=encoded_input["input_ids"].to(0						)       ,    max_new_tokens=1_0						)

           self.assertIn(self.tokenizer.decode(output_sequences[0]       ,    skip_special_tokens=__UpperCamelCase						)       ,    self.EXPECTED_OUTPUTS						)


      def 			__UpperCAmelCase		(						self						)->						Dict:
           __lowerCAmelCase             =       BitsAndBytesConfig()
           __lowerCAmelCase             =       True

           __lowerCAmelCase             =       AutoModelForCausalLM.from_pretrained(
               self.model_name       ,    quantization_config=__UpperCamelCase       ,    device_map="auto"						)

           __lowerCAmelCase             =       self.tokenizer(self.input_text       ,    return_tensors="pt"						)
           __lowerCAmelCase             =       model_abit_from_config.generate(
               input_ids=encoded_input["input_ids"].to(0						)       ,    max_new_tokens=1_0						)

           self.assertIn(self.tokenizer.decode(output_sequences[0]       ,    skip_special_tokens=__UpperCamelCase						)       ,    self.EXPECTED_OUTPUTS						)


      def 			__UpperCAmelCase		(						self						)->						Optional[Any]:
           with self.assertRaises(__UpperCamelCase						), tempfile.TemporaryDirectory() as tmpdirname:
                self.model_abit.save_pretrained(__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self						)->						Optional[int]:
           __lowerCAmelCase             =       BitsAndBytesConfig()

           with self.assertRaises(__UpperCamelCase						):
                __lowerCAmelCase             =       AutoModelForCausalLM.from_pretrained(
                    self.model_name       ,    quantization_config=__UpperCamelCase       ,    load_in_abit=__UpperCamelCase       ,    device_map="auto"       ,    bnb_abit_quant_type="nf4"       ,    )


      def 			__UpperCAmelCase		(						self						)->						Any:
           with self.assertRaises(__UpperCamelCase						):
                # Tries with `str`
                self.model_abit.to("cpu"						)

           with self.assertRaises(__UpperCamelCase						):
                # Tries with a `dtype``
                self.model_abit.to(torch.floataa						)

           with self.assertRaises(__UpperCamelCase						):
                # Tries with a `device`
                self.model_abit.to(torch.device("cuda:0"						)						)

           with self.assertRaises(__UpperCamelCase						):
                # Tries with a `device`
                self.model_abit.float()

           with self.assertRaises(__UpperCamelCase						):
                # Tries with a `device`
                self.model_abit.half()

           # Test if we did not break anything
           __lowerCAmelCase             =       self.tokenizer(self.input_text       ,    return_tensors="pt"						)

           __lowerCAmelCase             =       self.model_fpaa.to(torch.floataa						)
           __lowerCAmelCase             =       self.model_fpaa.generate(input_ids=encoded_input["input_ids"].to(0						)       ,    max_new_tokens=1_0						)

           # Check this does not throw an error
           __lowerCAmelCase             =       self.model_fpaa.to("cpu"						)

           # Check this does not throw an error
           __lowerCAmelCase             =       self.model_fpaa.half()

           # Check this does not throw an error
           __lowerCAmelCase             =       self.model_fpaa.float()


      def 			__UpperCAmelCase		(						self						)->						List[str]:
           __lowerCAmelCase             =       AutoModelForSeqaSeqLM.from_pretrained("t5-small"       ,    load_in_abit=__UpperCamelCase       ,    device_map="auto"						)
           self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa						)



@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 				_UpperCamelCase      (unittest.TestCase      ):


      @classmethod
      def 			__UpperCAmelCase		(						cls						)->						str:
           __lowerCAmelCase             =       "t5-small"
           __lowerCAmelCase             =       "google/flan-t5-small"  # flan-t5 uses dense-act instead of dense-relu-dense
           __lowerCAmelCase             =       AutoTokenizer.from_pretrained(cls.model_name						)
           __lowerCAmelCase             =       "Translate in German: Hello, my dog is cute"


      def 			__UpperCAmelCase		(						self						)->						Union[str, Any]:
           gc.collect()
           torch.cuda.empty_cache()


      def 			__UpperCAmelCase		(						self						)->						str:
           from transformers import TaForConditionalGeneration

           __lowerCAmelCase             =       TaForConditionalGeneration._keep_in_fpaa_modules
           __lowerCAmelCase             =       None

           # test with `t5-small`
           __lowerCAmelCase             =       TaForConditionalGeneration.from_pretrained(self.model_name       ,    load_in_abit=__UpperCamelCase       ,    device_map="auto"						)
           __lowerCAmelCase             =       self.tokenizer(self.input_text       ,    return_tensors="pt"						).to(0						)
           __lowerCAmelCase             =       model.generate(**__UpperCamelCase						)

           # test with `flan-t5-small`
           __lowerCAmelCase             =       TaForConditionalGeneration.from_pretrained(
               self.dense_act_model_name       ,    load_in_abit=__UpperCamelCase       ,    device_map="auto"						)
           __lowerCAmelCase             =       self.tokenizer(self.input_text       ,    return_tensors="pt"						).to(0						)
           __lowerCAmelCase             =       model.generate(**__UpperCamelCase						)
           __lowerCAmelCase             =       modules


      def 			__UpperCAmelCase		(						self						)->						Optional[int]:
           import bitsandbytes as bnb

           from transformers import TaForConditionalGeneration

           # test with `t5-small`
           __lowerCAmelCase             =       TaForConditionalGeneration.from_pretrained(self.model_name       ,    load_in_abit=__UpperCamelCase       ,    device_map="auto"						)

           # there was a bug with decoders - this test checks that it is fixed
           self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q       ,    bnb.nn.Linearabit						)						)

           __lowerCAmelCase             =       self.tokenizer(self.input_text       ,    return_tensors="pt"						).to(0						)
           __lowerCAmelCase             =       model.generate(**__UpperCamelCase						)

           # test with `flan-t5-small`
           __lowerCAmelCase             =       TaForConditionalGeneration.from_pretrained(
               self.dense_act_model_name       ,    load_in_abit=__UpperCamelCase       ,    device_map="auto"						)
           __lowerCAmelCase             =       self.tokenizer(self.input_text       ,    return_tensors="pt"						).to(0						)
           __lowerCAmelCase             =       model.generate(**__UpperCamelCase						)



class 				_UpperCamelCase      (a_      ):


      def 			__UpperCAmelCase		(						self						)->						Optional[Any]:
           super().setUp()
           # model_name
           __lowerCAmelCase             =       "bigscience/bloom-560m"
           __lowerCAmelCase             =       "t5-small"

           # Different types of model

           __lowerCAmelCase             =       AutoModel.from_pretrained(self.model_name       ,    load_in_abit=__UpperCamelCase       ,    device_map="auto"						)
           # Sequence classification model
           __lowerCAmelCase             =       AutoModelForSequenceClassification.from_pretrained(
               self.model_name       ,    load_in_abit=__UpperCamelCase       ,    device_map="auto"						)
           # CausalLM model
           __lowerCAmelCase             =       AutoModelForCausalLM.from_pretrained(self.model_name       ,    load_in_abit=__UpperCamelCase       ,    device_map="auto"						)
           # Seq2seq model
           __lowerCAmelCase             =       AutoModelForSeqaSeqLM.from_pretrained(
               self.seq_to_seq_name       ,    load_in_abit=__UpperCamelCase       ,    device_map="auto"						)


      def 			__UpperCAmelCase		(						self						)->						Optional[int]:
           del self.base_model
           del self.sequence_model
           del self.model_abit
           del self.seq_to_seq_model

           gc.collect()
           torch.cuda.empty_cache()


      def 			__UpperCAmelCase		(						self						)->						Optional[int]:
           from bitsandbytes.nn import Paramsabit

           self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit						)

           # Other heads should be nn.Parameter
           self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter						)
           self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter						)
           self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter						)



class 				_UpperCamelCase      (a_      ):


      def 			__UpperCAmelCase		(						self						)->						str:
           super().setUp()


      def 			__UpperCAmelCase		(						self						)->						Optional[Any]:
           del self.pipe

           gc.collect()
           torch.cuda.empty_cache()


      def 			__UpperCAmelCase		(						self						)->						int:
           __lowerCAmelCase             =       pipeline(
               "text-generation"       ,    model=self.model_name       ,    model_kwargs={"device_map": "auto", "load_in_4bit": True, "torch_dtype": torch.floataa}       ,    max_new_tokens=self.MAX_NEW_TOKENS       ,    )

           # Real second forward pass
           __lowerCAmelCase             =       self.pipe(self.input_text						)
           self.assertIn(pipeline_output[0]["generated_text"]       ,    self.EXPECTED_OUTPUTS						)



@require_torch_multi_gpu
class 				_UpperCamelCase      (a_      ):


      def 			__UpperCAmelCase		(						self						)->						int:
           super().setUp()


      def 			__UpperCAmelCase		(						self						)->						str:
           __lowerCAmelCase             =       AutoModelForCausalLM.from_pretrained(
               self.model_name       ,    load_in_abit=__UpperCamelCase       ,    device_map="balanced"						)

           # Check correct device map
           self.assertEqual(set(model_parallel.hf_device_map.values()						)       ,    {0, 1}						)

           # Check that inference pass works on the model
           __lowerCAmelCase             =       self.tokenizer(self.input_text       ,    return_tensors="pt"						)

           # Second real batch
           __lowerCAmelCase             =       model_parallel.generate(input_ids=encoded_input["input_ids"].to(0						)       ,    max_new_tokens=1_0						)
           self.assertIn(self.tokenizer.decode(output_parallel[0]       ,    skip_special_tokens=__UpperCamelCase						)       ,    self.EXPECTED_OUTPUTS						)



class 				_UpperCamelCase      (a_      ):


      def 			__UpperCAmelCase		(						self						)->						str:
           __lowerCAmelCase             =       "facebook/opt-350m"
           super().setUp()


      def 			__UpperCAmelCase		(						self						)->						Union[str, Any]:
           if version.parse(importlib.metadata.version("bitsandbytes"						)						) < version.parse("0.37.0"						):
                return

           # Step 1: freeze all parameters
           __lowerCAmelCase             =       AutoModelForCausalLM.from_pretrained(self.model_name       ,    load_in_abit=__UpperCamelCase						)

           self.assertEqual(set(model.hf_device_map.values()						)       ,    {torch.cuda.current_device()}						)

           for param in model.parameters():
                __lowerCAmelCase             =       False  # freeze the model - train adapters later
                if param.ndim == 1:
                     # cast the small parameters (e.g. layernorm) to fp32 for stability
                     __lowerCAmelCase             =       param.data.to(torch.floataa						)

        # Step 2: add adapters
           for _, module in model.named_modules():
                if "OPTAttention" in repr(type(__UpperCamelCase						)						):
                     __lowerCAmelCase             =       LoRALayer(module.q_proj       ,    rank=1_6						)
                     __lowerCAmelCase             =       LoRALayer(module.k_proj       ,    rank=1_6						)
                     __lowerCAmelCase             =       LoRALayer(module.v_proj       ,    rank=1_6						)

        # Step 3: dummy batch
           __lowerCAmelCase             =       self.tokenizer("Test batch "       ,    return_tensors="pt"						).to(0						)

           # Step 4: Check if the gradient is not None
           with torch.cuda.amp.autocast():
                __lowerCAmelCase             =       model.forward(**__UpperCamelCase						)
                out.logits.norm().backward()

           for module in model.modules():
                if isinstance(__UpperCamelCase       ,    __UpperCamelCase						):
                     self.assertTrue(module.adapter[1].weight.grad is not None						)
                     self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0						)
                elif isinstance(__UpperCamelCase       ,    nn.Embedding						):
                     self.assertTrue(module.weight.grad is None						)



class 				_UpperCamelCase      (a_      ):
      snake_case_	=				"""gpt2-xl"""
      snake_case_	=				3.3_1_9_1_8_5_4_8_5_4_1_5_2_1_8_7














import itertools
import string
from collections.abc import Generator, Iterable





def    __lowerCAmelCase   (    __snake_case       ,			__snake_case							):
     __lowerCAmelCase             =       iter(__snake_case							)
     while True:
          __lowerCAmelCase             =       tuple(itertools.islice(__snake_case       ,			__snake_case							)							)
          if not chunk:
               return
          yield chunk





def    __lowerCAmelCase   (    __snake_case							):
     __lowerCAmelCase             =       "".join([c.upper() for c in dirty if c in string.ascii_letters]							)
     __lowerCAmelCase             =       ""

     if len(__snake_case							) < 2:
          return dirty

     for i in range(len(__snake_case							) - 1							):
          clean += dirty[i]

          if dirty[i] == dirty[i + 1]:
               clean += "X"

     clean += dirty[-1]

     if len(__snake_case							) & 1:
          clean += "X"

     return clean





def    __lowerCAmelCase   (    __snake_case							):
     # I and J are used interchangeably to allow
     # us to use a 5x5 table (25 letters)
     __lowerCAmelCase             =       "ABCDEFGHIKLMNOPQRSTUVWXYZ"
     # we're using a list instead of a '2d' array because it makes the math
     # for setting up the table and doing the actual encoding/decoding simpler
     __lowerCAmelCase             =       []

     # copy key chars into the table if they are in `alphabet` ignoring duplicates
     for char in key.upper():
          if char not in table and char in alphabet:
               table.append(__snake_case							)

    # fill the rest of the table in with the remaining alphabet chars
     for char in alphabet:
          if char not in table:
               table.append(__snake_case							)

     return table





def    __lowerCAmelCase   (    __snake_case       ,			__snake_case							):
     __lowerCAmelCase             =       generate_table(__snake_case							)
     __lowerCAmelCase             =       prepare_input(__snake_case							)
     __lowerCAmelCase             =       ""

     # https://en.wikipedia.org/wiki/Playfair_cipher#Description
     for chara, chara in chunker(__snake_case       ,			2							):
          __lowerCAmelCase      ,			__lowerCAmelCase             =       divmod(table.index(__snake_case							)       ,			5							)
          __lowerCAmelCase      ,			__lowerCAmelCase             =       divmod(table.index(__snake_case							)       ,			5							)

          if rowa == rowa:
               ciphertext += table[rowa * 5 + (cola + 1) % 5]
               ciphertext += table[rowa * 5 + (cola + 1) % 5]
          elif cola == cola:
               ciphertext += table[((rowa + 1) % 5) * 5 + cola]
               ciphertext += table[((rowa + 1) % 5) * 5 + cola]
          else:  # rectangle
               ciphertext += table[rowa * 5 + cola]
               ciphertext += table[rowa * 5 + cola]

     return ciphertext





def    __lowerCAmelCase   (    __snake_case       ,			__snake_case							):
     __lowerCAmelCase             =       generate_table(__snake_case							)
     __lowerCAmelCase             =       ""

     # https://en.wikipedia.org/wiki/Playfair_cipher#Description
     for chara, chara in chunker(__snake_case       ,			2							):
          __lowerCAmelCase      ,			__lowerCAmelCase             =       divmod(table.index(__snake_case							)       ,			5							)
          __lowerCAmelCase      ,			__lowerCAmelCase             =       divmod(table.index(__snake_case							)       ,			5							)

          if rowa == rowa:
               plaintext += table[rowa * 5 + (cola - 1) % 5]
               plaintext += table[rowa * 5 + (cola - 1) % 5]
          elif cola == cola:
               plaintext += table[((rowa - 1) % 5) * 5 + cola]
               plaintext += table[((rowa - 1) % 5) * 5 + cola]
          else:  # rectangle
               plaintext += table[rowa * 5 + cola]
               plaintext += table[rowa * 5 + cola]

     return plaintext






