


'''simple docstring'''



import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)







def     _UpperCamelCase      (		UpperCamelCase__					):
			if model.config.model_type == "gpt2":
						return model.transformer.h[0].mlp.c_fc
			return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
		import torch
		import torch.nn as nn


		class 		_snake_case					(     nn.Module	):




							def __init__(      self							,							_lowerCamelCase							,							_lowerCamelCase):
										super().__init__()
										UpperCAmelCase__  :					List[Any]        =     module
										UpperCAmelCase__  :					str        =     nn.Sequential(
										    nn.Linear(module.in_features							,							_lowerCamelCase							,							bias=_lowerCamelCase)							,							nn.Linear(_lowerCamelCase							,							module.out_features							,							bias=_lowerCamelCase)							,							)
										UpperCAmelCase__  :					Any        =     (2.0 / (5 * min(module.in_features							,							module.out_features))) ** 0.5
										nn.init.normal_(self.adapter[0].weight							,							std=_lowerCamelCase)
										nn.init.zeros_(self.adapter[1].weight)
										self.adapter.to(module.weight.device)




							def 				snake_case__						(      self							,							_lowerCamelCase							,							*_lowerCamelCase							,							**_lowerCamelCase):
										return self.module(_lowerCamelCase							,							*_lowerCamelCase							,							**_lowerCamelCase) + self.adapter(_lowerCamelCase)


@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 		_snake_case					(     unittest.TestCase	):
					# We keep the constants inside the init function and model loading inside setUp function

					# We need to test on relatively large models (aka >1b parameters otherwise the quantiztion may not work as expected)
					# Therefore here we use only bloom-1b3 to test our module
					lowerCAmelCase      :str   =					'''bigscience/bloom-1b7'''

					# Constant values
					lowerCAmelCase      :Optional[int]   =					2.1_09_65_95_52_69_25_74

					lowerCAmelCase      :str   =					'''Hello my name is'''
					lowerCAmelCase      :Optional[int]   =					set()
					EXPECTED_OUTPUTS.add('''Hello my name is John and I am a professional photographer. I'''	)
					EXPECTED_OUTPUTS.add('''Hello my name is John.\nI am a friend of your father.\n'''	)
					EXPECTED_OUTPUTS.add('''Hello my name is John Doe, I am a student at the University'''	)
					lowerCAmelCase      :Any   =					10




					def 				snake_case__						(      self):
								# Models and tokenizer
								UpperCAmelCase__  :					str        =     AutoTokenizer.from_pretrained(self.model_name)


class 		_snake_case					(     a__	):




					def 				snake_case__						(      self):
								super().setUp()

								# Models and tokenizer
								UpperCAmelCase__  :					Dict        =     AutoModelForCausalLM.from_pretrained(
								    self.model_name							,							torch_dtype=torch.floataa							,							device_map="""auto""")
								UpperCAmelCase__  :					List[Any]        =     AutoModelForCausalLM.from_pretrained(self.model_name							,							load_in_abit=_lowerCamelCase							,							device_map="""auto""")




					def 				snake_case__						(      self):
								del self.model_fpaa
								del self.model_abit

								gc.collect()
								torch.cuda.empty_cache()




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[str]        =     self.model_abit.config

								self.assertTrue(hasattr(_lowerCamelCase							,							"""quantization_config"""))

								UpperCAmelCase__  :					Union[str, Any]        =     config.to_dict()
								UpperCAmelCase__  :					List[Any]        =     config.to_diff_dict()

								UpperCAmelCase__  :					int        =     config.to_json_string()




					def 				snake_case__						(      self):
								from bitsandbytes.nn import Paramsabit

								UpperCAmelCase__  :					Any        =     self.model_fpaa.get_memory_footprint()
								UpperCAmelCase__  :					Optional[int]        =     self.model_abit.get_memory_footprint()

								self.assertAlmostEqual(mem_fpaa / mem_abit							,							self.EXPECTED_RELATIVE_DIFFERENCE)
								UpperCAmelCase__  :					List[str]        =     get_some_linear_layer(self.model_abit)
								self.assertTrue(linear.weight.__class__ == Paramsabit)




					def 				snake_case__						(      self):
								from transformers import TaPreTrainedModel

								self.model_fpaa.get_memory_footprint()
								self.model_abit.get_memory_footprint()

								for name, module in self.model_abit.named_modules():
											if isinstance(_lowerCamelCase							,							torch.nn.Linear):
														if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
																	# 4-bit parameters are packed in uint8 variables
																	self.assertTrue(module.weight.dtype == torch.uinta)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Any        =     self.tokenizer(self.input_text							,							return_tensors="""pt""")
								UpperCAmelCase__  :					List[Any]        =     self.model_abit.generate(input_ids=encoded_input["""input_ids"""].to(0)							,							max_new_tokens=10)

								self.assertIn(self.tokenizer.decode(output_sequences[0]							,							skip_special_tokens=_lowerCamelCase)							,							self.EXPECTED_OUTPUTS)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[str]        =     BitsAndBytesConfig()
								UpperCAmelCase__  :					Optional[Any]        =     True

								UpperCAmelCase__  :					List[str]        =     AutoModelForCausalLM.from_pretrained(
								    self.model_name							,							quantization_config=_lowerCamelCase							,							device_map="""auto""")

								UpperCAmelCase__  :					List[str]        =     self.tokenizer(self.input_text							,							return_tensors="""pt""")
								UpperCAmelCase__  :					Dict        =     model_abit_from_config.generate(
								    input_ids=encoded_input["""input_ids"""].to(0)							,							max_new_tokens=10)

								self.assertIn(self.tokenizer.decode(output_sequences[0]							,							skip_special_tokens=_lowerCamelCase)							,							self.EXPECTED_OUTPUTS)




					def 				snake_case__						(      self):
								with self.assertRaises(_lowerCamelCase), tempfile.TemporaryDirectory() as tmpdirname:
											self.model_abit.save_pretrained(_lowerCamelCase)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Dict        =     BitsAndBytesConfig()

								with self.assertRaises(_lowerCamelCase):
											UpperCAmelCase__  :					Any        =     AutoModelForCausalLM.from_pretrained(
											    self.model_name							,							quantization_config=_lowerCamelCase							,							load_in_abit=_lowerCamelCase							,							device_map="""auto"""							,							bnb_abit_quant_type="""nf4"""							,							)




					def 				snake_case__						(      self):
								with self.assertRaises(_lowerCamelCase):
											# Tries with `str`
											self.model_abit.to("""cpu""")

								with self.assertRaises(_lowerCamelCase):
											# Tries with a `dtype``
											self.model_abit.to(torch.floataa)

								with self.assertRaises(_lowerCamelCase):
											# Tries with a `device`
											self.model_abit.to(torch.device("""cuda:0"""))

								with self.assertRaises(_lowerCamelCase):
											# Tries with a `device`
											self.model_abit.float()

								with self.assertRaises(_lowerCamelCase):
											# Tries with a `device`
											self.model_abit.half()

								# Test if we did not break anything
								UpperCAmelCase__  :					int        =     self.tokenizer(self.input_text							,							return_tensors="""pt""")

								UpperCAmelCase__  :					int        =     self.model_fpaa.to(torch.floataa)
								UpperCAmelCase__  :					int        =     self.model_fpaa.generate(input_ids=encoded_input["""input_ids"""].to(0)							,							max_new_tokens=10)

								# Check this does not throw an error
								UpperCAmelCase__  :					Optional[Any]        =     self.model_fpaa.to("""cpu""")

								# Check this does not throw an error
								UpperCAmelCase__  :					str        =     self.model_fpaa.half()

								# Check this does not throw an error
								UpperCAmelCase__  :					Optional[Any]        =     self.model_fpaa.float()




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					str        =     AutoModelForSeqaSeqLM.from_pretrained("""t5-small"""							,							load_in_abit=_lowerCamelCase							,							device_map="""auto""")
								self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa)


@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 		_snake_case					(     unittest.TestCase	):




					@classmethod
					def 				snake_case__						(      cls):
								UpperCAmelCase__  :					Optional[Any]        =     """t5-small"""
								UpperCAmelCase__  :					int        =     """google/flan-t5-small"""  # flan-t5 uses dense-act instead of dense-relu-dense
								UpperCAmelCase__  :					str        =     AutoTokenizer.from_pretrained(cls.model_name)
								UpperCAmelCase__  :					Optional[Any]        =     """Translate in German: Hello, my dog is cute"""




					def 				snake_case__						(      self):
								gc.collect()
								torch.cuda.empty_cache()




					def 				snake_case__						(      self):
								from transformers import TaForConditionalGeneration

								UpperCAmelCase__  :					Tuple        =     TaForConditionalGeneration._keep_in_fpaa_modules
								UpperCAmelCase__  :					Any        =     None

								# test with `t5-small`
								UpperCAmelCase__  :					int        =     TaForConditionalGeneration.from_pretrained(self.model_name							,							load_in_abit=_lowerCamelCase							,							device_map="""auto""")
								UpperCAmelCase__  :					Any        =     self.tokenizer(self.input_text							,							return_tensors="""pt""").to(0)
								UpperCAmelCase__  :					Tuple        =     model.generate(**_lowerCamelCase)

								# test with `flan-t5-small`
								UpperCAmelCase__  :					List[Any]        =     TaForConditionalGeneration.from_pretrained(
								    self.dense_act_model_name							,							load_in_abit=_lowerCamelCase							,							device_map="""auto""")
								UpperCAmelCase__  :					Union[str, Any]        =     self.tokenizer(self.input_text							,							return_tensors="""pt""").to(0)
								UpperCAmelCase__  :					List[Any]        =     model.generate(**_lowerCamelCase)
								UpperCAmelCase__  :					Any        =     modules




					def 				snake_case__						(      self):
								import bitsandbytes as bnb

								from transformers import TaForConditionalGeneration

								# test with `t5-small`
								UpperCAmelCase__  :					int        =     TaForConditionalGeneration.from_pretrained(self.model_name							,							load_in_abit=_lowerCamelCase							,							device_map="""auto""")

								# there was a bug with decoders - this test checks that it is fixed
								self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q							,							bnb.nn.Linearabit))

								UpperCAmelCase__  :					Tuple        =     self.tokenizer(self.input_text							,							return_tensors="""pt""").to(0)
								UpperCAmelCase__  :					Any        =     model.generate(**_lowerCamelCase)

								# test with `flan-t5-small`
								UpperCAmelCase__  :					List[str]        =     TaForConditionalGeneration.from_pretrained(
								    self.dense_act_model_name							,							load_in_abit=_lowerCamelCase							,							device_map="""auto""")
								UpperCAmelCase__  :					int        =     self.tokenizer(self.input_text							,							return_tensors="""pt""").to(0)
								UpperCAmelCase__  :					str        =     model.generate(**_lowerCamelCase)


class 		_snake_case					(     a__	):




					def 				snake_case__						(      self):
								super().setUp()
								# model_name
								UpperCAmelCase__  :					Union[str, Any]        =     """bigscience/bloom-560m"""
								UpperCAmelCase__  :					Optional[Any]        =     """t5-small"""

								# Different types of model

								UpperCAmelCase__  :					str        =     AutoModel.from_pretrained(self.model_name							,							load_in_abit=_lowerCamelCase							,							device_map="""auto""")
								# Sequence classification model
								UpperCAmelCase__  :					Optional[int]        =     AutoModelForSequenceClassification.from_pretrained(
								    self.model_name							,							load_in_abit=_lowerCamelCase							,							device_map="""auto""")
								# CausalLM model
								UpperCAmelCase__  :					List[Any]        =     AutoModelForCausalLM.from_pretrained(self.model_name							,							load_in_abit=_lowerCamelCase							,							device_map="""auto""")
								# Seq2seq model
								UpperCAmelCase__  :					str        =     AutoModelForSeqaSeqLM.from_pretrained(
								    self.seq_to_seq_name							,							load_in_abit=_lowerCamelCase							,							device_map="""auto""")




					def 				snake_case__						(      self):
								del self.base_model
								del self.sequence_model
								del self.model_abit
								del self.seq_to_seq_model

								gc.collect()
								torch.cuda.empty_cache()




					def 				snake_case__						(      self):
								from bitsandbytes.nn import Paramsabit

								self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit)

								# Other heads should be nn.Parameter
								self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter)
								self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter)
								self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter)


class 		_snake_case					(     a__	):




					def 				snake_case__						(      self):
								super().setUp()




					def 				snake_case__						(      self):
								del self.pipe

								gc.collect()
								torch.cuda.empty_cache()




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Union[str, Any]        =     pipeline(
								    """text-generation"""							,							model=self.model_name							,							model_kwargs={"""device_map""": """auto""", """load_in_4bit""": True, """torch_dtype""": torch.floataa}							,							max_new_tokens=self.MAX_NEW_TOKENS							,							)

								# Real second forward pass
								UpperCAmelCase__  :					Union[str, Any]        =     self.pipe(self.input_text)
								self.assertIn(pipeline_output[0]["""generated_text"""]							,							self.EXPECTED_OUTPUTS)


@require_torch_multi_gpu
class 		_snake_case					(     a__	):




					def 				snake_case__						(      self):
								super().setUp()




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[Any]        =     AutoModelForCausalLM.from_pretrained(
								    self.model_name							,							load_in_abit=_lowerCamelCase							,							device_map="""balanced""")

								# Check correct device map
								self.assertEqual(set(model_parallel.hf_device_map.values())							,							{0, 1})

								# Check that inference pass works on the model
								UpperCAmelCase__  :					Union[str, Any]        =     self.tokenizer(self.input_text							,							return_tensors="""pt""")

								# Second real batch
								UpperCAmelCase__  :					str        =     model_parallel.generate(input_ids=encoded_input["""input_ids"""].to(0)							,							max_new_tokens=10)
								self.assertIn(self.tokenizer.decode(output_parallel[0]							,							skip_special_tokens=_lowerCamelCase)							,							self.EXPECTED_OUTPUTS)


class 		_snake_case					(     a__	):




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Dict        =     """facebook/opt-350m"""
								super().setUp()




					def 				snake_case__						(      self):
								if version.parse(importlib.metadata.version("""bitsandbytes""")) < version.parse("""0.37.0"""):
											return

								# Step 1: freeze all parameters
								UpperCAmelCase__  :					str        =     AutoModelForCausalLM.from_pretrained(self.model_name							,							load_in_abit=_lowerCamelCase)

								self.assertEqual(set(model.hf_device_map.values())							,							{torch.cuda.current_device()})

								for param in model.parameters():
											UpperCAmelCase__  :					Dict        =     False  # freeze the model - train adapters later
											if param.ndim == 1:
														# cast the small parameters (e.g. layernorm) to fp32 for stability
														UpperCAmelCase__  :					str        =     param.data.to(torch.floataa)

        # Step 2: add adapters
								for _, module in model.named_modules():
											if "OPTAttention" in repr(type(_lowerCamelCase)):
														UpperCAmelCase__  :					Dict        =     LoRALayer(module.q_proj							,							rank=16)
														UpperCAmelCase__  :					Tuple        =     LoRALayer(module.k_proj							,							rank=16)
														UpperCAmelCase__  :					int        =     LoRALayer(module.v_proj							,							rank=16)

        # Step 3: dummy batch
								UpperCAmelCase__  :					Optional[Any]        =     self.tokenizer("""Test batch """							,							return_tensors="""pt""").to(0)

								# Step 4: Check if the gradient is not None
								with torch.cuda.amp.autocast():
											UpperCAmelCase__  :					Optional[Any]        =     model.forward(**_lowerCamelCase)
											out.logits.norm().backward()

								for module in model.modules():
											if isinstance(_lowerCamelCase							,							_lowerCamelCase):
														self.assertTrue(module.adapter[1].weight.grad is not None)
														self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0)
											elif isinstance(_lowerCamelCase							,							nn.Embedding):
														self.assertTrue(module.weight.grad is None)


class 		_snake_case					(     a__	):
					lowerCAmelCase      :List[str]   =					'''gpt2-xl'''
					lowerCAmelCase      :Dict   =					3.31_91_85_48_54_15_21_87



'''simple docstring'''



from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__A	=logging.get_logger(__name__)

__A	={
    'xlm-mlm-en-2048': 'https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json',
    'xlm-mlm-ende-1024': 'https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json',
    'xlm-mlm-enfr-1024': 'https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json',
    'xlm-mlm-enro-1024': 'https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json',
    'xlm-mlm-tlm-xnli15-1024': 'https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json',
    'xlm-mlm-xnli15-1024': 'https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json',
    'xlm-clm-enfr-1024': 'https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json',
    'xlm-clm-ende-1024': 'https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json',
    'xlm-mlm-17-1280': 'https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json',
    'xlm-mlm-100-1280': 'https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json',
}


class 		_snake_case					(     a__	):
					lowerCAmelCase      :Any   =					'''xlm'''
					lowerCAmelCase      :Any   =					{
					    '''hidden_size''': '''emb_dim''',
					    '''num_attention_heads''': '''n_heads''',
					    '''num_hidden_layers''': '''n_layers''',
					    '''n_words''': '''vocab_size''',  # For backward compatibility
					}




					def __init__(      self							,							_lowerCamelCase=3_0145							,							_lowerCamelCase=2048							,							_lowerCamelCase=12							,							_lowerCamelCase=16							,							_lowerCamelCase=0.1							,							_lowerCamelCase=0.1							,							_lowerCamelCase=True							,							_lowerCamelCase=False							,							_lowerCamelCase=False							,							_lowerCamelCase=False							,							_lowerCamelCase=1							,							_lowerCamelCase=True							,							_lowerCamelCase=512							,							_lowerCamelCase=2048**-0.5							,							_lowerCamelCase=1e-1_2							,							_lowerCamelCase=0.02							,							_lowerCamelCase=0							,							_lowerCamelCase=1							,							_lowerCamelCase=2							,							_lowerCamelCase=3							,							_lowerCamelCase=5							,							_lowerCamelCase=True							,							_lowerCamelCase="first"							,							_lowerCamelCase=True							,							_lowerCamelCase=None							,							_lowerCamelCase=True							,							_lowerCamelCase=0.1							,							_lowerCamelCase=5							,							_lowerCamelCase=5							,							_lowerCamelCase=0							,							_lowerCamelCase=0							,							_lowerCamelCase=2							,							_lowerCamelCase=0							,							**_lowerCamelCase							,							):
								UpperCAmelCase__  :					Tuple        =     vocab_size
								UpperCAmelCase__  :					Tuple        =     emb_dim
								UpperCAmelCase__  :					Optional[Any]        =     n_layers
								UpperCAmelCase__  :					List[str]        =     n_heads
								UpperCAmelCase__  :					List[str]        =     dropout
								UpperCAmelCase__  :					Optional[int]        =     attention_dropout
								UpperCAmelCase__  :					Tuple        =     gelu_activation
								UpperCAmelCase__  :					Optional[Any]        =     sinusoidal_embeddings
								UpperCAmelCase__  :					int        =     causal
								UpperCAmelCase__  :					Union[str, Any]        =     asm
								UpperCAmelCase__  :					Optional[Any]        =     n_langs
								UpperCAmelCase__  :					List[Any]        =     use_lang_emb
								UpperCAmelCase__  :					Union[str, Any]        =     layer_norm_eps
								UpperCAmelCase__  :					List[str]        =     bos_index
								UpperCAmelCase__  :					List[Any]        =     eos_index
								UpperCAmelCase__  :					int        =     pad_index
								UpperCAmelCase__  :					str        =     unk_index
								UpperCAmelCase__  :					Dict        =     mask_index
								UpperCAmelCase__  :					str        =     is_encoder
								UpperCAmelCase__  :					Dict        =     max_position_embeddings
								UpperCAmelCase__  :					Any        =     embed_init_std
								UpperCAmelCase__  :					List[Any]        =     init_std
								UpperCAmelCase__  :					List[str]        =     summary_type
								UpperCAmelCase__  :					Union[str, Any]        =     summary_use_proj
								UpperCAmelCase__  :					Any        =     summary_activation
								UpperCAmelCase__  :					List[str]        =     summary_proj_to_labels
								UpperCAmelCase__  :					Union[str, Any]        =     summary_first_dropout
								UpperCAmelCase__  :					str        =     start_n_top
								UpperCAmelCase__  :					str        =     end_n_top
								UpperCAmelCase__  :					Tuple        =     mask_token_id
								UpperCAmelCase__  :					Union[str, Any]        =     lang_id

								if "n_words" in kwargs:
											UpperCAmelCase__  :					List[str]        =     kwargs["""n_words"""]

								super().__init__(pad_token_id=_lowerCamelCase							,							bos_token_id=_lowerCamelCase							,							**_lowerCamelCase)


class 		_snake_case					(     a__	):




					@property
					def 				snake_case__						(      self):
								if self.task == "multiple-choice":
											UpperCAmelCase__  :					Any        =     {0: """batch""", 1: """choice""", 2: """sequence"""}
								else:
											UpperCAmelCase__  :					List[Any]        =     {0: """batch""", 1: """sequence"""}
								return OrderedDict(
								    [
								        ("""input_ids""", dynamic_axis),
								        ("""attention_mask""", dynamic_axis),
								        ("""token_type_ids""", dynamic_axis),
								    ])