



"""simple docstring"""






import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)






def 					snake_case__       (		__lowerCamelCase    :   List[Any]     ):
			"""simple docstring"""
			if model.config.model_type == "gpt2":
						return model.transformer.h[0].mlp.c_fc
			return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
		import torch
		import torch.nn as nn

		class 	__SCREAMING_SNAKE_CASE     (				nn.Module       ):



							def __init__(	self							:  Union[str, Any],  lowerCamelCase							:  Dict,  lowerCamelCase							:  Optional[Any]					)-> List[Any]:
										super().__init__()
										lowerCamelCase__		:		Tuple		     =module
										lowerCamelCase__		:		List[str]		     =nn.Sequential(
										    nn.Linear(module.in_features,  _UpperCAmelCase,  bias=_UpperCAmelCase					),  nn.Linear(_UpperCAmelCase,  module.out_features,  bias=_UpperCAmelCase					),  )
										lowerCamelCase__		:		Optional[int]		     =(2.0 / (5 * min(module.in_features,  module.out_features					))) ** 0.5
										nn.init.normal_(self.adapter[0].weight,  std=_UpperCAmelCase					)
										nn.init.zeros_(self.adapter[1].weight					)
										self.adapter.to(module.weight.device					)




							def 		snake_case	(	self							:  Any,  lowerCamelCase							:  Tuple,  *lowerCamelCase							:  Any,  **lowerCamelCase							:  Union[str, Any]					)-> Any:
										return self.module(_UpperCAmelCase,  *_UpperCAmelCase,  **_UpperCAmelCase					) + self.adapter(_UpperCAmelCase					)

@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 	__SCREAMING_SNAKE_CASE     (				unittest.TestCase       ):
					_a             =							"bigscience/bloom-1b7"

					# Constant values
					_a             =							2.109659552692574

					_a             =							"Hello my name is"
					_a             =							set()
					EXPECTED_OUTPUTS.add('Hello my name is John and I am a professional photographer. I'       )
					EXPECTED_OUTPUTS.add('Hello my name is John.\nI am a friend of your father.\n'       )
					EXPECTED_OUTPUTS.add('Hello my name is John Doe, I am a student at the University'       )
					_a             =							1_0



					def 		snake_case	(	self							:  Optional[int]					)-> Optional[Any]:
								# Models and tokenizer
								lowerCamelCase__		:		List[Any]		     =AutoTokenizer.from_pretrained(self.model_name					)





class 	__SCREAMING_SNAKE_CASE     (				_UpperCAmelCase       ):



					def 		snake_case	(	self							:  Optional[Any]					)-> Optional[Any]:
								super().setUp()

								# Models and tokenizer
								lowerCamelCase__		:		Optional[Any]		     =AutoModelForCausalLM.from_pretrained(
								    self.model_name,  torch_dtype=torch.floataa,  device_map='''auto'''					)
								lowerCamelCase__		:		List[Any]		     =AutoModelForCausalLM.from_pretrained(self.model_name,  load_in_abit=_UpperCAmelCase,  device_map='''auto'''					)



					def 		snake_case	(	self							:  Optional[Any]					)-> Any:
								del self.model_fpaa
								del self.model_abit

								gc.collect()
								torch.cuda.empty_cache()



					def 		snake_case	(	self							:  List[Any]					)-> List[Any]:
								lowerCamelCase__		:		List[str]		     =self.model_abit.config

								self.assertTrue(hasattr(_UpperCAmelCase,  '''quantization_config'''					)					)

								lowerCamelCase__		:		Union[str, Any]		     =config.to_dict()
								lowerCamelCase__		:		Union[str, Any]		     =config.to_diff_dict()

								lowerCamelCase__		:		Dict		     =config.to_json_string()



					def 		snake_case	(	self							:  Optional[Any]					)-> List[Any]:
								from bitsandbytes.nn import Paramsabit

								lowerCamelCase__		:		Optional[int]		     =self.model_fpaa.get_memory_footprint()
								lowerCamelCase__		:		int		     =self.model_abit.get_memory_footprint()

								self.assertAlmostEqual(mem_fpaa / mem_abit,  self.EXPECTED_RELATIVE_DIFFERENCE					)
								lowerCamelCase__		:		List[str]		     =get_some_linear_layer(self.model_abit					)
								self.assertTrue(linear.weight.__class__ == Paramsabit					)



					def 		snake_case	(	self							:  int					)-> Optional[int]:
								from transformers import TaPreTrainedModel

								self.model_fpaa.get_memory_footprint()
								self.model_abit.get_memory_footprint()

								for name, module in self.model_abit.named_modules():
											if isinstance(_UpperCAmelCase,  torch.nn.Linear					):
														if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
																	# 4-bit parameters are packed in uint8 variables
																	self.assertTrue(module.weight.dtype == torch.uinta					)



					def 		snake_case	(	self							:  str					)-> int:
								lowerCamelCase__		:		Tuple		     =self.tokenizer(self.input_text,  return_tensors='''pt'''					)
								lowerCamelCase__		:		List[Any]		     =self.model_abit.generate(input_ids=encoded_input['''input_ids'''].to(0					),  max_new_tokens=10					)

								self.assertIn(self.tokenizer.decode(output_sequences[0],  skip_special_tokens=_UpperCAmelCase					),  self.EXPECTED_OUTPUTS					)



					def 		snake_case	(	self							:  Tuple					)-> Union[str, Any]:
								lowerCamelCase__		:		str		     =BitsAndBytesConfig()
								lowerCamelCase__		:		str		     =True

								lowerCamelCase__		:		List[str]		     =AutoModelForCausalLM.from_pretrained(
								    self.model_name,  quantization_config=_UpperCAmelCase,  device_map='''auto'''					)

								lowerCamelCase__		:		Tuple		     =self.tokenizer(self.input_text,  return_tensors='''pt'''					)
								lowerCamelCase__		:		Any		     =model_abit_from_config.generate(
								    input_ids=encoded_input['''input_ids'''].to(0					),  max_new_tokens=10					)

								self.assertIn(self.tokenizer.decode(output_sequences[0],  skip_special_tokens=_UpperCAmelCase					),  self.EXPECTED_OUTPUTS					)



					def 		snake_case	(	self							:  Dict					)-> Any:
								with self.assertRaises(_UpperCAmelCase					), tempfile.TemporaryDirectory() as tmpdirname:
											self.model_abit.save_pretrained(_UpperCAmelCase					)



					def 		snake_case	(	self							:  Dict					)-> int:
								lowerCamelCase__		:		int		     =BitsAndBytesConfig()

								with self.assertRaises(_UpperCAmelCase					):
											lowerCamelCase__		:		str		     =AutoModelForCausalLM.from_pretrained(
											    self.model_name,  quantization_config=_UpperCAmelCase,  load_in_abit=_UpperCAmelCase,  device_map='''auto''',  bnb_abit_quant_type='''nf4''',  )



					def 		snake_case	(	self							:  int					)-> Optional[int]:
								with self.assertRaises(_UpperCAmelCase					):
											# Tries with `str`
											self.model_abit.to('''cpu'''					)

								with self.assertRaises(_UpperCAmelCase					):
											# Tries with a `dtype``
											self.model_abit.to(torch.floataa					)

								with self.assertRaises(_UpperCAmelCase					):
											# Tries with a `device`
											self.model_abit.to(torch.device('''cuda:0'''					)					)

								with self.assertRaises(_UpperCAmelCase					):
											# Tries with a `device`
											self.model_abit.float()

								with self.assertRaises(_UpperCAmelCase					):
											# Tries with a `device`
											self.model_abit.half()

								# Test if we did not break anything
								lowerCamelCase__		:		str		     =self.tokenizer(self.input_text,  return_tensors='''pt'''					)

								lowerCamelCase__		:		str		     =self.model_fpaa.to(torch.floataa					)
								lowerCamelCase__		:		int		     =self.model_fpaa.generate(input_ids=encoded_input['''input_ids'''].to(0					),  max_new_tokens=10					)

								# Check this does not throw an error
								lowerCamelCase__		:		List[Any]		     =self.model_fpaa.to('''cpu'''					)

								# Check this does not throw an error
								lowerCamelCase__		:		Tuple		     =self.model_fpaa.half()

								# Check this does not throw an error
								lowerCamelCase__		:		Any		     =self.model_fpaa.float()




					def 		snake_case	(	self							:  List[Any]					)-> str:
								lowerCamelCase__		:		int		     =AutoModelForSeqaSeqLM.from_pretrained('''t5-small''',  load_in_abit=_UpperCAmelCase,  device_map='''auto'''					)
								self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa					)





@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class 	__SCREAMING_SNAKE_CASE     (				unittest.TestCase       ):



					@classmethod
					def 		snake_case	(	cls							:  Tuple					)-> Dict:
								lowerCamelCase__		:		List[Any]		     ='''t5-small'''
								lowerCamelCase__		:		List[Any]		     ='''google/flan-t5-small'''  # flan-t5 uses dense-act instead of dense-relu-dense
								lowerCamelCase__		:		List[str]		     =AutoTokenizer.from_pretrained(cls.model_name					)
								lowerCamelCase__		:		Any		     ='''Translate in German: Hello, my dog is cute'''



					def 		snake_case	(	self							:  List[Any]					)-> Optional[int]:
								gc.collect()
								torch.cuda.empty_cache()



					def 		snake_case	(	self							:  Tuple					)-> List[Any]:
								from transformers import TaForConditionalGeneration

								lowerCamelCase__		:		List[str]		     =TaForConditionalGeneration._keep_in_fpaa_modules
								lowerCamelCase__		:		Union[str, Any]		     =None

								# test with `t5-small`
								lowerCamelCase__		:		Dict		     =TaForConditionalGeneration.from_pretrained(self.model_name,  load_in_abit=_UpperCAmelCase,  device_map='''auto'''					)
								lowerCamelCase__		:		Any		     =self.tokenizer(self.input_text,  return_tensors='''pt'''					).to(0					)
								lowerCamelCase__		:		str		     =model.generate(**_UpperCAmelCase					)

								# test with `flan-t5-small`
								lowerCamelCase__		:		int		     =TaForConditionalGeneration.from_pretrained(
								    self.dense_act_model_name,  load_in_abit=_UpperCAmelCase,  device_map='''auto'''					)
								lowerCamelCase__		:		Any		     =self.tokenizer(self.input_text,  return_tensors='''pt'''					).to(0					)
								lowerCamelCase__		:		str		     =model.generate(**_UpperCAmelCase					)
								lowerCamelCase__		:		List[str]		     =modules




					def 		snake_case	(	self							:  Dict					)-> Optional[int]:
								import bitsandbytes as bnb

								from transformers import TaForConditionalGeneration

								# test with `t5-small`
								lowerCamelCase__		:		int		     =TaForConditionalGeneration.from_pretrained(self.model_name,  load_in_abit=_UpperCAmelCase,  device_map='''auto'''					)

								# there was a bug with decoders - this test checks that it is fixed
								self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q,  bnb.nn.Linearabit					)					)

								lowerCamelCase__		:		str		     =self.tokenizer(self.input_text,  return_tensors='''pt'''					).to(0					)
								lowerCamelCase__		:		str		     =model.generate(**_UpperCAmelCase					)

								# test with `flan-t5-small`
								lowerCamelCase__		:		Optional[Any]		     =TaForConditionalGeneration.from_pretrained(
								    self.dense_act_model_name,  load_in_abit=_UpperCAmelCase,  device_map='''auto'''					)
								lowerCamelCase__		:		Optional[Any]		     =self.tokenizer(self.input_text,  return_tensors='''pt'''					).to(0					)
								lowerCamelCase__		:		List[Any]		     =model.generate(**_UpperCAmelCase					)





class 	__SCREAMING_SNAKE_CASE     (				_UpperCAmelCase       ):



					def 		snake_case	(	self							:  Any					)-> Optional[Any]:
								super().setUp()
								# model_name
								lowerCamelCase__		:		Optional[Any]		     ='''bigscience/bloom-560m'''
								lowerCamelCase__		:		int		     ='''t5-small'''

								# Different types of model

								lowerCamelCase__		:		str		     =AutoModel.from_pretrained(self.model_name,  load_in_abit=_UpperCAmelCase,  device_map='''auto'''					)
								# Sequence classification model
								lowerCamelCase__		:		int		     =AutoModelForSequenceClassification.from_pretrained(
								    self.model_name,  load_in_abit=_UpperCAmelCase,  device_map='''auto'''					)
								# CausalLM model
								lowerCamelCase__		:		int		     =AutoModelForCausalLM.from_pretrained(self.model_name,  load_in_abit=_UpperCAmelCase,  device_map='''auto'''					)
								# Seq2seq model
								lowerCamelCase__		:		List[Any]		     =AutoModelForSeqaSeqLM.from_pretrained(
								    self.seq_to_seq_name,  load_in_abit=_UpperCAmelCase,  device_map='''auto'''					)



					def 		snake_case	(	self							:  int					)-> str:
								del self.base_model
								del self.sequence_model
								del self.model_abit
								del self.seq_to_seq_model

								gc.collect()
								torch.cuda.empty_cache()




					def 		snake_case	(	self							:  Optional[Any]					)-> List[Any]:
								from bitsandbytes.nn import Paramsabit

								self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit					)

								# Other heads should be nn.Parameter
								self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter					)
								self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter					)
								self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter					)





class 	__SCREAMING_SNAKE_CASE     (				_UpperCAmelCase       ):



					def 		snake_case	(	self							:  List[Any]					)-> Optional[Any]:
								super().setUp()



					def 		snake_case	(	self							:  Any					)-> Optional[int]:
								del self.pipe

								gc.collect()
								torch.cuda.empty_cache()




					def 		snake_case	(	self							:  Tuple					)-> Union[str, Any]:
								lowerCamelCase__		:		List[str]		     =pipeline(
								    '''text-generation''',  model=self.model_name,  model_kwargs={'''device_map''': '''auto''', '''load_in_4bit''': True, '''torch_dtype''': torch.floataa},  max_new_tokens=self.MAX_NEW_TOKENS,  )

								# Real second forward pass
								lowerCamelCase__		:		Optional[int]		     =self.pipe(self.input_text					)
								self.assertIn(pipeline_output[0]['''generated_text'''],  self.EXPECTED_OUTPUTS					)





@require_torch_multi_gpu
class 	__SCREAMING_SNAKE_CASE     (				_UpperCAmelCase       ):



					def 		snake_case	(	self							:  Any					)-> str:
								super().setUp()




					def 		snake_case	(	self							:  Optional[int]					)-> int:
								lowerCamelCase__		:		Tuple		     =AutoModelForCausalLM.from_pretrained(
								    self.model_name,  load_in_abit=_UpperCAmelCase,  device_map='''balanced'''					)

								# Check correct device map
								self.assertEqual(set(model_parallel.hf_device_map.values()					),  {0, 1}					)

								# Check that inference pass works on the model
								lowerCamelCase__		:		str		     =self.tokenizer(self.input_text,  return_tensors='''pt'''					)

								# Second real batch
								lowerCamelCase__		:		str		     =model_parallel.generate(input_ids=encoded_input['''input_ids'''].to(0					),  max_new_tokens=10					)
								self.assertIn(self.tokenizer.decode(output_parallel[0],  skip_special_tokens=_UpperCAmelCase					),  self.EXPECTED_OUTPUTS					)





class 	__SCREAMING_SNAKE_CASE     (				_UpperCAmelCase       ):



					def 		snake_case	(	self							:  List[str]					)-> Optional[int]:
								lowerCamelCase__		:		Tuple		     ='''facebook/opt-350m'''
								super().setUp()




					def 		snake_case	(	self							:  Dict					)-> str:
								if version.parse(importlib.metadata.version('''bitsandbytes'''					)					) < version.parse('''0.37.0'''					):
											return

								# Step 1: freeze all parameters
								lowerCamelCase__		:		Tuple		     =AutoModelForCausalLM.from_pretrained(self.model_name,  load_in_abit=_UpperCAmelCase					)

								self.assertEqual(set(model.hf_device_map.values()					),  {torch.cuda.current_device()}					)

								for param in model.parameters():
											lowerCamelCase__		:		Optional[int]		     =False  # freeze the model - train adapters later
											if param.ndim == 1:
														# cast the small parameters (e.g. layernorm) to fp32 for stability
														lowerCamelCase__		:		int		     =param.data.to(torch.floataa					)

        # Step 2: add adapters
								for _, module in model.named_modules():
											if "OPTAttention" in repr(type(_UpperCAmelCase					)					):
														lowerCamelCase__		:		int		     =LoRALayer(module.q_proj,  rank=16					)
														lowerCamelCase__		:		List[str]		     =LoRALayer(module.k_proj,  rank=16					)
														lowerCamelCase__		:		Optional[Any]		     =LoRALayer(module.v_proj,  rank=16					)

        # Step 3: dummy batch
								lowerCamelCase__		:		int		     =self.tokenizer('''Test batch ''',  return_tensors='''pt'''					).to(0					)

								# Step 4: Check if the gradient is not None
								with torch.cuda.amp.autocast():
											lowerCamelCase__		:		List[str]		     =model.forward(**_UpperCAmelCase					)
											out.logits.norm().backward()

								for module in model.modules():
											if isinstance(_UpperCAmelCase,  _UpperCAmelCase					):
														self.assertTrue(module.adapter[1].weight.grad is not None					)
														self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0					)
											elif isinstance(_UpperCAmelCase,  nn.Embedding					):
														self.assertTrue(module.weight.grad is None					)





class 	__SCREAMING_SNAKE_CASE     (				_UpperCAmelCase       ):
					_a             =							"gpt2-xl"
					_a             =							3.3191854854152187









"""simple docstring"""






def 					snake_case__       (		__lowerCamelCase    :   int     ):
			"""simple docstring"""
			lowerCamelCase__		:		List[Any]		     =[0] * len(__lowerCamelCase     )
			lowerCamelCase__		:		List[Any]		     =[]
			lowerCamelCase__		:		List[Any]		     =[1] * len(__lowerCamelCase     )

			for values in graph.values():
						for i in values:
									indegree[i] += 1

			for i in range(len(__lowerCamelCase     )     ):
						if indegree[i] == 0:
									queue.append(__lowerCamelCase     )

			while queue:
						lowerCamelCase__		:		Tuple		     =queue.pop(0     )
						for x in graph[vertex]:
									indegree[x] -= 1

									if long_dist[vertex] + 1 > long_dist[x]:
												lowerCamelCase__		:		Optional[Any]		     =long_dist[vertex] + 1

									if indegree[x] == 0:
												queue.append(__lowerCamelCase     )

			print(max(__lowerCamelCase     )     )


# Adjacency list of Graph
_lowercase			:			Optional[Any]										=						{0: [2, 3, 4], 1: [2, 7], 2: [5], 3: [5, 7], 4: [7], 5: [6], 6: [7], 7: []}

longest_distance(graph)




