


import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)







def 	_UpperCamelCase   (    UpperCamelCase_       :					int		)  ->   Optional[Any]:

						"""simple docstring"""



						if model.config.model_type == "gpt2":
												return model.transformer.h[0].mlp.c_fc
						return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
					import torch
					import torch.nn as nn


					class     __SCREAMING_SNAKE_CASE					(   nn.Module):






											def __init__(	self     ,						_UpperCamelCase     ,						_UpperCamelCase   ):






																	"""simple docstring"""
																	super().__init__()
																	lowerCAmelCase__    						=						module
																	lowerCAmelCase__    						=						nn.Sequential(
																	    nn.Linear(module.in_features     ,						_lowerCamelCase     ,						bias=_lowerCamelCase   )     ,						nn.Linear(_lowerCamelCase     ,						module.out_features     ,						bias=_lowerCamelCase   )     ,						)
																	lowerCAmelCase__    						=						(2.0 / (5 * min(module.in_features     ,						module.out_features   ))) ** 0.5
																	nn.init.normal_(self.adapter[0].weight     ,						std=_lowerCamelCase   )
																	nn.init.zeros_(self.adapter[1].weight   )
																	self.adapter.to(module.weight.device   )







											def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						*_UpperCamelCase     ,						**_UpperCamelCase   ):






																	"""simple docstring"""
																	return self.module(_lowerCamelCase     ,						*_lowerCamelCase     ,						**_lowerCamelCase   ) + self.adapter(_lowerCamelCase   )


@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class     __SCREAMING_SNAKE_CASE					(   unittest.TestCase):
						_SCREAMING_SNAKE_CASE	:       Any   =					"bigscience/bloom-1b7"

						# Constant values
						_SCREAMING_SNAKE_CASE	:       str   =					2.1_0_9_6_5_9_5_5_2_6_9_2_5_7_4

						_SCREAMING_SNAKE_CASE	:       Tuple   =					"Hello my name is"
						_SCREAMING_SNAKE_CASE	:       Any   =					set()
						EXPECTED_OUTPUTS.add('''Hello my name is John and I am a professional photographer. I''')
						EXPECTED_OUTPUTS.add('''Hello my name is John.\nI am a friend of your father.\n''')
						EXPECTED_OUTPUTS.add('''Hello my name is John Doe, I am a student at the University''')
						_SCREAMING_SNAKE_CASE	:       List[str]   =					10






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						AutoTokenizer.from_pretrained(self.model_name   )




class     __SCREAMING_SNAKE_CASE					(   lowerCAmelCase__):






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												super().setUp()

												# Models and tokenizer
												lowerCAmelCase__    						=						AutoModelForCausalLM.from_pretrained(
												    self.model_name     ,						torch_dtype=torch.floataa     ,						device_map='auto'   )
												lowerCAmelCase__    						=						AutoModelForCausalLM.from_pretrained(self.model_name     ,						load_in_abit=_lowerCamelCase     ,						device_map='auto'   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												del self.model_fpaa
												del self.model_abit

												gc.collect()
												torch.cuda.empty_cache()






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.model_abit.config

												self.assertTrue(hasattr(_lowerCamelCase     ,						'quantization_config'   )   )

												lowerCAmelCase__    						=						config.to_dict()
												lowerCAmelCase__    						=						config.to_diff_dict()

												lowerCAmelCase__    						=						config.to_json_string()






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												from bitsandbytes.nn import Paramsabit

												lowerCAmelCase__    						=						self.model_fpaa.get_memory_footprint()
												lowerCAmelCase__    						=						self.model_abit.get_memory_footprint()

												self.assertAlmostEqual(mem_fpaa / mem_abit     ,						self.EXPECTED_RELATIVE_DIFFERENCE   )
												lowerCAmelCase__    						=						get_some_linear_layer(self.model_abit   )
												self.assertTrue(linear.weight.__class__ == Paramsabit   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												from transformers import TaPreTrainedModel

												self.model_fpaa.get_memory_footprint()
												self.model_abit.get_memory_footprint()

												for name, module in self.model_abit.named_modules():
																		if isinstance(_lowerCamelCase     ,						torch.nn.Linear   ):
																								if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
																														# 4-bit parameters are packed in uint8 variables
																														self.assertTrue(module.weight.dtype == torch.uinta   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.tokenizer(self.input_text     ,						return_tensors='pt'   )
												lowerCAmelCase__    						=						self.model_abit.generate(input_ids=encoded_input['input_ids'].to(0   )     ,						max_new_tokens=10   )

												self.assertIn(self.tokenizer.decode(output_sequences[0]     ,						skip_special_tokens=_lowerCamelCase   )     ,						self.EXPECTED_OUTPUTS   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						BitsAndBytesConfig()
												lowerCAmelCase__    						=						True

												lowerCAmelCase__    						=						AutoModelForCausalLM.from_pretrained(
												    self.model_name     ,						quantization_config=_lowerCamelCase     ,						device_map='auto'   )

												lowerCAmelCase__    						=						self.tokenizer(self.input_text     ,						return_tensors='pt'   )
												lowerCAmelCase__    						=						model_abit_from_config.generate(
												    input_ids=encoded_input['input_ids'].to(0   )     ,						max_new_tokens=10   )

												self.assertIn(self.tokenizer.decode(output_sequences[0]     ,						skip_special_tokens=_lowerCamelCase   )     ,						self.EXPECTED_OUTPUTS   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""





												with self.assertRaises(_lowerCamelCase   ), tempfile.TemporaryDirectory() as tmpdirname:
																		self.model_abit.save_pretrained(_lowerCamelCase   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						BitsAndBytesConfig()

												with self.assertRaises(_lowerCamelCase   ):
																		lowerCAmelCase__    						=						AutoModelForCausalLM.from_pretrained(
																		    self.model_name     ,						quantization_config=_lowerCamelCase     ,						load_in_abit=_lowerCamelCase     ,						device_map='auto'     ,						bnb_abit_quant_type='nf4'     ,						)






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""





												with self.assertRaises(_lowerCamelCase   ):
																		# Tries with `str`
																		self.model_abit.to('cpu'   )

												with self.assertRaises(_lowerCamelCase   ):
																		# Tries with a `dtype``
																		self.model_abit.to(torch.floataa   )

												with self.assertRaises(_lowerCamelCase   ):
																		# Tries with a `device`
																		self.model_abit.to(torch.device('cuda:0'   )   )

												with self.assertRaises(_lowerCamelCase   ):
																		# Tries with a `device`
																		self.model_abit.float()

												with self.assertRaises(_lowerCamelCase   ):
																		# Tries with a `device`
																		self.model_abit.half()

												# Test if we did not break anything
												lowerCAmelCase__    						=						self.tokenizer(self.input_text     ,						return_tensors='pt'   )

												lowerCAmelCase__    						=						self.model_fpaa.to(torch.floataa   )
												lowerCAmelCase__    						=						self.model_fpaa.generate(input_ids=encoded_input['input_ids'].to(0   )     ,						max_new_tokens=10   )

												# Check this does not throw an error
												lowerCAmelCase__    						=						self.model_fpaa.to('cpu'   )

												# Check this does not throw an error
												lowerCAmelCase__    						=						self.model_fpaa.half()

												# Check this does not throw an error
												lowerCAmelCase__    						=						self.model_fpaa.float()







						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						AutoModelForSeqaSeqLM.from_pretrained('t5-small'     ,						load_in_abit=_lowerCamelCase     ,						device_map='auto'   )
												self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa   )




@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class     __SCREAMING_SNAKE_CASE					(   unittest.TestCase):






						@classmethod
						def       UpperCamelCase__      (	cls   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						't5-small'
												lowerCAmelCase__    						=						'google/flan-t5-small'  # flan-t5 uses dense-act instead of dense-relu-dense
												lowerCAmelCase__    						=						AutoTokenizer.from_pretrained(cls.model_name   )
												lowerCAmelCase__    						=						'Translate in German: Hello, my dog is cute'






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												gc.collect()
												torch.cuda.empty_cache()






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												from transformers import TaForConditionalGeneration

												lowerCAmelCase__    						=						TaForConditionalGeneration._keep_in_fpaa_modules
												lowerCAmelCase__    						=						None

												# test with `t5-small`
												lowerCAmelCase__    						=						TaForConditionalGeneration.from_pretrained(self.model_name     ,						load_in_abit=_lowerCamelCase     ,						device_map='auto'   )
												lowerCAmelCase__    						=						self.tokenizer(self.input_text     ,						return_tensors='pt'   ).to(0   )
												lowerCAmelCase__    						=						model.generate(**_lowerCamelCase   )

												# test with `flan-t5-small`
												lowerCAmelCase__    						=						TaForConditionalGeneration.from_pretrained(
												    self.dense_act_model_name     ,						load_in_abit=_lowerCamelCase     ,						device_map='auto'   )
												lowerCAmelCase__    						=						self.tokenizer(self.input_text     ,						return_tensors='pt'   ).to(0   )
												lowerCAmelCase__    						=						model.generate(**_lowerCamelCase   )
												lowerCAmelCase__    						=						modules







						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												import bitsandbytes as bnb

												from transformers import TaForConditionalGeneration

												# test with `t5-small`
												lowerCAmelCase__    						=						TaForConditionalGeneration.from_pretrained(self.model_name     ,						load_in_abit=_lowerCamelCase     ,						device_map='auto'   )

												# there was a bug with decoders - this test checks that it is fixed
												self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q     ,						bnb.nn.Linearabit   )   )

												lowerCAmelCase__    						=						self.tokenizer(self.input_text     ,						return_tensors='pt'   ).to(0   )
												lowerCAmelCase__    						=						model.generate(**_lowerCamelCase   )

												# test with `flan-t5-small`
												lowerCAmelCase__    						=						TaForConditionalGeneration.from_pretrained(
												    self.dense_act_model_name     ,						load_in_abit=_lowerCamelCase     ,						device_map='auto'   )
												lowerCAmelCase__    						=						self.tokenizer(self.input_text     ,						return_tensors='pt'   ).to(0   )
												lowerCAmelCase__    						=						model.generate(**_lowerCamelCase   )




class     __SCREAMING_SNAKE_CASE					(   lowerCAmelCase__):






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												super().setUp()
												# model_name
												lowerCAmelCase__    						=						'bigscience/bloom-560m'
												lowerCAmelCase__    						=						't5-small'

												# Different types of model

												lowerCAmelCase__    						=						AutoModel.from_pretrained(self.model_name     ,						load_in_abit=_lowerCamelCase     ,						device_map='auto'   )
												# Sequence classification model
												lowerCAmelCase__    						=						AutoModelForSequenceClassification.from_pretrained(
												    self.model_name     ,						load_in_abit=_lowerCamelCase     ,						device_map='auto'   )
												# CausalLM model
												lowerCAmelCase__    						=						AutoModelForCausalLM.from_pretrained(self.model_name     ,						load_in_abit=_lowerCamelCase     ,						device_map='auto'   )
												# Seq2seq model
												lowerCAmelCase__    						=						AutoModelForSeqaSeqLM.from_pretrained(
												    self.seq_to_seq_name     ,						load_in_abit=_lowerCamelCase     ,						device_map='auto'   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												del self.base_model
												del self.sequence_model
												del self.model_abit
												del self.seq_to_seq_model

												gc.collect()
												torch.cuda.empty_cache()







						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												from bitsandbytes.nn import Paramsabit

												self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit   )

												# Other heads should be nn.Parameter
												self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter   )
												self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter   )
												self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter   )




class     __SCREAMING_SNAKE_CASE					(   lowerCAmelCase__):






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												super().setUp()






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												del self.pipe

												gc.collect()
												torch.cuda.empty_cache()







						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						pipeline(
												    'text-generation'     ,						model=self.model_name     ,						model_kwargs={'device_map': 'auto', 'load_in_4bit': True, 'torch_dtype': torch.floataa}     ,						max_new_tokens=self.MAX_NEW_TOKENS     ,						)

												# Real second forward pass
												lowerCAmelCase__    						=						self.pipe(self.input_text   )
												self.assertIn(pipeline_output[0]['generated_text']     ,						self.EXPECTED_OUTPUTS   )




@require_torch_multi_gpu
class     __SCREAMING_SNAKE_CASE					(   lowerCAmelCase__):






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												super().setUp()







						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						AutoModelForCausalLM.from_pretrained(
												    self.model_name     ,						load_in_abit=_lowerCamelCase     ,						device_map='balanced'   )

												# Check correct device map
												self.assertEqual(set(model_parallel.hf_device_map.values()   )     ,						{0, 1}   )

												# Check that inference pass works on the model
												lowerCAmelCase__    						=						self.tokenizer(self.input_text     ,						return_tensors='pt'   )

												# Second real batch
												lowerCAmelCase__    						=						model_parallel.generate(input_ids=encoded_input['input_ids'].to(0   )     ,						max_new_tokens=10   )
												self.assertIn(self.tokenizer.decode(output_parallel[0]     ,						skip_special_tokens=_lowerCamelCase   )     ,						self.EXPECTED_OUTPUTS   )




class     __SCREAMING_SNAKE_CASE					(   lowerCAmelCase__):






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						'facebook/opt-350m'
												super().setUp()







						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""





												if version.parse(importlib.metadata.version('bitsandbytes'   )   ) < version.parse('0.37.0'   ):
																		return

												# Step 1: freeze all parameters
												lowerCAmelCase__    						=						AutoModelForCausalLM.from_pretrained(self.model_name     ,						load_in_abit=_lowerCamelCase   )

												self.assertEqual(set(model.hf_device_map.values()   )     ,						{torch.cuda.current_device()}   )

												for param in model.parameters():
																		lowerCAmelCase__    						=						False  # freeze the model - train adapters later
																		if param.ndim == 1:
																								# cast the small parameters (e.g. layernorm) to fp32 for stability
																								lowerCAmelCase__    						=						param.data.to(torch.floataa   )

        # Step 2: add adapters
												for _, module in model.named_modules():
																		if "OPTAttention" in repr(type(_lowerCamelCase   )   ):
																								lowerCAmelCase__    						=						LoRALayer(module.q_proj     ,						rank=16   )
																								lowerCAmelCase__    						=						LoRALayer(module.k_proj     ,						rank=16   )
																								lowerCAmelCase__    						=						LoRALayer(module.v_proj     ,						rank=16   )

        # Step 3: dummy batch
												lowerCAmelCase__    						=						self.tokenizer('Test batch '     ,						return_tensors='pt'   ).to(0   )

												# Step 4: Check if the gradient is not None
												with torch.cuda.amp.autocast():
																		lowerCAmelCase__    						=						model.forward(**_lowerCamelCase   )
																		out.logits.norm().backward()

												for module in model.modules():
																		if isinstance(_lowerCamelCase     ,						_lowerCamelCase   ):
																								self.assertTrue(module.adapter[1].weight.grad is not None   )
																								self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0   )
																		elif isinstance(_lowerCamelCase     ,						nn.Embedding   ):
																								self.assertTrue(module.weight.grad is None   )




class     __SCREAMING_SNAKE_CASE					(   lowerCAmelCase__):
						_SCREAMING_SNAKE_CASE	:       Dict   =					"gpt2-xl"
						_SCREAMING_SNAKE_CASE	:       Union[str, Any]   =					3.3_1_9_1_8_5_4_8_5_4_1_5_2_1_8_7













import argparse

import torch

from transformers import BertConfig, BertForPreTraining, load_tf_weights_in_bert
from transformers.utils import logging


logging.set_verbosity_info()







def 	_UpperCamelCase   (    UpperCamelCase_       :					Dict    ,       UpperCamelCase_       :					str    ,       UpperCamelCase_       :					Optional[Any]		)  ->   Any:

						"""simple docstring"""



						lowerCAmelCase__    						=						BertConfig.from_json_file(UpperCamelCase_		)
						print(F"Building PyTorch model from configuration: {config}"		)
						lowerCAmelCase__    						=						BertForPreTraining(UpperCamelCase_		)

						# Load weights from tf checkpoint
						load_tf_weights_in_bert(UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_		)

						# Save pytorch-model
						print(F"Save PyTorch model to {pytorch_dump_path}"		)
						torch.save(model.state_dict()    ,       UpperCamelCase_		)


if __name__ == "__main__":
					__snake_case    :    Union[str, Any]         =     argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--tf_checkpoint_path""", default=None, type=str, required=True, help="""Path to the TensorFlow checkpoint path."""
					)
					parser.add_argument(
					    """--bert_config_file""",
					    default=None,
					    type=str,
					    required=True,
					    help=(
					        """The config json file corresponding to the pre-trained BERT model. \n"""
					        """This specifies the model architecture."""
					    ),
					)
					parser.add_argument(
					    """--pytorch_dump_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
					)
					__snake_case    :    List[str]         =     parser.parse_args()
					convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)





