

from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import (
			    TFRoFormerForCausalLM,
			    TFRoFormerForMaskedLM,
			    TFRoFormerForMultipleChoice,
			    TFRoFormerForQuestionAnswering,
			    TFRoFormerForSequenceClassification,
			    TFRoFormerForTokenClassification,
			    TFRoFormerModel,
			)
			from transformers.models.roformer.modeling_tf_roformer import (
			    TFRoFormerSelfAttention,
			    TFRoFormerSinusoidalPositionalEmbedding,
			)

class        a	:




							"""simple docstring"""


							def __init__(							self :						Any					,			lowerCamelCase :						List[Any]					,			lowerCamelCase :						Optional[Any]=13					,			lowerCamelCase :						Optional[int]=7					,			lowerCamelCase :						str=True					,			lowerCamelCase :						Optional[int]=True					,			lowerCamelCase :						List[Any]=True					,			lowerCamelCase :						Dict=True					,			lowerCamelCase :						Union[str, Any]=99					,			lowerCamelCase :						Any=32					,			lowerCamelCase :						List[str]=2					,			lowerCamelCase :						Any=4					,			lowerCamelCase :						List[str]=37					,			lowerCamelCase :						Dict="gelu"					,			lowerCamelCase :						Optional[int]=0.1					,			lowerCamelCase :						List[Any]=0.1					,			lowerCamelCase :						Tuple=512					,			lowerCamelCase :						Any=16					,			lowerCamelCase :						str=2					,			lowerCamelCase :						List[Any]=0.02					,			lowerCamelCase :						Tuple=3					,			lowerCamelCase :						Any=4					,			lowerCamelCase :						Optional[int]=None					,			)			->					int:
														__snake_case	:    Any											=     parent
														__snake_case	:    int											=     13
														__snake_case	:    Dict											=     7
														__snake_case	:    str											=     True
														__snake_case	:    Tuple											=     True
														__snake_case	:    Optional[Any]											=     True
														__snake_case	:    List[Any]											=     True
														__snake_case	:    Union[str, Any]											=     99
														__snake_case	:    Union[str, Any]											=     32
														__snake_case	:    Optional[Any]											=     2
														__snake_case	:    Tuple											=     4
														__snake_case	:    Union[str, Any]											=     37
														__snake_case	:    int											=     "gelu"
														__snake_case	:    Union[str, Any]											=     0.1
														__snake_case	:    Tuple											=     0.1
														__snake_case	:    str											=     512
														__snake_case	:    Any											=     16
														__snake_case	:    int											=     2
														__snake_case	:    int											=     0.02
														__snake_case	:    Optional[Any]											=     3
														__snake_case	:    Dict											=     4
														__snake_case	:    int											=     None

							def 					__snake_case      (							self :						Dict       )			->					Union[str, Any]:
														__snake_case	:    Optional[int]											=     ids_tensor([self.batch_size, self.seq_length]					,			self.vocab_size       )

														__snake_case	:    Optional[int]											=     None
														if self.use_input_mask:
																					__snake_case	:    Optional[Any]											=     random_attention_mask([self.batch_size, self.seq_length]       )

														__snake_case	:    List[Any]											=     None
														if self.use_token_type_ids:
																					__snake_case	:    int											=     ids_tensor([self.batch_size, self.seq_length]					,			self.type_vocab_size       )

														__snake_case	:    List[Any]											=     None
														__snake_case	:    Tuple											=     None
														__snake_case	:    Any											=     None
														if self.use_labels:
																					__snake_case	:    List[str]											=     ids_tensor([self.batch_size]					,			self.type_sequence_label_size       )
																					__snake_case	:    Any											=     ids_tensor([self.batch_size, self.seq_length]					,			self.num_labels       )
																					__snake_case	:    Optional[Any]											=     ids_tensor([self.batch_size]					,			self.num_choices       )

														__snake_case	:    Union[str, Any]											=     RoFormerConfig(
														    vocab_size=self.vocab_size					,			hidden_size=self.hidden_size					,			num_hidden_layers=self.num_hidden_layers					,			num_attention_heads=self.num_attention_heads					,			intermediate_size=self.intermediate_size					,			hidden_act=self.hidden_act					,			hidden_dropout_prob=self.hidden_dropout_prob					,			attention_probs_dropout_prob=self.attention_probs_dropout_prob					,			max_position_embeddings=self.max_position_embeddings					,			type_vocab_size=self.type_vocab_size					,			initializer_range=self.initializer_range					,			return_dict=lowerCamelCase					,			)

														return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

							def 					__snake_case      (							self :						Optional[int]					,			lowerCamelCase :						Optional[int]					,			lowerCamelCase :						Tuple					,			lowerCamelCase :						Any					,			lowerCamelCase :						str					,			lowerCamelCase :						List[str]					,			lowerCamelCase :						int					,			lowerCamelCase :						Optional[int]       )			->					str:
														__snake_case	:    List[Any]											=     TFRoFormerModel(config=lowerCamelCase       )
														__snake_case	:    Optional[Any]											=     {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}

														__snake_case	:    Union[str, Any]											=     [input_ids, input_mask]
														__snake_case	:    Optional[int]											=     model(lowerCamelCase       )

														__snake_case	:    Optional[Any]											=     model(lowerCamelCase       )

														self.parent.assertEqual(result.last_hidden_state.shape					,			(self.batch_size, self.seq_length, self.hidden_size)       )

							def 					__snake_case      (							self :						Optional[int]					,			lowerCamelCase :						Optional[int]					,			lowerCamelCase :						List[str]					,			lowerCamelCase :						Tuple					,			lowerCamelCase :						Optional[int]					,			lowerCamelCase :						Optional[Any]					,			lowerCamelCase :						Tuple					,			lowerCamelCase :						Optional[int]       )			->					List[Any]:
														__snake_case	:    Any											=     True
														__snake_case	:    str											=     TFRoFormerForCausalLM(config=lowerCamelCase       )
														__snake_case	:    Optional[int]											=     {
														    "input_ids": input_ids,
														    "attention_mask": input_mask,
														    "token_type_ids": token_type_ids,
														}
														__snake_case	:    List[Any]											=     model(lowerCamelCase       )["logits"]
														self.parent.assertListEqual(
														    list(prediction_scores.numpy().shape       )					,			[self.batch_size, self.seq_length, self.vocab_size]       )

							def 					__snake_case      (							self :						Optional[Any]					,			lowerCamelCase :						Union[str, Any]					,			lowerCamelCase :						Any					,			lowerCamelCase :						str					,			lowerCamelCase :						Optional[int]					,			lowerCamelCase :						List[Any]					,			lowerCamelCase :						Optional[Any]					,			lowerCamelCase :						Any       )			->					Union[str, Any]:
														__snake_case	:    List[Any]											=     TFRoFormerForMaskedLM(config=lowerCamelCase       )
														__snake_case	:    Dict											=     {
														    "input_ids": input_ids,
														    "attention_mask": input_mask,
														    "token_type_ids": token_type_ids,
														}
														__snake_case	:    str											=     model(lowerCamelCase       )
														self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.seq_length, self.vocab_size)       )

							def 					__snake_case      (							self :						Dict					,			lowerCamelCase :						Optional[Any]					,			lowerCamelCase :						Tuple					,			lowerCamelCase :						Optional[Any]					,			lowerCamelCase :						Dict					,			lowerCamelCase :						Any					,			lowerCamelCase :						Union[str, Any]					,			lowerCamelCase :						Union[str, Any]       )			->					Optional[Any]:
														__snake_case	:    Dict											=     self.num_labels
														__snake_case	:    Tuple											=     TFRoFormerForSequenceClassification(config=lowerCamelCase       )
														__snake_case	:    List[Any]											=     {
														    "input_ids": input_ids,
														    "attention_mask": input_mask,
														    "token_type_ids": token_type_ids,
														}

														__snake_case	:    Tuple											=     model(lowerCamelCase       )
														self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.num_labels)       )

							def 					__snake_case      (							self :						Optional[int]					,			lowerCamelCase :						List[Any]					,			lowerCamelCase :						Optional[Any]					,			lowerCamelCase :						Tuple					,			lowerCamelCase :						int					,			lowerCamelCase :						Optional[int]					,			lowerCamelCase :						Optional[Any]					,			lowerCamelCase :						Union[str, Any]       )			->					str:
														__snake_case	:    List[str]											=     self.num_choices
														__snake_case	:    Optional[Any]											=     TFRoFormerForMultipleChoice(config=lowerCamelCase       )
														__snake_case	:    Optional[int]											=     tf.tile(tf.expand_dims(lowerCamelCase					,			1       )					,			(1, self.num_choices, 1)       )
														__snake_case	:    Optional[Any]											=     tf.tile(tf.expand_dims(lowerCamelCase					,			1       )					,			(1, self.num_choices, 1)       )
														__snake_case	:    Dict											=     tf.tile(tf.expand_dims(lowerCamelCase					,			1       )					,			(1, self.num_choices, 1)       )
														__snake_case	:    Optional[Any]											=     {
														    "input_ids": multiple_choice_inputs_ids,
														    "attention_mask": multiple_choice_input_mask,
														    "token_type_ids": multiple_choice_token_type_ids,
														}
														__snake_case	:    Optional[int]											=     model(lowerCamelCase       )
														self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.num_choices)       )

							def 					__snake_case      (							self :						Optional[int]					,			lowerCamelCase :						Tuple					,			lowerCamelCase :						Union[str, Any]					,			lowerCamelCase :						List[Any]					,			lowerCamelCase :						Optional[Any]					,			lowerCamelCase :						List[str]					,			lowerCamelCase :						Dict					,			lowerCamelCase :						List[str]       )			->					Union[str, Any]:
														__snake_case	:    Optional[int]											=     self.num_labels
														__snake_case	:    Union[str, Any]											=     TFRoFormerForTokenClassification(config=lowerCamelCase       )
														__snake_case	:    List[str]											=     {
														    "input_ids": input_ids,
														    "attention_mask": input_mask,
														    "token_type_ids": token_type_ids,
														}
														__snake_case	:    Any											=     model(lowerCamelCase       )
														self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.seq_length, self.num_labels)       )

							def 					__snake_case      (							self :						List[Any]					,			lowerCamelCase :						Dict					,			lowerCamelCase :						List[Any]					,			lowerCamelCase :						Dict					,			lowerCamelCase :						List[str]					,			lowerCamelCase :						int					,			lowerCamelCase :						List[Any]					,			lowerCamelCase :						int       )			->					Optional[Any]:
														__snake_case	:    List[str]											=     TFRoFormerForQuestionAnswering(config=lowerCamelCase       )
														__snake_case	:    Optional[int]											=     {
														    "input_ids": input_ids,
														    "attention_mask": input_mask,
														    "token_type_ids": token_type_ids,
														}

														__snake_case	:    str											=     model(lowerCamelCase       )
														self.parent.assertEqual(result.start_logits.shape					,			(self.batch_size, self.seq_length)       )
														self.parent.assertEqual(result.end_logits.shape					,			(self.batch_size, self.seq_length)       )






							def 					__snake_case      (							self :						Dict       )			->					str:
														__snake_case	:    Tuple											=     self.prepare_config_and_inputs()
														(
														    __snake_case    
														)	:    Optional[int]											=     config_and_inputs
														__snake_case	:    Tuple											=     {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
														return config, inputs_dict

@require_tf
class        a	(_lowerCAmelCase  ,       _lowerCAmelCase  ,       unittest.TestCase   ):




							"""simple docstring"""


							__UpperCAmelCase					:							Union[str, Any]  						=						(
							    (
							        TFRoFormerModel,
							        TFRoFormerForCausalLM,
							        TFRoFormerForMaskedLM,
							        TFRoFormerForQuestionAnswering,
							        TFRoFormerForSequenceClassification,
							        TFRoFormerForTokenClassification,
							        TFRoFormerForMultipleChoice,
							    )
							    if is_tf_available()
							    else ()
							)
							__UpperCAmelCase					:							Optional[int]  						=						(
							    {
							        "feature-extraction": TFRoFormerModel,
							        "fill-mask": TFRoFormerForMaskedLM,
							        "question-answering": TFRoFormerForQuestionAnswering,
							        "text-classification": TFRoFormerForSequenceClassification,
							        "text-generation": TFRoFormerForCausalLM,
							        "token-classification": TFRoFormerForTokenClassification,
							        "zero-shot": TFRoFormerForSequenceClassification,
							    }
							    if is_tf_available()
							    else {}
							)

							__UpperCAmelCase					:							Dict  						=						False
							__UpperCAmelCase					:							Optional[Any]  						=						False

							def 					__snake_case      (							self :						int					,			lowerCamelCase :						Any					,			lowerCamelCase :						Tuple					,			lowerCamelCase :						Optional[Any]					,			lowerCamelCase :						int					,			lowerCamelCase :						Tuple       )			->					List[Any]:
														if pipeline_test_casse_name == "TextGenerationPipelineTests":
																					return True

														return False

							def 					__snake_case      (							self :						Union[str, Any]       )			->					str:
														__snake_case	:    List[Any]											=     TFRoFormerModelTester(self       )
														__snake_case	:    Dict											=     ConfigTester(self					,			config_class=lowerCamelCase					,			hidden_size=37       )

							def 					__snake_case      (							self :						Union[str, Any]       )			->					Optional[Any]:
														self.config_tester.run_common_tests()

							def 					__snake_case      (							self :						str       )			->					str:
														__snake_case	:    List[str]											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(*lowerCamelCase       )

							def 					__snake_case      (							self :						Optional[int]       )			->					Tuple:
														__snake_case	:    Optional[int]											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase       )

							def 					__snake_case      (							self :						List[str]       )			->					Dict:
														__snake_case	:    List[str]											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_lm_head(*lowerCamelCase       )

							def 					__snake_case      (							self :						Tuple       )			->					Optional[int]:
														__snake_case	:    Dict											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase       )

							def 					__snake_case      (							self :						int       )			->					Optional[Any]:
														__snake_case	:    Optional[Any]											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_question_answering(*lowerCamelCase       )

							def 					__snake_case      (							self :						Union[str, Any]       )			->					List[str]:
														__snake_case	:    Optional[int]											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase       )

							def 					__snake_case      (							self :						int       )			->					Optional[int]:
														__snake_case	:    Tuple											=     self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_token_classification(*lowerCamelCase       )






							@slow
							def 					__snake_case      (							self :						Union[str, Any]       )			->					Optional[int]:
														__snake_case	:    Optional[int]											=     TFRoFormerModel.from_pretrained("junnyu/roformer_chinese_base"       )
														self.assertIsNotNone(lowerCamelCase       )

@require_tf
class        a	(unittest.TestCase   ):




							"""simple docstring"""


							@slow
							def 					__snake_case      (							self :						str       )			->					Union[str, Any]:
														__snake_case	:    Optional[Any]											=     TFRoFormerForMaskedLM.from_pretrained("junnyu/roformer_chinese_base"       )
														__snake_case	:    int											=     tf.constant([[0, 1, 2, 3, 4, 5]]       )
														__snake_case	:    Optional[Any]											=     model(lowerCamelCase       )[0]

														# TODO Replace vocab size
														__snake_case	:    Union[str, Any]											=     50000

														__snake_case	:    int											=     [1, 6, vocab_size]
														self.assertEqual(output.shape					,			lowerCamelCase       )

														print(output[:, :3, :3]       )

														# TODO Replace values below with what was printed above.
														__snake_case	:    int											=     tf.constant(
														    [
														        [
														            [-0.12_05_33_41, -1.0_26_49_01, 0.29_22_19_46],
														            [-1.5_13_37_83, 0.19_74_33, 0.15_19_06_07],
														            [-5.0_13_54_03, -3.90_02_56, -0.84_03_87_64],
														        ]
														    ]       )
														tf.debugging.assert_near(output[:, :3, :3]					,			lowerCamelCase					,			atol=1E-4       )

@require_tf
class        a	(unittest.TestCase   ):




							"""simple docstring"""


							__UpperCAmelCase					:							int  						=						1e-4

							def 					__snake_case      (							self :						Any       )			->					List[Any]:
														__snake_case	:    Optional[int]											=     tf.constant([[4, 10]]       )
														__snake_case	:    Optional[Any]											=     TFRoFormerSinusoidalPositionalEmbedding(num_positions=6					,			embedding_dim=6       )

														__snake_case	:    List[str]											=     emba(input_ids.shape       )
														__snake_case	:    List[Any]											=     tf.constant(
														    [[0.00_00, 0.00_00, 0.00_00, 1.00_00, 1.00_00, 1.00_00], [0.84_15, 0.04_64, 0.00_22, 0.54_03, 0.99_89, 1.00_00]]       )

														tf.debugging.assert_near(lowerCamelCase					,			lowerCamelCase					,			atol=self.tolerance       )






							def 					__snake_case      (							self :						Optional[Any]       )			->					Dict:
														__snake_case	:    str											=     tf.constant(
														    [
														        [0.00_00, 0.00_00, 0.00_00, 0.00_00, 0.00_00],
														        [0.84_15, 0.82_19, 0.80_20, 0.78_19, 0.76_17],
														        [0.90_93, 0.93_64, 0.95_81, 0.97_49, 0.98_70],
														    ]       )
														__snake_case	:    Tuple											=     TFRoFormerSinusoidalPositionalEmbedding(num_positions=512					,			embedding_dim=512       )
														emba([2, 16, 512]       )
														__snake_case	:    Union[str, Any]											=     emba.weight[:3, :5]

														tf.debugging.assert_near(lowerCamelCase					,			lowerCamelCase					,			atol=self.tolerance       )

@require_tf
class        a	(unittest.TestCase   ):




							"""simple docstring"""


							__UpperCAmelCase					:							Union[str, Any]  						=						1e-4

							def 					__snake_case      (							self :						Any       )			->					Dict:
														# 2,12,16,64
														__snake_case	:    List[Any]											=     tf.reshape(tf.range(2 * 12 * 16 * 64					,			dtype=tf.floataa       )					,			shape=(2, 12, 16, 64)       ) / 100

														__snake_case	:    Tuple											=     -tf.reshape(tf.range(2 * 12 * 16 * 64					,			dtype=tf.floataa       )					,			shape=(2, 12, 16, 64)       ) / 100

														__snake_case	:    Optional[Any]											=     TFRoFormerSinusoidalPositionalEmbedding(num_positions=32					,			embedding_dim=64       )
														__snake_case	:    Any											=     embed_positions([2, 16, 768]       )[None, None, :, :]

														__snake_case	:    int											=     TFRoFormerSelfAttention.apply_rotary_position_embeddings(
														    lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase       )

														__snake_case	:    str											=     tf.constant(
														    [
														        [0.00_00, 0.01_00, 0.02_00, 0.03_00, 0.04_00, 0.05_00, 0.06_00, 0.07_00],
														        [-0.20_12, 0.88_97, 0.02_63, 0.94_01, 0.20_74, 0.94_63, 0.34_81, 0.93_43],
														        [-1.70_57, 0.62_71, -1.21_45, 1.38_97, -0.63_03, 1.76_47, -0.11_73, 1.89_85],
														        [-2.17_31, -1.63_97, -2.73_58, 0.28_54, -2.18_40, 1.71_83, -1.30_18, 2.48_71],
														        [0.27_17, -3.61_73, -2.92_06, -2.19_88, -3.66_38, 0.38_58, -2.91_55, 2.29_80],
														        [3.98_59, -2.15_80, -0.79_84, -4.49_04, -4.11_81, -2.02_52, -4.47_82, 1.12_53],
														    ]       )
														__snake_case	:    str											=     tf.constant(
														    [
														        [0.00_00, -0.01_00, -0.02_00, -0.03_00, -0.04_00, -0.05_00, -0.06_00, -0.07_00],
														        [0.20_12, -0.88_97, -0.02_63, -0.94_01, -0.20_74, -0.94_63, -0.34_81, -0.93_43],
														        [1.70_57, -0.62_71, 1.21_45, -1.38_97, 0.63_03, -1.76_47, 0.11_73, -1.89_85],
														        [2.17_31, 1.63_97, 2.73_58, -0.28_54, 2.18_40, -1.71_83, 1.30_18, -2.48_71],
														        [-0.27_17, 3.61_73, 2.92_06, 2.19_88, 3.66_38, -0.38_58, 2.91_55, -2.29_80],
														        [-3.98_59, 2.15_80, 0.79_84, 4.49_04, 4.11_81, 2.02_52, 4.47_82, -1.12_53],
														    ]       )

														tf.debugging.assert_near(query_layer[0, 0, :6, :8]					,			lowerCamelCase					,			atol=self.tolerance       )
														tf.debugging.assert_near(key_layer[0, 0, :6, :8]					,			lowerCamelCase					,			atol=self.tolerance       )





import importlib.metadata
import warnings
from copy import deepcopy

from packaging import version

from ..utils import logging
from .import_utils import is_accelerate_available, is_bitsandbytes_available


if is_bitsandbytes_available():
			import bitsandbytes as bnb
			import torch
			import torch.nn as nn

			from ..pytorch_utils import ConvaD

if is_accelerate_available():
			from accelerate import init_empty_weights
			from accelerate.utils import find_tied_parameters

_snake_case  :	Any			      =       logging.get_logger(__name__)







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase=None							,			__lowerCamelCase=None			):
							# Recurse if needed
							if "." in tensor_name:
														__snake_case	:    Tuple											=     tensor_name.split("."			)
														for split in splits[:-1]:
																					__snake_case	:    List[str]											=     getattr(__lowerCamelCase							,			__lowerCamelCase			)
																					if new_module is None:
																												raise ValueError(F'{module} has no attribute {split}.'			)
																					__snake_case	:    Optional[int]											=     new_module
														__snake_case	:    Union[str, Any]											=     splits[-1]

							if tensor_name not in module._parameters and tensor_name not in module._buffers:
														raise ValueError(F'{module} does not have a parameter or a buffer named {tensor_name}.'			)
							__snake_case	:    Optional[int]											=     tensor_name in module._buffers
							__snake_case	:    List[str]											=     getattr(__lowerCamelCase							,			__lowerCamelCase			)

							if old_value.device == torch.device("meta"			) and device not in ["meta", torch.device("meta"			)] and value is None:
														raise ValueError(F'{tensor_name} is on the meta device, we need a `value` to put in on {device}.'			)

							__snake_case	:    List[Any]											=     False
							__snake_case	:    Optional[int]											=     False
							if is_buffer or not is_bitsandbytes_available():
														__snake_case	:    Dict											=     False
														__snake_case	:    Optional[int]											=     False
							else:
														__snake_case	:    Any											=     hasattr(bnb.nn							,			"Params4bit"			) and isinstance(module._parameters[tensor_name]							,			bnb.nn.Paramsabit			)
														__snake_case	:    Union[str, Any]											=     isinstance(module._parameters[tensor_name]							,			bnb.nn.IntaParams			)

							if is_abit or is_abit:
														__snake_case	:    Union[str, Any]											=     module._parameters[tensor_name]
														if param.device.type != "cuda":
																					if value is None:
																												__snake_case	:    Any											=     old_value.to(__lowerCamelCase			)
																					elif isinstance(__lowerCamelCase							,			torch.Tensor			):
																												__snake_case	:    int											=     value.to("cpu"			)
																												if value.dtype == torch.inta:
																																			__snake_case	:    Optional[int]											=     version.parse(importlib.metadata.version("bitsandbytes"			)			) > version.parse(
																																			    "0.37.2"			)
																																			if not is_abit_serializable:
																																										raise ValueError(
																																										    "Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. "
																																										    "Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`."			)
																					else:
																												__snake_case	:    Optional[int]											=     torch.tensor(__lowerCamelCase							,			device="cpu"			)

																					# Support models using `Conv1D` in place of `nn.Linear` (e.g. gpt2) by transposing the weight matrix prior to quantization.
																					# Since weights are saved in the correct "orientation", we skip transposing when loading.
																					if issubclass(module.source_cls							,			__lowerCamelCase			) and fpaa_statistics is None:
																												__snake_case	:    List[Any]											=     new_value.T

																					__snake_case	:    Tuple											=     old_value.__dict__
																					if is_abit:
																												__snake_case	:    List[str]											=     bnb.nn.IntaParams(__lowerCamelCase							,			requires_grad=__lowerCamelCase							,			**__lowerCamelCase			).to(__lowerCamelCase			)
																					elif is_abit:
																												__snake_case	:    Optional[int]											=     bnb.nn.Paramsabit(__lowerCamelCase							,			requires_grad=__lowerCamelCase							,			**__lowerCamelCase			).to(__lowerCamelCase			)

																					__snake_case	:    Dict											=     new_value
																					if fpaa_statistics is not None:
																												setattr(module.weight							,			"SCB"							,			fpaa_statistics.to(__lowerCamelCase			)			)

							else:
														if value is None:
																					__snake_case	:    Tuple											=     old_value.to(__lowerCamelCase			)
														elif isinstance(__lowerCamelCase							,			torch.Tensor			):
																					__snake_case	:    Dict											=     value.to(__lowerCamelCase			)
														else:
																					__snake_case	:    List[Any]											=     torch.tensor(__lowerCamelCase							,			device=__lowerCamelCase			)

														if is_buffer:
																					__snake_case	:    Optional[Any]											=     new_value
														else:
																					__snake_case	:    int											=     nn.Parameter(__lowerCamelCase							,			requires_grad=old_value.requires_grad			)
																					__snake_case	:    Tuple											=     new_value







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase=None							,			__lowerCamelCase=None							,			__lowerCamelCase=None							,			__lowerCamelCase=False			):
							for name, module in model.named_children():
														if current_key_name is None:
																					__snake_case	:    List[Any]											=     []
														current_key_name.append(__lowerCamelCase			)

														if (isinstance(__lowerCamelCase							,			nn.Linear			) or isinstance(__lowerCamelCase							,			__lowerCamelCase			)) and name not in modules_to_not_convert:
																					# Check if the current key is not in the `modules_to_not_convert`
																					if not any(key in ".".join(__lowerCamelCase			) for key in modules_to_not_convert			):
																												with init_empty_weights():
																																			if isinstance(__lowerCamelCase							,			__lowerCamelCase			):
																																										__snake_case				,    __snake_case	:    Union[str, Any]											=     module.weight.shape
																																			else:
																																										__snake_case	:    List[Any]											=     module.in_features
																																										__snake_case	:    Tuple											=     module.out_features

																																			if quantization_config.quantization_method() == "llm_int8":
																																										__snake_case	:    Tuple											=     bnb.nn.LinearabitLt(
																																										    __lowerCamelCase							,			__lowerCamelCase							,			module.bias is not None							,			has_fpaa_weights=quantization_config.llm_inta_has_fpaa_weight							,			threshold=quantization_config.llm_inta_threshold							,			)
																																										__snake_case	:    List[str]											=     True
																																			else:
																																										if (
																																										    quantization_config.llm_inta_skip_modules is not None
																																										    and name in quantization_config.llm_inta_skip_modules
																																										):
																																																	pass
																																										else:
																																																	__snake_case	:    Optional[int]											=     bnb.nn.Linearabit(
																																																	    __lowerCamelCase							,			__lowerCamelCase							,			module.bias is not None							,			quantization_config.bnb_abit_compute_dtype							,			compress_statistics=quantization_config.bnb_abit_use_double_quant							,			quant_type=quantization_config.bnb_abit_quant_type							,			)
																																																	__snake_case	:    int											=     True
                    # Store the module class in case we need to transpose the weight later
																																			__snake_case	:    str											=     type(__lowerCamelCase			)
																																			# Force requires grad to False to avoid unexpected errors
																																			model._modules[name].requires_grad_(__lowerCamelCase			)
														if len(list(module.children()			)			) > 0:
																					__snake_case				,    __snake_case	:    Tuple											=     _replace_with_bnb_linear(
																					    __lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase							,			has_been_replaced=__lowerCamelCase							,			)
														# Remove the last key for recursion
														current_key_name.pop(-1			)
							return model, has_been_replaced







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase=None							,			__lowerCamelCase=None							,			__lowerCamelCase=None			):
							__snake_case	:    List[str]											=     ["lm_head"] if modules_to_not_convert is None else modules_to_not_convert
							__snake_case				,    __snake_case	:    Any											=     _replace_with_bnb_linear(
							    __lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase			)

							if not has_been_replaced:
														logger.warning(
														    "You are loading your model in 8bit or 4bit but no linear modules were found in your model."
														    " Please double check your model architecture, or submit an issue on github if you think this is"
														    " a bug."			)

							return model







def        lowerCAmelCase_  (      *__lowerCamelCase							,			**__lowerCamelCase			):
							warnings.warn(
							    "`replace_8bit_linear` will be deprecated in a future version, please use `replace_with_bnb_linear` instead"							,			__lowerCamelCase							,			)
							return replace_with_bnb_linear(*__lowerCamelCase							,			**__lowerCamelCase			)







def        lowerCAmelCase_  (      *__lowerCamelCase							,			**__lowerCamelCase			):
							warnings.warn(
							    "`set_module_8bit_tensor_to_device` will be deprecated in a future version, please use `set_module_quantized_tensor_to_device` instead"							,			__lowerCamelCase							,			)
							return set_module_quantized_tensor_to_device(*__lowerCamelCase							,			**__lowerCamelCase			)







def        lowerCAmelCase_  (      __lowerCamelCase			):
							__snake_case	:    Optional[int]											=     deepcopy(__lowerCamelCase			)  # this has 0 cost since it is done inside `init_empty_weights` context manager`
							tied_model.tie_weights()

							__snake_case	:    Any											=     find_tied_parameters(__lowerCamelCase			)
							# For compatibility with Accelerate < 0.18
							if isinstance(__lowerCamelCase							,			__lowerCamelCase			):
														__snake_case	:    Dict											=     sum(list(tied_params.values()			)							,			[]			) + list(tied_params.keys()			)
							else:
														__snake_case	:    Any											=     sum(__lowerCamelCase							,			[]			)
							__snake_case	:    Optional[int]											=     len(__lowerCamelCase			) > 0

							# Check if it is a base model
							__snake_case	:    Dict											=     not hasattr(__lowerCamelCase							,			model.base_model_prefix			)

							# Ignore this for base models (BertModel, GPT2Model, etc.)
							if (not has_tied_params) and is_base_model:
														return []

							# otherwise they have an attached head
							__snake_case	:    Optional[int]											=     list(model.named_children()			)
							__snake_case	:    List[str]											=     [list_modules[-1][0]]

							# add last module together with tied weights
							__snake_case	:    int											=     set(__lowerCamelCase			) - set(__lowerCamelCase			)
							__snake_case	:    Optional[Any]											=     list(set(__lowerCamelCase			)			) + list(__lowerCamelCase			)

							# remove ".weight" from the keys
							__snake_case	:    Optional[Any]											=     [".weight", ".bias"]
							__snake_case	:    Any											=     []
							for name in list_untouched:
														for name_to_remove in names_to_remove:
																					if name_to_remove in name:
																												__snake_case	:    Optional[int]											=     name.replace(__lowerCamelCase							,			""			)
														filtered_module_names.append(__lowerCamelCase			)

							return filtered_module_names

