



"""simple docstring"""






import importlib.metadata
import warnings
from copy import deepcopy

from packaging import version

from ..utils import logging
from .import_utils import is_accelerate_available, is_bitsandbytes_available


if is_bitsandbytes_available():
	import bitsandbytes as bnb
	import torch
	import torch.nn as nn

	from ..pytorch_utils import ConvaD

if is_accelerate_available():
	from accelerate import init_empty_weights
	from accelerate.utils import find_tied_parameters

__snake_case				:  Dict =       logging.get_logger(__name__)


def      _lowercase							(    __snake_case   ,__snake_case   ,__snake_case   ,__snake_case=None   ,__snake_case=None      )		->  Tuple:
			# Recurse if needed
			if "." in tensor_name:
						__lowerCAmelCase :    Union[str, Any]				=			tensor_name.split("."      )
						for split in splits[:-1]:
									__lowerCAmelCase :    List[str]				=			getattr(_UpperCAmelCase   ,_UpperCAmelCase      )
									if new_module is None:
												raise ValueError(F"""{module} has no attribute {split}."""      )
									__lowerCAmelCase :    Optional[Any]				=			new_module
						__lowerCAmelCase :    str				=			splits[-1]

			if tensor_name not in module._parameters and tensor_name not in module._buffers:
						raise ValueError(F"""{module} does not have a parameter or a buffer named {tensor_name}."""      )
			__lowerCAmelCase :    int				=			tensor_name in module._buffers
			__lowerCAmelCase :    List[Any]				=			getattr(_UpperCAmelCase   ,_UpperCAmelCase      )

			if old_value.device == torch.device("meta"      ) and device not in ["meta", torch.device("meta"      )] and value is None:
						raise ValueError(F"""{tensor_name} is on the meta device, we need a `value` to put in on {device}."""      )

			__lowerCAmelCase :    str				=			False
			__lowerCAmelCase :    str				=			False
			if is_buffer or not is_bitsandbytes_available():
						__lowerCAmelCase :    int				=			False
						__lowerCAmelCase :    str				=			False
			else:
						__lowerCAmelCase :    Union[str, Any]				=			hasattr(bnb.nn   ,"Params4bit"      ) and isinstance(module._parameters[tensor_name]   ,bnb.nn.Paramsabit      )
						__lowerCAmelCase :    List[Any]				=			isinstance(module._parameters[tensor_name]   ,bnb.nn.IntaParams      )

			if is_abit or is_abit:
						__lowerCAmelCase :    List[str]				=			module._parameters[tensor_name]
						if param.device.type != "cuda":
									if value is None:
												__lowerCAmelCase :    List[str]				=			old_value.to(_UpperCAmelCase      )
									elif isinstance(_UpperCAmelCase   ,torch.Tensor      ):
												__lowerCAmelCase :    Optional[int]				=			value.to("cpu"      )
												if value.dtype == torch.inta:
															__lowerCAmelCase :    List[Any]				=			version.parse(importlib.metadata.version("bitsandbytes"      )      ) > version.parse(
															    "0.37.2"      )
															if not is_abit_serializable:
																		raise ValueError(
																		    "Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. "
																		    "Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`."      )
									else:
												__lowerCAmelCase :    int				=			torch.tensor(_UpperCAmelCase   ,device="cpu"      )

									# Support models using `Conv1D` in place of `nn.Linear` (e.g. gpt2) by transposing the weight matrix prior to quantization.
									# Since weights are saved in the correct "orientation", we skip transposing when loading.
									if issubclass(module.source_cls   ,_UpperCAmelCase      ) and fpaa_statistics is None:
												__lowerCAmelCase :    Tuple				=			new_value.T

									__lowerCAmelCase :    int				=			old_value.__dict__
									if is_abit:
												__lowerCAmelCase :    Any				=			bnb.nn.IntaParams(_UpperCAmelCase   ,requires_grad=_UpperCAmelCase   ,**_UpperCAmelCase      ).to(_UpperCAmelCase      )
									elif is_abit:
												__lowerCAmelCase :    Dict				=			bnb.nn.Paramsabit(_UpperCAmelCase   ,requires_grad=_UpperCAmelCase   ,**_UpperCAmelCase      ).to(_UpperCAmelCase      )

									__lowerCAmelCase :    int				=			new_value
									if fpaa_statistics is not None:
												setattr(module.weight   ,"SCB"   ,fpaa_statistics.to(_UpperCAmelCase      )      )

			else:
						if value is None:
									__lowerCAmelCase :    Optional[Any]				=			old_value.to(_UpperCAmelCase      )
						elif isinstance(_UpperCAmelCase   ,torch.Tensor      ):
									__lowerCAmelCase :    Optional[int]				=			value.to(_UpperCAmelCase      )
						else:
									__lowerCAmelCase :    Tuple				=			torch.tensor(_UpperCAmelCase   ,device=_UpperCAmelCase      )

						if is_buffer:
									__lowerCAmelCase :    Dict				=			new_value
						else:
									__lowerCAmelCase :    Optional[Any]				=			nn.Parameter(_UpperCAmelCase   ,requires_grad=old_value.requires_grad      )
									__lowerCAmelCase :    Optional[Any]				=			new_value


def      _lowercase							(    __snake_case   ,__snake_case=None   ,__snake_case=None   ,__snake_case=None   ,__snake_case=False      )		->  Optional[int]:
			for name, module in model.named_children():
						if current_key_name is None:
									__lowerCAmelCase :    Any				=			[]
						current_key_name.append(_UpperCAmelCase      )

						if (isinstance(_UpperCAmelCase   ,nn.Linear      ) or isinstance(_UpperCAmelCase   ,_UpperCAmelCase      )) and name not in modules_to_not_convert:
									# Check if the current key is not in the `modules_to_not_convert`
									if not any(key in ".".join(_UpperCAmelCase      ) for key in modules_to_not_convert      ):
												with init_empty_weights():
															if isinstance(_UpperCAmelCase   ,_UpperCAmelCase      ):
																		__lowerCAmelCase		,      __lowerCAmelCase :    Any				=			module.weight.shape
															else:
																		__lowerCAmelCase :    Optional[int]				=			module.in_features
																		__lowerCAmelCase :    List[Any]				=			module.out_features

															if quantization_config.quantization_method() == "llm_int8":
																		__lowerCAmelCase :    int				=			bnb.nn.LinearabitLt(
																		    _UpperCAmelCase   ,_UpperCAmelCase   ,module.bias is not None   ,has_fpaa_weights=quantization_config.llm_inta_has_fpaa_weight   ,threshold=quantization_config.llm_inta_threshold   ,)
																		__lowerCAmelCase :    Optional[int]				=			True
															else:
																		if (
																		    quantization_config.llm_inta_skip_modules is not None
																		    and name in quantization_config.llm_inta_skip_modules
																		):
																					pass
																		else:
																					__lowerCAmelCase :    Dict				=			bnb.nn.Linearabit(
																					    _UpperCAmelCase   ,_UpperCAmelCase   ,module.bias is not None   ,quantization_config.bnb_abit_compute_dtype   ,compress_statistics=quantization_config.bnb_abit_use_double_quant   ,quant_type=quantization_config.bnb_abit_quant_type   ,)
																					__lowerCAmelCase :    Tuple				=			True
																					             # Store the module class in case we need to transpose the weight later
															__lowerCAmelCase :    str				=			type(_UpperCAmelCase      )
															# Force requires grad to False to avoid unexpected errors
															model._modules[name].requires_grad_(_UpperCAmelCase      )
						if len(list(module.children()      )      ) > 0:
									__lowerCAmelCase		,      __lowerCAmelCase :    int				=			_replace_with_bnb_linear(
									    _UpperCAmelCase   ,_UpperCAmelCase   ,_UpperCAmelCase   ,_UpperCAmelCase   ,has_been_replaced=_UpperCAmelCase   ,)
						# Remove the last key for recursion
						current_key_name.pop(-1      )
			return model, has_been_replaced


def      _lowercase							(    __snake_case   ,__snake_case=None   ,__snake_case=None   ,__snake_case=None      )		->  List[Any]:
			__lowerCAmelCase :    List[Any]				=			["lm_head"] if modules_to_not_convert is None else modules_to_not_convert
			__lowerCAmelCase		,      __lowerCAmelCase :    str				=			_replace_with_bnb_linear(
			    _UpperCAmelCase   ,_UpperCAmelCase   ,_UpperCAmelCase   ,_UpperCAmelCase      )

			if not has_been_replaced:
						logger.warning(
						    "You are loading your model in 8bit or 4bit but no linear modules were found in your model."
						    " Please double check your model architecture, or submit an issue on github if you think this is"
						    " a bug."      )

			return model


def      _lowercase							(    *__snake_case   ,**__snake_case      )		->  str:
			warnings.warn(
			    "`replace_8bit_linear` will be deprecated in a future version, please use `replace_with_bnb_linear` instead"   ,_UpperCAmelCase   ,)
			return replace_with_bnb_linear(*_UpperCAmelCase   ,**_UpperCAmelCase      )


def      _lowercase							(    *__snake_case   ,**__snake_case      )		->  List[str]:
			warnings.warn(
			    "`set_module_8bit_tensor_to_device` will be deprecated in a future version, please use `set_module_quantized_tensor_to_device` instead"   ,_UpperCAmelCase   ,)
			return set_module_quantized_tensor_to_device(*_UpperCAmelCase   ,**_UpperCAmelCase      )





def      _lowercase							(    __snake_case      )		->  Any:
			__lowerCAmelCase :    Dict				=			deepcopy(_UpperCAmelCase      )  # this has 0 cost since it is done inside `init_empty_weights` context manager`
			tied_model.tie_weights()

			__lowerCAmelCase :    Optional[Any]				=			find_tied_parameters(_UpperCAmelCase      )
			# For compatibility with Accelerate < 0.18
			if isinstance(_UpperCAmelCase   ,_UpperCAmelCase      ):
						__lowerCAmelCase :    Any				=			sum(list(tied_params.values()      )   ,[]      ) + list(tied_params.keys()      )
			else:
						__lowerCAmelCase :    Tuple				=			sum(_UpperCAmelCase   ,[]      )
			__lowerCAmelCase :    List[str]				=			len(_UpperCAmelCase      ) > 0

			# Check if it is a base model
			__lowerCAmelCase :    Optional[Any]				=			not hasattr(_UpperCAmelCase   ,model.base_model_prefix      )

			# Ignore this for base models (BertModel, GPT2Model, etc.)
			if (not has_tied_params) and is_base_model:
						return []

			# otherwise they have an attached head
			__lowerCAmelCase :    Any				=			list(model.named_children()      )
			__lowerCAmelCase :    Optional[int]				=			[list_modules[-1][0]]

			# add last module together with tied weights
			__lowerCAmelCase :    List[Any]				=			set(_UpperCAmelCase      ) - set(_UpperCAmelCase      )
			__lowerCAmelCase :    Any				=			list(set(_UpperCAmelCase      )      ) + list(_UpperCAmelCase      )

			# remove ".weight" from the keys
			__lowerCAmelCase :    Optional[Any]				=			[".weight", ".bias"]
			__lowerCAmelCase :    Dict				=			[]
			for name in list_untouched:
						for name_to_remove in names_to_remove:
									if name_to_remove in name:
												__lowerCAmelCase :    Any				=			name.replace(_UpperCAmelCase   ,""      )
						filtered_module_names.append(_UpperCAmelCase      )

			return filtered_module_names






"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase     :						Optional[Any]						       =  logging.get_logger(__name__)

__UpperCamelCase     :						List[Any]						       =  {
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}







class       a							(   a__  ):
 snake_case__             =       '''megatron-bert'''



 def __init__(     self				,       _snake_case=2_90_56				,       _snake_case=10_24				,       _snake_case=24				,       _snake_case=16				,       _snake_case=40_96				,       _snake_case="gelu"				,       _snake_case=0.1				,       _snake_case=0.1				,       _snake_case=5_12				,       _snake_case=2				,       _snake_case=0.02				,       _snake_case=1E-12				,       _snake_case=0				,       _snake_case="absolute"				,       _snake_case=True				,       **_snake_case				,       ):







  """simple docstring"""






  super().__init__(pad_token_id=_snake_case				,       **_snake_case     )

  lowerCAmelCase    				= vocab_size
  lowerCAmelCase    				= hidden_size
  lowerCAmelCase    				= num_hidden_layers
  lowerCAmelCase    				= num_attention_heads
  lowerCAmelCase    				= hidden_act
  lowerCAmelCase    				= intermediate_size
  lowerCAmelCase    				= hidden_dropout_prob
  lowerCAmelCase    				= attention_probs_dropout_prob
  lowerCAmelCase    				= max_position_embeddings
  lowerCAmelCase    				= type_vocab_size
  lowerCAmelCase    				= initializer_range
  lowerCAmelCase    				= layer_norm_eps
  lowerCAmelCase    				= position_embedding_type
  lowerCAmelCase    				= use_cache





