

import importlib.metadata
import warnings
from copy import deepcopy

from packaging import version

from ..utils import logging
from .import_utils import is_accelerate_available, is_bitsandbytes_available


if is_bitsandbytes_available():
					import bitsandbytes as bnb
					import torch
					import torch.nn as nn

					from ..pytorch_utils import ConvaD

if is_accelerate_available():
					from accelerate import init_empty_weights
					from accelerate.utils import find_tied_parameters

_UpperCAmelCase	:     Tuple		       = logging.get_logger(__name__)





def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__=None     ,							UpperCamelCase__=None   ):







							'''simple docstring'''






							if "." in tensor_name:
														snake_case_           =					tensor_name.split('.'   )
														for split in splits[:-1]:
																					snake_case_           =					getattr(UpperCamelCase__     ,							UpperCamelCase__   )
																					if new_module is None:
																												raise ValueError(F'''{module} has no attribute {split}.'''   )
																					snake_case_           =					new_module
														snake_case_           =					splits[-1]

							if tensor_name not in module._parameters and tensor_name not in module._buffers:
														raise ValueError(F'''{module} does not have a parameter or a buffer named {tensor_name}.'''   )
							snake_case_           =					tensor_name in module._buffers
							snake_case_           =					getattr(UpperCamelCase__     ,							UpperCamelCase__   )

							if old_value.device == torch.device('meta'   ) and device not in ["meta", torch.device('meta'   )] and value is None:
														raise ValueError(F'''{tensor_name} is on the meta device, we need a `value` to put in on {device}.'''   )

							snake_case_           =					False
							snake_case_           =					False
							if is_buffer or not is_bitsandbytes_available():
														snake_case_           =					False
														snake_case_           =					False
							else:
														snake_case_           =					hasattr(bnb.nn     ,							'Params4bit'   ) and isinstance(module._parameters[tensor_name]     ,							bnb.nn.Paramsabit   )
														snake_case_           =					isinstance(module._parameters[tensor_name]     ,							bnb.nn.IntaParams   )

							if is_abit or is_abit:
														snake_case_           =					module._parameters[tensor_name]
														if param.device.type != "cuda":
																					if value is None:
																												snake_case_           =					old_value.to(UpperCamelCase__   )
																					elif isinstance(UpperCamelCase__     ,							torch.Tensor   ):
																												snake_case_           =					value.to('cpu'   )
																												if value.dtype == torch.inta:
																																			snake_case_           =					version.parse(importlib.metadata.version('bitsandbytes'   )   ) > version.parse(
																																			    '0.37.2'   )
																																			if not is_abit_serializable:
																																										raise ValueError(
																																										    'Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. '
																																										    'Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`.'   )
																					else:
																												snake_case_           =					torch.tensor(UpperCamelCase__     ,							device='cpu'   )

																					# Support models using `Conv1D` in place of `nn.Linear` (e.g. gpt2) by transposing the weight matrix prior to quantization.
																					# Since weights are saved in the correct "orientation", we skip transposing when loading.
																					if issubclass(module.source_cls     ,							UpperCamelCase__   ) and fpaa_statistics is None:
																												snake_case_           =					new_value.T

																					snake_case_           =					old_value.__dict__
																					if is_abit:
																												snake_case_           =					bnb.nn.IntaParams(UpperCamelCase__     ,							requires_grad=UpperCamelCase__     ,							**UpperCamelCase__   ).to(UpperCamelCase__   )
																					elif is_abit:
																												snake_case_           =					bnb.nn.Paramsabit(UpperCamelCase__     ,							requires_grad=UpperCamelCase__     ,							**UpperCamelCase__   ).to(UpperCamelCase__   )

																					snake_case_           =					new_value
																					if fpaa_statistics is not None:
																												setattr(module.weight     ,							'SCB'     ,							fpaa_statistics.to(UpperCamelCase__   )   )

							else:
														if value is None:
																					snake_case_           =					old_value.to(UpperCamelCase__   )
														elif isinstance(UpperCamelCase__     ,							torch.Tensor   ):
																					snake_case_           =					value.to(UpperCamelCase__   )
														else:
																					snake_case_           =					torch.tensor(UpperCamelCase__     ,							device=UpperCamelCase__   )

														if is_buffer:
																					snake_case_           =					new_value
														else:
																					snake_case_           =					nn.Parameter(UpperCamelCase__     ,							requires_grad=old_value.requires_grad   )
																					snake_case_           =					new_value





def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__=None     ,							UpperCamelCase__=None     ,							UpperCamelCase__=None     ,							UpperCamelCase__=False   ):







							'''simple docstring'''






							for name, module in model.named_children():
														if current_key_name is None:
																					snake_case_           =					[]
														current_key_name.append(UpperCamelCase__   )

														if (isinstance(UpperCamelCase__     ,							nn.Linear   ) or isinstance(UpperCamelCase__     ,							UpperCamelCase__   )) and name not in modules_to_not_convert:
																					# Check if the current key is not in the `modules_to_not_convert`
																					if not any(key in '.'.join(UpperCamelCase__   ) for key in modules_to_not_convert   ):
																												with init_empty_weights():
																																			if isinstance(UpperCamelCase__     ,							UpperCamelCase__   ):
																																										snake_case_           =					module.weight.shape
																																			else:
																																										snake_case_           =					module.in_features
																																										snake_case_           =					module.out_features

																																			if quantization_config.quantization_method() == "llm_int8":
																																										snake_case_           =					bnb.nn.LinearabitLt(
																																										    UpperCamelCase__     ,							UpperCamelCase__     ,							module.bias is not None     ,							has_fpaa_weights=quantization_config.llm_inta_has_fpaa_weight     ,							threshold=quantization_config.llm_inta_threshold     ,							)
																																										snake_case_           =					True
																																			else:
																																										if (
																																										    quantization_config.llm_inta_skip_modules is not None
																																										    and name in quantization_config.llm_inta_skip_modules
																																										):
																																																	pass
																																										else:
																																																	snake_case_           =					bnb.nn.Linearabit(
																																																	    UpperCamelCase__     ,							UpperCamelCase__     ,							module.bias is not None     ,							quantization_config.bnb_abit_compute_dtype     ,							compress_statistics=quantization_config.bnb_abit_use_double_quant     ,							quant_type=quantization_config.bnb_abit_quant_type     ,							)
																																																	snake_case_           =					True
                    # Store the module class in case we need to transpose the weight later
																																			snake_case_           =					type(UpperCamelCase__   )
																																			# Force requires grad to False to avoid unexpected errors
																																			model._modules[name].requires_grad_(UpperCamelCase__   )
														if len(list(module.children()   )   ) > 0:
																					snake_case_           =					_replace_with_bnb_linear(
																					    UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__     ,							has_been_replaced=UpperCamelCase__     ,							)
														# Remove the last key for recursion
														current_key_name.pop(-1   )
							return model, has_been_replaced





def       __lowerCamelCase			(				UpperCamelCase__     ,							UpperCamelCase__=None     ,							UpperCamelCase__=None     ,							UpperCamelCase__=None   ):







							'''simple docstring'''






							snake_case_           =					["""lm_head"""] if modules_to_not_convert is None else modules_to_not_convert
							snake_case_           =					_replace_with_bnb_linear(
							    UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__     ,							UpperCamelCase__   )

							if not has_been_replaced:
														logger.warning(
														    'You are loading your model in 8bit or 4bit but no linear modules were found in your model.'
														    ' Please double check your model architecture, or submit an issue on github if you think this is'
														    ' a bug.'   )

							return model





def       __lowerCamelCase			(				*UpperCamelCase__     ,							**UpperCamelCase__   ):







							'''simple docstring'''






							warnings.warn(
							    '`replace_8bit_linear` will be deprecated in a future version, please use `replace_with_bnb_linear` instead'     ,							UpperCamelCase__     ,							)
							return replace_with_bnb_linear(*UpperCamelCase__     ,							**UpperCamelCase__   )





def       __lowerCamelCase			(				*UpperCamelCase__     ,							**UpperCamelCase__   ):







							'''simple docstring'''






							warnings.warn(
							    '`set_module_8bit_tensor_to_device` will be deprecated in a future version, please use `set_module_quantized_tensor_to_device` instead'     ,							UpperCamelCase__     ,							)
							return set_module_quantized_tensor_to_device(*UpperCamelCase__     ,							**UpperCamelCase__   )





def       __lowerCamelCase			(				UpperCamelCase__   ):







							'''simple docstring'''






							snake_case_           =					deepcopy(UpperCamelCase__   )  # this has 0 cost since it is done inside `init_empty_weights` context manager`
							tied_model.tie_weights()

							snake_case_           =					find_tied_parameters(UpperCamelCase__   )
							# For compatibility with Accelerate < 0.18
							if isinstance(UpperCamelCase__     ,							UpperCamelCase__   ):
														snake_case_           =					sum(list(tied_params.values()   )     ,							[]   ) + list(tied_params.keys()   )
							else:
														snake_case_           =					sum(UpperCamelCase__     ,							[]   )
							snake_case_           =					len(UpperCamelCase__   ) > 0

							# Check if it is a base model
							snake_case_           =					not hasattr(UpperCamelCase__     ,							model.base_model_prefix   )

							# Ignore this for base models (BertModel, GPT2Model, etc.)
							if (not has_tied_params) and is_base_model:
														return []

							# otherwise they have an attached head
							snake_case_           =					list(model.named_children()   )
							snake_case_           =					[list_modules[-1][0]]

							# add last module together with tied weights
							snake_case_           =					set(UpperCamelCase__   ) - set(UpperCamelCase__   )
							snake_case_           =					list(set(UpperCamelCase__   )   ) + list(UpperCamelCase__   )

							# remove ".weight" from the keys
							snake_case_           =					[""".weight""", """.bias"""]
							snake_case_           =					[]
							for name in list_untouched:
														for name_to_remove in names_to_remove:
																					if name_to_remove in name:
																												snake_case_           =					name.replace(UpperCamelCase__     ,							''   )
														filtered_module_names.append(UpperCamelCase__   )

							return filtered_module_names









import json
import os
import unittest

from transformers import BatchEncoding, MvpTokenizer, MvpTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin, filter_roberta_detectors







@require_tokenizers
class 				lowercase       ( lowercase_			,						unittest.TestCase    ):
				__SCREAMING_SNAKE_CASE      :				Optional[int]     =    MvpTokenizer
				__SCREAMING_SNAKE_CASE      :				Optional[int]     =    MvpTokenizerFast
				__SCREAMING_SNAKE_CASE      :				Optional[int]     =    True
				__SCREAMING_SNAKE_CASE      :				Tuple     =    filter_roberta_detectors




				def 					a							(      self ):
											super().setUp()
											snake_case_           =					[
											    'l',
											    'o',
											    'w',
											    'e',
											    'r',
											    's',
											    't',
											    'i',
											    'd',
											    'n',
											    '\u0120',
											    '\u0120l',
											    '\u0120n',
											    '\u0120lo',
											    '\u0120low',
											    'er',
											    '\u0120lowest',
											    '\u0120newer',
											    '\u0120wider',
											    '<unk>',
											]
											snake_case_           =					dict(zip(snake_case   ,     range(len(snake_case ) ) ) )
											snake_case_           =					['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
											snake_case_           =					{'unk_token': '<unk>'}

											snake_case_           =					os.path.join(self.tmpdirname   ,     VOCAB_FILES_NAMES['vocab_file'] )
											snake_case_           =					os.path.join(self.tmpdirname   ,     VOCAB_FILES_NAMES['merges_file'] )
											with open(self.vocab_file   ,     'w'   ,     encoding='utf-8' ) as fp:
																		fp.write(json.dumps(snake_case ) + '\n' )
											with open(self.merges_file   ,     'w'   ,     encoding='utf-8' ) as fp:
																		fp.write('\n'.join(snake_case ) )




				def 					a							(      self   ,     **snake_case ):
											kwargs.update(self.special_tokens_map )
											return self.tokenizer_class.from_pretrained(self.tmpdirname   ,     **snake_case )




				def 					a							(      self   ,     **snake_case ):
											kwargs.update(self.special_tokens_map )
											return self.rust_tokenizer_class.from_pretrained(self.tmpdirname   ,     **snake_case )




				def 					a							(      self   ,     snake_case ):
											return "lower newer", "lower newer"




				@cached_property
				def 					a							(      self ):
											return MvpTokenizer.from_pretrained('RUCAIBox/mvp' )




				@cached_property
				def 					a							(      self ):
											return MvpTokenizerFast.from_pretrained('RUCAIBox/mvp' )




				@require_torch
				def 					a							(      self ):
											snake_case_           =					['A long paragraph for summarization.', 'Another paragraph for summarization.']
											snake_case_           =					[0, 250, 251, 1_7818, 13, 3_9186, 1938, 4, 2]

											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		snake_case_           =					tokenizer(snake_case   ,     max_length=len(snake_case )   ,     padding=snake_case   ,     return_tensors='pt' )
																		self.assertIsInstance(snake_case   ,     snake_case )

																		self.assertEqual((2, 9)   ,     batch.input_ids.shape )
																		self.assertEqual((2, 9)   ,     batch.attention_mask.shape )
																		snake_case_           =					batch.input_ids.tolist()[0]
																		self.assertListEqual(snake_case   ,     snake_case )
																		# Test that special tokens are reset




				@require_torch
				def 					a							(      self ):
											snake_case_           =					['A long paragraph for summarization.', 'Another paragraph for summarization.']
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		snake_case_           =					tokenizer(snake_case   ,     padding=snake_case   ,     return_tensors='pt' )
																		# check if input_ids are returned and no labels
																		self.assertIn('input_ids'   ,     snake_case )
																		self.assertIn('attention_mask'   ,     snake_case )
																		self.assertNotIn('labels'   ,     snake_case )
																		self.assertNotIn('decoder_attention_mask'   ,     snake_case )




				@require_torch
				def 					a							(      self ):
											snake_case_           =					[
											    'Summary of the text.',
											    'Another summary.',
											]
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		snake_case_           =					tokenizer(text_target=snake_case   ,     max_length=32   ,     padding='max_length'   ,     return_tensors='pt' )
																		self.assertEqual(32   ,     targets['input_ids'].shape[1] )




				@require_torch
				def 					a							(      self ):
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		snake_case_           =					tokenizer(
																		    ['I am a small frog' * 1024, 'I am a small frog']   ,     padding=snake_case   ,     truncation=snake_case   ,     return_tensors='pt' )
																		self.assertIsInstance(snake_case   ,     snake_case )
																		self.assertEqual(batch.input_ids.shape   ,     (2, 1024) )




				@require_torch
				def 					a							(      self ):
											snake_case_           =					['A long paragraph for summarization.']
											snake_case_           =					[
											    'Summary of the text.',
											]
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		snake_case_           =					tokenizer(snake_case   ,     text_target=snake_case   ,     return_tensors='pt' )
																		snake_case_           =					inputs['input_ids']
																		snake_case_           =					inputs['labels']
																		self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item() )
																		self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item() )
																		self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item() )
																		self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item() )




				def 					a							(      self ):
											pass




				def 					a							(      self ):
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
																									snake_case_           =					self.rust_tokenizer_class.from_pretrained(snake_case   ,     **snake_case )
																									snake_case_           =					self.tokenizer_class.from_pretrained(snake_case   ,     **snake_case )
																									snake_case_           =					'A, <mask> AllenNLP sentence.'
																									snake_case_           =					tokenizer_r.encode_plus(snake_case   ,     add_special_tokens=snake_case   ,     return_token_type_ids=snake_case )
																									snake_case_           =					tokenizer_p.encode_plus(snake_case   ,     add_special_tokens=snake_case   ,     return_token_type_ids=snake_case )

																									# token_type_ids should put 0 everywhere
																									self.assertEqual(sum(tokens_r['token_type_ids'] )   ,     sum(tokens_p['token_type_ids'] ) )

																									# attention_mask should put 1 everywhere, so sum over length should be 1
																									self.assertEqual(
																									    sum(tokens_r['attention_mask'] ) / len(tokens_r['attention_mask'] )   ,     sum(tokens_p['attention_mask'] ) / len(tokens_p['attention_mask'] )   ,     )

																									snake_case_           =					tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids'] )
																									snake_case_           =					tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids'] )

																									# Rust correctly handles the space before the mask while python doesnt
																									self.assertSequenceEqual(tokens_p['input_ids']   ,     [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2] )
																									self.assertSequenceEqual(tokens_r['input_ids']   ,     [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2] )

																									self.assertSequenceEqual(
																									    snake_case   ,     ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>'] )
																									self.assertSequenceEqual(
																									    snake_case   ,     ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>'] )




