




"""simple docstring"""




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a_        =       logging.get_logger(__name__)
a_        =       {'vocab_file': 'spiece.model'}

a_        =       {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

a_        =       {
    'albert-base-v1': 5_1_2,
    'albert-large-v1': 5_1_2,
    'albert-xlarge-v1': 5_1_2,
    'albert-xxlarge-v1': 5_1_2,
    'albert-base-v2': 5_1_2,
    'albert-large-v2': 5_1_2,
    'albert-xlarge-v2': 5_1_2,
    'albert-xxlarge-v2': 5_1_2,
}

a_        =       '▁'

class   UpperCAmelCase_    (			_UpperCamelCase		):
		UpperCamelCase							=VOCAB_FILES_NAMES
		UpperCamelCase							=PRETRAINED_VOCAB_FILES_MAP
		UpperCamelCase							=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		def __init__(						self				,						UpperCamelCase_				,						UpperCamelCase_=True				,						UpperCamelCase_=True				,						UpperCamelCase_=False				,						UpperCamelCase_="[CLS]"				,						UpperCamelCase_="[SEP]"				,						UpperCamelCase_="<unk>"				,						UpperCamelCase_="[SEP]"				,						UpperCamelCase_="<pad>"				,						UpperCamelCase_="[CLS]"				,						UpperCamelCase_="[MASK]"				,						UpperCamelCase_ = None				,						**UpperCamelCase_				,						)						->       int:
					# Mask token behave like a normal word, i.e. include the space before it and
					# is included in the raw text, there should be a match in a non-normalized sentence.
					__lowercase			:  Any								=       (
					    AddedToken(lowercase__				,						lstrip=lowercase__				,						rstrip=lowercase__				,						normalized=lowercase__       )
					    if isinstance(lowercase__				,						lowercase__       )
					    else mask_token
					)

					__lowercase			:  Optional[int]								=       {} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    do_lower_case=lowercase__				,						remove_space=lowercase__				,						keep_accents=lowercase__				,						bos_token=lowercase__				,						eos_token=lowercase__				,						unk_token=lowercase__				,						sep_token=lowercase__				,						pad_token=lowercase__				,						cls_token=lowercase__				,						mask_token=lowercase__				,						sp_model_kwargs=self.sp_model_kwargs				,						**lowercase__				,						)

					__lowercase			:  Optional[Any]								=       do_lower_case
					__lowercase			:  List[str]								=       remove_space
					__lowercase			:  Dict								=       keep_accents
					__lowercase			:  str								=       vocab_file

					__lowercase			:  Optional[int]								=       spm.SentencePieceProcessor(**self.sp_model_kwargs       )
					self.sp_model.Load(lowercase__       )
		@property
		def 	_lowerCamelCase							(						self       )						->       Optional[Any]:
					return len(self.sp_model       )
		def 	_lowerCamelCase							(						self       )						->       Union[str, Any]:
					__lowercase			:  Union[str, Any]								=       {self.convert_ids_to_tokens(lowercase__       ): i for i in range(self.vocab_size       )}
					vocab.update(self.added_tokens_encoder       )
					return vocab
		def __getstate__(						self       )						->       List[str]:
					__lowercase			:  List[Any]								=       self.__dict__.copy()
					__lowercase			:  str								=       None
					return state
		def __setstate__(						self				,						UpperCamelCase_       )						->       str:
					__lowercase			:  Optional[Any]								=       d

					# for backward compatibility
					if not hasattr(self				,						'''sp_model_kwargs'''       ):
								__lowercase			:  List[Any]								=       {}

					__lowercase			:  Dict								=       spm.SentencePieceProcessor(**self.sp_model_kwargs       )
					self.sp_model.Load(self.vocab_file       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Optional[int]:
					if self.remove_space:
								__lowercase			:  Optional[Any]								=       ''' '''.join(inputs.strip().split()       )
					else:
								__lowercase			:  str								=       inputs
					__lowercase			:  Tuple								=       outputs.replace('''``'''				,						'''"'''       ).replace('''\'\''''				,						'''"'''       )

					if not self.keep_accents:
								__lowercase			:  Dict								=       unicodedata.normalize('''NFKD'''				,						lowercase__       )
								__lowercase			:  int								=       ''''''.join([c for c in outputs if not unicodedata.combining(lowercase__       )]       )
					if self.do_lower_case:
								__lowercase			:  Dict								=       outputs.lower()

					return outputs
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Optional[int]:
					__lowercase			:  Tuple								=       self.preprocess_text(lowercase__       )
					__lowercase			:  Tuple								=       self.sp_model.encode(lowercase__				,						out_type=lowercase__       )
					__lowercase			:  Any								=       []
					for piece in pieces:
								if len(lowercase__       ) > 1 and piece[-1] == str(''','''       ) and piece[-2].isdigit():
											__lowercase			:  List[Any]								=       self.sp_model.EncodeAsPieces(piece[:-1].replace(lowercase__				,						''''''       )       )
											if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
														if len(cur_pieces[0]       ) == 1:
																	__lowercase			:  Optional[int]								=       cur_pieces[1:]
														else:
																	__lowercase			:  List[str]								=       cur_pieces[0][1:]
											cur_pieces.append(piece[-1]       )
											new_pieces.extend(lowercase__       )
								else:
											new_pieces.append(lowercase__       )

					return new_pieces
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Tuple:
					return self.sp_model.PieceToId(lowercase__       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       str:
					return self.sp_model.IdToPiece(lowercase__       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       List[Any]:
					__lowercase			:  Optional[Any]								=       []
					__lowercase			:  List[Any]								=       ''''''
					__lowercase			:  Tuple								=       False
					for token in tokens:
								# make sure that special tokens are not decoded using sentencepiece model
								if token in self.all_special_tokens:
											if not prev_is_special:
														out_string += " "
											out_string += self.sp_model.decode(lowercase__       ) + token
											__lowercase			:  List[Any]								=       True
											__lowercase			:  str								=       []
								else:
											current_sub_tokens.append(lowercase__       )
											__lowercase			:  Dict								=       False
					out_string += self.sp_model.decode(lowercase__       )
					return out_string.strip()
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_ = None       )						->       int:
					__lowercase			:  Optional[int]								=       [self.sep_token_id]
					__lowercase			:  Union[str, Any]								=       [self.cls_token_id]
					if token_ids_a is None:
								return cls + token_ids_a + sep
					return cls + token_ids_a + sep + token_ids_a + sep
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_ = None				,						UpperCamelCase_ = False       )						->       int:

					if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=lowercase__				,						token_ids_a=lowercase__				,						already_has_special_tokens=lowercase__       )

					if token_ids_a is not None:
								return [1] + ([0] * len(lowercase__       )) + [1] + ([0] * len(lowercase__       )) + [1]
					return [1] + ([0] * len(lowercase__       )) + [1]
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_ = None       )						->       str:
					__lowercase			:  Any								=       [self.sep_token_id]
					__lowercase			:  Any								=       [self.cls_token_id]

					if token_ids_a is None:
								return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_ = None       )						->       int:
					if not os.path.isdir(lowercase__       ):
								logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
								return
					__lowercase			:  Any								=       os.path.join(
					    lowercase__				,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )

					if os.path.abspath(self.vocab_file       ) != os.path.abspath(lowercase__       ) and os.path.isfile(self.vocab_file       ):
								copyfile(self.vocab_file				,						lowercase__       )
					elif not os.path.isfile(self.vocab_file       ):
								with open(lowercase__				,						'''wb'''       ) as fi:
											__lowercase			:  Dict								=       self.sp_model.serialized_model_proto()
											fi.write(lowercase__       )

					return (out_vocab_file,)







import copy
import inspect
import unittest

from transformers import AutoBackbone
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import require_timm, require_torch, torch_device
from transformers.utils.import_utils import is_torch_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor


if is_torch_available():
				import torch

				from transformers import TimmBackbone, TimmBackboneConfig

from ...test_pipeline_mixin import PipelineTesterMixin




class 							lowerCamelCase   :


	def __init__(    self				,     lowercase__				,     lowercase__=None				,     lowercase__=None				,     lowercase__=None				,     lowercase__="resnet50"				,     lowercase__=3				,     lowercase__=3_2				,     lowercase__=3				,     lowercase__=True				,     lowercase__=True				,     ):
							__UpperCAmelCase			:							List[str]	    =	parent
							__UpperCAmelCase			:							Dict	    =	out_indices if out_indices is not None else [4]
							__UpperCAmelCase			:							List[Any]	    =	stage_names
							__UpperCAmelCase			:							int	    =	out_features
							__UpperCAmelCase			:							Union[str, Any]	    =	backbone
							__UpperCAmelCase			:							Optional[Any]	    =	batch_size
							__UpperCAmelCase			:							Optional[Any]	    =	image_size
							__UpperCAmelCase			:							Optional[int]	    =	num_channels
							__UpperCAmelCase			:							List[Any]	    =	use_pretrained_backbone
							__UpperCAmelCase			:							Dict	    =	is_training


	def  A(    self):
							__UpperCAmelCase			:							Optional[Any]	    =	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])
							__UpperCAmelCase			:							Tuple	    =	self.get_config()

							return config, pixel_values


	def  A(    self):
							return TimmBackboneConfig(
							    image_size=self.image_size				,     num_channels=self.num_channels				,     out_features=self.out_features				,     out_indices=self.out_indices				,     stage_names=self.stage_names				,     use_pretrained_backbone=self.use_pretrained_backbone				,     backbone=self.backbone				,     )


	def  A(    self				,     lowercase__				,     lowercase__):
							__UpperCAmelCase			:							Tuple	    =	TimmBackbone(config=lowercase__)
							model.to(lowercase__)
							model.eval()
							with torch.no_grad():
													__UpperCAmelCase			:							Union[str, Any]	    =	model(lowercase__)
							self.parent.assertEqual(
							    result.feature_map[-1].shape				,     (self.batch_size, model.channels[-1], 1_4, 1_4)				,     )


	def  A(    self):
							__UpperCAmelCase			:							Dict	    =	self.prepare_config_and_inputs()
							__UpperCAmelCase    ,      __UpperCAmelCase			:							List[Any]	    =	config_and_inputs
							__UpperCAmelCase			:							List[Any]	    =	{'''pixel_values''': pixel_values}
							return config, inputs_dict






@require_torch
@require_timm
class 							lowerCamelCase   (  _UpperCamelCase							,						_UpperCamelCase							,						_UpperCamelCase							,						unittest.TestCase ):
	_lowerCAmelCase     :		Dict										=						(TimmBackbone,) if is_torch_available() else ()
	_lowerCAmelCase     :		str										=						{'''feature-extraction''': TimmBackbone} if is_torch_available() else {}
	_lowerCAmelCase     :		List[str]										=						False
	_lowerCAmelCase     :		str										=						False
	_lowerCAmelCase     :		List[Any]										=						False
	_lowerCAmelCase     :		List[str]										=						False


	def  A(    self):
							__UpperCAmelCase			:							List[Any]	    =	TimmBackboneModelTester(self)
							__UpperCAmelCase			:							int	    =	ConfigTester(self				,     config_class=lowercase__				,     has_text_modality=lowercase__)


	def  A(    self):
							self.config_tester.create_and_test_config_to_json_string()
							self.config_tester.create_and_test_config_to_json_file()
							self.config_tester.create_and_test_config_from_and_save_pretrained()
							self.config_tester.create_and_test_config_with_num_labels()
							self.config_tester.check_config_can_be_init_without_params()
							self.config_tester.check_config_arguments_init()


	def  A(    self):
							__UpperCAmelCase			:							int	    =	'''resnet18'''
							__UpperCAmelCase			:							List[str]	    =	'''microsoft/resnet-18'''

							__UpperCAmelCase			:							Any	    =	AutoBackbone.from_pretrained(lowercase__				,     use_timm_backbone=lowercase__)
							__UpperCAmelCase			:							Union[str, Any]	    =	AutoBackbone.from_pretrained(lowercase__)

							self.assertEqual(len(timm_model.out_features)				,     len(transformers_model.out_features))
							self.assertEqual(len(timm_model.stage_names)				,     len(transformers_model.stage_names))
							self.assertEqual(timm_model.channels				,     transformers_model.channels)
							# Out indices are set to the last layer by default. For timm models, we don't know
							# the number of layers in advance, so we set it to (-1,), whereas for transformers
							# models, we set it to [len(stage_names) - 1] (kept for backward compatibility).
							self.assertEqual(timm_model.out_indices				,     (-1,))
							self.assertEqual(transformers_model.out_indices				,     [len(timm_model.stage_names) - 1])

							__UpperCAmelCase			:							Union[str, Any]	    =	AutoBackbone.from_pretrained(lowercase__				,     use_timm_backbone=lowercase__				,     out_indices=[1, 2, 3])
							__UpperCAmelCase			:							Any	    =	AutoBackbone.from_pretrained(lowercase__				,     out_indices=[1, 2, 3])

							self.assertEqual(timm_model.out_indices				,     transformers_model.out_indices)
							self.assertEqual(len(timm_model.out_features)				,     len(transformers_model.out_features))
							self.assertEqual(timm_model.channels				,     transformers_model.channels)


	@unittest.skip('''TimmBackbone doesn\'t support feed forward chunking''')
	def  A(    self):
							pass


	@unittest.skip('''TimmBackbone doesn\'t have num_hidden_layers attribute''')
	def  A(    self):
							pass


	@unittest.skip('''TimmBackbone initialization is managed on the timm side''')
	def  A(    self):
							pass


	@unittest.skip('''TimmBackbone models doesn\'t have inputs_embeds''')
	def  A(    self):
							pass


	@unittest.skip('''TimmBackbone models doesn\'t have inputs_embeds''')
	def  A(    self):
							pass


	@unittest.skip('''TimmBackbone model cannot be created without specifying a backbone checkpoint''')
	def  A(    self):
							pass


	@unittest.skip('''Only checkpoints on timm can be loaded into TimmBackbone''')
	def  A(    self):
							pass


	@unittest.skip('''model weights aren\'t tied in TimmBackbone.''')
	def  A(    self):
							pass


	@unittest.skip('''model weights aren\'t tied in TimmBackbone.''')
	def  A(    self):
							pass


	@unittest.skip('''Only checkpoints on timm can be loaded into TimmBackbone''')
	def  A(    self):
							pass


	@unittest.skip('''Only checkpoints on timm can be loaded into TimmBackbone''')
	def  A(    self):
							pass


	@unittest.skip('''TimmBackbone doesn\'t have hidden size info in its configuration.''')
	def  A(    self):
							pass


	@unittest.skip('''TimmBackbone doesn\'t support output_attentions.''')
	def  A(    self):
							pass


	@unittest.skip('''Safetensors is not supported by timm.''')
	def  A(    self):
							pass


	@unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.''')
	def  A(    self):
							pass


	def  A(    self):
							__UpperCAmelCase    ,      __UpperCAmelCase			:							Optional[int]	    =	self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
													__UpperCAmelCase			:							Any	    =	model_class(lowercase__)
													__UpperCAmelCase			:							Optional[int]	    =	inspect.signature(model.forward)
													# signature.parameters is an OrderedDict => so arg_names order is deterministic
													__UpperCAmelCase			:							Optional[int]	    =	[*signature.parameters.keys()]

													__UpperCAmelCase			:							Dict	    =	['''pixel_values''']
													self.assertListEqual(arg_names[:1]				,     lowercase__)


	def  A(    self):
							__UpperCAmelCase    ,      __UpperCAmelCase			:							str	    =	self.model_tester.prepare_config_and_inputs_for_common()
							__UpperCAmelCase			:							Tuple	    =	True
							__UpperCAmelCase			:							Optional[int]	    =	self.has_attentions

							# no need to test all models as different heads yield the same functionality
							__UpperCAmelCase			:							Optional[Any]	    =	self.all_model_classes[0]
							__UpperCAmelCase			:							Optional[int]	    =	model_class(lowercase__)
							model.to(lowercase__)

							__UpperCAmelCase			:							Tuple	    =	self._prepare_for_class(lowercase__				,     lowercase__)
							__UpperCAmelCase			:							Optional[int]	    =	model(**lowercase__)
							__UpperCAmelCase			:							List[str]	    =	outputs[0][-1]

							# Encoder-/Decoder-only models
							__UpperCAmelCase			:							Dict	    =	outputs.hidden_states[0]
							hidden_states.retain_grad()

							if self.has_attentions:
													__UpperCAmelCase			:							Optional[int]	    =	outputs.attentions[0]
													attentions.retain_grad()

							output.flatten()[0].backward(retain_graph=lowercase__)

							self.assertIsNotNone(hidden_states.grad)

							if self.has_attentions:
													self.assertIsNotNone(attentions.grad)


	def  A(    self):
							__UpperCAmelCase    ,      __UpperCAmelCase			:							Dict	    =	self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
													__UpperCAmelCase			:							Optional[Any]	    =	model_class(lowercase__)
													model.to(lowercase__)
													model.eval()
													__UpperCAmelCase			:							Union[str, Any]	    =	model(**lowercase__)

													self.assertEqual(len(result.feature_maps)				,     len(config.out_indices))
													self.assertEqual(len(model.channels)				,     len(config.out_indices))

													# Check output of last stage is taken if out_features=None, out_indices=None
													__UpperCAmelCase			:							List[str]	    =	copy.deepcopy(lowercase__)
													__UpperCAmelCase			:							str	    =	None
													__UpperCAmelCase			:							Optional[int]	    =	model_class(lowercase__)
													model.to(lowercase__)
													model.eval()
													__UpperCAmelCase			:							Optional[Any]	    =	model(**lowercase__)

													self.assertEqual(len(result.feature_maps)				,     1)
													self.assertEqual(len(model.channels)				,     1)

													# Check backbone can be initialized with fresh weights
													__UpperCAmelCase			:							Tuple	    =	copy.deepcopy(lowercase__)
													__UpperCAmelCase			:							Optional[int]	    =	False
													__UpperCAmelCase			:							Optional[int]	    =	model_class(lowercase__)
													model.to(lowercase__)
													model.eval()
													__UpperCAmelCase			:							List[Any]	    =	model(**lowercase__)

