






"""simple docstring"""






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE						:						List[str]					   =       logging.get_logger(__name__)
SCREAMING_SNAKE_CASE						:						Optional[int]					   =       {'vocab_file': 'spiece.model'}

SCREAMING_SNAKE_CASE						:						List[str]					   =       {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

SCREAMING_SNAKE_CASE						:						List[str]					   =       {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

SCREAMING_SNAKE_CASE						:						int					   =       '▁'





class       snake_case_       (    snake_case__       ):
	SCREAMING_SNAKE_CASE_:							Optional[Any]   			=					VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE_:							Optional[int]   			=					PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE_:							Tuple   			=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


	def __init__(      self  ,					__a  ,					__a=True  ,					__a=True  ,					__a=False  ,					__a="[CLS]"  ,					__a="[SEP]"  ,					__a="<unk>"  ,					__a="[SEP]"  ,					__a="<pad>"  ,					__a="[CLS]"  ,					__a="[MASK]"  ,					__a = None  ,					**__a  ,					):


					"""simple docstring"""

					A__							 =       (
					    AddedToken(UpperCAmelCase_  ,					lstrip=UpperCAmelCase_  ,					rstrip=UpperCAmelCase_  ,					normalized=UpperCAmelCase_						)
					    if isinstance(UpperCAmelCase_  ,					UpperCAmelCase_						)
					    else mask_token
					)

					A__							 =       {} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    do_lower_case=UpperCAmelCase_  ,					remove_space=UpperCAmelCase_  ,					keep_accents=UpperCAmelCase_  ,					bos_token=UpperCAmelCase_  ,					eos_token=UpperCAmelCase_  ,					unk_token=UpperCAmelCase_  ,					sep_token=UpperCAmelCase_  ,					pad_token=UpperCAmelCase_  ,					cls_token=UpperCAmelCase_  ,					mask_token=UpperCAmelCase_  ,					sp_model_kwargs=self.sp_model_kwargs  ,					**UpperCAmelCase_  ,					)

					A__							 =       do_lower_case
					A__							 =       remove_space
					A__							 =       keep_accents
					A__							 =       vocab_file

					A__							 =       spm.SentencePieceProcessor(**self.sp_model_kwargs						)
					self.sp_model.Load(UpperCAmelCase_						)


	@property
	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					return len(self.sp_model						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       {self.convert_ids_to_tokens(UpperCAmelCase_						): i for i in range(self.vocab_size						)}
					vocab.update(self.added_tokens_encoder						)
					return vocab


	def __getstate__(      self						):


					"""simple docstring"""

					A__							 =       self.__dict__.copy()
					A__							 =       None
					return state


	def __setstate__(      self  ,					__a						):


					"""simple docstring"""

					A__							 =       d

					# for backward compatibility
					if not hasattr(self  ,					'sp_model_kwargs'						):
									A__							 =       {}

					A__							 =       spm.SentencePieceProcessor(**self.sp_model_kwargs						)
					self.sp_model.Load(self.vocab_file						)


	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					if self.remove_space:
									A__							 =       ' '.join(inputs.strip().split()						)
					else:
									A__							 =       inputs
					A__							 =       outputs.replace('``'  ,					'\"'						).replace('\'\''  ,					'\"'						)

					if not self.keep_accents:
									A__							 =       unicodedata.normalize('NFKD'  ,					UpperCAmelCase_						)
									A__							 =       ''.join([c for c in outputs if not unicodedata.combining(UpperCAmelCase_						)]						)
					if self.do_lower_case:
									A__							 =       outputs.lower()

					return outputs


	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					A__							 =       self.preprocess_text(UpperCAmelCase_						)
					A__							 =       self.sp_model.encode(UpperCAmelCase_  ,					out_type=UpperCAmelCase_						)
					A__							 =       []
					for piece in pieces:
									if len(UpperCAmelCase_						) > 1 and piece[-1] == str(','						) and piece[-2].isdigit():
													A__							 =       self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase_  ,					''						)						)
													if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																	if len(cur_pieces[0]						) == 1:
																					A__							 =       cur_pieces[1:]
																	else:
																					A__							 =       cur_pieces[0][1:]
													cur_pieces.append(piece[-1]						)
													new_pieces.extend(UpperCAmelCase_						)
									else:
													new_pieces.append(UpperCAmelCase_						)

					return new_pieces


	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					return self.sp_model.PieceToId(UpperCAmelCase_						)


	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					return self.sp_model.IdToPiece(UpperCAmelCase_						)


	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					A__							 =       []
					A__							 =       ''
					A__							 =       False
					for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
													if not prev_is_special:
																	out_string += " "
													out_string += self.sp_model.decode(UpperCAmelCase_						) + token
													A__							 =       True
													A__							 =       []
									else:
													current_sub_tokens.append(UpperCAmelCase_						)
													A__							 =       False
					out_string += self.sp_model.decode(UpperCAmelCase_						)
					return out_string.strip()


	def       _UpperCAmelCase						(      self  ,					__a  ,					__a = None						):


					"""simple docstring"""

					A__							 =       [self.sep_token_id]
					A__							 =       [self.cls_token_id]
					if token_ids_a is None:
									return cls + token_ids_a + sep
					return cls + token_ids_a + sep + token_ids_a + sep


	def       _UpperCAmelCase						(      self  ,					__a  ,					__a = None  ,					__a = False						):


					"""simple docstring"""

					if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=UpperCAmelCase_  ,					token_ids_a=UpperCAmelCase_  ,					already_has_special_tokens=UpperCAmelCase_						)

					if token_ids_a is not None:
									return [1] + ([0] * len(UpperCAmelCase_						)) + [1] + ([0] * len(UpperCAmelCase_						)) + [1]
					return [1] + ([0] * len(UpperCAmelCase_						)) + [1]


	def       _UpperCAmelCase						(      self  ,					__a  ,					__a = None						):


					"""simple docstring"""

					A__							 =       [self.sep_token_id]
					A__							 =       [self.cls_token_id]

					if token_ids_a is None:
									return len(cls + token_ids_a + sep						) * [0]
					return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






	def       _UpperCAmelCase						(      self  ,					__a  ,					__a = None						):


					"""simple docstring"""

					if not os.path.isdir(UpperCAmelCase_						):
									logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
									return
					A__							 =       os.path.join(
					    UpperCAmelCase_  ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

					if os.path.abspath(self.vocab_file						) != os.path.abspath(UpperCAmelCase_						) and os.path.isfile(self.vocab_file						):
									copyfile(self.vocab_file  ,					UpperCAmelCase_						)
					elif not os.path.isfile(self.vocab_file						):
									with open(UpperCAmelCase_  ,					'wb'						) as fi:
													A__							 =       self.sp_model.serialized_model_proto()
													fi.write(UpperCAmelCase_						)

					return (out_vocab_file,)










"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE						:						Any					   =       logging.get_logger(__name__)





class       snake_case_       (    _lowerCamelCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							Union[str, Any]   			=					"""timm_backbone"""


	def __init__(      self  ,					__a=None  ,					__a=3  ,					__a=True  ,					__a=True  ,					__a=None  ,					**__a  ,					):


					"""simple docstring"""

					super().__init__(**__a						)
					A__							 =       backbone
					A__							 =       num_channels
					A__							 =       features_only
					A__							 =       use_pretrained_backbone
					A__							 =       True
					A__							 =       out_indices if out_indices is not None else (-1,)


