

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a_      =     logging.get_logger(__name__)
a_      =     {'vocab_file': 'spiece.model'}

a_      =     {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

a_      =     {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

a_      =     '▁'






class _UpperCamelCase				(				__A ):
			'''simple docstring'''







			lowerCamelCase__							     =VOCAB_FILES_NAMES
			lowerCamelCase__							     =PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__							     =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			def __init__(      self				:			Any   ,   a				:			List[str]   ,   a				:			List[str]=True   ,   a				:			str=True   ,   a				:			int=False   ,   a				:			List[Any]="[CLS]"   ,   a				:			str="[SEP]"   ,   a				:			Tuple="<unk>"   ,   a				:			Any="[SEP]"   ,   a				:			List[str]="<pad>"   ,   a				:			str="[CLS]"   ,   a				:			Optional[int]="[MASK]"   ,   a				:			Optional[Dict[str, Any]] = None   ,   **a				:			Optional[int]   ,   )				->							None:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							(
				    AddedToken(a   ,   lstrip=a   ,   rstrip=a   ,   normalized=a						)
				    if isinstance(a   ,   a						)
				    else mask_token
				)

				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							{} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    do_lower_case=a   ,   remove_space=a   ,   keep_accents=a   ,   bos_token=a   ,   eos_token=a   ,   unk_token=a   ,   sep_token=a   ,   pad_token=a   ,   cls_token=a   ,   mask_token=a   ,   sp_model_kwargs=self.sp_model_kwargs   ,   **a   ,   )

				SCREAMING_SNAKE_CASE :   Any					  =							do_lower_case
				SCREAMING_SNAKE_CASE :   Any					  =							remove_space
				SCREAMING_SNAKE_CASE :   List[Any]					  =							keep_accents
				SCREAMING_SNAKE_CASE :   Tuple					  =							vocab_file

				SCREAMING_SNAKE_CASE :   int					  =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
				self.sp_model.Load(a						)
			@property
			def 				__UpperCamelCase			(      self				:			Optional[Any]						)				->							Tuple:

				"""simple docstring"""

				return len(self.sp_model						)
			def 				__UpperCamelCase			(      self				:			Union[str, Any]						)				->							List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   Any					  =							{self.convert_ids_to_tokens(a						): i for i in range(self.vocab_size						)}
				vocab.update(self.added_tokens_encoder						)
				return vocab
			def __getstate__(      self				:			Optional[int]						)				->							Dict:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   Optional[int]					  =							self.__dict__.copy()
				SCREAMING_SNAKE_CASE :   List[Any]					  =							None
				return state
			def __setstate__(      self				:			Any   ,   a				:			Tuple						)				->							List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							d

				# for backward compatibility
				if not hasattr(self   ,   "sp_model_kwargs"						):
					SCREAMING_SNAKE_CASE :   str					  =							{}

				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
				self.sp_model.Load(self.vocab_file						)
			def 				__UpperCamelCase			(      self				:			Dict   ,   a				:			List[Any]						)				->							Tuple:

				"""simple docstring"""

				if self.remove_space:
					SCREAMING_SNAKE_CASE :   Any					  =							" ".join(inputs.strip().split()						)
				else:
					SCREAMING_SNAKE_CASE :   List[str]					  =							inputs
				SCREAMING_SNAKE_CASE :   Tuple					  =							outputs.replace("``"   ,   "\""						).replace("''"   ,   "\""						)

				if not self.keep_accents:
					SCREAMING_SNAKE_CASE :   Optional[int]					  =							unicodedata.normalize("NFKD"   ,   a						)
					SCREAMING_SNAKE_CASE :   Tuple					  =							"".join([c for c in outputs if not unicodedata.combining(a						)]						)
				if self.do_lower_case:
					SCREAMING_SNAKE_CASE :   Optional[int]					  =							outputs.lower()

				return outputs
			def 				__UpperCamelCase			(      self				:			str   ,   a				:			str						)				->							List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   int					  =							self.preprocess_text(a						)
				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							self.sp_model.encode(a   ,   out_type=a						)
				SCREAMING_SNAKE_CASE :   List[str]					  =							[]
				for piece in pieces:
					if len(a						) > 1 and piece[-1] == str(","						) and piece[-2].isdigit():
						SCREAMING_SNAKE_CASE :   Optional[Any]					  =							self.sp_model.EncodeAsPieces(piece[:-1].replace(a   ,   ""						)						)
						if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
							if len(cur_pieces[0]						) == 1:
								SCREAMING_SNAKE_CASE :   Tuple					  =							cur_pieces[1:]
							else:
								SCREAMING_SNAKE_CASE :   Optional[Any]					  =							cur_pieces[0][1:]
						cur_pieces.append(piece[-1]						)
						new_pieces.extend(a						)
					else:
						new_pieces.append(a						)

				return new_pieces
			def 				__UpperCamelCase			(      self				:			str   ,   a				:			List[str]						)				->							Union[str, Any]:

				"""simple docstring"""

				return self.sp_model.PieceToId(a						)
			def 				__UpperCamelCase			(      self				:			Union[str, Any]   ,   a				:			Optional[Any]						)				->							Union[str, Any]:

				"""simple docstring"""

				return self.sp_model.IdToPiece(a						)
			def 				__UpperCamelCase			(      self				:			List[str]   ,   a				:			Union[str, Any]						)				->							str:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   List[Any]					  =							[]
				SCREAMING_SNAKE_CASE :   Any					  =							""
				SCREAMING_SNAKE_CASE :   List[Any]					  =							False
				for token in tokens:
					# make sure that special tokens are not decoded using sentencepiece model
					if token in self.all_special_tokens:
						if not prev_is_special:
							out_string += " "
						out_string += self.sp_model.decode(a						) + token
						SCREAMING_SNAKE_CASE :   Any					  =							True
						SCREAMING_SNAKE_CASE :   Tuple					  =							[]
					else:
						current_sub_tokens.append(a						)
						SCREAMING_SNAKE_CASE :   Dict					  =							False
				out_string += self.sp_model.decode(a						)
				return out_string.strip()
			def 				__UpperCamelCase			(      self				:			List[Any]   ,   a				:			List[int]   ,   a				:			Optional[List[int]] = None						)				->							List[int]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   List[str]					  =							[self.sep_token_id]
				SCREAMING_SNAKE_CASE :   Optional[int]					  =							[self.cls_token_id]
				if token_ids_a is None:
					return cls + token_ids_a + sep
				return cls + token_ids_a + sep + token_ids_a + sep
			def 				__UpperCamelCase			(      self				:			str   ,   a				:			List[int]   ,   a				:			Optional[List[int]] = None   ,   a				:			bool = False						)				->							List[int]:

				"""simple docstring"""

				if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=a   ,   token_ids_a=a   ,   already_has_special_tokens=a						)

				if token_ids_a is not None:
					return [1] + ([0] * len(a						)) + [1] + ([0] * len(a						)) + [1]
				return [1] + ([0] * len(a						)) + [1]
			def 				__UpperCamelCase			(      self				:			Any   ,   a				:			List[int]   ,   a				:			Optional[List[int]] = None						)				->							List[int]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							[self.sep_token_id]
				SCREAMING_SNAKE_CASE :   Any					  =							[self.cls_token_id]

				if token_ids_a is None:
					return len(cls + token_ids_a + sep						) * [0]
				return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






			def 				__UpperCamelCase			(      self				:			Optional[int]   ,   a				:			str   ,   a				:			Optional[str] = None						)				->							Tuple[str]:

				"""simple docstring"""

				if not os.path.isdir(a						):
					logger.error(F"Vocabulary path ({save_directory}) should be a directory"						)
					return
				SCREAMING_SNAKE_CASE :   Any					  =							os.path.join(
				    a   ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

				if os.path.abspath(self.vocab_file						) != os.path.abspath(a						) and os.path.isfile(self.vocab_file						):
					copyfile(self.vocab_file   ,   a						)
				elif not os.path.isfile(self.vocab_file						):
					with open(a   ,   "wb"						) as fi:
						SCREAMING_SNAKE_CASE :   Any					  =							self.sp_model.serialized_model_proto()
						fi.write(a						)

				return (out_vocab_file,)


from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


a_      =     {'configuration_van': ['VAN_PRETRAINED_CONFIG_ARCHIVE_MAP', 'VanConfig']}


try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			a_      =     [
			    'VAN_PRETRAINED_MODEL_ARCHIVE_LIST',
			    'VanForImageClassification',
			    'VanModel',
			    'VanPreTrainedModel',
			]

if TYPE_CHECKING:
			from .configuration_van import VAN_PRETRAINED_CONFIG_ARCHIVE_MAP, VanConfig

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_van import (
						    VAN_PRETRAINED_MODEL_ARCHIVE_LIST,
						    VanForImageClassification,
						    VanModel,
						    VanPreTrainedModel,
						)

else:
			import sys

			a_      =     _LazyModule(__name__, globals()['__file__'], _import_structure)