



"""simple docstring"""







import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A    =			logging.get_logger(__name__)
A    =			{'vocab_file': 'spiece.model'}

A    =			{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

A    =			{
    'albert-base-v1': 5_1_2,
    'albert-large-v1': 5_1_2,
    'albert-xlarge-v1': 5_1_2,
    'albert-xxlarge-v1': 5_1_2,
    'albert-base-v2': 5_1_2,
    'albert-large-v2': 5_1_2,
    'albert-xlarge-v2': 5_1_2,
    'albert-xxlarge-v2': 5_1_2,
}

A    =			'▁'






class      UpperCAmelCase__		(      UpperCamelCase     ):
		lowerCAmelCase_       :	int 							=     VOCAB_FILES_NAMES
		lowerCAmelCase_       :	Optional[int] 							=     PRETRAINED_VOCAB_FILES_MAP
		lowerCAmelCase_       :	List[str] 							=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



		def __init__( self  :    List[str]      ,							snake_case  :    int      ,							snake_case  :    Union[str, Any]=True      ,							snake_case  :    Union[str, Any]=True      ,							snake_case  :    Tuple=False      ,							snake_case  :    Tuple="[CLS]"      ,							snake_case  :    int="[SEP]"      ,							snake_case  :    Any="<unk>"      ,							snake_case  :    List[str]="[SEP]"      ,							snake_case  :    Optional[Any]="<pad>"      ,							snake_case  :    int="[CLS]"      ,							snake_case  :    List[str]="[MASK]"      ,							snake_case  :    Optional[Dict[str, Any]] = None      ,							**snake_case  :    Optional[int]      ,							)			->						None:






									'''simple docstring'''

									A         =					(
									    AddedToken(snake_case      ,							lstrip=snake_case      ,							rstrip=snake_case      ,							normalized=snake_case				)
									    if isinstance(snake_case      ,							snake_case				)
									    else mask_token
									)

									A         =					{} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    do_lower_case=snake_case      ,							remove_space=snake_case      ,							keep_accents=snake_case      ,							bos_token=snake_case      ,							eos_token=snake_case      ,							unk_token=snake_case      ,							sep_token=snake_case      ,							pad_token=snake_case      ,							cls_token=snake_case      ,							mask_token=snake_case      ,							sp_model_kwargs=self.sp_model_kwargs      ,							**snake_case      ,							)

									A         =					do_lower_case
									A         =					remove_space
									A         =					keep_accents
									A         =					vocab_file

									A         =					spm.SentencePieceProcessor(**self.sp_model_kwargs				)
									self.sp_model.Load(snake_case				)



		@property
		def 					A_		( self  :    Tuple				)			->						Tuple:






									'''simple docstring'''

									return len(self.sp_model				)



		def 					A_		( self  :    Dict				)			->						List[Any]:






									'''simple docstring'''

									A         =					{self.convert_ids_to_tokens(snake_case				): i for i in range(self.vocab_size				)}
									vocab.update(self.added_tokens_encoder				)
									return vocab



		def __getstate__( self  :    Union[str, Any]				)			->						Union[str, Any]:






									'''simple docstring'''

									A         =					self.__dict__.copy()
									A         =					None
									return state



		def __setstate__( self  :    List[str]      ,							snake_case  :    int				)			->						str:






									'''simple docstring'''

									A         =					d

									# for backward compatibility
									if not hasattr(self      ,							'sp_model_kwargs'				):
																A         =					{}

									A         =					spm.SentencePieceProcessor(**self.sp_model_kwargs				)
									self.sp_model.Load(self.vocab_file				)



		def 					A_		( self  :    str      ,							snake_case  :    List[str]				)			->						Any:






									'''simple docstring'''

									if self.remove_space:
																A         =					' '.join(inputs.strip().split()				)
									else:
																A         =					inputs
									A         =					outputs.replace('``'      ,							'"'				).replace('\'\''      ,							'"'				)

									if not self.keep_accents:
																A         =					unicodedata.normalize('NFKD'      ,							snake_case				)
																A         =					''.join([c for c in outputs if not unicodedata.combining(snake_case				)]				)
									if self.do_lower_case:
																A         =					outputs.lower()

									return outputs



		def 					A_		( self  :    int      ,							snake_case  :    str				)			->						List[str]:






									'''simple docstring'''

									A         =					self.preprocess_text(snake_case				)
									A         =					self.sp_model.encode(snake_case      ,							out_type=snake_case				)
									A         =					[]
									for piece in pieces:
																if len(snake_case				) > 1 and piece[-1] == str(','				) and piece[-2].isdigit():
																							A         =					self.sp_model.EncodeAsPieces(piece[:-1].replace(snake_case      ,							''				)				)
																							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																														if len(cur_pieces[0]				) == 1:
																																					A         =					cur_pieces[1:]
																														else:
																																					A         =					cur_pieces[0][1:]
																							cur_pieces.append(piece[-1]				)
																							new_pieces.extend(snake_case				)
																else:
																							new_pieces.append(snake_case				)

									return new_pieces



		def 					A_		( self  :    Tuple      ,							snake_case  :    List[Any]				)			->						List[str]:






									'''simple docstring'''

									return self.sp_model.PieceToId(snake_case				)



		def 					A_		( self  :    Dict      ,							snake_case  :    List[str]				)			->						Tuple:






									'''simple docstring'''

									return self.sp_model.IdToPiece(snake_case				)



		def 					A_		( self  :    Optional[int]      ,							snake_case  :    List[str]				)			->						List[Any]:






									'''simple docstring'''

									A         =					[]
									A         =					''
									A         =					False
									for token in tokens:
																# make sure that special tokens are not decoded using sentencepiece model
																if token in self.all_special_tokens:
																							if not prev_is_special:
																														out_string += " "
																							out_string += self.sp_model.decode(snake_case				) + token
																							A         =					True
																							A         =					[]
																else:
																							current_sub_tokens.append(snake_case				)
																							A         =					False
									out_string += self.sp_model.decode(snake_case				)
									return out_string.strip()



		def 					A_		( self  :    List[str]      ,							snake_case  :    List[int]      ,							snake_case  :    Optional[List[int]] = None				)			->						List[int]:






									'''simple docstring'''

									A         =					[self.sep_token_id]
									A         =					[self.cls_token_id]
									if token_ids_a is None:
																return cls + token_ids_a + sep
									return cls + token_ids_a + sep + token_ids_a + sep



		def 					A_		( self  :    Optional[Any]      ,							snake_case  :    List[int]      ,							snake_case  :    Optional[List[int]] = None      ,							snake_case  :    bool = False				)			->						List[int]:






									'''simple docstring'''

									if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=snake_case      ,							token_ids_a=snake_case      ,							already_has_special_tokens=snake_case				)

									if token_ids_a is not None:
																return [1] + ([0] * len(snake_case				)) + [1] + ([0] * len(snake_case				)) + [1]
									return [1] + ([0] * len(snake_case				)) + [1]



		def 					A_		( self  :    str      ,							snake_case  :    List[int]      ,							snake_case  :    Optional[List[int]] = None				)			->						List[int]:






									'''simple docstring'''

									A         =					[self.sep_token_id]
									A         =					[self.cls_token_id]

									if token_ids_a is None:
																return len(cls + token_ids_a + sep				) * [0]
									return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]



		def 					A_		( self  :    List[str]      ,							snake_case  :    str      ,							snake_case  :    Optional[str] = None				)			->						Tuple[str]:






									'''simple docstring'''

									if not os.path.isdir(snake_case				):
																logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
																return
									A         =					os.path.join(
									    snake_case      ,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)

									if os.path.abspath(self.vocab_file				) != os.path.abspath(snake_case				) and os.path.isfile(self.vocab_file				):
																copyfile(self.vocab_file      ,							snake_case				)
									elif not os.path.isfile(self.vocab_file				):
																with open(snake_case      ,							'wb'				) as fi:
																							A         =					self.sp_model.serialized_model_proto()
																							fi.write(snake_case				)

									return (out_vocab_file,)








"""simple docstring"""







from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_batched,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


A    =			logging.get_logger(__name__)






class      UpperCAmelCase__		(      UpperCamelCase     ):
		lowerCAmelCase_       :	Tuple 							=     ["""pixel_values"""]



		def __init__( self  :    int      ,							snake_case  :    bool = True      ,							snake_case  :    Optional[Dict[str, int]] = None      ,							snake_case  :    PILImageResampling = PILImageResampling.BICUBIC      ,							snake_case  :    bool = True      ,							snake_case  :    bool = True      ,							snake_case  :    Union[int, float] = 1 / 255      ,							snake_case  :    Dict[str, int] = None      ,							snake_case  :    bool = True      ,							snake_case  :    Optional[Union[float, List[float]]] = None      ,							snake_case  :    Optional[Union[float, List[float]]] = None      ,							**snake_case  :    Union[str, Any]      ,							)			->						None:






									'''simple docstring'''

									super().__init__(**snake_case				)
									A         =					size if size is not None else {'height': 224, 'width': 224}
									A         =					get_size_dict(snake_case				)
									A         =					crop_size if crop_size is not None else {'height': 224, 'width': 224}
									A         =					get_size_dict(snake_case      ,							default_to_square=snake_case      ,							param_name='crop_size'				)

									A         =					do_resize
									A         =					do_rescale
									A         =					do_normalize
									A         =					do_center_crop
									A         =					crop_size
									A         =					size
									A         =					resample
									A         =					rescale_factor
									A         =					image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
									A         =					image_std if image_std is not None else IMAGENET_DEFAULT_STD



		def 					A_		( self  :    Union[str, Any]      ,							snake_case  :    np.ndarray      ,							snake_case  :    Dict[str, int]      ,							snake_case  :    PILImageResampling = PILImageResampling.BILINEAR      ,							snake_case  :    Optional[Union[str, ChannelDimension]] = None      ,							**snake_case  :    Dict      ,							)			->						np.ndarray:






									'''simple docstring'''

									A         =					get_size_dict(snake_case				)

									if "shortest_edge" in size:
																A         =					get_resize_output_image_size(snake_case      ,							size=size['shortest_edge']      ,							default_to_square=snake_case				)
																# size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"])
									elif "height" in size and "width" in size:
																A         =					(size['height'], size['width'])
									else:
																raise ValueError(f"""Size must contain 'height' and 'width' keys or 'shortest_edge' key. Got {size.keys()}"""				)
									return resize(snake_case      ,							size=snake_case      ,							resample=snake_case      ,							data_format=snake_case      ,							**snake_case				)



		def 					A_		( self  :    Union[str, Any]      ,							snake_case  :    np.ndarray      ,							snake_case  :    Dict[str, int]      ,							snake_case  :    Optional[Union[str, ChannelDimension]] = None      ,							**snake_case  :    List[Any]      ,							)			->						np.ndarray:






									'''simple docstring'''

									A         =					get_size_dict(snake_case				)
									if "height" not in size or "width" not in size:
																raise ValueError(f"""The `size` parameter must contain the keys (height, width). Got {size.keys()}"""				)
									return center_crop(snake_case      ,							size=(size['height'], size['width'])      ,							data_format=snake_case      ,							**snake_case				)



		def 					A_		( self  :    Dict      ,							snake_case  :    np.ndarray      ,							snake_case  :    float      ,							snake_case  :    Optional[Union[str, ChannelDimension]] = None      ,							**snake_case  :    Dict				)			->						np.ndarray:






									'''simple docstring'''

									return rescale(snake_case      ,							scale=snake_case      ,							data_format=snake_case      ,							**snake_case				)



		def 					A_		( self  :    Union[str, Any]      ,							snake_case  :    np.ndarray      ,							snake_case  :    Union[float, List[float]]      ,							snake_case  :    Union[float, List[float]]      ,							snake_case  :    Optional[Union[str, ChannelDimension]] = None      ,							**snake_case  :    Union[str, Any]      ,							)			->						np.ndarray:






									'''simple docstring'''

									return normalize(snake_case      ,							mean=snake_case      ,							std=snake_case      ,							data_format=snake_case      ,							**snake_case				)



		def 					A_		( self  :    int      ,							snake_case  :    ImageInput      ,							snake_case  :    Optional[bool] = None      ,							snake_case  :    Dict[str, int] = None      ,							snake_case  :    PILImageResampling = None      ,							snake_case  :    bool = None      ,							snake_case  :    int = None      ,							snake_case  :    Optional[bool] = None      ,							snake_case  :    Optional[float] = None      ,							snake_case  :    Optional[bool] = None      ,							snake_case  :    Optional[Union[float, List[float]]] = None      ,							snake_case  :    Optional[Union[float, List[float]]] = None      ,							snake_case  :    Optional[Union[str, TensorType]] = None      ,							snake_case  :    Union[str, ChannelDimension] = ChannelDimension.FIRST      ,							**snake_case  :    List[str]      ,							)			->						BatchFeature:






									'''simple docstring'''

									A         =					do_resize if do_resize is not None else self.do_resize
									A         =					do_rescale if do_rescale is not None else self.do_rescale
									A         =					do_normalize if do_normalize is not None else self.do_normalize
									A         =					do_center_crop if do_center_crop is not None else self.do_center_crop
									A         =					crop_size if crop_size is not None else self.crop_size
									A         =					get_size_dict(snake_case      ,							param_name='crop_size'      ,							default_to_square=snake_case				)
									A         =					resample if resample is not None else self.resample
									A         =					rescale_factor if rescale_factor is not None else self.rescale_factor
									A         =					image_mean if image_mean is not None else self.image_mean
									A         =					image_std if image_std is not None else self.image_std

									A         =					size if size is not None else self.size
									A         =					get_size_dict(snake_case				)

									if not is_batched(snake_case				):
																A         =					[images]

									if not valid_images(snake_case				):
																raise ValueError(
																    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
																    'torch.Tensor, tf.Tensor or jax.ndarray.'				)

									if do_resize and size is None:
																raise ValueError('Size must be specified if do_resize is True.'				)

									if do_center_crop and crop_size is None:
																raise ValueError('Crop size must be specified if do_center_crop is True.'				)

									if do_rescale and rescale_factor is None:
																raise ValueError('Rescale factor must be specified if do_rescale is True.'				)

									# All transformations expect numpy arrays.
									A         =					[to_numpy_array(snake_case				) for image in images]

									if do_resize:
																A         =					[self.resize(image=snake_case      ,							size=snake_case      ,							resample=snake_case				) for image in images]

									if do_center_crop:
																A         =					[self.center_crop(image=snake_case      ,							size=snake_case				) for image in images]

									if do_rescale:
																A         =					[self.rescale(image=snake_case      ,							scale=snake_case				) for image in images]

									if do_normalize:
																A         =					[self.normalize(image=snake_case      ,							mean=snake_case      ,							std=snake_case				) for image in images]

									A         =					[to_channel_dimension_format(snake_case      ,							snake_case				) for image in images]

									A         =					{'pixel_values': images}
									return BatchFeature(data=snake_case      ,							tensor_type=snake_case				)



