






"""simple docstring"""







import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


SCREAMING_SNAKE_CASE									=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE									=						{'vocab_file': 'spiece.model'}

SCREAMING_SNAKE_CASE									=						{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

SCREAMING_SNAKE_CASE									=						{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
SCREAMING_SNAKE_CASE									=						0
SCREAMING_SNAKE_CASE									=						1
SCREAMING_SNAKE_CASE									=						2
SCREAMING_SNAKE_CASE									=						3
SCREAMING_SNAKE_CASE									=						4



class  _lowerCamelCase			(_A     ):
				_snake_case     =  VOCAB_FILES_NAMES
				_snake_case     =  PRETRAINED_VOCAB_FILES_MAP
				_snake_case     =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_snake_case     =  "left"







				def __init__( self    :       List[str]			,					lowerCamelCase_    :       Any			,					lowerCamelCase_    :       str=False			,					lowerCamelCase_    :       Dict=True			,					lowerCamelCase_    :       List[str]=False			,					lowerCamelCase_    :       List[str]="<s>"			,					lowerCamelCase_    :       int="</s>"			,					lowerCamelCase_    :       str="<unk>"			,					lowerCamelCase_    :       int="<sep>"			,					lowerCamelCase_    :       Optional[Any]="<pad>"			,					lowerCamelCase_    :       Tuple="<cls>"			,					lowerCamelCase_    :       Optional[Any]="<mask>"			,					lowerCamelCase_    :       int=["<eop>", "<eod>"]			,					lowerCamelCase_    :       Optional[Dict[str, Any]] = None			,					**lowerCamelCase_    :       Optional[int]			,					):

					"""simple docstring"""






					_lowercase :    str							=							AddedToken(__lowerCamelCase			,					lstrip=__lowerCamelCase			,					rstrip=__lowerCamelCase						) if isinstance(__lowerCamelCase			,					__lowerCamelCase						) else mask_token

					_lowercase :    Optional[int]							=							{} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    do_lower_case=__lowerCamelCase			,					remove_space=__lowerCamelCase			,					keep_accents=__lowerCamelCase			,					bos_token=__lowerCamelCase			,					eos_token=__lowerCamelCase			,					unk_token=__lowerCamelCase			,					sep_token=__lowerCamelCase			,					pad_token=__lowerCamelCase			,					cls_token=__lowerCamelCase			,					mask_token=__lowerCamelCase			,					additional_special_tokens=__lowerCamelCase			,					sp_model_kwargs=self.sp_model_kwargs			,					**__lowerCamelCase			,					)

					_lowercase :    int							=							3

					_lowercase :    Union[str, Any]							=							do_lower_case
					_lowercase :    List[Any]							=							remove_space
					_lowercase :    List[Any]							=							keep_accents
					_lowercase :    str							=							vocab_file

					_lowercase :    Optional[Any]							=							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
					self.sp_model.Load(__lowerCamelCase						)







				@property
				def 							__UpperCAmelCase					( self    :       str						):

					"""simple docstring"""






					return len(self.sp_model						)







				def 							__UpperCAmelCase					( self    :       Union[str, Any]						):

					"""simple docstring"""






					_lowercase :    str							=							{self.convert_ids_to_tokens(__lowerCamelCase						): i for i in range(self.vocab_size						)}
					vocab.update(self.added_tokens_encoder						)
					return vocab







				def __getstate__( self    :       Tuple						):

					"""simple docstring"""






					_lowercase :    Optional[int]							=							self.__dict__.copy()
					_lowercase :    Dict							=							None
					return state







				def __setstate__( self    :       Any			,					lowerCamelCase_    :       Any						):

					"""simple docstring"""






					_lowercase :    Union[str, Any]							=							d

					# for backward compatibility
					if not hasattr(self			,					'sp_model_kwargs'						):
						_lowercase :    Optional[int]							=							{}

					_lowercase :    Optional[int]							=							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
					self.sp_model.Load(self.vocab_file						)







				def 							__UpperCAmelCase					( self    :       Union[str, Any]			,					lowerCamelCase_    :       int						):

					"""simple docstring"""






					if self.remove_space:
						_lowercase :    Union[str, Any]							=							" ".join(inputs.strip().split()						)
					else:
						_lowercase :    str							=							inputs
					_lowercase :    Dict							=							outputs.replace('``'			,					'\"'						).replace('\'\''			,					'\"'						)

					if not self.keep_accents:
						_lowercase :    Optional[int]							=							unicodedata.normalize('NFKD'			,					__lowerCamelCase						)
						_lowercase :    Tuple							=							"".join([c for c in outputs if not unicodedata.combining(__lowerCamelCase						)]						)
					if self.do_lower_case:
						_lowercase :    Tuple							=							outputs.lower()

					return outputs







				def 							__UpperCAmelCase					( self    :       Optional[int]			,					lowerCamelCase_    :       str						):

					"""simple docstring"""






					_lowercase :    Dict							=							self.preprocess_text(__lowerCamelCase						)
					_lowercase :    Tuple							=							self.sp_model.encode(__lowerCamelCase			,					out_type=__lowerCamelCase						)
					_lowercase :    Any							=							[]
					for piece in pieces:
						if len(__lowerCamelCase						) > 1 and piece[-1] == str(','						) and piece[-2].isdigit():
							_lowercase :    Optional[Any]							=							self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCamelCase			,					''						)						)
							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
								if len(cur_pieces[0]						) == 1:
									_lowercase :    Optional[Any]							=							cur_pieces[1:]
								else:
									_lowercase :    List[str]							=							cur_pieces[0][1:]
							cur_pieces.append(piece[-1]						)
							new_pieces.extend(__lowerCamelCase						)
						else:
							new_pieces.append(__lowerCamelCase						)

					return new_pieces







				def 							__UpperCAmelCase					( self    :       Any			,					lowerCamelCase_    :       Tuple						):

					"""simple docstring"""






					return self.sp_model.PieceToId(__lowerCamelCase						)







				def 							__UpperCAmelCase					( self    :       int			,					lowerCamelCase_    :       Optional[int]						):

					"""simple docstring"""






					return self.sp_model.IdToPiece(__lowerCamelCase						)







				def 							__UpperCAmelCase					( self    :       Optional[Any]			,					lowerCamelCase_    :       Tuple						):

					"""simple docstring"""






					_lowercase :    Any							=							"".join(__lowerCamelCase						).replace(__lowerCamelCase			,					' '						).strip()
					return out_string







				def 							__UpperCAmelCase					( self    :       Union[str, Any]			,					lowerCamelCase_    :       List[int]			,					lowerCamelCase_    :       bool = False			,					lowerCamelCase_    :       bool = None			,					lowerCamelCase_    :       bool = True			,					**lowerCamelCase_    :       Union[str, Any]			,					):

					"""simple docstring"""






					_lowercase :    Any							=							kwargs.pop('use_source_tokenizer'			,					__lowerCamelCase						)

					_lowercase :    Any							=							self.convert_ids_to_tokens(__lowerCamelCase			,					skip_special_tokens=__lowerCamelCase						)

					# To avoid mixing byte-level and unicode for byte-level BPT
					# we need to build string separately for added tokens and byte-level tokens
					# cf. https://github.com/huggingface/transformers/issues/1133
					_lowercase :    Dict							=							[]
					_lowercase :    Dict							=							[]
					for token in filtered_tokens:
						if skip_special_tokens and token in self.all_special_ids:
							continue
						if token in self.added_tokens_encoder:
							if current_sub_text:
								sub_texts.append(self.convert_tokens_to_string(__lowerCamelCase						)						)
								_lowercase :    Optional[Any]							=							[]
							sub_texts.append(__lowerCamelCase						)
						else:
							current_sub_text.append(__lowerCamelCase						)
					if current_sub_text:
						sub_texts.append(self.convert_tokens_to_string(__lowerCamelCase						)						)

					# Mimic the behavior of the Rust tokenizer:
					# By default, there are no spaces between special tokens
					_lowercase :    Any							=							"".join(__lowerCamelCase						)

					_lowercase :    Optional[int]							=							(
					    clean_up_tokenization_spaces
					    if clean_up_tokenization_spaces is not None
					    else self.clean_up_tokenization_spaces
					)
					if clean_up_tokenization_spaces:
						_lowercase :    Dict							=							self.clean_up_tokenization(__lowerCamelCase						)
						return clean_text
					else:
						return text







				def 							__UpperCAmelCase					( self    :       Any			,					lowerCamelCase_    :       List[int]			,					lowerCamelCase_    :       Optional[List[int]] = None						):

					"""simple docstring"""






					_lowercase :    Dict							=							[self.sep_token_id]
					_lowercase :    Dict							=							[self.cls_token_id]
					if token_ids_a is None:
						return token_ids_a + sep + cls
					return token_ids_a + sep + token_ids_a + sep + cls







				def 							__UpperCAmelCase					( self    :       str			,					lowerCamelCase_    :       List[int]			,					lowerCamelCase_    :       Optional[List[int]] = None			,					lowerCamelCase_    :       bool = False						):

					"""simple docstring"""






					if already_has_special_tokens:
						return super().get_special_tokens_mask(
						    token_ids_a=__lowerCamelCase			,					token_ids_a=__lowerCamelCase			,					already_has_special_tokens=__lowerCamelCase						)

					if token_ids_a is not None:
						return ([0] * len(__lowerCamelCase						)) + [1] + ([0] * len(__lowerCamelCase						)) + [1, 1]
					return ([0] * len(__lowerCamelCase						)) + [1, 1]







				def 							__UpperCAmelCase					( self    :       List[Any]			,					lowerCamelCase_    :       List[int]			,					lowerCamelCase_    :       Optional[List[int]] = None						):

					"""simple docstring"""






					_lowercase :    Optional[Any]							=							[self.sep_token_id]
					_lowercase :    Dict							=							[2]

					if token_ids_a is None:
						return len(token_ids_a + sep						) * [0] + cls_segment_id
					return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1] + cls_segment_id







				def 							__UpperCAmelCase					( self    :       Union[str, Any]			,					lowerCamelCase_    :       str			,					lowerCamelCase_    :       Optional[str] = None						):

					"""simple docstring"""






					if not os.path.isdir(__lowerCamelCase						):
						logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
						return
					_lowercase :    str							=							os.path.join(
					    __lowerCamelCase			,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

					if os.path.abspath(self.vocab_file						) != os.path.abspath(__lowerCamelCase						) and os.path.isfile(self.vocab_file						):
						copyfile(self.vocab_file			,					__lowerCamelCase						)
					elif not os.path.isfile(self.vocab_file						):
						with open(__lowerCamelCase			,					'wb'						) as fi:
							_lowercase :    Tuple							=							self.sp_model.serialized_model_proto()
							fi.write(__lowerCamelCase						)

					return (out_vocab_file,)




"""simple docstring"""







from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


SCREAMING_SNAKE_CASE									=						logging.get_logger(__name__)



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  ["pixel_values"]







				def __init__( self    :       Optional[int]			,					lowerCamelCase_    :       bool = True			,					lowerCamelCase_    :       Optional[Dict[str, int]] = None			,					lowerCamelCase_    :       PILImageResampling = PILImageResampling.BILINEAR			,					lowerCamelCase_    :       bool = True			,					lowerCamelCase_    :       Dict[str, int] = None			,					lowerCamelCase_    :       bool = True			,					lowerCamelCase_    :       Union[int, float] = 1 / 2_5_5			,					lowerCamelCase_    :       bool = True			,					lowerCamelCase_    :       Optional[Union[float, List[float]]] = None			,					lowerCamelCase_    :       Optional[Union[float, List[float]]] = None			,					**lowerCamelCase_    :       int			,					):

					"""simple docstring"""






					super().__init__(**lowerCamelCase_						)
					_lowercase :    Optional[Any]							=							size if size is not None else {'shortest_edge': 2_5_6}
					_lowercase :    Optional[int]							=							get_size_dict(lowerCamelCase_			,					default_to_square=lowerCamelCase_						)
					_lowercase :    List[str]							=							crop_size if crop_size is not None else {'height': 2_2_4, 'width': 2_2_4}
					_lowercase :    Any							=							get_size_dict(lowerCamelCase_						)
					_lowercase :    Optional[int]							=							do_resize
					_lowercase :    Optional[int]							=							size
					_lowercase :    Union[str, Any]							=							resample
					_lowercase :    Optional[Any]							=							do_center_crop
					_lowercase :    Union[str, Any]							=							crop_size
					_lowercase :    Any							=							do_rescale
					_lowercase :    Tuple							=							rescale_factor
					_lowercase :    Optional[int]							=							do_normalize
					_lowercase :    int							=							image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
					_lowercase :    Any							=							image_std if image_std is not None else IMAGENET_STANDARD_STD







				def 							__UpperCAmelCase					( self    :       List[Any]			,					lowerCamelCase_    :       np.ndarray			,					lowerCamelCase_    :       Dict[str, int]			,					lowerCamelCase_    :       PILImageResampling = PILImageResampling.BICUBIC			,					lowerCamelCase_    :       Optional[Union[str, ChannelDimension]] = None			,					**lowerCamelCase_    :       Tuple			,					):

					"""simple docstring"""






					_lowercase :    Optional[Any]							=							get_size_dict(lowerCamelCase_			,					default_to_square=lowerCamelCase_						)
					if "shortest_edge" not in size:
						raise ValueError(F'''The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}'''						)
					_lowercase :    Dict							=							get_resize_output_image_size(lowerCamelCase_			,					size=size['shortest_edge']			,					default_to_square=lowerCamelCase_						)
					return resize(lowerCamelCase_			,					size=lowerCamelCase_			,					resample=lowerCamelCase_			,					data_format=lowerCamelCase_			,					**lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       Dict			,					lowerCamelCase_    :       np.ndarray			,					lowerCamelCase_    :       Dict[str, int]			,					lowerCamelCase_    :       Optional[Union[str, ChannelDimension]] = None			,					**lowerCamelCase_    :       str			,					):

					"""simple docstring"""






					_lowercase :    Tuple							=							get_size_dict(lowerCamelCase_						)
					return center_crop(lowerCamelCase_			,					size=(size['height'], size['width'])			,					data_format=lowerCamelCase_			,					**lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       Any			,					lowerCamelCase_    :       np.ndarray			,					lowerCamelCase_    :       float			,					lowerCamelCase_    :       Optional[Union[str, ChannelDimension]] = None			,					**lowerCamelCase_    :       Optional[Any]						):

					"""simple docstring"""






					return rescale(lowerCamelCase_			,					scale=lowerCamelCase_			,					data_format=lowerCamelCase_			,					**lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       List[Any]			,					lowerCamelCase_    :       np.ndarray			,					lowerCamelCase_    :       Union[float, List[float]]			,					lowerCamelCase_    :       Union[float, List[float]]			,					lowerCamelCase_    :       Optional[Union[str, ChannelDimension]] = None			,					**lowerCamelCase_    :       List[Any]			,					):

					"""simple docstring"""






					return normalize(lowerCamelCase_			,					mean=lowerCamelCase_			,					std=lowerCamelCase_			,					data_format=lowerCamelCase_			,					**lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       List[Any]			,					lowerCamelCase_    :       ImageInput			,					lowerCamelCase_    :       Optional[bool] = None			,					lowerCamelCase_    :       Dict[str, int] = None			,					lowerCamelCase_    :       PILImageResampling = None			,					lowerCamelCase_    :       bool = None			,					lowerCamelCase_    :       Dict[str, int] = None			,					lowerCamelCase_    :       Optional[bool] = None			,					lowerCamelCase_    :       Optional[float] = None			,					lowerCamelCase_    :       Optional[bool] = None			,					lowerCamelCase_    :       Optional[Union[float, List[float]]] = None			,					lowerCamelCase_    :       Optional[Union[float, List[float]]] = None			,					lowerCamelCase_    :       Optional[Union[str, TensorType]] = None			,					lowerCamelCase_    :       Union[str, ChannelDimension] = ChannelDimension.FIRST			,					**lowerCamelCase_    :       int			,					):

					"""simple docstring"""






					_lowercase :    Optional[Any]							=							do_resize if do_resize is not None else self.do_resize
					_lowercase :    List[Any]							=							size if size is not None else self.size
					_lowercase :    Union[str, Any]							=							get_size_dict(lowerCamelCase_			,					default_to_square=lowerCamelCase_						)
					_lowercase :    Any							=							resample if resample is not None else self.resample
					_lowercase :    Tuple							=							do_center_crop if do_center_crop is not None else self.do_center_crop
					_lowercase :    List[Any]							=							crop_size if crop_size is not None else self.crop_size
					_lowercase :    Tuple							=							get_size_dict(lowerCamelCase_						)
					_lowercase :    List[Any]							=							do_rescale if do_rescale is not None else self.do_rescale
					_lowercase :    int							=							rescale_factor if rescale_factor is not None else self.rescale_factor
					_lowercase :    int							=							do_normalize if do_normalize is not None else self.do_normalize
					_lowercase :    str							=							image_mean if image_mean is not None else self.image_mean
					_lowercase :    List[Any]							=							image_std if image_std is not None else self.image_std

					_lowercase :    Union[str, Any]							=							make_list_of_images(lowerCamelCase_						)

					if not valid_images(lowerCamelCase_						):
						raise ValueError(
						    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
						    'torch.Tensor, tf.Tensor or jax.ndarray.'						)

					if do_resize and size is None:
						raise ValueError('Size must be specified if do_resize is True.'						)

					if do_center_crop and crop_size is None:
						raise ValueError('Crop size must be specified if do_center_crop is True.'						)

					if do_rescale and rescale_factor is None:
						raise ValueError('Rescale factor must be specified if do_rescale is True.'						)

					if do_normalize and (image_mean is None or image_std is None):
						raise ValueError('Image mean and std must be specified if do_normalize is True.'						)

					# All transformations expect numpy arrays.
					_lowercase :    Optional[Any]							=							[to_numpy_array(lowerCamelCase_						) for image in images]

					if do_resize:
						_lowercase :    Optional[Any]							=							[self.resize(image=lowerCamelCase_			,					size=lowerCamelCase_			,					resample=lowerCamelCase_						) for image in images]

					if do_center_crop:
						_lowercase :    List[Any]							=							[self.center_crop(image=lowerCamelCase_			,					size=lowerCamelCase_						) for image in images]

					if do_rescale:
						_lowercase :    str							=							[self.rescale(image=lowerCamelCase_			,					scale=lowerCamelCase_						) for image in images]

					if do_normalize:
						_lowercase :    Any							=							[self.normalize(image=lowerCamelCase_			,					mean=lowerCamelCase_			,					std=lowerCamelCase_						) for image in images]

					_lowercase :    int							=							[to_channel_dimension_format(lowerCamelCase_			,					lowerCamelCase_						) for image in images]

					_lowercase :    Tuple							=							{'pixel_values': images}
					return BatchFeature(data=lowerCamelCase_			,					tensor_type=lowerCamelCase_						)



