



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCamelCase__			= logging.get_logger(__name__)

lowerCamelCase__			= {"""vocab_file""": """spiece.model"""}

lowerCamelCase__			= {
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    }
}

lowerCamelCase__			= {
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

# Segments (not really needed)
lowerCamelCase__			= 0
lowerCamelCase__			= 1
lowerCamelCase__			= 2
lowerCamelCase__			= 3
lowerCamelCase__			= 4






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):
							__lowerCamelCase	:	List[str]    =VOCAB_FILES_NAMES
							__lowerCamelCase	:	str    =PRETRAINED_VOCAB_FILES_MAP
							__lowerCamelCase	:	List[str]    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowerCamelCase	:	Optional[Any]    ='left'





							def __init__(   self		:		List[str]      ,				__lowercase		:		List[Any]      ,				__lowercase		:		Tuple=False      ,				__lowercase		:		str=True      ,				__lowercase		:		Dict=False      ,				__lowercase		:		Optional[int]="<s>"      ,				__lowercase		:		List[Any]="</s>"      ,				__lowercase		:		Optional[Any]="<unk>"      ,				__lowercase		:		List[str]="<sep>"      ,				__lowercase		:		List[Any]="<pad>"      ,				__lowercase		:		Tuple="<cls>"      ,				__lowercase		:		Any="<mask>"      ,				__lowercase		:		Any=["<eop>", "<eod>"]      ,				__lowercase		:		Optional[Dict[str, Any]] = None      ,				**__lowercase		:		Tuple      ,				):

											'''simple docstring'''
											# Mask token behave like a normal word, i.e. include the space before it
											__a       =		AddedToken(__lowercase      ,				lstrip=__lowercase      ,				rstrip=__lowercase	) if isinstance(__lowercase      ,				__lowercase	) else mask_token

											__a       =		{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    do_lower_case=__lowercase      ,				remove_space=__lowercase      ,				keep_accents=__lowercase      ,				bos_token=__lowercase      ,				eos_token=__lowercase      ,				unk_token=__lowercase      ,				sep_token=__lowercase      ,				pad_token=__lowercase      ,				cls_token=__lowercase      ,				mask_token=__lowercase      ,				additional_special_tokens=__lowercase      ,				sp_model_kwargs=self.sp_model_kwargs      ,				**__lowercase      ,				)

											__a       =		3

											__a       =		do_lower_case
											__a       =		remove_space
											__a       =		keep_accents
											__a       =		vocab_file

											__a       =		spm.SentencePieceProcessor(**self.sp_model_kwargs	)
											self.sp_model.Load(__lowercase	)





							@property
							def 						UpperCamelCase_					(   self		:		Any	):

											'''simple docstring'''
											return len(self.sp_model	)





							def 						UpperCamelCase_					(   self		:		Dict	):

											'''simple docstring'''
											__a       =		{self.convert_ids_to_tokens(__lowercase	): i for i in range(self.vocab_size	)}
											vocab.update(self.added_tokens_encoder	)
											return vocab





							def __getstate__(   self		:		Union[str, Any]	):

											'''simple docstring'''
											__a       =		self.__dict__.copy()
											__a       =		None
											return state





							def __setstate__(   self		:		Optional[Any]      ,				__lowercase		:		Any	):

											'''simple docstring'''
											__a       =		d

											# for backward compatibility
											if not hasattr(self      ,				"""sp_model_kwargs"""	):
															__a       =		{}

											__a       =		spm.SentencePieceProcessor(**self.sp_model_kwargs	)
											self.sp_model.Load(self.vocab_file	)





							def 						UpperCamelCase_					(   self		:		Any      ,				__lowercase		:		str	):

											'''simple docstring'''
											if self.remove_space:
															__a       =		""" """.join(inputs.strip().split()	)
											else:
															__a       =		inputs
											__a       =		outputs.replace("""``"""      ,				"""\""""	).replace("""''"""      ,				"""\""""	)

											if not self.keep_accents:
															__a       =		unicodedata.normalize("""NFKD"""      ,				__lowercase	)
															__a       =		"""""".join([c for c in outputs if not unicodedata.combining(__lowercase	)]	)
											if self.do_lower_case:
															__a       =		outputs.lower()

											return outputs





							def 						UpperCamelCase_					(   self		:		Dict      ,				__lowercase		:		str	):

											'''simple docstring'''
											__a       =		self.preprocess_text(__lowercase	)
											__a       =		self.sp_model.encode(__lowercase      ,				out_type=__lowercase	)
											__a       =		[]
											for piece in pieces:
															if len(__lowercase	) > 1 and piece[-1] == str(""","""	) and piece[-2].isdigit():
																			__a       =		self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowercase      ,				""""""	)	)
																			if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																							if len(cur_pieces[0]	) == 1:
																											__a       =		cur_pieces[1:]
																							else:
																											__a       =		cur_pieces[0][1:]
																			cur_pieces.append(piece[-1]	)
																			new_pieces.extend(__lowercase	)
															else:
																			new_pieces.append(__lowercase	)

											return new_pieces





							def 						UpperCamelCase_					(   self		:		Dict      ,				__lowercase		:		Any	):

											'''simple docstring'''
											return self.sp_model.PieceToId(__lowercase	)





							def 						UpperCamelCase_					(   self		:		Optional[int]      ,				__lowercase		:		List[Any]	):

											'''simple docstring'''
											return self.sp_model.IdToPiece(__lowercase	)





							def 						UpperCamelCase_					(   self		:		Optional[int]      ,				__lowercase		:		Optional[Any]	):

											'''simple docstring'''
											__a       =		"""""".join(__lowercase	).replace(__lowercase      ,				""" """	).strip()
											return out_string





							def 						UpperCamelCase_					(   self		:		List[Any]      ,				__lowercase		:		List[int]      ,				__lowercase		:		bool = False      ,				__lowercase		:		bool = None      ,				__lowercase		:		bool = True      ,				**__lowercase		:		Union[str, Any]      ,				):

											'''simple docstring'''
											__a       =		kwargs.pop("""use_source_tokenizer"""      ,				__lowercase	)

											__a       =		self.convert_ids_to_tokens(__lowercase      ,				skip_special_tokens=__lowercase	)

											# To avoid mixing byte-level and unicode for byte-level BPT
											# we need to build string separately for added tokens and byte-level tokens
											# cf. https://github.com/huggingface/transformers/issues/1133
											__a       =		[]
											__a       =		[]
											for token in filtered_tokens:
															if skip_special_tokens and token in self.all_special_ids:
																			continue
															if token in self.added_tokens_encoder:
																			if current_sub_text:
																							sub_texts.append(self.convert_tokens_to_string(__lowercase	)	)
																							__a       =		[]
																			sub_texts.append(__lowercase	)
															else:
																			current_sub_text.append(__lowercase	)
											if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(__lowercase	)	)

											# Mimic the behavior of the Rust tokenizer:
											# By default, there are no spaces between special tokens
											__a       =		"""""".join(__lowercase	)

											__a       =		(
											    clean_up_tokenization_spaces
											    if clean_up_tokenization_spaces is not None
											    else self.clean_up_tokenization_spaces
											)
											if clean_up_tokenization_spaces:
															__a       =		self.clean_up_tokenization(__lowercase	)
															return clean_text
											else:
															return text





							def 						UpperCamelCase_					(   self		:		Union[str, Any]      ,				__lowercase		:		List[int]      ,				__lowercase		:		Optional[List[int]] = None	):

											'''simple docstring'''
											__a       =		[self.sep_token_id]
											__a       =		[self.cls_token_id]
											if token_ids_a is None:
															return token_ids_a + sep + cls
											return token_ids_a + sep + token_ids_a + sep + cls





							def 						UpperCamelCase_					(   self		:		Any      ,				__lowercase		:		List[int]      ,				__lowercase		:		Optional[List[int]] = None      ,				__lowercase		:		bool = False	):

											'''simple docstring'''

											if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=__lowercase      ,				token_ids_a=__lowercase      ,				already_has_special_tokens=__lowercase	)

											if token_ids_a is not None:
															return ([0] * len(__lowercase	)) + [1] + ([0] * len(__lowercase	)) + [1, 1]
											return ([0] * len(__lowercase	)) + [1, 1]





							def 						UpperCamelCase_					(   self		:		int      ,				__lowercase		:		List[int]      ,				__lowercase		:		Optional[List[int]] = None	):

											'''simple docstring'''
											__a       =		[self.sep_token_id]
											__a       =		[2]

											if token_ids_a is None:
															return len(token_ids_a + sep	) * [0] + cls_segment_id
											return len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1] + cls_segment_id







							def 						UpperCamelCase_					(   self		:		Optional[Any]      ,				__lowercase		:		str      ,				__lowercase		:		Optional[str] = None	):

											'''simple docstring'''
											if not os.path.isdir(__lowercase	):
															logger.error(F"Vocabulary path ({save_directory}) should be a directory"	)
															return
											__a       =		os.path.join(
											    __lowercase      ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

											if os.path.abspath(self.vocab_file	) != os.path.abspath(__lowercase	) and os.path.isfile(self.vocab_file	):
															copyfile(self.vocab_file      ,				__lowercase	)
											elif not os.path.isfile(self.vocab_file	):
															with open(__lowercase      ,				"""wb"""	) as fi:
																			__a       =		self.sp_model.serialized_model_proto()
																			fi.write(__lowercase	)

											return (out_vocab_file,)





from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
						import PIL


lowerCamelCase__			= logging.get_logger(__name__)






class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__					):
							__lowerCamelCase	:	str    =['pixel_values']





							def __init__(   self		:		str      ,				__lowercase		:		bool = True      ,				__lowercase		:		Dict[str, int] = None      ,				__lowercase		:		PILImageResampling = PIL.Image.BICUBIC      ,				__lowercase		:		bool = True      ,				__lowercase		:		Dict[str, int] = None      ,				__lowercase		:		Union[int, float] = 1 / 255      ,				__lowercase		:		bool = True      ,				__lowercase		:		bool = True      ,				__lowercase		:		Optional[Union[float, List[float]]] = None      ,				__lowercase		:		Optional[Union[float, List[float]]] = None      ,				**__lowercase		:		Any      ,				):

											'''simple docstring'''
											super().__init__(**__lowercase	)
											__a       =		size if size is not None else {"""height""": 256, """width""": 256}
											__a       =		get_size_dict(__lowercase	)
											__a       =		crop_size if crop_size is not None else {"""height""": 224, """width""": 224}
											__a       =		get_size_dict(__lowercase      ,				param_name="""crop_size"""	)

											__a       =		do_resize
											__a       =		size
											__a       =		resample
											__a       =		do_center_crop
											__a       =		crop_size
											__a       =		do_rescale
											__a       =		rescale_factor
											__a       =		do_normalize
											__a       =		image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
											__a       =		image_std if image_std is not None else IMAGENET_STANDARD_STD





							def 						UpperCamelCase_					(   self		:		List[Any]      ,				__lowercase		:		np.ndarray      ,				__lowercase		:		Dict[str, int]      ,				__lowercase		:		PILImageResampling = PIL.Image.BICUBIC      ,				__lowercase		:		Optional[Union[str, ChannelDimension]] = None      ,				**__lowercase		:		Union[str, Any]      ,				):

											'''simple docstring'''
											__a       =		get_size_dict(__lowercase	)
											if "height" not in size or "width" not in size:
															raise ValueError(F"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"	)
											return resize(
											    __lowercase      ,				size=(size["""height"""], size["""width"""])      ,				resample=__lowercase      ,				data_format=__lowercase      ,				**__lowercase	)





							def 						UpperCamelCase_					(   self		:		Dict      ,				__lowercase		:		np.ndarray      ,				__lowercase		:		Dict[str, int]      ,				__lowercase		:		Optional[Union[str, ChannelDimension]] = None      ,				**__lowercase		:		str      ,				):

											'''simple docstring'''
											__a       =		get_size_dict(__lowercase	)
											if "height" not in size or "width" not in size:
															raise ValueError(F"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"	)
											return center_crop(__lowercase      ,				size=(size["""height"""], size["""width"""])      ,				data_format=__lowercase      ,				**__lowercase	)





							def 						UpperCamelCase_					(   self		:		int      ,				__lowercase		:		np.ndarray      ,				__lowercase		:		Union[int, float]      ,				__lowercase		:		Optional[Union[str, ChannelDimension]] = None      ,				**__lowercase		:		List[str]      ,				):

											'''simple docstring'''
											return rescale(__lowercase      ,				scale=__lowercase      ,				data_format=__lowercase      ,				**__lowercase	)





							def 						UpperCamelCase_					(   self		:		Tuple      ,				__lowercase		:		np.ndarray      ,				__lowercase		:		Union[float, List[float]]      ,				__lowercase		:		Union[float, List[float]]      ,				__lowercase		:		Optional[Union[str, ChannelDimension]] = None      ,				**__lowercase		:		str      ,				):

											'''simple docstring'''
											return normalize(__lowercase      ,				mean=__lowercase      ,				std=__lowercase      ,				data_format=__lowercase      ,				**__lowercase	)







							def 						UpperCamelCase_					(   self		:		Optional[Any]      ,				__lowercase		:		ImageInput      ,				__lowercase		:		bool = None      ,				__lowercase		:		Dict[str, int] = None      ,				__lowercase		:		Dict=None      ,				__lowercase		:		bool = None      ,				__lowercase		:		Dict[str, int] = None      ,				__lowercase		:		bool = None      ,				__lowercase		:		float = None      ,				__lowercase		:		bool = None      ,				__lowercase		:		Optional[Union[float, List[float]]] = None      ,				__lowercase		:		Optional[Union[float, List[float]]] = None      ,				__lowercase		:		Optional[Union[str, TensorType]] = None      ,				__lowercase		:		ChannelDimension = ChannelDimension.FIRST      ,				**__lowercase		:		Optional[Any]      ,				):

											'''simple docstring'''
											__a       =		do_resize if do_resize is not None else self.do_resize
											__a       =		resample if resample is not None else self.resample
											__a       =		do_center_crop if do_center_crop is not None else self.do_center_crop
											__a       =		do_rescale if do_rescale is not None else self.do_rescale
											__a       =		rescale_factor if rescale_factor is not None else self.rescale_factor
											__a       =		do_normalize if do_normalize is not None else self.do_normalize
											__a       =		image_mean if image_mean is not None else self.image_mean
											__a       =		image_std if image_std is not None else self.image_std

											__a       =		size if size is not None else self.size
											__a       =		get_size_dict(__lowercase	)
											__a       =		crop_size if crop_size is not None else self.crop_size
											__a       =		get_size_dict(__lowercase      ,				param_name="""crop_size"""	)

											__a       =		make_list_of_images(__lowercase	)

											if not valid_images(__lowercase	):
															raise ValueError(
															    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
															    """torch.Tensor, tf.Tensor or jax.ndarray."""	)

											if do_resize and size is None or resample is None:
															raise ValueError("""Size and resample must be specified if do_resize is True."""	)

											if do_center_crop and crop_size is None:
															raise ValueError("""Crop size must be specified if do_center_crop is True."""	)

											if do_rescale and rescale_factor is None:
															raise ValueError("""Rescale factor must be specified if do_rescale is True."""	)

											if do_normalize and (image_mean is None or image_std is None):
															raise ValueError("""Image mean and std must be specified if do_normalize is True."""	)

											# All transformations expect numpy arrays.
											__a       =		[to_numpy_array(__lowercase	) for image in images]

											if do_resize:
															__a       =		[self.resize(image=__lowercase      ,				size=__lowercase      ,				resample=__lowercase	) for image in images]

											if do_center_crop:
															__a       =		[self.center_crop(image=__lowercase      ,				size=__lowercase	) for image in images]

											if do_rescale:
															__a       =		[self.rescale(image=__lowercase      ,				scale=__lowercase	) for image in images]

											if do_normalize:
															__a       =		[self.normalize(image=__lowercase      ,				mean=__lowercase      ,				std=__lowercase	) for image in images]

											__a       =		[to_channel_dimension_format(__lowercase      ,				__lowercase	) for image in images]

											__a       =		{"""pixel_values""": images}
											return BatchFeature(data=__lowercase      ,				tensor_type=__lowercase	)
