





"""simple docstring"""





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase					      =   logging.get_logger(__name__)

_UpperCamelCase					      =   '▁'

_UpperCamelCase					      =   {'vocab_file': 'sentencepiece.bpe.model'}

_UpperCamelCase					      =   {
    'vocab_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model',
    }
}

_UpperCamelCase					      =   {
    'facebook/xglm-564M': 2048,
}
class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
				"""simple docstring"""
				__snake_case :							str									=		VOCAB_FILES_NAMES
				__snake_case :							str									=		PRETRAINED_VOCAB_FILES_MAP
				__snake_case :							Tuple									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__snake_case :							Dict									=		["""input_ids""", """attention_mask"""]




				def __init__(  self	:Dict    ,     __lowercase	:Tuple    ,     __lowercase	:Optional[Any]="<s>"    ,     __lowercase	:List[Any]="</s>"    ,     __lowercase	:Optional[Any]="</s>"    ,     __lowercase	:Optional[int]="<s>"    ,     __lowercase	:int="<unk>"    ,     __lowercase	:Optional[Any]="<pad>"    ,     __lowercase	:Optional[Dict[str, Any]] = None    ,     **__lowercase	:List[str]    ,     ):
											__lowerCamelCase   :   Tuple       ={} if sp_model_kwargs is None else sp_model_kwargs

											# Compatibility with the original tokenizer
											__lowerCamelCase   :   List[str]       =7
											__lowerCamelCase   :   int       =[f'<madeupword{i}>' for i in range(self.num_madeup_words      )]

											__lowerCamelCase   :   Optional[int]       =kwargs.get('''additional_special_tokens'''    ,     []      )
											kwargs["additional_special_tokens"] += [
											    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
											]

											super().__init__(
											    bos_token=__lowercase    ,     eos_token=__lowercase    ,     unk_token=__lowercase    ,     sep_token=__lowercase    ,     cls_token=__lowercase    ,     pad_token=__lowercase    ,     sp_model_kwargs=self.sp_model_kwargs    ,     **__lowercase    ,     )

											__lowerCamelCase   :   Union[str, Any]       =spm.SentencePieceProcessor(**self.sp_model_kwargs      )
											self.sp_model.Load(str(__lowercase      )      )
											__lowerCamelCase   :   Union[str, Any]       =vocab_file

											# Original fairseq vocab and spm vocab must be "aligned":
											# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
											# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
											# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
											# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

											# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
											__lowerCamelCase   :   int       =1

											# Mimic fairseq token-to-id alignment for the first 4 token
											__lowerCamelCase   :   List[str]       ={'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

											__lowerCamelCase   :   int       =len(self.sp_model      )
											__lowerCamelCase   :   Dict       ={f'<madeupword{i}>': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words      )}
											self.fairseq_tokens_to_ids.update(__lowercase      )

											__lowerCamelCase   :   Optional[Any]       ={v: k for k, v in self.fairseq_tokens_to_ids.items()}




				def __getstate__(  self	:Any      ):
											__lowerCamelCase   :   Optional[int]       =self.__dict__.copy()
											__lowerCamelCase   :   List[str]       =None
											__lowerCamelCase   :   List[str]       =self.sp_model.serialized_model_proto()
											return state




				def __setstate__(  self	:Dict    ,     __lowercase	:Optional[Any]      ):
											__lowerCamelCase   :   Union[str, Any]       =d

											# for backward compatibility
											if not hasattr(self    ,     '''sp_model_kwargs'''      ):
																		__lowerCamelCase   :   List[Any]       ={}

											__lowerCamelCase   :   Optional[int]       =spm.SentencePieceProcessor(**self.sp_model_kwargs      )
											self.sp_model.LoadFromSerializedProto(self.sp_model_proto      )




				def        __lowercase	(  self	:List[Any]    ,     __lowercase	:List[int]    ,     __lowercase	:Optional[List[int]] = None      ):

											if token_ids_a is None:
																		return [self.sep_token_id] + token_ids_a
											__lowerCamelCase   :   int       =[self.sep_token_id]
											return sep + token_ids_a + sep + sep + token_ids_a




				def        __lowercase	(  self	:Any    ,     __lowercase	:List[int]    ,     __lowercase	:Optional[List[int]] = None    ,     __lowercase	:bool = False      ):

											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=__lowercase    ,     token_ids_a=__lowercase    ,     already_has_special_tokens=__lowercase      )

											if token_ids_a is None:
																		return [1] + ([0] * len(__lowercase      ))
											return [1] + ([0] * len(__lowercase      )) + [1, 1] + ([0] * len(__lowercase      ))




				def        __lowercase	(  self	:str    ,     __lowercase	:List[int]    ,     __lowercase	:Optional[List[int]] = None      ):
											__lowerCamelCase   :   List[str]       =[self.sep_token_id]

											if token_ids_a is None:
																		return len(sep + token_ids_a      ) * [0]
											return len(sep + token_ids_a + sep + sep + token_ids_a      ) * [0]




				@property
				def        __lowercase	(  self	:str      ):
											return len(self.sp_model      ) + self.fairseq_offset + self.num_madeup_words




				def        __lowercase	(  self	:Optional[int]      ):
											__lowerCamelCase   :   List[Any]       ={self.convert_ids_to_tokens(__lowercase      ): i for i in range(self.vocab_size      )}
											vocab.update(self.added_tokens_encoder      )
											return vocab




				def        __lowercase	(  self	:int    ,     __lowercase	:str      ):
											return self.sp_model.encode(__lowercase    ,     out_type=__lowercase      )




				def        __lowercase	(  self	:Optional[int]    ,     __lowercase	:str      ):
											if token in self.fairseq_tokens_to_ids:
																		return self.fairseq_tokens_to_ids[token]
											__lowerCamelCase   :   Optional[Any]       =self.sp_model.PieceToId(__lowercase      )

											# Need to return unknown token if the SP model returned 0
											return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




				def        __lowercase	(  self	:Dict    ,     __lowercase	:Tuple      ):
											if index in self.fairseq_ids_to_tokens:
																		return self.fairseq_ids_to_tokens[index]
											return self.sp_model.IdToPiece(index - self.fairseq_offset      )




				def        __lowercase	(  self	:Optional[int]    ,     __lowercase	:str      ):
											__lowerCamelCase   :   Tuple       =''''''.join(__lowercase      ).replace(__lowercase    ,     ''' '''      ).strip()
											return out_string




				def        __lowercase	(  self	:Optional[int]    ,     __lowercase	:str    ,     __lowercase	:Optional[str] = None      ):
											if not os.path.isdir(__lowercase      ):
																		logger.error(f'Vocabulary path ({save_directory}) should be a directory'      )
																		return
											__lowerCamelCase   :   Optional[Any]       =os.path.join(
											    __lowercase    ,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

											if os.path.abspath(self.vocab_file      ) != os.path.abspath(__lowercase      ) and os.path.isfile(self.vocab_file      ):
																		copyfile(self.vocab_file    ,     __lowercase      )
											elif not os.path.isfile(self.vocab_file      ):
																		with open(__lowercase    ,     '''wb'''      ) as fi:
																									__lowerCamelCase   :   Optional[Any]       =self.sp_model.serialized_model_proto()
																									fi.write(__lowercase      )

											return (out_vocab_file,)











"""simple docstring"""





from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
							import PIL


_UpperCamelCase					      =   logging.get_logger(__name__)
class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
				"""simple docstring"""
				__snake_case :							str									=		["""pixel_values"""]




				def __init__(  self	:str    ,     __lowercase	:bool = True    ,     __lowercase	:Dict[str, int] = None    ,     __lowercase	:float = None    ,     __lowercase	:PILImageResampling = PILImageResampling.BILINEAR    ,     __lowercase	:bool = True    ,     __lowercase	:Union[int, float] = 1 / 255    ,     __lowercase	:bool = True    ,     __lowercase	:Optional[Union[float, List[float]]] = None    ,     __lowercase	:Optional[Union[float, List[float]]] = None    ,     **__lowercase	:Tuple    ,     ):
											super().__init__(**__lowercase      )
											__lowerCamelCase   :   int       =size if size is not None else {'''shortest_edge''': 384}
											__lowerCamelCase   :   List[str]       =get_size_dict(__lowercase    ,     default_to_square=__lowercase      )

											__lowerCamelCase   :   Any       =do_resize
											__lowerCamelCase   :   List[str]       =size
											# Default value set here for backwards compatibility where the value in config is None
											__lowerCamelCase   :   Tuple       =crop_pct if crop_pct is not None else 224 / 256
											__lowerCamelCase   :   Any       =resample
											__lowerCamelCase   :   List[str]       =do_rescale
											__lowerCamelCase   :   Dict       =rescale_factor
											__lowerCamelCase   :   Any       =do_normalize
											__lowerCamelCase   :   int       =image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
											__lowerCamelCase   :   List[Any]       =image_std if image_std is not None else IMAGENET_STANDARD_STD




				def        __lowercase	(  self	:Any    ,     __lowercase	:np.ndarray    ,     __lowercase	:Dict[str, int]    ,     __lowercase	:float    ,     __lowercase	:PILImageResampling = PILImageResampling.BICUBIC    ,     __lowercase	:Optional[Union[str, ChannelDimension]] = None    ,     **__lowercase	:int    ,     ):
											__lowerCamelCase   :   int       =get_size_dict(__lowercase    ,     default_to_square=__lowercase      )
											if "shortest_edge" not in size:
																		raise ValueError(f'Size dictionary must contain \'shortest_edge\' key. Got {size.keys()}'      )
											__lowerCamelCase   :   Optional[int]       =size['''shortest_edge''']

											if shortest_edge < 384:
																		# maintain same ratio, resizing shortest edge to shortest_edge/crop_pct
																		__lowerCamelCase   :   int       =int(shortest_edge / crop_pct      )
																		__lowerCamelCase   :   str       =get_resize_output_image_size(__lowercase    ,     size=__lowercase    ,     default_to_square=__lowercase      )
																		__lowerCamelCase   :   str       =resize(image=__lowercase    ,     size=__lowercase    ,     resample=__lowercase    ,     data_format=__lowercase    ,     **__lowercase      )
																		# then crop to (shortest_edge, shortest_edge)
																		return center_crop(image=__lowercase    ,     size=(shortest_edge, shortest_edge)    ,     data_format=__lowercase    ,     **__lowercase      )
											else:
																		# warping (no cropping) when evaluated at 384 or larger
																		return resize(
																		    __lowercase    ,     size=(shortest_edge, shortest_edge)    ,     resample=__lowercase    ,     data_format=__lowercase    ,     **__lowercase      )




				def        __lowercase	(  self	:int    ,     __lowercase	:np.ndarray    ,     __lowercase	:Union[int, float]    ,     __lowercase	:Optional[Union[str, ChannelDimension]] = None    ,     **__lowercase	:List[Any]    ,     ):
											return rescale(__lowercase    ,     scale=__lowercase    ,     data_format=__lowercase    ,     **__lowercase      )




				def        __lowercase	(  self	:str    ,     __lowercase	:np.ndarray    ,     __lowercase	:Union[float, List[float]]    ,     __lowercase	:Union[float, List[float]]    ,     __lowercase	:Optional[Union[str, ChannelDimension]] = None    ,     **__lowercase	:Optional[int]    ,     ):
											return normalize(__lowercase    ,     mean=__lowercase    ,     std=__lowercase    ,     data_format=__lowercase    ,     **__lowercase      )




				def        __lowercase	(  self	:int    ,     __lowercase	:ImageInput    ,     __lowercase	:bool = None    ,     __lowercase	:Dict[str, int] = None    ,     __lowercase	:float = None    ,     __lowercase	:PILImageResampling = None    ,     __lowercase	:bool = None    ,     __lowercase	:float = None    ,     __lowercase	:bool = None    ,     __lowercase	:Optional[Union[float, List[float]]] = None    ,     __lowercase	:Optional[Union[float, List[float]]] = None    ,     __lowercase	:Optional[Union[str, TensorType]] = None    ,     __lowercase	:ChannelDimension = ChannelDimension.FIRST    ,     **__lowercase	:Optional[Any]    ,     ):
											__lowerCamelCase   :   Optional[Any]       =do_resize if do_resize is not None else self.do_resize
											__lowerCamelCase   :   int       =crop_pct if crop_pct is not None else self.crop_pct
											__lowerCamelCase   :   List[Any]       =resample if resample is not None else self.resample
											__lowerCamelCase   :   List[Any]       =do_rescale if do_rescale is not None else self.do_rescale
											__lowerCamelCase   :   List[Any]       =rescale_factor if rescale_factor is not None else self.rescale_factor
											__lowerCamelCase   :   List[Any]       =do_normalize if do_normalize is not None else self.do_normalize
											__lowerCamelCase   :   Optional[Any]       =image_mean if image_mean is not None else self.image_mean
											__lowerCamelCase   :   str       =image_std if image_std is not None else self.image_std

											__lowerCamelCase   :   str       =size if size is not None else self.size
											__lowerCamelCase   :   Optional[int]       =get_size_dict(__lowercase    ,     default_to_square=__lowercase      )

											__lowerCamelCase   :   Optional[int]       =make_list_of_images(__lowercase      )

											if not valid_images(__lowercase      ):
																		raise ValueError(
																		    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
																		    '''torch.Tensor, tf.Tensor or jax.ndarray.'''      )

											if do_resize and size is None or resample is None:
																		raise ValueError('''Size and resample must be specified if do_resize is True.'''      )

											if do_resize and size["shortest_edge"] < 384 and crop_pct is None:
																		raise ValueError('''crop_pct must be specified if size < 384.'''      )

											if do_rescale and rescale_factor is None:
																		raise ValueError('''Rescale factor must be specified if do_rescale is True.'''      )

											if do_normalize and (image_mean is None or image_std is None):
																		raise ValueError('''Image mean and std must be specified if do_normalize is True.'''      )

											# All transformations expect numpy arrays.
											__lowerCamelCase   :   List[Any]       =[to_numpy_array(__lowercase      ) for image in images]

											if do_resize:
																		__lowerCamelCase   :   Union[str, Any]       =[self.resize(image=__lowercase    ,     size=__lowercase    ,     crop_pct=__lowercase    ,     resample=__lowercase      ) for image in images]

											if do_rescale:
																		__lowerCamelCase   :   int       =[self.rescale(image=__lowercase    ,     scale=__lowercase      ) for image in images]

											if do_normalize:
																		__lowerCamelCase   :   Union[str, Any]       =[self.normalize(image=__lowercase    ,     mean=__lowercase    ,     std=__lowercase      ) for image in images]

											__lowerCamelCase   :   Union[str, Any]       =[to_channel_dimension_format(__lowercase    ,     __lowercase      ) for image in images]

											__lowerCamelCase   :   Tuple       ={'''pixel_values''': images}
											return BatchFeature(data=__lowercase    ,     tensor_type=__lowercase      )




