
"""simple docstring"""





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_a								=						logging.get_logger(__name__)

_a								=						"""▁"""

_a								=						{"""vocab_file""": """sentencepiece.bpe.model"""}

_a								=						{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

_a								=						{
    """facebook/mbart-large-en-ro""": 1024,
    """facebook/mbart-large-cc25""": 1024,
}

# fmt: off
_a								=						["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]






class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = VOCAB_FILES_NAMES
		lowercase__         = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowercase__         = PRETRAINED_VOCAB_FILES_MAP
		lowercase__         = ['input_ids', 'attention_mask']

		lowercase__         = []
		lowercase__         = []



		def __init__(   self							,						__a							,						__a="<s>"							,						__a="</s>"							,						__a="</s>"							,						__a="<s>"							,						__a="<unk>"							,						__a="<pad>"							,						__a="<mask>"							,						__a=None							,						__a=None							,						__a=None							,						__a = None							,						__a=None							,						**__a							,						)			->			Optional[Any]:






							'''simple docstring'''
							# Mask token behave like a normal word, i.e. include the space before it
							_UpperCamelCase									=    AddedToken(__a							,						lstrip=__a							,						rstrip=__a) if isinstance(__a							,						__a) else mask_token

							_UpperCamelCase									=    {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=__a							,						eos_token=__a							,						unk_token=__a							,						sep_token=__a							,						cls_token=__a							,						pad_token=__a							,						mask_token=__a							,						tokenizer_file=__a							,						src_lang=__a							,						tgt_lang=__a							,						additional_special_tokens=__a							,						sp_model_kwargs=self.sp_model_kwargs							,						**__a							,						)

							_UpperCamelCase									=    spm.SentencePieceProcessor(**self.sp_model_kwargs)
							self.sp_model.Load(str(__a))
							_UpperCamelCase									=    vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							_UpperCamelCase									=    {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							_UpperCamelCase									=    1

							_UpperCamelCase									=    len(self.sp_model)
							_UpperCamelCase									=    {
							    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(__a)
							}
							_UpperCamelCase									=    {v: k for k, v in self.lang_code_to_id.items()}
							_UpperCamelCase									=    len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset

							self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
							_UpperCamelCase									=    {v: k for k, v in self.fairseq_tokens_to_ids.items()}
							_UpperCamelCase									=    list(self.lang_code_to_id.keys())

							if additional_special_tokens is not None:
												# Only add those special tokens if they are not already there.
												self._additional_special_tokens.extend(
												    [t for t in additional_special_tokens if t not in self._additional_special_tokens])

							_UpperCamelCase									=    src_lang if src_lang is not None else '''en_XX'''
							_UpperCamelCase									=    self.lang_code_to_id[self._src_lang]
							_UpperCamelCase									=    tgt_lang
							self.set_src_lang_special_tokens(self._src_lang)



		def __getstate__(   self)			->			Optional[int]:






							'''simple docstring'''
							_UpperCamelCase									=    self.__dict__.copy()
							_UpperCamelCase									=    None
							_UpperCamelCase									=    self.sp_model.serialized_model_proto()
							return state



		def __setstate__(   self							,						__a)			->			int:






							'''simple docstring'''
							_UpperCamelCase									=    d

							# for backward compatibility
							if not hasattr(self							,						'''sp_model_kwargs'''):
												_UpperCamelCase									=    {}

							_UpperCamelCase									=    spm.SentencePieceProcessor(**self.sp_model_kwargs)
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto)



		@property
		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							return len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset + 1  # Plus 1 for the mask token



		@property
		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							return self._src_lang



		@src_lang.setter
		def 		UpperCAmelCase				(   self							,						__a)			->			None:






							'''simple docstring'''
							_UpperCamelCase									=    new_src_lang
							self.set_src_lang_special_tokens(self._src_lang)



		def 		UpperCAmelCase				(   self							,						__a							,						__a = None							,						__a = False)			->			List[int]:






							'''simple docstring'''

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__a							,						token_ids_a=__a							,						already_has_special_tokens=__a)

							_UpperCamelCase									=    [1] * len(self.prefix_tokens)
							_UpperCamelCase									=    [1] * len(self.suffix_tokens)
							if token_ids_a is None:
												return prefix_ones + ([0] * len(__a)) + suffix_ones
							return prefix_ones + ([0] * len(__a)) + ([0] * len(__a)) + suffix_ones



		def 		UpperCAmelCase				(   self							,						__a							,						__a = None)			->			List[int]:






							'''simple docstring'''
							if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



		def 		UpperCAmelCase				(   self							,						__a							,						__a = None)			->			List[int]:






							'''simple docstring'''
							_UpperCamelCase									=    [self.sep_token_id]
							_UpperCamelCase									=    [self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a							,						__a							,						**__a)			->			List[Any]:






							'''simple docstring'''
							if src_lang is None or tgt_lang is None:
												raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model''')
							_UpperCamelCase									=    src_lang
							_UpperCamelCase									=    self(__a							,						add_special_tokens=__a							,						return_tensors=__a							,						**__a)
							_UpperCamelCase									=    self.convert_tokens_to_ids(__a)
							_UpperCamelCase									=    tgt_lang_id
							return inputs



		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    {self.convert_ids_to_tokens(__a): i for i in range(self.vocab_size)}
							vocab.update(self.added_tokens_encoder)
							return vocab



		def 		UpperCAmelCase				(   self							,						__a)			->			List[str]:






							'''simple docstring'''
							return self.sp_model.encode(__a							,						out_type=__a)



		def 		UpperCAmelCase				(   self							,						__a)			->			List[Any]:






							'''simple docstring'''
							if token in self.fairseq_tokens_to_ids:
												return self.fairseq_tokens_to_ids[token]
							_UpperCamelCase									=    self.sp_model.PieceToId(__a)

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



		def 		UpperCAmelCase				(   self							,						__a)			->			Union[str, Any]:






							'''simple docstring'''
							if index in self.fairseq_ids_to_tokens:
												return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset)



		def 		UpperCAmelCase				(   self							,						__a)			->			int:






							'''simple docstring'''
							_UpperCamelCase									=    ''''''.join(__a).replace(__a							,						''' ''').strip()
							return out_string



		def 		UpperCAmelCase				(   self							,						__a							,						__a = None)			->			Tuple[str]:






							'''simple docstring'''
							if not os.path.isdir(__a):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
												return
							_UpperCamelCase									=    os.path.join(
							    __a							,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

							if os.path.abspath(self.vocab_file) != os.path.abspath(__a) and os.path.isfile(self.vocab_file):
												copyfile(self.vocab_file							,						__a)
							elif not os.path.isfile(self.vocab_file):
												with open(__a							,						'''wb''') as fi:
																	_UpperCamelCase									=    self.sp_model.serialized_model_proto()
																	fi.write(__a)

							return (out_vocab_file,)



		def 		UpperCAmelCase				(   self							,						__a							,						__a = "en_XX"							,						__a = None							,						__a = "ro_RO"							,						**__a							,						)			->			BatchEncoding:






							'''simple docstring'''
							_UpperCamelCase									=    src_lang
							_UpperCamelCase									=    tgt_lang
							return super().prepare_seqaseq_batch(__a							,						__a							,						**__a)



		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							return self.set_src_lang_special_tokens(self.src_lang)



		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							return self.set_tgt_lang_special_tokens(self.tgt_lang)



		def 		UpperCAmelCase				(   self							,						__a)			->			None:






							'''simple docstring'''
							_UpperCamelCase									=    self.lang_code_to_id[src_lang]
							_UpperCamelCase									=    []
							_UpperCamelCase									=    [self.eos_token_id, self.cur_lang_code]




		def 		UpperCAmelCase				(   self							,						__a)			->			None:






							'''simple docstring'''
							_UpperCamelCase									=    self.lang_code_to_id[lang]
							_UpperCamelCase									=    []
							_UpperCamelCase									=    [self.eos_token_id, self.cur_lang_code]








"""simple docstring"""





from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_batched,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


_a								=						logging.get_logger(__name__)






class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = ['pixel_values']



		def __init__(   self							,						__a = True							,						__a = None							,						__a = PILImageResampling.BICUBIC							,						__a = True							,						__a = True							,						__a = 1 / 2_55							,						__a = None							,						__a = True							,						__a = None							,						__a = None							,						**__a							,						)			->			None:






							'''simple docstring'''
							super().__init__(**__a)
							_UpperCamelCase									=    size if size is not None else {'''height''': 2_24, '''width''': 2_24}
							_UpperCamelCase									=    get_size_dict(__a)
							_UpperCamelCase									=    crop_size if crop_size is not None else {'''height''': 2_24, '''width''': 2_24}
							_UpperCamelCase									=    get_size_dict(__a							,						default_to_square=__a							,						param_name='''crop_size''')

							_UpperCamelCase									=    do_resize
							_UpperCamelCase									=    do_rescale
							_UpperCamelCase									=    do_normalize
							_UpperCamelCase									=    do_center_crop
							_UpperCamelCase									=    crop_size
							_UpperCamelCase									=    size
							_UpperCamelCase									=    resample
							_UpperCamelCase									=    rescale_factor
							_UpperCamelCase									=    image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
							_UpperCamelCase									=    image_std if image_std is not None else IMAGENET_DEFAULT_STD



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a = PILImageResampling.BILINEAR							,						__a = None							,						**__a							,						)			->			np.ndarray:






							'''simple docstring'''
							_UpperCamelCase									=    get_size_dict(__a)

							if "shortest_edge" in size:
												_UpperCamelCase									=    get_resize_output_image_size(__a							,						size=size['''shortest_edge''']							,						default_to_square=__a)
												# size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"])
							elif "height" in size and "width" in size:
												_UpperCamelCase									=    (size['''height'''], size['''width'''])
							else:
												raise ValueError(F'''Size must contain \'height\' and \'width\' keys or \'shortest_edge\' key. Got {size.keys()}''')
							return resize(__a							,						size=__a							,						resample=__a							,						data_format=__a							,						**__a)



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a = None							,						**__a							,						)			->			np.ndarray:






							'''simple docstring'''
							_UpperCamelCase									=    get_size_dict(__a)
							if "height" not in size or "width" not in size:
												raise ValueError(F'''The `size` parameter must contain the keys (height, width). Got {size.keys()}''')
							return center_crop(__a							,						size=(size['''height'''], size['''width'''])							,						data_format=__a							,						**__a)



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a = None							,						**__a)			->			np.ndarray:






							'''simple docstring'''
							return rescale(__a							,						scale=__a							,						data_format=__a							,						**__a)



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a							,						__a = None							,						**__a							,						)			->			np.ndarray:






							'''simple docstring'''
							return normalize(__a							,						mean=__a							,						std=__a							,						data_format=__a							,						**__a)




		def 		UpperCAmelCase				(   self							,						__a							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = ChannelDimension.FIRST							,						**__a							,						)			->			BatchFeature:






							'''simple docstring'''
							_UpperCamelCase									=    do_resize if do_resize is not None else self.do_resize
							_UpperCamelCase									=    do_rescale if do_rescale is not None else self.do_rescale
							_UpperCamelCase									=    do_normalize if do_normalize is not None else self.do_normalize
							_UpperCamelCase									=    do_center_crop if do_center_crop is not None else self.do_center_crop
							_UpperCamelCase									=    crop_size if crop_size is not None else self.crop_size
							_UpperCamelCase									=    get_size_dict(__a							,						param_name='''crop_size'''							,						default_to_square=__a)
							_UpperCamelCase									=    resample if resample is not None else self.resample
							_UpperCamelCase									=    rescale_factor if rescale_factor is not None else self.rescale_factor
							_UpperCamelCase									=    image_mean if image_mean is not None else self.image_mean
							_UpperCamelCase									=    image_std if image_std is not None else self.image_std

							_UpperCamelCase									=    size if size is not None else self.size
							_UpperCamelCase									=    get_size_dict(__a)

							if not is_batched(__a):
												_UpperCamelCase									=    [images]

							if not valid_images(__a):
												raise ValueError(
												    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
												    '''torch.Tensor, tf.Tensor or jax.ndarray.''')

							if do_resize and size is None:
												raise ValueError('''Size must be specified if do_resize is True.''')

							if do_center_crop and crop_size is None:
												raise ValueError('''Crop size must be specified if do_center_crop is True.''')

							if do_rescale and rescale_factor is None:
												raise ValueError('''Rescale factor must be specified if do_rescale is True.''')

							# All transformations expect numpy arrays.
							_UpperCamelCase									=    [to_numpy_array(__a) for image in images]

							if do_resize:
												_UpperCamelCase									=    [self.resize(image=__a							,						size=__a							,						resample=__a) for image in images]

							if do_center_crop:
												_UpperCamelCase									=    [self.center_crop(image=__a							,						size=__a) for image in images]

							if do_rescale:
												_UpperCamelCase									=    [self.rescale(image=__a							,						scale=__a) for image in images]

							if do_normalize:
												_UpperCamelCase									=    [self.normalize(image=__a							,						mean=__a							,						std=__a) for image in images]

							_UpperCamelCase									=    [to_channel_dimension_format(__a							,						__a) for image in images]

							_UpperCamelCase									=    {'''pixel_values''': images}
							return BatchFeature(data=__a							,						tensor_type=__a)






