


'''simple docstring'''






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowercase				=logging.get_logger(__name__)

lowercase				='▁'

lowercase				={'vocab_file': 'sentencepiece.bpe.model'}

lowercase				={
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

lowercase				={
    'facebook/mbart-large-en-ro': 1024,
    'facebook/mbart-large-cc25': 1024,
}

# fmt: off
lowercase				=['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']






class    __magic_name__				(						lowerCAmelCase							):
       UpperCAmelCase		 =VOCAB_FILES_NAMES
       UpperCAmelCase		 =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCAmelCase		 =PRETRAINED_VOCAB_FILES_MAP
       UpperCAmelCase		 =["input_ids", "attention_mask"]

       UpperCAmelCase		 =[]
       UpperCAmelCase		 =[]






       def __init__(     self    ,				snake_case    ,				snake_case="<s>"    ,				snake_case="</s>"    ,				snake_case="</s>"    ,				snake_case="<s>"    ,				snake_case="<unk>"    ,				snake_case="<pad>"    ,				snake_case="<mask>"    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case = None    ,				snake_case=None    ,				**snake_case    ,				)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[int]				       =AddedToken(__a    ,				lstrip=__a    ,				rstrip=__a) if isinstance(__a    ,				__a) else mask_token

          _UpperCAmelCase  :  List[Any]				       ={} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              bos_token=__a    ,				eos_token=__a    ,				unk_token=__a    ,				sep_token=__a    ,				cls_token=__a    ,				pad_token=__a    ,				mask_token=__a    ,				tokenizer_file=__a    ,				src_lang=__a    ,				tgt_lang=__a    ,				additional_special_tokens=__a    ,				sp_model_kwargs=self.sp_model_kwargs    ,				**__a    ,				)

          _UpperCAmelCase  :  str				       =spm.SentencePieceProcessor(**self.sp_model_kwargs)
          self.sp_model.Load(str(__a))
          _UpperCAmelCase  :  Tuple				       =vocab_file

          # Original fairseq vocab and spm vocab must be "aligned":
          # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
          # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
          # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
          # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

          # Mimic fairseq token-to-id alignment for the first 4 token
          _UpperCAmelCase  :  Tuple				       ={'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

          # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
          _UpperCAmelCase  :  Optional[int]				       =1

          _UpperCAmelCase  :  Any				       =len(self.sp_model)
          _UpperCAmelCase  :  List[Any]				       ={
              code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(__a)
          }
          _UpperCAmelCase  :  List[Any]				       ={v: k for k, v in self.lang_code_to_id.items()}
          _UpperCAmelCase  :  Optional[Any]				       =len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset

          self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
          _UpperCAmelCase  :  List[Any]				       ={v: k for k, v in self.fairseq_tokens_to_ids.items()}
          _UpperCAmelCase  :  List[str]				       =list(self.lang_code_to_id.keys())

          if additional_special_tokens is not None:
             # Only add those special tokens if they are not already there.
             self._additional_special_tokens.extend(
                 [t for t in additional_special_tokens if t not in self._additional_special_tokens])

          _UpperCAmelCase  :  Tuple				       =src_lang if src_lang is not None else 'en_XX'
          _UpperCAmelCase  :  List[Any]				       =self.lang_code_to_id[self._src_lang]
          _UpperCAmelCase  :  Optional[Any]				       =tgt_lang
          self.set_src_lang_special_tokens(self._src_lang)






       def __getstate__(     self)		->    Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase  :  Dict				       =self.__dict__.copy()
          _UpperCAmelCase  :  Optional[int]				       =None
          _UpperCAmelCase  :  int				       =self.sp_model.serialized_model_proto()
          return state






       def __setstate__(     self    ,				snake_case)		->    int:




          '''simple docstring'''
          _UpperCAmelCase  :  List[Any]				       =d

          # for backward compatibility
          if not hasattr(self    ,				'sp_model_kwargs'):
             _UpperCAmelCase  :  int				       ={}

          _UpperCAmelCase  :  int				       =spm.SentencePieceProcessor(**self.sp_model_kwargs)
          self.sp_model.LoadFromSerializedProto(self.sp_model_proto)






       @property
       def      lowerCAmelCase						(     self)		->    Dict:




          '''simple docstring'''
          return len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset + 1  # Plus 1 for the mask token






       @property
       def      lowerCAmelCase						(     self)		->    str:




          '''simple docstring'''
          return self._src_lang






       @src_lang.setter
       def      lowerCAmelCase						(     self    ,				snake_case)		->    None:




          '''simple docstring'''
          _UpperCAmelCase  :  List[Any]				       =new_src_lang
          self.set_src_lang_special_tokens(self._src_lang)






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case = None    ,				snake_case = False)		->    List[int]:




          '''simple docstring'''

          if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=__a    ,				token_ids_a=__a    ,				already_has_special_tokens=__a)

          _UpperCAmelCase  :  Any				       =[1] * len(self.prefix_tokens)
          _UpperCAmelCase  :  Union[str, Any]				       =[1] * len(self.suffix_tokens)
          if token_ids_a is None:
             return prefix_ones + ([0] * len(__a)) + suffix_ones
          return prefix_ones + ([0] * len(__a)) + ([0] * len(__a)) + suffix_ones






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case = None)		->    List[int]:




          '''simple docstring'''
          if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case = None)		->    List[int]:




          '''simple docstring'''
          _UpperCAmelCase  :  List[str]				       =[self.sep_token_id]
          _UpperCAmelCase  :  Dict				       =[self.cls_token_id]

          if token_ids_a is None:
             return len(cls + token_ids_a + sep) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case    ,				snake_case    ,				snake_case    ,				**snake_case)		->    List[Any]:




          '''simple docstring'''
          if src_lang is None or tgt_lang is None:
             raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model')
          _UpperCAmelCase  :  str				       =src_lang
          _UpperCAmelCase  :  str				       =self(__a    ,				add_special_tokens=__a    ,				return_tensors=__a    ,				**__a)
          _UpperCAmelCase  :  Union[str, Any]				       =self.convert_tokens_to_ids(__a)
          _UpperCAmelCase  :  Tuple				       =tgt_lang_id
          return inputs






       def      lowerCAmelCase						(     self)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  int				       ={self.convert_ids_to_tokens(__a): i for i in range(self.vocab_size)}
          vocab.update(self.added_tokens_encoder)
          return vocab






       def      lowerCAmelCase						(     self    ,				snake_case)		->    List[str]:




          '''simple docstring'''
          return self.sp_model.encode(__a    ,				out_type=__a)






       def      lowerCAmelCase						(     self    ,				snake_case)		->    List[Any]:




          '''simple docstring'''
          if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
          _UpperCAmelCase  :  Tuple				       =self.sp_model.PieceToId(__a)

          # Need to return unknown token if the SP model returned 0
          return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






       def      lowerCAmelCase						(     self    ,				snake_case)		->    Union[str, Any]:




          '''simple docstring'''
          if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
          return self.sp_model.IdToPiece(index - self.fairseq_offset)






       def      lowerCAmelCase						(     self    ,				snake_case)		->    int:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[int]				       =''.join(__a).replace(__a    ,				' ').strip()
          return out_string






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case = None)		->    Tuple[str]:




          '''simple docstring'''
          if not os.path.isdir(__a):
             logger.error(f"Vocabulary path ({save_directory}) should be a directory")
             return
          _UpperCAmelCase  :  Optional[Any]				       =os.path.join(
              __a    ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

          if os.path.abspath(self.vocab_file) != os.path.abspath(__a) and os.path.isfile(self.vocab_file):
             copyfile(self.vocab_file    ,				__a)
          elif not os.path.isfile(self.vocab_file):
             with open(__a    ,				'wb') as fi:
                _UpperCAmelCase  :  Tuple				       =self.sp_model.serialized_model_proto()
                fi.write(__a)

          return (out_vocab_file,)






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case = "en_XX"    ,				snake_case = None    ,				snake_case = "ro_RO"    ,				**snake_case    ,				)		->    BatchEncoding:




          '''simple docstring'''
          _UpperCAmelCase  :  Any				       =src_lang
          _UpperCAmelCase  :  int				       =tgt_lang
          return super().prepare_seqaseq_batch(__a    ,				__a    ,				**__a)






       def      lowerCAmelCase						(     self)		->    str:




          '''simple docstring'''
          return self.set_src_lang_special_tokens(self.src_lang)






       def      lowerCAmelCase						(     self)		->    str:




          '''simple docstring'''
          return self.set_tgt_lang_special_tokens(self.tgt_lang)






       def      lowerCAmelCase						(     self    ,				snake_case)		->    None:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[Any]				       =self.lang_code_to_id[src_lang]
          _UpperCAmelCase  :  Optional[Any]				       =[]
          _UpperCAmelCase  :  Dict				       =[self.eos_token_id, self.cur_lang_code]






       def      lowerCAmelCase						(     self    ,				snake_case)		->    None:




          '''simple docstring'''
          _UpperCAmelCase  :  List[str]				       =self.lang_code_to_id[lang]
          _UpperCAmelCase  :  Tuple				       =[]
          _UpperCAmelCase  :  Union[str, Any]				       =[self.eos_token_id, self.cur_lang_code]



"""simple docstring"""





from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_batched,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


_a								=						logging.get_logger(__name__)






class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = ['pixel_values']



		def __init__(   self							,						__a = True							,						__a = None							,						__a = PILImageResampling.BICUBIC							,						__a = True							,						__a = True							,						__a = 1 / 2_55							,						__a = None							,						__a = True							,						__a = None							,						__a = None							,						**__a							,						)			->			None:






							'''simple docstring'''
							super().__init__(**__a)
							_UpperCamelCase									=    size if size is not None else {'''height''': 2_24, '''width''': 2_24}
							_UpperCamelCase									=    get_size_dict(__a)
							_UpperCamelCase									=    crop_size if crop_size is not None else {'''height''': 2_24, '''width''': 2_24}
							_UpperCamelCase									=    get_size_dict(__a							,						default_to_square=__a							,						param_name='''crop_size''')

							_UpperCamelCase									=    do_resize
							_UpperCamelCase									=    do_rescale
							_UpperCamelCase									=    do_normalize
							_UpperCamelCase									=    do_center_crop
							_UpperCamelCase									=    crop_size
							_UpperCamelCase									=    size
							_UpperCamelCase									=    resample
							_UpperCamelCase									=    rescale_factor
							_UpperCamelCase									=    image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
							_UpperCamelCase									=    image_std if image_std is not None else IMAGENET_DEFAULT_STD



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a = PILImageResampling.BILINEAR							,						__a = None							,						**__a							,						)			->			np.ndarray:






							'''simple docstring'''
							_UpperCamelCase									=    get_size_dict(__a)

							if "shortest_edge" in size:
												_UpperCamelCase									=    get_resize_output_image_size(__a							,						size=size['''shortest_edge''']							,						default_to_square=__a)
												# size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"])
							elif "height" in size and "width" in size:
												_UpperCamelCase									=    (size['''height'''], size['''width'''])
							else:
												raise ValueError(F'''Size must contain \'height\' and \'width\' keys or \'shortest_edge\' key. Got {size.keys()}''')
							return resize(__a							,						size=__a							,						resample=__a							,						data_format=__a							,						**__a)



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a = None							,						**__a							,						)			->			np.ndarray:






							'''simple docstring'''
							_UpperCamelCase									=    get_size_dict(__a)
							if "height" not in size or "width" not in size:
												raise ValueError(F'''The `size` parameter must contain the keys (height, width). Got {size.keys()}''')
							return center_crop(__a							,						size=(size['''height'''], size['''width'''])							,						data_format=__a							,						**__a)



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a = None							,						**__a)			->			np.ndarray:






							'''simple docstring'''
							return rescale(__a							,						scale=__a							,						data_format=__a							,						**__a)



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a							,						__a = None							,						**__a							,						)			->			np.ndarray:






							'''simple docstring'''
							return normalize(__a							,						mean=__a							,						std=__a							,						data_format=__a							,						**__a)




		def 		UpperCAmelCase				(   self							,						__a							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						__a = ChannelDimension.FIRST							,						**__a							,						)			->			BatchFeature:






							'''simple docstring'''
							_UpperCamelCase									=    do_resize if do_resize is not None else self.do_resize
							_UpperCamelCase									=    do_rescale if do_rescale is not None else self.do_rescale
							_UpperCamelCase									=    do_normalize if do_normalize is not None else self.do_normalize
							_UpperCamelCase									=    do_center_crop if do_center_crop is not None else self.do_center_crop
							_UpperCamelCase									=    crop_size if crop_size is not None else self.crop_size
							_UpperCamelCase									=    get_size_dict(__a							,						param_name='''crop_size'''							,						default_to_square=__a)
							_UpperCamelCase									=    resample if resample is not None else self.resample
							_UpperCamelCase									=    rescale_factor if rescale_factor is not None else self.rescale_factor
							_UpperCamelCase									=    image_mean if image_mean is not None else self.image_mean
							_UpperCamelCase									=    image_std if image_std is not None else self.image_std

							_UpperCamelCase									=    size if size is not None else self.size
							_UpperCamelCase									=    get_size_dict(__a)

							if not is_batched(__a):
												_UpperCamelCase									=    [images]

							if not valid_images(__a):
												raise ValueError(
												    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
												    '''torch.Tensor, tf.Tensor or jax.ndarray.''')

							if do_resize and size is None:
												raise ValueError('''Size must be specified if do_resize is True.''')

							if do_center_crop and crop_size is None:
												raise ValueError('''Crop size must be specified if do_center_crop is True.''')

							if do_rescale and rescale_factor is None:
												raise ValueError('''Rescale factor must be specified if do_rescale is True.''')

							# All transformations expect numpy arrays.
							_UpperCamelCase									=    [to_numpy_array(__a) for image in images]

							if do_resize:
												_UpperCamelCase									=    [self.resize(image=__a							,						size=__a							,						resample=__a) for image in images]

							if do_center_crop:
												_UpperCamelCase									=    [self.center_crop(image=__a							,						size=__a) for image in images]

							if do_rescale:
												_UpperCamelCase									=    [self.rescale(image=__a							,						scale=__a) for image in images]

							if do_normalize:
												_UpperCamelCase									=    [self.normalize(image=__a							,						mean=__a							,						std=__a) for image in images]

							_UpperCamelCase									=    [to_channel_dimension_format(__a							,						__a) for image in images]

							_UpperCamelCase									=    {'''pixel_values''': images}
							return BatchFeature(data=__a							,						tensor_type=__a)






