


import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case_						:			List[str]													=							logging.get_logger(__name__)

snake_case_						:			List[Any]													=							'''▁'''

snake_case_						:			Optional[Any]													=							{
    '''vocab_file''': '''vocab.json''',
    '''spm_file''': '''sentencepiece.bpe.model''',
}

snake_case_						:			Optional[int]													=							{
    '''vocab_file''': {
        '''facebook/s2t-small-librispeech-asr''': (
            '''https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json'''
        ),
    },
    '''spm_file''': {
        '''facebook/s2t-small-librispeech-asr''': (
            '''https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model'''
        )
    },
}

snake_case_						:			int													=							{
    '''facebook/s2t-small-librispeech-asr''': 10_24,
}

snake_case_						:			List[Any]													=							['''pt''', '''fr''', '''ru''', '''nl''', '''ro''', '''it''', '''es''', '''de''']

snake_case_						:			Any													=							{'''mustc''': MUSTC_LANGS}

class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						VOCAB_FILES_NAMES
						UpperCAmelCase             =						PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase             =						MAX_MODEL_INPUT_SIZES
						UpperCAmelCase             =						["input_ids", "attention_mask"]

						UpperCAmelCase             =						[]




						def __init__(       self					:      Optional[Any]				,    _a					:      Any				,    _a					:      Union[str, Any]				,    _a					:      str="<s>"				,    _a					:      Any="</s>"				,    _a					:      Dict="<pad>"				,    _a					:      List[str]="<unk>"				,    _a					:      Union[str, Any]=False				,    _a					:      Optional[Any]=False				,    _a					:      Optional[int]=None				,    _a					:      List[str]=None				,    _a					:      Optional[Dict[str, Any]] = None				,    **_a					:      int				,    )							->     None:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ={} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    bos_token=_a				,    eos_token=_a				,    unk_token=_a				,    pad_token=_a				,    do_upper_case=_a				,    do_lower_case=_a				,    tgt_lang=_a				,    lang_codes=_a				,    sp_model_kwargs=self.sp_model_kwargs				,    **_a				,    )
										_SCREAMING_SNAKE_CASE			  =do_upper_case
										_SCREAMING_SNAKE_CASE			  =do_lower_case

										_SCREAMING_SNAKE_CASE			  =load_json(_a  )
										_SCREAMING_SNAKE_CASE			  ={v: k for k, v in self.encoder.items()}
										_SCREAMING_SNAKE_CASE			  =spm_file
										_SCREAMING_SNAKE_CASE			  =load_spm(_a				,    self.sp_model_kwargs  )

										if lang_codes is not None:
														_SCREAMING_SNAKE_CASE			  =lang_codes
														_SCREAMING_SNAKE_CASE			  =LANGUAGES[lang_codes]
														_SCREAMING_SNAKE_CASE			  =[f"<lang:{lang}>" for lang in self.langs]
														_SCREAMING_SNAKE_CASE			  ={lang: self.sp_model.PieceToId(f"<lang:{lang}>"  ) for lang in self.langs}

														_SCREAMING_SNAKE_CASE			  =self.lang_tokens
														_SCREAMING_SNAKE_CASE			  =tgt_lang if tgt_lang is not None else self.langs[0]

														self.set_tgt_lang_special_tokens(self._tgt_lang  )
										else:
														_SCREAMING_SNAKE_CASE			  ={}




						@property
						def 							__UpperCamelCase				(       self					:      Dict  )							->     int:


										"""simple docstring"""





										return len(self.encoder  )




						@property
						def 							__UpperCamelCase				(       self					:      Any  )							->     str:


										"""simple docstring"""





										return self._tgt_lang




						@tgt_lang.setter
						def 							__UpperCamelCase				(       self					:      Any				,    _a					:      List[str]  )							->     None:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =new_tgt_lang
										self.set_tgt_lang_special_tokens(_a  )




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      str  )							->     None:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.lang_code_to_id[tgt_lang]
										_SCREAMING_SNAKE_CASE			  =[lang_code_id]




						def 							__UpperCamelCase				(       self					:      Any				,    _a					:      str  )							->     List[str]:


										"""simple docstring"""





										return self.sp_model.encode(_a				,    out_type=_a  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      Union[str, Any]  )							->     Tuple:


										"""simple docstring"""





										return self.encoder.get(_a				,    self.encoder[self.unk_token]  )




						def 							__UpperCamelCase				(       self					:      List[Any]				,    _a					:      int  )							->     str:


										"""simple docstring"""





										return self.decoder.get(_a				,    self.unk_token  )




						def 							__UpperCamelCase				(       self					:      Optional[int]				,    _a					:      List[str]  )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =[]
										_SCREAMING_SNAKE_CASE			  =''''''
										for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																		_SCREAMING_SNAKE_CASE			  =self.sp_model.decode(_a  )
																		out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
																		_SCREAMING_SNAKE_CASE			  =[]
														else:
																		current_sub_tokens.append(_a  )
										_SCREAMING_SNAKE_CASE			  =self.sp_model.decode(_a  )
										out_string += decoded.upper() if self.do_upper_case else decoded
										return out_string.strip()




						def 							__UpperCamelCase				(       self					:      Union[str, Any]				,    _a					:      Optional[Any]				,    _a					:      Union[str, Any]=None  )							->     List[int]:


										"""simple docstring"""





										if token_ids_a is None:
														return self.prefix_tokens + token_ids_a + [self.eos_token_id]
										# We don't expect to process pairs, but leave the pair logic for API consistency
										return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      List[int]				,    _a					:      Optional[List[int]] = None				,    _a					:      bool = False  )							->     List[int]:


										"""simple docstring"""





										if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=_a				,    token_ids_a=_a				,    already_has_special_tokens=_a  )

										_SCREAMING_SNAKE_CASE			  =[1] * len(self.prefix_tokens  )
										_SCREAMING_SNAKE_CASE			  =[1]
										if token_ids_a is None:
														return prefix_ones + ([0] * len(_a  )) + suffix_ones
										return prefix_ones + ([0] * len(_a  )) + ([0] * len(_a  )) + suffix_ones




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.encoder.copy()
										vocab.update(self.added_tokens_encoder  )
										return vocab




						def __getstate__(       self					:      str  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.__dict__.copy()
										_SCREAMING_SNAKE_CASE			  =None
										return state




						def __setstate__(       self					:      Dict				,    _a					:      Dict  )							->     None:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =d

										# for backward compatibility
										if not hasattr(self				,    '''sp_model_kwargs'''  ):
														_SCREAMING_SNAKE_CASE			  ={}

										_SCREAMING_SNAKE_CASE			  =load_spm(self.spm_file				,    self.sp_model_kwargs  )




						def 							__UpperCamelCase				(       self					:      Any				,    _a					:      str				,    _a					:      Optional[str] = None  )							->     Tuple[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =Path(_a  )
										assert save_dir.is_dir(), f"{save_directory} should be a directory"
										_SCREAMING_SNAKE_CASE			  =save_dir / (
										    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
										)
										_SCREAMING_SNAKE_CASE			  =save_dir / (
										    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
										)

										save_json(self.encoder				,    _a  )

										if os.path.abspath(self.spm_file  ) != os.path.abspath(_a  ) and os.path.isfile(self.spm_file  ):
														copyfile(self.spm_file				,    _a  )
										elif not os.path.isfile(self.spm_file  ):
														with open(_a				,    '''wb'''  ) as fi:
																		_SCREAMING_SNAKE_CASE			  =self.sp_model.serialized_model_proto()
																		fi.write(_a  )

										return (str(_a  ), str(_a  ))







def 						lowerCamelCase(   a__	,a__):
				_SCREAMING_SNAKE_CASE			  =sentencepiece.SentencePieceProcessor(**a__)
				spm.Load(str(a__))
				return spm







def 						lowerCamelCase(   a__):
				with open(a__	,'''r''') as f:
								return json.load(a__)







def 						lowerCamelCase(   a__	,a__):
				with open(a__	,'''w''') as f:
								json.dump(a__	,a__	,indent=2)



from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
							import PIL

snake_case_						:			Union[str, Any]													=							logging.get_logger(__name__)







def 						lowerCamelCase(   a__):
				if isinstance(a__	,(list, tuple)) and isinstance(videos[0]	,(list, tuple)) and is_valid_image(videos[0][0]):
								return videos

				elif isinstance(a__	,(list, tuple)) and is_valid_image(videos[0]):
								return [videos]

				elif is_valid_image(a__):
								return [[videos]]

				raise ValueError(f"Could not make batched video from {videos}")


class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						["pixel_values"]




						def __init__(       self					:      Tuple				,    _a					:      bool = True				,    _a					:      Dict[str, int] = None				,    _a					:      PILImageResampling = PILImageResampling.BILINEAR				,    _a					:      bool = True				,    _a					:      Dict[str, int] = None				,    _a					:      bool = True				,    _a					:      Union[int, float] = 1 / 255				,    _a					:      bool = True				,    _a					:      bool = True				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    **_a					:      Any				,    )							->     None:


										"""simple docstring"""





										super().__init__(**_a  )
										_SCREAMING_SNAKE_CASE			  =size if size is not None else {'''shortest_edge''': 256}
										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    default_to_square=_a  )
										_SCREAMING_SNAKE_CASE			  =crop_size if crop_size is not None else {'''height''': 224, '''width''': 224}
										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    param_name='''crop_size'''  )

										_SCREAMING_SNAKE_CASE			  =do_resize
										_SCREAMING_SNAKE_CASE			  =size
										_SCREAMING_SNAKE_CASE			  =do_center_crop
										_SCREAMING_SNAKE_CASE			  =crop_size
										_SCREAMING_SNAKE_CASE			  =resample
										_SCREAMING_SNAKE_CASE			  =do_rescale
										_SCREAMING_SNAKE_CASE			  =rescale_factor
										_SCREAMING_SNAKE_CASE			  =offset
										_SCREAMING_SNAKE_CASE			  =do_normalize
										_SCREAMING_SNAKE_CASE			  =image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
										_SCREAMING_SNAKE_CASE			  =image_std if image_std is not None else IMAGENET_STANDARD_STD




						def 							__UpperCamelCase				(       self					:      List[Any]				,    _a					:      np.ndarray				,    _a					:      Dict[str, int]				,    _a					:      PILImageResampling = PILImageResampling.BILINEAR				,    _a					:      Optional[Union[str, ChannelDimension]] = None				,    **_a					:      List[Any]				,    )							->     np.ndarray:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    default_to_square=_a  )
										if "shortest_edge" in size:
														_SCREAMING_SNAKE_CASE			  =get_resize_output_image_size(_a				,    size['''shortest_edge''']				,    default_to_square=_a  )
										elif "height" in size and "width" in size:
														_SCREAMING_SNAKE_CASE			  =(size['''height'''], size['''width'''])
										else:
														raise ValueError(f"Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}"  )
										return resize(_a				,    size=_a				,    resample=_a				,    data_format=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      int				,    _a					:      np.ndarray				,    _a					:      Dict[str, int]				,    _a					:      Optional[Union[str, ChannelDimension]] = None				,    **_a					:      Dict				,    )							->     np.ndarray:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a  )
										if "height" not in size or "width" not in size:
														raise ValueError(f"Size must have 'height' and 'width' as keys. Got {size.keys()}"  )
										return center_crop(_a				,    size=(size['''height'''], size['''width'''])				,    data_format=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      np.ndarray				,    _a					:      Union[int, float]				,    _a					:      bool = True				,    _a					:      Optional[Union[str, ChannelDimension]] = None				,    **_a					:      List[str]				,    )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =image.astype(np.floataa  )
										if offset:
														_SCREAMING_SNAKE_CASE			  =image - (scale / 2)
										return rescale(_a				,    scale=_a				,    data_format=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      List[str]				,    _a					:      np.ndarray				,    _a					:      Union[float, List[float]]				,    _a					:      Union[float, List[float]]				,    _a					:      Optional[Union[str, ChannelDimension]] = None				,    **_a					:      Any				,    )							->     np.ndarray:


										"""simple docstring"""





										return normalize(_a				,    mean=_a				,    std=_a				,    data_format=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      ImageInput				,    _a					:      bool = None				,    _a					:      Dict[str, int] = None				,    _a					:      PILImageResampling = None				,    _a					:      bool = None				,    _a					:      Dict[str, int] = None				,    _a					:      bool = None				,    _a					:      float = None				,    _a					:      bool = None				,    _a					:      bool = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[ChannelDimension] = ChannelDimension.FIRST				,    )							->     np.ndarray:


										"""simple docstring"""





										if do_resize and size is None or resample is None:
														raise ValueError('''Size and resample must be specified if do_resize is True.'''  )

										if do_center_crop and crop_size is None:
														raise ValueError('''Crop size must be specified if do_center_crop is True.'''  )

										if do_rescale and rescale_factor is None:
														raise ValueError('''Rescale factor must be specified if do_rescale is True.'''  )

										if do_normalize and (image_mean is None or image_std is None):
														raise ValueError('''Image mean and std must be specified if do_normalize is True.'''  )

										if offset and not do_rescale:
														raise ValueError('''For offset, do_rescale must also be set to True.'''  )

										# All transformations expect numpy arrays.
										_SCREAMING_SNAKE_CASE			  =to_numpy_array(_a  )

										if do_resize:
														_SCREAMING_SNAKE_CASE			  =self.resize(image=_a				,    size=_a				,    resample=_a  )

										if do_center_crop:
														_SCREAMING_SNAKE_CASE			  =self.center_crop(_a				,    size=_a  )

										if do_rescale:
														_SCREAMING_SNAKE_CASE			  =self.rescale(image=_a				,    scale=_a				,    offset=_a  )

										if do_normalize:
														_SCREAMING_SNAKE_CASE			  =self.normalize(image=_a				,    mean=_a				,    std=_a  )

										_SCREAMING_SNAKE_CASE			  =to_channel_dimension_format(_a				,    _a  )
										return image




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      ImageInput				,    _a					:      bool = None				,    _a					:      Dict[str, int] = None				,    _a					:      PILImageResampling = None				,    _a					:      bool = None				,    _a					:      Dict[str, int] = None				,    _a					:      bool = None				,    _a					:      float = None				,    _a					:      bool = None				,    _a					:      bool = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[Union[str, TensorType]] = None				,    _a					:      ChannelDimension = ChannelDimension.FIRST				,    **_a					:      str				,    )							->     PIL.Image.Image:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =do_resize if do_resize is not None else self.do_resize
										_SCREAMING_SNAKE_CASE			  =resample if resample is not None else self.resample
										_SCREAMING_SNAKE_CASE			  =do_center_crop if do_center_crop is not None else self.do_center_crop
										_SCREAMING_SNAKE_CASE			  =do_rescale if do_rescale is not None else self.do_rescale
										_SCREAMING_SNAKE_CASE			  =rescale_factor if rescale_factor is not None else self.rescale_factor
										_SCREAMING_SNAKE_CASE			  =offset if offset is not None else self.offset
										_SCREAMING_SNAKE_CASE			  =do_normalize if do_normalize is not None else self.do_normalize
										_SCREAMING_SNAKE_CASE			  =image_mean if image_mean is not None else self.image_mean
										_SCREAMING_SNAKE_CASE			  =image_std if image_std is not None else self.image_std

										_SCREAMING_SNAKE_CASE			  =size if size is not None else self.size
										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    default_to_square=_a  )
										_SCREAMING_SNAKE_CASE			  =crop_size if crop_size is not None else self.crop_size
										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    param_name='''crop_size'''  )

										if not valid_images(_a  ):
														raise ValueError(
														    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
														    '''torch.Tensor, tf.Tensor or jax.ndarray.'''  )

										_SCREAMING_SNAKE_CASE			  =make_batched(_a  )

										_SCREAMING_SNAKE_CASE			  =[
										    [
										        self._preprocess_image(
										            image=_a				,    do_resize=_a				,    size=_a				,    resample=_a				,    do_center_crop=_a				,    crop_size=_a				,    do_rescale=_a				,    rescale_factor=_a				,    offset=_a				,    do_normalize=_a				,    image_mean=_a				,    image_std=_a				,    data_format=_a				,    )
										        for img in video
										    ]
										    for video in videos
										]

										_SCREAMING_SNAKE_CASE			  ={'''pixel_values''': videos}
										return BatchFeature(data=_a				,    tensor_type=_a  )