





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a     :    List[str]         =						logging.get_logger(__name__)

_a     :    Union[str, Any]         =						{'vocab_file': 'spm_char.model'}

_a     :    Dict         =						{
    'vocab_file': {
        'microsoft/speecht5_asr': 'https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model',
        'microsoft/speecht5_tts': 'https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model',
        'microsoft/speecht5_vc': 'https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model',
    }
}

_a     :    List[Any]         =						{
    'microsoft/speecht5_asr': 1_024,
    'microsoft/speecht5_tts': 1_024,
    'microsoft/speecht5_vc': 1_024,
}
class   a_ (			snake_case__      ):
			A__      :			int       	=  VOCAB_FILES_NAMES
			A__      :			Optional[int]       	=  PRETRAINED_VOCAB_FILES_MAP
			A__      :			Tuple       	=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			A__      :			Union[str, Any]       	=  ['input_ids', 'attention_mask']

			def __init__(   self   :  Any     , UpperCAmelCase__   :  Optional[int]     , UpperCAmelCase__   :  Optional[int]="<s>"     , UpperCAmelCase__   :  Dict="</s>"     , UpperCAmelCase__   :  List[Any]="<unk>"     , UpperCAmelCase__   :  Dict="<pad>"     , UpperCAmelCase__   :  str = None     , **UpperCAmelCase__   :  Dict     , ):






							"""simple docstring"""
							snake_case   :					Optional[int]    						=				{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=_UpperCamelCase     , eos_token=_UpperCamelCase     , unk_token=_UpperCamelCase     , pad_token=_UpperCamelCase     , sp_model_kwargs=self.sp_model_kwargs     , **_UpperCamelCase     , )

							snake_case   :					str    						=				vocab_file

							snake_case   :					Dict    						=				spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(_UpperCamelCase						)

			@property
			def      lowerCAmelCase(   self   :  Union[str, Any]						):






							"""simple docstring"""
							return self.sp_model.get_piece_size()

			def      lowerCAmelCase(   self   :  Union[str, Any]						):






							"""simple docstring"""
							snake_case   :					Dict    						=				{self.convert_ids_to_tokens(_UpperCamelCase						): i for i in range(self.vocab_size						)}
							vocab.update(self.added_tokens_encoder						)
							return vocab

			def __getstate__(   self   :  Dict						):






							"""simple docstring"""
							snake_case   :					List[Any]    						=				self.__dict__.copy()
							snake_case   :					int    						=				None
							return state

			def __setstate__(   self   :  Optional[int]     , UpperCAmelCase__   :  Union[str, Any]						):






							"""simple docstring"""
							snake_case   :					List[Any]    						=				d

							# for backward compatibility
							if not hasattr(self     , '''sp_model_kwargs'''						):
											snake_case   :					Any    						=				{}

							snake_case   :					Union[str, Any]    						=				spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(self.vocab_file						)

			def      lowerCAmelCase(   self   :  int     , UpperCAmelCase__   :  List[str]						):






							"""simple docstring"""
							return self.sp_model.encode(_UpperCamelCase     , out_type=_UpperCamelCase						)

			def      lowerCAmelCase(   self   :  Any     , UpperCAmelCase__   :  Optional[Any]						):






							"""simple docstring"""
							return self.sp_model.piece_to_id(_UpperCamelCase						)

			def      lowerCAmelCase(   self   :  Tuple     , UpperCAmelCase__   :  Tuple						):






							"""simple docstring"""
							snake_case   :					Union[str, Any]    						=				self.sp_model.IdToPiece(_UpperCamelCase						)
							return token

			def      lowerCAmelCase(   self   :  str     , UpperCAmelCase__   :  Union[str, Any]						):






							"""simple docstring"""
							snake_case   :					str    						=				[]
							snake_case   :					Any    						=				''''''
							for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
															out_string += self.sp_model.decode(_UpperCamelCase						) + token
															snake_case   :					Dict    						=				[]
											else:
															current_sub_tokens.append(_UpperCamelCase						)
							out_string += self.sp_model.decode(_UpperCamelCase						)
							return out_string.strip()

			def      lowerCAmelCase(   self   :  Union[str, Any]     , UpperCAmelCase__   :  Optional[Any]     , UpperCAmelCase__   :  str=None						):






							"""simple docstring"""


							if token_ids_a is None:
											return token_ids_a + [self.eos_token_id]
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return token_ids_a + token_ids_a + [self.eos_token_id]

			def      lowerCAmelCase(   self   :  Any     , UpperCAmelCase__   :  List[Any]     , UpperCAmelCase__   :  Any = None     , UpperCAmelCase__   :  List[str] = False						):






							"""simple docstring"""


							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=_UpperCamelCase     , token_ids_a=_UpperCamelCase     , already_has_special_tokens=_UpperCamelCase						)

							snake_case   :					Union[str, Any]    						=				[1]
							if token_ids_a is None:
											return ([0] * len(_UpperCamelCase						)) + suffix_ones
							return ([0] * len(_UpperCamelCase						)) + ([0] * len(_UpperCamelCase						)) + suffix_ones



			def      lowerCAmelCase(   self   :  Any     , UpperCAmelCase__   :  str     , UpperCAmelCase__   :  Tuple = None						):






							"""simple docstring"""


							if not os.path.isdir(_UpperCamelCase						):
											logger.error(F"Vocabulary path ({save_directory}) should be a directory"						)
											return
							snake_case   :					int    						=				os.path.join(
							    _UpperCamelCase     , (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)

							if os.path.abspath(self.vocab_file						) != os.path.abspath(_UpperCamelCase						) and os.path.isfile(self.vocab_file						):
											copyfile(self.vocab_file     , _UpperCamelCase						)
							elif not os.path.isfile(self.vocab_file						):
											with open(_UpperCamelCase     , '''wb'''						) as fi:
															snake_case   :					Optional[int]    						=				self.sp_model.serialized_model_proto()
															fi.write(_UpperCamelCase						)

							return (out_vocab_file,)



import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_video_inputs


if is_torch_available():
    import torch

if is_vision_available():
    from PIL import Image

    from transformers import VivitImageProcessor

class     __snake_case    (				unittest.TestCase  ):
       """simple docstring"""







       def __init__(		self      , _UpperCamelCase      , _UpperCamelCase=7      , _UpperCamelCase=3      , _UpperCamelCase=10      , _UpperCamelCase=18      , _UpperCamelCase=30      , _UpperCamelCase=4_00      , _UpperCamelCase=True      , _UpperCamelCase=None      , _UpperCamelCase=True      , _UpperCamelCase=[0.5, 0.5, 0.5]      , _UpperCamelCase=[0.5, 0.5, 0.5]      , _UpperCamelCase=None      , )      ->							str:






              """simple docstring"""


              __snake_case       	=							size if size is not None else {"""shortest_edge""": 18}
              __snake_case       	=							crop_size if crop_size is not None else {"""height""": 18, """width""": 18}

              __snake_case       	=							parent
              __snake_case       	=							batch_size
              __snake_case       	=							num_channels
              __snake_case       	=							num_frames
              __snake_case       	=							image_size
              __snake_case       	=							min_resolution
              __snake_case       	=							max_resolution
              __snake_case       	=							do_resize
              __snake_case       	=							size
              __snake_case       	=							do_normalize
              __snake_case       	=							image_mean
              __snake_case       	=							image_std
              __snake_case       	=							crop_size






       def 			a		(		self   )      ->							Optional[Any]:






              """simple docstring"""


              return {
                  "image_mean": self.image_mean,
                  "image_std": self.image_std,
                  "do_normalize": self.do_normalize,
                  "do_resize": self.do_resize,
                  "size": self.size,
                  "crop_size": self.crop_size,
              }






@require_torch
@require_vision
class     __snake_case    (				snake_case__			,	unittest.TestCase  ):
       """simple docstring"""







       __SCREAMING_SNAKE_CASE				    =							VivitImageProcessor if is_vision_available() else None


       def 			a		(		self   )      ->							int:






              """simple docstring"""


              __snake_case       	=							VivitImageProcessingTester(self   )


       @property
       def 			a		(		self   )      ->							str:






              """simple docstring"""


              return self.image_processor_tester.prepare_image_processor_dict()


       def 			a		(		self   )      ->							List[Any]:






              """simple docstring"""


              __snake_case       	=							self.image_processing_class(**self.image_processor_dict   )
              self.assertTrue(hasattr(_UpperCamelCase      , """image_mean"""   )   )
              self.assertTrue(hasattr(_UpperCamelCase      , """image_std"""   )   )
              self.assertTrue(hasattr(_UpperCamelCase      , """do_normalize"""   )   )
              self.assertTrue(hasattr(_UpperCamelCase      , """do_resize"""   )   )
              self.assertTrue(hasattr(_UpperCamelCase      , """do_center_crop"""   )   )
              self.assertTrue(hasattr(_UpperCamelCase      , """size"""   )   )


       def 			a		(		self   )      ->							str:






              """simple docstring"""


              __snake_case       	=							self.image_processing_class.from_dict(self.image_processor_dict   )
              self.assertEqual(image_processor.size      , {"""shortest_edge""": 18}   )
              self.assertEqual(image_processor.crop_size      , {"""height""": 18, """width""": 18}   )

              __snake_case       	=							self.image_processing_class.from_dict(self.image_processor_dict      , size=42      , crop_size=84   )
              self.assertEqual(image_processor.size      , {"""shortest_edge""": 42}   )
              self.assertEqual(image_processor.crop_size      , {"""height""": 84, """width""": 84}   )


       def 			a		(		self   )      ->							List[str]:






              """simple docstring"""


              __snake_case       	=							self.image_processing_class(**self.image_processor_dict   )
              # create random PIL videos
              __snake_case       	=							prepare_video_inputs(self.image_processor_tester      , equal_resolution=_UpperCamelCase   )
              for video in video_inputs:
                     self.assertIsInstance(_UpperCamelCase      , _UpperCamelCase   )
                     self.assertIsInstance(video[0]      , Image.Image   )

              # Test not batched input
              __snake_case       	=							image_processing(video_inputs[0]      , return_tensors="""pt"""   ).pixel_values
              self.assertEqual(
                  encoded_videos.shape      , (
                      1,
                      self.image_processor_tester.num_frames,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["""height"""],
                      self.image_processor_tester.crop_size["""width"""],
                  )      , )

              # Test batched
              __snake_case       	=							image_processing(_UpperCamelCase      , return_tensors="""pt"""   ).pixel_values
              self.assertEqual(
                  encoded_videos.shape      , (
                      self.image_processor_tester.batch_size,
                      self.image_processor_tester.num_frames,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["""height"""],
                      self.image_processor_tester.crop_size["""width"""],
                  )      , )


       def 			a		(		self   )      ->							Union[str, Any]:






              """simple docstring"""


              __snake_case       	=							self.image_processing_class(**self.image_processor_dict   )
              # create random numpy tensors
              __snake_case       	=							prepare_video_inputs(self.image_processor_tester      , equal_resolution=_UpperCamelCase      , numpify=_UpperCamelCase   )
              for video in video_inputs:
                     self.assertIsInstance(_UpperCamelCase      , _UpperCamelCase   )
                     self.assertIsInstance(video[0]      , np.ndarray   )

              # Test not batched input
              __snake_case       	=							image_processing(video_inputs[0]      , return_tensors="""pt"""   ).pixel_values
              self.assertEqual(
                  encoded_videos.shape      , (
                      1,
                      self.image_processor_tester.num_frames,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["""height"""],
                      self.image_processor_tester.crop_size["""width"""],
                  )      , )

              # Test batched
              __snake_case       	=							image_processing(_UpperCamelCase      , return_tensors="""pt"""   ).pixel_values
              self.assertEqual(
                  encoded_videos.shape      , (
                      self.image_processor_tester.batch_size,
                      self.image_processor_tester.num_frames,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["""height"""],
                      self.image_processor_tester.crop_size["""width"""],
                  )      , )






       def 			a		(		self   )      ->							List[Any]:






              """simple docstring"""


              __snake_case       	=							self.image_processing_class(**self.image_processor_dict   )
              # create random PyTorch tensors
              __snake_case       	=							prepare_video_inputs(self.image_processor_tester      , equal_resolution=_UpperCamelCase      , torchify=_UpperCamelCase   )
              for video in video_inputs:
                     self.assertIsInstance(_UpperCamelCase      , _UpperCamelCase   )
                     self.assertIsInstance(video[0]      , torch.Tensor   )

              # Test not batched input
              __snake_case       	=							image_processing(video_inputs[0]      , return_tensors="""pt"""   ).pixel_values
              self.assertEqual(
                  encoded_videos.shape      , (
                      1,
                      self.image_processor_tester.num_frames,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["""height"""],
                      self.image_processor_tester.crop_size["""width"""],
                  )      , )

              # Test batched
              __snake_case       	=							image_processing(_UpperCamelCase      , return_tensors="""pt"""   ).pixel_values
              self.assertEqual(
                  encoded_videos.shape      , (
                      self.image_processor_tester.batch_size,
                      self.image_processor_tester.num_frames,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["""height"""],
                      self.image_processor_tester.crop_size["""width"""],
                  )      , )

