





"""simple docstring"""





import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase					      =   logging.get_logger(__name__)

_UpperCamelCase					      =   '▁'

_UpperCamelCase					      =   {
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
}

_UpperCamelCase					      =   {
    'vocab_file': {
        'facebook/s2t-small-librispeech-asr': (
            'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json'
        ),
    },
    'spm_file': {
        'facebook/s2t-small-librispeech-asr': (
            'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model'
        )
    },
}

_UpperCamelCase					      =   {
    'facebook/s2t-small-librispeech-asr': 1024,
}

_UpperCamelCase					      =   ['pt', 'fr', 'ru', 'nl', 'ro', 'it', 'es', 'de']

_UpperCamelCase					      =   {'mustc': MUSTC_LANGS}
class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
				"""simple docstring"""
				__snake_case :							List[str]									=		VOCAB_FILES_NAMES
				__snake_case :							str									=		PRETRAINED_VOCAB_FILES_MAP
				__snake_case :							Optional[int]									=		MAX_MODEL_INPUT_SIZES
				__snake_case :							int									=		["""input_ids""", """attention_mask"""]

				__snake_case :							List[int]									=		[]




				def __init__(  self	:Union[str, Any]    ,     __lowercase	:Dict    ,     __lowercase	:Union[str, Any]    ,     __lowercase	:Optional[Any]="<s>"    ,     __lowercase	:Dict="</s>"    ,     __lowercase	:Any="<pad>"    ,     __lowercase	:Union[str, Any]="<unk>"    ,     __lowercase	:Union[str, Any]=False    ,     __lowercase	:str=False    ,     __lowercase	:Optional[int]=None    ,     __lowercase	:Tuple=None    ,     __lowercase	:Optional[Dict[str, Any]] = None    ,     **__lowercase	:Union[str, Any]    ,     ):
											__lowerCamelCase   :   Optional[Any]       ={} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    bos_token=__lowercase    ,     eos_token=__lowercase    ,     unk_token=__lowercase    ,     pad_token=__lowercase    ,     do_upper_case=__lowercase    ,     do_lower_case=__lowercase    ,     tgt_lang=__lowercase    ,     lang_codes=__lowercase    ,     sp_model_kwargs=self.sp_model_kwargs    ,     **__lowercase    ,     )
											__lowerCamelCase   :   Optional[int]       =do_upper_case
											__lowerCamelCase   :   Tuple       =do_lower_case

											__lowerCamelCase   :   Optional[Any]       =load_json(__lowercase      )
											__lowerCamelCase   :   Dict       ={v: k for k, v in self.encoder.items()}
											__lowerCamelCase   :   Tuple       =spm_file
											__lowerCamelCase   :   Optional[int]       =load_spm(__lowercase    ,     self.sp_model_kwargs      )

											if lang_codes is not None:
																		__lowerCamelCase   :   int       =lang_codes
																		__lowerCamelCase   :   Dict       =LANGUAGES[lang_codes]
																		__lowerCamelCase   :   List[Any]       =[f'<lang:{lang}>' for lang in self.langs]
																		__lowerCamelCase   :   Dict       ={lang: self.sp_model.PieceToId(f'<lang:{lang}>'      ) for lang in self.langs}

																		__lowerCamelCase   :   Dict       =self.lang_tokens
																		__lowerCamelCase   :   Optional[int]       =tgt_lang if tgt_lang is not None else self.langs[0]

																		self.set_tgt_lang_special_tokens(self._tgt_lang      )
											else:
																		__lowerCamelCase   :   int       ={}




				@property
				def        __lowercase	(  self	:Any      ):
											return len(self.encoder      )




				@property
				def        __lowercase	(  self	:Tuple      ):
											return self._tgt_lang




				@tgt_lang.setter
				def        __lowercase	(  self	:List[Any]    ,     __lowercase	:Any      ):
											__lowerCamelCase   :   Any       =new_tgt_lang
											self.set_tgt_lang_special_tokens(__lowercase      )




				def        __lowercase	(  self	:str    ,     __lowercase	:str      ):
											__lowerCamelCase   :   Optional[int]       =self.lang_code_to_id[tgt_lang]
											__lowerCamelCase   :   Dict       =[lang_code_id]




				def        __lowercase	(  self	:List[str]    ,     __lowercase	:str      ):
											return self.sp_model.encode(__lowercase    ,     out_type=__lowercase      )




				def        __lowercase	(  self	:str    ,     __lowercase	:Optional[Any]      ):
											return self.encoder.get(__lowercase    ,     self.encoder[self.unk_token]      )




				def        __lowercase	(  self	:List[Any]    ,     __lowercase	:int      ):
											return self.decoder.get(__lowercase    ,     self.unk_token      )




				def        __lowercase	(  self	:Any    ,     __lowercase	:List[str]      ):
											__lowerCamelCase   :   Tuple       =[]
											__lowerCamelCase   :   Optional[Any]       =''''''
											for token in tokens:
																		# make sure that special tokens are not decoded using sentencepiece model
																		if token in self.all_special_tokens:
																									__lowerCamelCase   :   List[str]       =self.sp_model.decode(__lowercase      )
																									out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
																									__lowerCamelCase   :   Any       =[]
																		else:
																									current_sub_tokens.append(__lowercase      )
											__lowerCamelCase   :   Optional[int]       =self.sp_model.decode(__lowercase      )
											out_string += decoded.upper() if self.do_upper_case else decoded
											return out_string.strip()




				def        __lowercase	(  self	:List[Any]    ,     __lowercase	:Optional[Any]    ,     __lowercase	:Tuple=None      ):
											if token_ids_a is None:
																		return self.prefix_tokens + token_ids_a + [self.eos_token_id]
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]




				def        __lowercase	(  self	:Tuple    ,     __lowercase	:List[int]    ,     __lowercase	:Optional[List[int]] = None    ,     __lowercase	:bool = False      ):

											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=__lowercase    ,     token_ids_a=__lowercase    ,     already_has_special_tokens=__lowercase      )

											__lowerCamelCase   :   Optional[Any]       =[1] * len(self.prefix_tokens      )
											__lowerCamelCase   :   Tuple       =[1]
											if token_ids_a is None:
																		return prefix_ones + ([0] * len(__lowercase      )) + suffix_ones
											return prefix_ones + ([0] * len(__lowercase      )) + ([0] * len(__lowercase      )) + suffix_ones




				def        __lowercase	(  self	:Dict      ):
											__lowerCamelCase   :   Any       =self.encoder.copy()
											vocab.update(self.added_tokens_encoder      )
											return vocab




				def __getstate__(  self	:Optional[Any]      ):
											__lowerCamelCase   :   Any       =self.__dict__.copy()
											__lowerCamelCase   :   Optional[Any]       =None
											return state




				def __setstate__(  self	:Union[str, Any]    ,     __lowercase	:Dict      ):
											__lowerCamelCase   :   int       =d

											# for backward compatibility
											if not hasattr(self    ,     '''sp_model_kwargs'''      ):
																		__lowerCamelCase   :   Tuple       ={}

											__lowerCamelCase   :   Optional[int]       =load_spm(self.spm_file    ,     self.sp_model_kwargs      )




				def        __lowercase	(  self	:Any    ,     __lowercase	:str    ,     __lowercase	:Optional[str] = None      ):
											__lowerCamelCase   :   Dict       =Path(__lowercase      )
											assert save_dir.is_dir(), f'{save_directory} should be a directory'
											__lowerCamelCase   :   Dict       =save_dir / (
											    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
											)
											__lowerCamelCase   :   str       =save_dir / (
											    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
											)

											save_json(self.encoder    ,     __lowercase      )

											if os.path.abspath(self.spm_file      ) != os.path.abspath(__lowercase      ) and os.path.isfile(self.spm_file      ):
																		copyfile(self.spm_file    ,     __lowercase      )
											elif not os.path.isfile(self.spm_file      ):
																		with open(__lowercase    ,     '''wb'''      ) as fi:
																									__lowerCamelCase   :   Tuple       =self.sp_model.serialized_model_proto()
																									fi.write(__lowercase      )

											return (str(__lowercase      ), str(__lowercase      ))



def 					lowerCAmelCase_      (   SCREAMING_SNAKE_CASE							:				str     ,    SCREAMING_SNAKE_CASE							:				Dict[str, Any] ):


							'''simple docstring'''







							__lowerCamelCase   :   List[Any]       =sentencepiece.SentencePieceProcessor(**SCREAMING_SNAKE_CASE )
							spm.Load(str(SCREAMING_SNAKE_CASE ) )
							return spm



def 					lowerCAmelCase_      (   SCREAMING_SNAKE_CASE							:				str ):


							'''simple docstring'''







							with open(SCREAMING_SNAKE_CASE     ,    '''r''' ) as f:
														return json.load(SCREAMING_SNAKE_CASE )



def 					lowerCAmelCase_      (   SCREAMING_SNAKE_CASE							:				List[str]     ,    SCREAMING_SNAKE_CASE							:				str ):


							'''simple docstring'''







							with open(SCREAMING_SNAKE_CASE     ,    '''w''' ) as f:
														json.dump(SCREAMING_SNAKE_CASE     ,    SCREAMING_SNAKE_CASE     ,    indent=2 )











"""simple docstring"""





import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
							from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
							from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
							from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


_UpperCamelCase					      =   DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

_UpperCamelCase					      =   'main'
# Default branch name
_UpperCamelCase					      =   'f2c752cfc5c0ab6f4bdec59acea69eefbee381c2'
# One particular commit (not the top of `main`)
_UpperCamelCase					      =   'aaaaaaa'
# This commit does not exist, so we should 404.

_UpperCamelCase					      =   'd9e9f15bc825e4b2c9249e9578f884bbcb5e3684'
# Sha-1 of config.json on the top of `main`, for checking purposes
_UpperCamelCase					      =   '4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3'



@contextlib.contextmanager
def 					lowerCAmelCase_      (   ):


							'''simple docstring'''







							print('''Welcome!''' )
							yield
							print('''Bye!''' )



@contextlib.contextmanager
def 					lowerCAmelCase_      (   ):


							'''simple docstring'''







							print('''Bonjour!''' )
							yield
							print('''Au revoir!''' )
class 		SCREAMING_SNAKE_CASE_				(     unittest.TestCase ):
				"""simple docstring"""




				def        __lowercase	(  self	:str      ):
											# If the spec is missing, importlib would not be able to import the module dynamically.
											assert transformers.__spec__ is not None
											assert importlib.util.find_spec('''transformers'''      ) is not None

class 		SCREAMING_SNAKE_CASE_				(     unittest.TestCase ):
				"""simple docstring"""




				@unittest.mock.patch('''sys.stdout'''    ,     new_callable=io.StringIO      )
				def        __lowercase	(  self	:str    ,     __lowercase	:str      ):
											with ContextManagers([]      ):
																		print('''Transformers are awesome!'''      )
											# The print statement adds a new line at the end of the output
											self.assertEqual(mock_stdout.getvalue()    ,     '''Transformers are awesome!\n'''      )




				@unittest.mock.patch('''sys.stdout'''    ,     new_callable=io.StringIO      )
				def        __lowercase	(  self	:Any    ,     __lowercase	:Tuple      ):
											with ContextManagers([context_en()]      ):
																		print('''Transformers are awesome!'''      )
											# The output should be wrapped with an English welcome and goodbye
											self.assertEqual(mock_stdout.getvalue()    ,     '''Welcome!\nTransformers are awesome!\nBye!\n'''      )




				@unittest.mock.patch('''sys.stdout'''    ,     new_callable=io.StringIO      )
				def        __lowercase	(  self	:List[Any]    ,     __lowercase	:Optional[Any]      ):
											with ContextManagers([context_fr(), context_en()]      ):
																		print('''Transformers are awesome!'''      )
											# The output should be wrapped with an English and French welcome and goodbye
											self.assertEqual(mock_stdout.getvalue()    ,     '''Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n'''      )




				@require_torch
				def        __lowercase	(  self	:Tuple      ):
											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''']      )
											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''', '''next_sentence_label''']      )
											self.assertEqual(find_labels(__lowercase      )    ,     ['''start_positions''', '''end_positions''']      )
											class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
															"""simple docstring"""
															pass

											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''']      )




				@require_tf
				def        __lowercase	(  self	:str      ):
											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''']      )
											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''', '''next_sentence_label''']      )
											self.assertEqual(find_labels(__lowercase      )    ,     ['''start_positions''', '''end_positions''']      )
											class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
															"""simple docstring"""
															pass

											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''']      )




				@require_flax
				def        __lowercase	(  self	:str      ):
											# Flax models don't have labels
											self.assertEqual(find_labels(__lowercase      )    ,     []      )
											self.assertEqual(find_labels(__lowercase      )    ,     []      )
											self.assertEqual(find_labels(__lowercase      )    ,     []      )
											class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
															"""simple docstring"""
															pass

											self.assertEqual(find_labels(__lowercase      )    ,     []      )




