




import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin







class 				lowerCAmelCase_       (					lowerCamelCase_						,     unittest.TestCase							):
				__a     :       List[Any]						=     XLMTokenizer
				__a     :       int						=     False






				def 		snake_case	(			self ):
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							SCREAMING_SNAKE_CASE_   : Any					  =      [
							    'l',
							    'o',
							    'w',
							    'e',
							    'r',
							    's',
							    't',
							    'i',
							    'd',
							    'n',
							    'w</w>',
							    'r</w>',
							    't</w>',
							    'lo',
							    'low',
							    'er</w>',
							    'low</w>',
							    'lowest</w>',
							    'newer</w>',
							    'wider</w>',
							    '<unk>',
							]
							SCREAMING_SNAKE_CASE_   : str					  =      dict(zip(snake_case__      ,range(len(snake_case__ ) ) ) )
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      ['l o 123', 'lo w 1456', 'e r</w> 1789', '']

							SCREAMING_SNAKE_CASE_   : int					  =      os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['vocab_file'] )
							SCREAMING_SNAKE_CASE_   : str					  =      os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['merges_file'] )
							with open(self.vocab_file      ,'w' ) as fp:
										fp.write(json.dumps(snake_case__ ) )
							with open(self.merges_file      ,'w' ) as fp:
										fp.write('\n'.join(snake_case__ ) )






				def 		snake_case	(			self      ,snake_case__ ):
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      'lower newer'
							SCREAMING_SNAKE_CASE_   : Dict					  =      'lower newer'
							return input_text, output_text






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      XLMTokenizer(self.vocab_file      ,self.merges_file )

							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      'lower'
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      ['low', 'er</w>']
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer.tokenize(snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )

							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      tokens + ['<unk>']
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      [14, 15, 20]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__ )      ,snake_case__ )






				@slow
				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : int					  =      XLMTokenizer.from_pretrained('xlm-mlm-en-2048' )

							SCREAMING_SNAKE_CASE_   : List[Any]					  =      tokenizer.encode('sequence builders'      ,add_special_tokens=snake_case__ )
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      tokenizer.encode('multi-sequence build'      ,add_special_tokens=snake_case__ )

							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer.build_inputs_with_special_tokens(snake_case__ )
							SCREAMING_SNAKE_CASE_   : List[str]					  =      tokenizer.build_inputs_with_special_tokens(snake_case__      ,snake_case__ )

							assert encoded_sentence == [0] + text + [1]
							assert encoded_pair == [0] + text + [1] + text_a + [1]









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase__ :   Optional[Any]						=							{
    '''configuration_whisper''': ['''WHISPER_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''WhisperConfig''', '''WhisperOnnxConfig'''],
    '''feature_extraction_whisper''': ['''WhisperFeatureExtractor'''],
    '''processing_whisper''': ['''WhisperProcessor'''],
    '''tokenization_whisper''': ['''WhisperTokenizer'''],
}

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   str						=							['''WhisperTokenizerFast''']

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   List[Any]						=							[
						    '''WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''WhisperForConditionalGeneration''',
						    '''WhisperModel''',
						    '''WhisperPreTrainedModel''',
						    '''WhisperForAudioClassification''',
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   Tuple						=							[
						    '''TF_WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''TFWhisperForConditionalGeneration''',
						    '''TFWhisperModel''',
						    '''TFWhisperPreTrainedModel''',
						]

try:
						if not is_flax_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   List[str]						=							[
						    '''FlaxWhisperForConditionalGeneration''',
						    '''FlaxWhisperModel''',
						    '''FlaxWhisperPreTrainedModel''',
						    '''FlaxWhisperForAudioClassification''',
						]


if TYPE_CHECKING:
						from .configuration_whisper import WHISPER_PRETRAINED_CONFIG_ARCHIVE_MAP, WhisperConfig, WhisperOnnxConfig
						from .feature_extraction_whisper import WhisperFeatureExtractor
						from .processing_whisper import WhisperProcessor
						from .tokenization_whisper import WhisperTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_whisper_fast import WhisperTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_whisper import (
												    WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST,
												    WhisperForAudioClassification,
												    WhisperForConditionalGeneration,
												    WhisperModel,
												    WhisperPreTrainedModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_whisper import (
												    TF_WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFWhisperForConditionalGeneration,
												    TFWhisperModel,
												    TFWhisperPreTrainedModel,
												)

						try:
												if not is_flax_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_flax_whisper import (
												    FlaxWhisperForAudioClassification,
												    FlaxWhisperForConditionalGeneration,
												    FlaxWhisperModel,
												    FlaxWhisperPreTrainedModel,
												)

else:
						import sys

						UpperCamelCase__ :   Optional[Any]						=							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



