






import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 				lowerCAmelCase_	(		__SCREAMING_SNAKE_CASE   ,		unittest.TestCase			):
				_UpperCamelCase					:       List[str]							       = BertTokenizer
				_UpperCamelCase					:       List[Any]							       = BertTokenizerFast
				_UpperCamelCase					:       str							       = True
				_UpperCamelCase					:       Optional[Any]							       = True
				_UpperCamelCase					:       str							       = filter_non_english


				def 						__a					(					self      ):
											super().setUp()

											_lowercase							:   int 							=   [
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											_lowercase							:   int 							=   os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['vocab_file']      )
											with open(self.vocab_file ,						'w' ,						encoding='utf-8'      ) as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )


				def 						__a					(					self ,						_lowerCAmelCase      ):
											_lowercase							:   Any 							=   'UNwant\u00E9d,running'
											_lowercase							:   Tuple 							=   'unwanted, running'
											return input_text, output_text


				def 						__a					(					self      ):
											_lowercase							:   List[Any] 							=   self.tokenizer_class(self.vocab_file      )

											_lowercase							:   Dict 							=   tokenizer.tokenize('UNwant\u00E9d,running'      )
											self.assertListEqual(lowerCAmelCase__ ,						['un', '##want', '##ed', ',', 'runn', '##ing']      )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__      ) ,						[9, 6, 7, 1_2, 1_0, 1_1]      )


				def 						__a					(					self      ):




											if not self.test_rust_tokenizer:
																		return

											_lowercase							:   Optional[int] 							=   self.get_tokenizer()
											_lowercase							:   List[Any] 							=   self.get_rust_tokenizer()

											_lowercase							:   Any 							=   'UNwant\u00E9d,running'

											_lowercase							:   int 							=   tokenizer.tokenize(lowerCAmelCase__      )
											_lowercase							:   Dict 							=   rust_tokenizer.tokenize(lowerCAmelCase__      )
											self.assertListEqual(lowerCAmelCase__ ,						lowerCAmelCase__      )

											_lowercase							:   Optional[Any] 							=   tokenizer.encode(lowerCAmelCase__ ,						add_special_tokens=lowerCAmelCase__      )
											_lowercase							:   Optional[int] 							=   rust_tokenizer.encode(lowerCAmelCase__ ,						add_special_tokens=lowerCAmelCase__      )
											self.assertListEqual(lowerCAmelCase__ ,						lowerCAmelCase__      )

											_lowercase							:   Tuple 							=   self.get_rust_tokenizer()
											_lowercase							:   Optional[Any] 							=   tokenizer.encode(lowerCAmelCase__      )
											_lowercase							:   str 							=   rust_tokenizer.encode(lowerCAmelCase__      )
											self.assertListEqual(lowerCAmelCase__ ,						lowerCAmelCase__      )

											# With lower casing
											_lowercase							:   Dict 							=   self.get_tokenizer(do_lower_case=lowerCAmelCase__      )
											_lowercase							:   Dict 							=   self.get_rust_tokenizer(do_lower_case=lowerCAmelCase__      )

											_lowercase							:   Any 							=   'UNwant\u00E9d,running'

											_lowercase							:   Tuple 							=   tokenizer.tokenize(lowerCAmelCase__      )
											_lowercase							:   List[str] 							=   rust_tokenizer.tokenize(lowerCAmelCase__      )
											self.assertListEqual(lowerCAmelCase__ ,						lowerCAmelCase__      )

											_lowercase							:   List[str] 							=   tokenizer.encode(lowerCAmelCase__ ,						add_special_tokens=lowerCAmelCase__      )
											_lowercase							:   List[Any] 							=   rust_tokenizer.encode(lowerCAmelCase__ ,						add_special_tokens=lowerCAmelCase__      )
											self.assertListEqual(lowerCAmelCase__ ,						lowerCAmelCase__      )

											_lowercase							:   Tuple 							=   self.get_rust_tokenizer()
											_lowercase							:   Any 							=   tokenizer.encode(lowerCAmelCase__      )
											_lowercase							:   List[Any] 							=   rust_tokenizer.encode(lowerCAmelCase__      )
											self.assertListEqual(lowerCAmelCase__ ,						lowerCAmelCase__      )


				def 						__a					(					self      ):
											_lowercase							:   List[Any] 							=   BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'      ) ,						['ah', '\u535A', '\u63A8', 'zz']      )


				def 						__a					(					self      ):
											_lowercase							:   Union[str, Any] 							=   BasicTokenizer(do_lower_case=lowerCAmelCase__      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '      ) ,						['hello', '!', 'how', 'are', 'you', '?']      )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      ) ,						['hello']      )


				def 						__a					(					self      ):
											_lowercase							:   List[Any] 							=   BasicTokenizer(do_lower_case=lowerCAmelCase__ ,						strip_accents=lowerCAmelCase__      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      ) ,						['hällo', '!', 'how', 'are', 'you', '?']      )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      ) ,						['h\u00E9llo']      )


				def 						__a					(					self      ):
											_lowercase							:   Optional[int] 							=   BasicTokenizer(do_lower_case=lowerCAmelCase__ ,						strip_accents=lowerCAmelCase__      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      ) ,						['hallo', '!', 'how', 'are', 'you', '?']      )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      ) ,						['hello']      )


				def 						__a					(					self      ):
											_lowercase							:   Optional[Any] 							=   BasicTokenizer(do_lower_case=lowerCAmelCase__      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      ) ,						['hallo', '!', 'how', 'are', 'you', '?']      )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      ) ,						['hello']      )


				def 						__a					(					self      ):
											_lowercase							:   Optional[Any] 							=   BasicTokenizer(do_lower_case=lowerCAmelCase__      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '      ) ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?']      )


				def 						__a					(					self      ):
											_lowercase							:   str 							=   BasicTokenizer(do_lower_case=lowerCAmelCase__ ,						strip_accents=lowerCAmelCase__      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      ) ,						['HäLLo', '!', 'how', 'Are', 'yoU', '?']      )


				def 						__a					(					self      ):
											_lowercase							:   Tuple 							=   BasicTokenizer(do_lower_case=lowerCAmelCase__ ,						strip_accents=lowerCAmelCase__      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      ) ,						['HaLLo', '!', 'how', 'Are', 'yoU', '?']      )


				def 						__a					(					self      ):
											_lowercase							:   Any 							=   BasicTokenizer(do_lower_case=lowerCAmelCase__ ,						never_split=['[UNK]']      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'      ) ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']      )


				def 						__a					(					self      ):
											_lowercase							:   Tuple 							=   BasicTokenizer()
											_lowercase							:   Optional[Any] 							=   'a\n\'ll !!to?\'d of, can\'t.'
											_lowercase							:   Tuple 							=   ['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
											self.assertListEqual(tokenizer.tokenize(lowerCAmelCase__      ) ,						lowerCAmelCase__      )


				def 						__a					(					self      ):
											_lowercase							:   List[str] 							=   ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

											_lowercase							:   List[str] 							=   {}
											for i, token in enumerate(lowerCAmelCase__      ):
																		_lowercase							:   Optional[Any] 							=   i
											_lowercase							:   List[str] 							=   WordpieceTokenizer(vocab=lowerCAmelCase__ ,						unk_token='[UNK]'      )

											self.assertListEqual(tokenizer.tokenize(''      ) ,						[]      )

											self.assertListEqual(tokenizer.tokenize('unwanted running'      ) ,						['un', '##want', '##ed', 'runn', '##ing']      )

											self.assertListEqual(tokenizer.tokenize('unwantedX running'      ) ,						['[UNK]', 'runn', '##ing']      )


				def 						__a					(					self      ):
											self.assertTrue(_is_whitespace(' '      )      )
											self.assertTrue(_is_whitespace('\t'      )      )
											self.assertTrue(_is_whitespace('\r'      )      )
											self.assertTrue(_is_whitespace('\n'      )      )
											self.assertTrue(_is_whitespace('\u00A0'      )      )

											self.assertFalse(_is_whitespace('A'      )      )
											self.assertFalse(_is_whitespace('-'      )      )


				def 						__a					(					self      ):
											self.assertTrue(_is_control('\u0005'      )      )

											self.assertFalse(_is_control('A'      )      )
											self.assertFalse(_is_control(' '      )      )
											self.assertFalse(_is_control('\t'      )      )
											self.assertFalse(_is_control('\r'      )      )


				def 						__a					(					self      ):
											self.assertTrue(_is_punctuation('-'      )      )
											self.assertTrue(_is_punctuation('$'      )      )
											self.assertTrue(_is_punctuation('`'      )      )
											self.assertTrue(_is_punctuation('.'      )      )

											self.assertFalse(_is_punctuation('A'      )      )
											self.assertFalse(_is_punctuation(' '      )      )


				def 						__a					(					self      ):
											_lowercase							:   Optional[Any] 							=   self.get_tokenizer()
											_lowercase							:   List[Any] 							=   self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(lowerCAmelCase__      ) for t in ['Test', '\xad', 'test']] ,						[['[UNK]'], [], ['[UNK]']]      )

											self.assertListEqual(
											    [rust_tokenizer.tokenize(lowerCAmelCase__      ) for t in ['Test', '\xad', 'test']] ,						[['[UNK]'], [], ['[UNK]']]      )


				@slow
				def 						__a					(					self      ):
											_lowercase							:   str 							=   self.tokenizer_class.from_pretrained('bert-base-uncased'      )

											_lowercase							:   str 							=   tokenizer.encode('sequence builders' ,						add_special_tokens=lowerCAmelCase__      )
											_lowercase							:   Dict 							=   tokenizer.encode('multi-sequence build' ,						add_special_tokens=lowerCAmelCase__      )

											_lowercase							:   int 							=   tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__      )
											_lowercase							:   int 							=   tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__ ,						lowerCAmelCase__      )

											assert encoded_sentence == [1_0_1] + text + [1_0_2]
											assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]


				def 						__a					(					self      ):




											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
																									_lowercase							:   List[str] 							=   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__ ,						**lowerCAmelCase__      )

																									_lowercase							:   int 							=   F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																									_lowercase							:   Optional[Any] 							=   tokenizer_r.encode_plus(
																									    lowerCAmelCase__ ,						return_attention_mask=lowerCAmelCase__ ,						return_token_type_ids=lowerCAmelCase__ ,						return_offsets_mapping=lowerCAmelCase__ ,						add_special_tokens=lowerCAmelCase__ ,						)

																									_lowercase							:   List[str] 							=   tokenizer_r.do_lower_case if hasattr(lowerCAmelCase__ ,						'do_lower_case'      ) else False
																									_lowercase							:   int 							=   (
																									    [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), 'A'),
																									        ((1, 2), ','),
																									        ((3, 5), 'na'),
																									        ((5, 6), '##ï'),
																									        ((6, 8), '##ve'),
																									        ((9, 1_5), tokenizer_r.mask_token),
																									        ((1_6, 2_1), 'Allen'),
																									        ((2_1, 2_3), '##NL'),
																									        ((2_3, 2_4), '##P'),
																									        ((2_5, 3_3), 'sentence'),
																									        ((3_3, 3_4), '.'),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									    if not do_lower_case
																									    else [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), 'a'),
																									        ((1, 2), ','),
																									        ((3, 8), 'naive'),
																									        ((9, 1_5), tokenizer_r.mask_token),
																									        ((1_6, 2_1), 'allen'),
																									        ((2_1, 2_3), '##nl'),
																									        ((2_3, 2_4), '##p'),
																									        ((2_5, 3_3), 'sentence'),
																									        ((3_3, 3_4), '.'),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									)

																									self.assertEqual(
																									    [e[1] for e in expected_results] ,						tokenizer_r.convert_ids_to_tokens(tokens['input_ids']      )      )
																									self.assertEqual([e[0] for e in expected_results] ,						tokens['offset_mapping']      )


				def 						__a					(					self      ):
											_lowercase							:   Union[str, Any] 							=   ['的', '人', '有']
											_lowercase							:   Union[str, Any] 							=   ''.join(lowerCAmelCase__      )
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
																									_lowercase							:   str 							=   True
																									_lowercase							:   List[Any] 							=   self.tokenizer_class.from_pretrained(lowerCAmelCase__ ,						**lowerCAmelCase__      )
																									_lowercase							:   Optional[int] 							=   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__ ,						**lowerCAmelCase__      )

																									_lowercase							:   Any 							=   tokenizer_p.encode(lowerCAmelCase__ ,						add_special_tokens=lowerCAmelCase__      )
																									_lowercase							:   int 							=   tokenizer_r.encode(lowerCAmelCase__ ,						add_special_tokens=lowerCAmelCase__      )

																									_lowercase							:   Optional[int] 							=   tokenizer_r.convert_ids_to_tokens(lowerCAmelCase__      )
																									_lowercase							:   List[Any] 							=   tokenizer_p.convert_ids_to_tokens(lowerCAmelCase__      )

																									# it is expected that each Chinese character is not preceded by "##"
																									self.assertListEqual(lowerCAmelCase__ ,						lowerCAmelCase__      )
																									self.assertListEqual(lowerCAmelCase__ ,						lowerCAmelCase__      )

																									_lowercase							:   Any 							=   False
																									_lowercase							:   Dict 							=   self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__ ,						**lowerCAmelCase__      )
																									_lowercase							:   Any 							=   self.tokenizer_class.from_pretrained(lowerCAmelCase__ ,						**lowerCAmelCase__      )

																									_lowercase							:   List[Any] 							=   tokenizer_r.encode(lowerCAmelCase__ ,						add_special_tokens=lowerCAmelCase__      )
																									_lowercase							:   int 							=   tokenizer_p.encode(lowerCAmelCase__ ,						add_special_tokens=lowerCAmelCase__      )

																									_lowercase							:   Any 							=   tokenizer_r.convert_ids_to_tokens(lowerCAmelCase__      )
																									_lowercase							:   Any 							=   tokenizer_p.convert_ids_to_tokens(lowerCAmelCase__      )

																									# it is expected that only the first Chinese character is not preceded by "##".
																									_lowercase							:   Optional[int] 							=   [
																									    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCAmelCase__      )
																									]
																									self.assertListEqual(lowerCAmelCase__ ,						lowerCAmelCase__      )
																									self.assertListEqual(lowerCAmelCase__ ,						lowerCAmelCase__      )








from ...processing_utils import ProcessorMixin







class 	__lowercase    (__SCREAMING_SNAKE_CASE  ):





							"""simple docstring"""







							_UpperCAmelCase      =						"""WhisperFeatureExtractor"""
							_UpperCAmelCase      =						"""WhisperTokenizer"""






							def __init__(      self			,						lowerCAmelCase__			,						lowerCAmelCase__       ):





										"""simple docstring"""




										super().__init__(lowerCAmelCase__			,						lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					self.feature_extractor
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					False






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__=None			,						lowerCAmelCase__=None			,						lowerCAmelCase__=True       ):





										"""simple docstring"""




										return self.tokenizer.get_decoder_prompt_ids(task=lowerCAmelCase__			,						language=lowerCAmelCase__			,						no_timestamps=lowerCAmelCase__       )






							def __call__(      self			,						*lowerCAmelCase__			,						**lowerCAmelCase__       ):





										"""simple docstring"""




										if self._in_target_context_manager:
													return self.current_processor(*lowerCAmelCase__			,						**lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     Tuple					=					kwargs.pop('audio'			,						lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     str					=					kwargs.pop('sampling_rate'			,						lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					kwargs.pop('text'			,						lowerCAmelCase__       )
										if len(lowerCAmelCase__       ) > 0:
													SCREAMING_SNAKE_CASE_		:     List[Any]					=					args[0]
													SCREAMING_SNAKE_CASE_		:     List[Any]					=					args[1:]

										if audio is None and text is None:
													raise ValueError('You need to specify either an `audio` or `text` input to process.'       )

										if audio is not None:
													SCREAMING_SNAKE_CASE_		:     List[Any]					=					self.feature_extractor(lowerCAmelCase__			,						*lowerCAmelCase__			,						sampling_rate=lowerCAmelCase__			,						**lowerCAmelCase__       )
										if text is not None:
													SCREAMING_SNAKE_CASE_		:     Optional[int]					=					self.tokenizer(lowerCAmelCase__			,						**lowerCAmelCase__       )

										if text is None:
													return inputs

										elif audio is None:
													return encodings
										else:
													SCREAMING_SNAKE_CASE_		:     Optional[int]					=					encodings['input_ids']
													return inputs






							def 					UpperCamelCase__     (      self			,						*lowerCAmelCase__			,						**lowerCAmelCase__       ):





										"""simple docstring"""




										return self.tokenizer.batch_decode(*lowerCAmelCase__			,						**lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self			,						*lowerCAmelCase__			,						**lowerCAmelCase__       ):





										"""simple docstring"""




										return self.tokenizer.decode(*lowerCAmelCase__			,						**lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__			,						lowerCAmelCase__="np"       ):





										"""simple docstring"""




										return self.tokenizer.get_prompt_ids(lowerCAmelCase__			,						return_tensors=lowerCAmelCase__       )





